Felipe97/llama-cpp-compiled
01.1k
1#ifndef LLAMA_H2#define LLAMA_H3 4#include "ggml.h"5#include "ggml-cpu.h"6#include "ggml-backend.h"7#include "ggml-opt.h"8#include "gguf.h"9 10#include <stddef.h>11#include <stdint.h>12#include <stdio.h>13#include <stdbool.h>14 15#ifdef LLAMA_SHARED16# if defined(_WIN32) && !defined(__MINGW32__)17# ifdef LLAMA_BUILD18# define LLAMA_API __declspec(dllexport)19# else20# define LLAMA_API __declspec(dllimport)21# endif22# else23# define LLAMA_API __attribute__ ((visibility ("default")))24# endif25#else26# define LLAMA_API27#endif28 29#ifdef __GNUC__30# define DEPRECATED(func, hint) func __attribute__((deprecated(hint)))31#elif defined(_MSC_VER)32# define DEPRECATED(func, hint) __declspec(deprecated(hint)) func33#else34# define DEPRECATED(func, hint) func35#endif36 37#define LLAMA_DEFAULT_SEED 0xFFFFFFFF38 39#define LLAMA_TOKEN_NULL -140 41#define LLAMA_FILE_MAGIC_GGLA 0x67676c61u // 'ggla'42#define LLAMA_FILE_MAGIC_GGSN 0x6767736eu // 'ggsn'43#define LLAMA_FILE_MAGIC_GGSQ 0x67677371u // 'ggsq'44 45#define LLAMA_SESSION_MAGIC LLAMA_FILE_MAGIC_GGSN46#define LLAMA_SESSION_VERSION 1047 48#define LLAMA_STATE_SEQ_MAGIC LLAMA_FILE_MAGIC_GGSQ49#define LLAMA_STATE_SEQ_VERSION 350 51#ifdef __cplusplus52extern "C" {53#endif54 55 //56 // C interface57 //58 // TODO: show sample usage59 //60 61 struct llama_vocab;62 struct llama_model;63 struct llama_context;64 struct llama_sampler;65 66 typedef struct llama_memory_i * llama_memory_t;67 68 typedef int32_t llama_pos;69 typedef int32_t llama_token;70 typedef int32_t llama_seq_id;71 72 enum llama_vocab_type {73 LLAMA_VOCAB_TYPE_NONE = 0, // For models without vocab74 LLAMA_VOCAB_TYPE_SPM = 1, // LLaMA tokenizer based on byte-level BPE with byte fallback75 LLAMA_VOCAB_TYPE_BPE = 2, // GPT-2 tokenizer based on byte-level BPE76 LLAMA_VOCAB_TYPE_WPM = 3, // BERT tokenizer based on WordPiece77 LLAMA_VOCAB_TYPE_UGM = 4, // T5 tokenizer based on Unigram78 LLAMA_VOCAB_TYPE_RWKV = 5, // RWKV tokenizer based on greedy tokenization79 LLAMA_VOCAB_TYPE_PLAMO2 = 6, // PLaMo-2 tokenizer based on Aho-Corasick with dynamic programming80 LLAMA_VOCAB_TYPE_TEST = 7, // Dummy tokenizer for testing: rolling hash of fixed-size chunks -> tokens, tokens -> hex81 };82 83 enum llama_rope_type {84 LLAMA_ROPE_TYPE_NONE = -1,85 LLAMA_ROPE_TYPE_NORM = 0,86 LLAMA_ROPE_TYPE_NEOX = GGML_ROPE_TYPE_NEOX,87 LLAMA_ROPE_TYPE_MROPE = GGML_ROPE_TYPE_MROPE,88 LLAMA_ROPE_TYPE_IMROPE = GGML_ROPE_TYPE_IMROPE,89 LLAMA_ROPE_TYPE_VISION = GGML_ROPE_TYPE_VISION,90 };91 92 enum llama_token_type { //TODO: remove, required until per token attributes are available from GGUF file93 LLAMA_TOKEN_TYPE_UNDEFINED = 0,94 LLAMA_TOKEN_TYPE_NORMAL = 1,95 LLAMA_TOKEN_TYPE_UNKNOWN = 2,96 LLAMA_TOKEN_TYPE_CONTROL = 3,97 LLAMA_TOKEN_TYPE_USER_DEFINED = 4,98 LLAMA_TOKEN_TYPE_UNUSED = 5,99 LLAMA_TOKEN_TYPE_BYTE = 6,100 };101 102 enum llama_token_attr {103 LLAMA_TOKEN_ATTR_UNDEFINED = 0,104 LLAMA_TOKEN_ATTR_UNKNOWN = 1 << 0,105 LLAMA_TOKEN_ATTR_UNUSED = 1 << 1,106 LLAMA_TOKEN_ATTR_NORMAL = 1 << 2,107 LLAMA_TOKEN_ATTR_CONTROL = 1 << 3, // SPECIAL?108 LLAMA_TOKEN_ATTR_USER_DEFINED = 1 << 4,109 LLAMA_TOKEN_ATTR_BYTE = 1 << 5,110 LLAMA_TOKEN_ATTR_NORMALIZED = 1 << 6,111 LLAMA_TOKEN_ATTR_LSTRIP = 1 << 7,112 LLAMA_TOKEN_ATTR_RSTRIP = 1 << 8,113 LLAMA_TOKEN_ATTR_SINGLE_WORD = 1 << 9,114 };115 116 // model file types117 enum llama_ftype {118 LLAMA_FTYPE_ALL_F32 = 0,119 LLAMA_FTYPE_MOSTLY_F16 = 1, // except 1d tensors120 LLAMA_FTYPE_MOSTLY_Q4_0 = 2, // except 1d tensors121 LLAMA_FTYPE_MOSTLY_Q4_1 = 3, // except 1d tensors122 // LLAMA_FTYPE_MOSTLY_Q4_1_SOME_F16 = 4, // tok_embeddings.weight and output.weight are F16123 // LLAMA_FTYPE_MOSTLY_Q4_2 = 5, // support has been removed124 // LLAMA_FTYPE_MOSTLY_Q4_3 = 6, // support has been removed125 LLAMA_FTYPE_MOSTLY_Q8_0 = 7, // except 1d tensors126 LLAMA_FTYPE_MOSTLY_Q5_0 = 8, // except 1d tensors127 LLAMA_FTYPE_MOSTLY_Q5_1 = 9, // except 1d tensors128 LLAMA_FTYPE_MOSTLY_Q2_K = 10, // except 1d tensors129 LLAMA_FTYPE_MOSTLY_Q3_K_S = 11, // except 1d tensors130 LLAMA_FTYPE_MOSTLY_Q3_K_M = 12, // except 1d tensors131 LLAMA_FTYPE_MOSTLY_Q3_K_L = 13, // except 1d tensors132 LLAMA_FTYPE_MOSTLY_Q4_K_S = 14, // except 1d tensors133 LLAMA_FTYPE_MOSTLY_Q4_K_M = 15, // except 1d tensors134 LLAMA_FTYPE_MOSTLY_Q5_K_S = 16, // except 1d tensors135 LLAMA_FTYPE_MOSTLY_Q5_K_M = 17, // except 1d tensors136 LLAMA_FTYPE_MOSTLY_Q6_K = 18, // except 1d tensors137 LLAMA_FTYPE_MOSTLY_IQ2_XXS = 19, // except 1d tensors138 LLAMA_FTYPE_MOSTLY_IQ2_XS = 20, // except 1d tensors139 LLAMA_FTYPE_MOSTLY_Q2_K_S = 21, // except 1d tensors140 LLAMA_FTYPE_MOSTLY_IQ3_XS = 22, // except 1d tensors141 LLAMA_FTYPE_MOSTLY_IQ3_XXS = 23, // except 1d tensors142 LLAMA_FTYPE_MOSTLY_IQ1_S = 24, // except 1d tensors143 LLAMA_FTYPE_MOSTLY_IQ4_NL = 25, // except 1d tensors144 LLAMA_FTYPE_MOSTLY_IQ3_S = 26, // except 1d tensors145 LLAMA_FTYPE_MOSTLY_IQ3_M = 27, // except 1d tensors146 LLAMA_FTYPE_MOSTLY_IQ2_S = 28, // except 1d tensors147 LLAMA_FTYPE_MOSTLY_IQ2_M = 29, // except 1d tensors148 LLAMA_FTYPE_MOSTLY_IQ4_XS = 30, // except 1d tensors149 LLAMA_FTYPE_MOSTLY_IQ1_M = 31, // except 1d tensors150 LLAMA_FTYPE_MOSTLY_BF16 = 32, // except 1d tensors151 //LLAMA_FTYPE_MOSTLY_Q4_0_4_4 = 33, // removed from gguf files, use Q4_0 and runtime repack152 //LLAMA_FTYPE_MOSTLY_Q4_0_4_8 = 34, // removed from gguf files, use Q4_0 and runtime repack153 //LLAMA_FTYPE_MOSTLY_Q4_0_8_8 = 35, // removed from gguf files, use Q4_0 and runtime repack154 LLAMA_FTYPE_MOSTLY_TQ1_0 = 36, // except 1d tensors155 LLAMA_FTYPE_MOSTLY_TQ2_0 = 37, // except 1d tensors156 LLAMA_FTYPE_MOSTLY_MXFP4_MOE = 38, // except 1d tensors157 LLAMA_FTYPE_MOSTLY_NVFP4 = 39, // except 1d tensors158 LLAMA_FTYPE_MOSTLY_Q1_0 = 40, // except 1d tensors159 LLAMA_FTYPE_MOSTLY_Q2_0 = 41, // except 1d tensors160 161 LLAMA_FTYPE_GUESSED = 1024, // not specified in the model file162 };163 164 // Get the model file type (quantization) as a string, e.g. "Q8_0" or "Q4_K - Medium"165 LLAMA_API const char * llama_ftype_name(enum llama_ftype ftype);166 167 enum llama_rope_scaling_type {168 LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED = -1,169 LLAMA_ROPE_SCALING_TYPE_NONE = 0,170 LLAMA_ROPE_SCALING_TYPE_LINEAR = 1,171 LLAMA_ROPE_SCALING_TYPE_YARN = 2,172 LLAMA_ROPE_SCALING_TYPE_LONGROPE = 3,173 LLAMA_ROPE_SCALING_TYPE_MAX_VALUE = LLAMA_ROPE_SCALING_TYPE_LONGROPE,174 };175 176 enum llama_pooling_type {177 LLAMA_POOLING_TYPE_UNSPECIFIED = -1,178 LLAMA_POOLING_TYPE_NONE = 0,179 LLAMA_POOLING_TYPE_MEAN = 1,180 LLAMA_POOLING_TYPE_CLS = 2,181 LLAMA_POOLING_TYPE_LAST = 3,182 LLAMA_POOLING_TYPE_RANK = 4, // used by reranking models to attach the classification head to the graph183 };184 185 enum llama_attention_type {186 LLAMA_ATTENTION_TYPE_UNSPECIFIED = -1,187 LLAMA_ATTENTION_TYPE_CAUSAL = 0,188 LLAMA_ATTENTION_TYPE_NON_CAUSAL = 1,189 };190 191 enum llama_flash_attn_type {192 LLAMA_FLASH_ATTN_TYPE_AUTO = -1,193 LLAMA_FLASH_ATTN_TYPE_DISABLED = 0,194 LLAMA_FLASH_ATTN_TYPE_ENABLED = 1,195 };196 197 LLAMA_API const char * llama_flash_attn_type_name(enum llama_flash_attn_type flash_attn_type);198 199 enum llama_split_mode {200 LLAMA_SPLIT_MODE_NONE = 0, // single GPU201 LLAMA_SPLIT_MODE_LAYER = 1, // split layers and KV across GPUs202 LLAMA_SPLIT_MODE_ROW = 2, // split layers and KV across GPUs, use tensor parallelism if supported203 LLAMA_SPLIT_MODE_TENSOR = 3,204 };205 206 enum llama_load_mode {207 LLAMA_LOAD_MODE_AUTO = -1, // auto-detect based on device capabilities208 LLAMA_LOAD_MODE_NONE = 0, // no special loading mode209 LLAMA_LOAD_MODE_MMAP = 1, // memory map the model210 LLAMA_LOAD_MODE_MLOCK = 2, // force system to keep model in RAM rather than swapping or compressing211 LLAMA_LOAD_MODE_MMAP_MLOCK = 3, // mmap + force system to keep model in RAM rather than swapping or compressing212 LLAMA_LOAD_MODE_DIRECT_IO = 4, // use direct I/O if available213 };214 215 LLAMA_API const char * llama_load_mode_name(enum llama_load_mode load_mode);216 LLAMA_API enum llama_load_mode llama_load_mode_from_str(const char * str);217 218 enum llama_lazy_mode {219 LLAMA_LAZY_MODE_OFF = 0, // always read the whole tensor up front220 LLAMA_LAZY_MODE_AUTO = 1, // lazy only for marked tensors larger than 4 GiB (requires mmap)221 LLAMA_LAZY_MODE_ON = 2, // read the rows of tensors marked by the arch on demand (requires mmap)222 };223 224 enum llama_context_type {225 LLAMA_CONTEXT_TYPE_DEFAULT = 0,226 LLAMA_CONTEXT_TYPE_MTP = 1,227 };228 229 // TODO: simplify (https://github.com/ggml-org/llama.cpp/pull/9294#pullrequestreview-2286561979)230 typedef struct llama_token_data {231 llama_token id; // token id232 float logit; // log-odds of the token233 float p; // probability of the token234 } llama_token_data;235 236 typedef struct llama_token_data_array {237 // TODO: consider SoA238 // NOTE: this pointer can be modified by the samplers239 llama_token_data * data;240 size_t size;241 int64_t selected; // this is the index in the data array (i.e. not the token id)242 bool sorted; // note: do not assume the data is sorted - always check this flag243 } llama_token_data_array;244 245 typedef bool (*llama_progress_callback)(float progress, void * user_data);246 247 // Input data for llama_encode/llama_decode248 // A llama_batch object can contain input about one or many sequences249 // The provided arrays (i.e. token, embd, pos, etc.) must have size of n_tokens250 //251 // - token : the token ids of the input (used when embd is NULL)252 // - embd : token embeddings (i.e. float vector of size n_embd) (used when token is NULL)253 // - pos : the positions of the respective token in the sequence254 // (if set to NULL, the token position will be tracked automatically by llama_encode/llama_decode)255 // - seq_id : the sequence to which the respective token belongs256 // (if set to NULL, the sequence ID will be assumed to be 0)257 // - logits : if zero, the logits (and/or the embeddings) for the respective token will not be output258 // (if set to NULL:259 // - if embeddings: all tokens are output260 // - if not: only the last token is output261 // )262 //263 typedef struct llama_batch {264 int32_t n_tokens;265 266 llama_token * token;267 float * embd;268 llama_pos * pos;269 int32_t * n_seq_id;270 llama_seq_id ** seq_id;271 int8_t * logits; // TODO: rename this to "output"272 } llama_batch;273 274 enum llama_model_kv_override_type {275 LLAMA_KV_OVERRIDE_TYPE_INT,276 LLAMA_KV_OVERRIDE_TYPE_FLOAT,277 LLAMA_KV_OVERRIDE_TYPE_BOOL,278 LLAMA_KV_OVERRIDE_TYPE_STR,279 };280 281 enum llama_model_meta_key {282 LLAMA_MODEL_META_KEY_SAMPLING_SEQUENCE,283 LLAMA_MODEL_META_KEY_SAMPLING_TOP_K,284 LLAMA_MODEL_META_KEY_SAMPLING_TOP_P,285 LLAMA_MODEL_META_KEY_SAMPLING_MIN_P,286 LLAMA_MODEL_META_KEY_SAMPLING_XTC_PROBABILITY,287 LLAMA_MODEL_META_KEY_SAMPLING_XTC_THRESHOLD,288 LLAMA_MODEL_META_KEY_SAMPLING_TEMP,289 LLAMA_MODEL_META_KEY_SAMPLING_PENALTY_LAST_N,290 LLAMA_MODEL_META_KEY_SAMPLING_PENALTY_REPEAT,291 LLAMA_MODEL_META_KEY_SAMPLING_MIROSTAT,292 LLAMA_MODEL_META_KEY_SAMPLING_MIROSTAT_TAU,293 LLAMA_MODEL_META_KEY_SAMPLING_MIROSTAT_ETA,294 };295 296 struct llama_model_kv_override {297 enum llama_model_kv_override_type tag;298 299 char key[128];300 301 union {302 int64_t val_i64;303 double val_f64;304 bool val_bool;305 char val_str[128];306 };307 };308 309 struct llama_model_tensor_buft_override {310 const char * pattern;311 ggml_backend_buffer_type_t buft;312 };313 314 struct llama_model_params {315 // NULL-terminated list of devices to use for offloading (if NULL, all available devices are used)316 ggml_backend_dev_t * devices;317 318 // NULL-terminated list of buffer types to use for tensors that match a pattern319 const struct llama_model_tensor_buft_override * tensor_buft_overrides;320 321 int32_t n_gpu_layers; // number of layers to store in VRAM, a negative value means all layers322 enum llama_split_mode split_mode; // how to split the model across multiple GPUs323 enum llama_load_mode load_mode; // how to load the model324 325 enum llama_lazy_mode lazy_mode; // on-demand reading of tensors marked by the arch326 327 // the GPU that is used for the entire model when split_mode is LLAMA_SPLIT_MODE_NONE328 int32_t main_gpu;329 330 // proportion of the model (layers or rows) to offload to each GPU, size: llama_max_devices()331 const float * tensor_split;332 333 // Called with a progress value between 0.0 and 1.0. Pass NULL to disable.334 // If the provided progress_callback returns true, model loading continues.335 // If it returns false, model loading is immediately aborted.336 llama_progress_callback progress_callback;337 338 // context pointer passed to the progress callback339 void * progress_callback_user_data;340 341 // override key-value pairs of the model meta data342 const struct llama_model_kv_override * kv_overrides;343 344 // Keep the booleans together to avoid misalignment during copy-by-value.345 bool vocab_only; // only load the vocabulary, no weights346 bool check_tensors; // validate model tensor data347 bool use_extra_bufts; // use extra buffer types (used for weight repacking)348 bool no_host; // bypass host buffer allowing extra buffers to be used349 bool no_alloc; // only load metadata and simulate memory allocations350 bool load_mtp; // whether to load MTP layers351 };352 353 struct llama_sampler_seq_config {354 llama_seq_id seq_id;355 struct llama_sampler * sampler;356 };357 358 // NOTE: changing the default values of parameters marked as [EXPERIMENTAL] may cause crashes or incorrect results in certain configurations359 // https://github.com/ggml-org/llama.cpp/pull/7544360 struct llama_context_params {361 uint32_t n_ctx; // text context, 0 = from model362 uint32_t n_batch; // logical maximum batch size that can be submitted to llama_decode363 uint32_t n_ubatch; // physical maximum batch size364 uint32_t n_seq_max; // max number of sequences (i.e. distinct states for recurrent models)365 uint32_t n_rs_seq; // number of recurrent-state snapshots per seq for rollback (0 = no rollback) [EXPERIMENTAL]366 uint32_t n_outputs_max; // max outputs in a ubatch (0 = n_batch)367 uint32_t n_outputs_max_per_seq; // max outputs per sequence (0 = n_outputs_max)368 int32_t n_threads; // number of threads to use for generation369 int32_t n_threads_batch; // number of threads to use for batch processing370 371 enum llama_context_type ctx_type; // set the context type (e.g. MTP)372 enum llama_rope_scaling_type rope_scaling_type; // RoPE scaling type, from `enum llama_rope_scaling_type`373 enum llama_pooling_type pooling_type; // whether to pool (sum) embedding results by sequence id374 enum llama_attention_type attention_type; // attention type to use for embeddings375 enum llama_flash_attn_type flash_attn_type; // when to enable Flash Attention376 377 // ref: https://github.com/ggml-org/llama.cpp/pull/2054378 float rope_freq_base; // RoPE base frequency, 0 = from model379 float rope_freq_scale; // RoPE frequency scaling factor, 0 = from model380 float yarn_ext_factor; // YaRN extrapolation mix factor, negative = from model381 float yarn_attn_factor; // YaRN magnitude scaling factor382 float yarn_beta_fast; // YaRN low correction dim383 float yarn_beta_slow; // YaRN high correction dim384 uint32_t yarn_orig_ctx; // YaRN original context size385 float defrag_thold; // [DEPRECATED] defragment the KV cache if holes/size > thold, <= 0 disabled (default)386 387 ggml_backend_sched_eval_callback cb_eval;388 void * cb_eval_user_data;389 390 enum ggml_type type_k; // data type for K cache [EXPERIMENTAL]391 enum ggml_type type_v; // data type for V cache [EXPERIMENTAL]392 393 // Abort callback394 // if it returns true, execution of llama_decode() will be aborted395 // currently works only with CPU execution396 ggml_abort_callback abort_callback;397 void * abort_callback_data;398 399 // Keep the booleans together and at the end of the struct to avoid misalignment during copy-by-value.400 bool embeddings; // if true, extract embeddings (together with logits)401 bool offload_kqv; // offload the KQV ops (including the KV cache) to GPU402 bool no_perf; // measure performance timings403 bool op_offload; // offload host tensor operations to device404 bool swa_full; // use full-size SWA cache (https://github.com/ggml-org/llama.cpp/pull/13194#issuecomment-2868343055)405 // NOTE: setting to false when n_seq_max > 1 can cause bad performance in some cases406 // ref: https://github.com/ggml-org/llama.cpp/pull/13845#issuecomment-2924800573407 bool kv_unified; // use a unified buffer across the input sequences when computing the attention408 // try to disable when n_seq_max > 1 for improved performance when the sequences do not share a large prefix409 // ref: https://github.com/ggml-org/llama.cpp/pull/14363410 411 // [EXPERIMENTAL]412 // backend sampler chain configuration (make sure the caller keeps the sampler chains alive)413 // note: the samplers must be sampler chains (i.e. use llama_sampler_chain_init)414 struct llama_sampler_seq_config * samplers;415 size_t n_samplers;416 417 // a source/target/parent context418 // can be utilized in various ways, for example by sharing results or llama_memory between 2 contexts419 struct llama_context * ctx_other;420 };421 422 struct llama_model_tensor_override {423 const char * pattern;424 enum ggml_type type;425 };426 427 struct llama_model_imatrix_data {428 const char * name;429 const float * data;430 size_t size;431 };432 433 // model quantization parameters434 typedef struct llama_model_quantize_params {435 int32_t nthread; // number of threads to use for quantizing, if <=0 will use std::thread::hardware_concurrency()436 enum llama_ftype ftype; // quantize to this llama_ftype437 enum ggml_type output_tensor_type; // output tensor type438 enum ggml_type token_embedding_type; // token embeddings tensor type439 bool allow_requantize; // allow quantizing non-f32/f16 tensors440 bool quantize_output_tensor; // quantize output.weight441 bool only_copy; // only copy tensors - ftype, allow_requantize and quantize_output_tensor are ignored442 bool pure; // quantize all tensors to the default type443 bool keep_split; // quantize to the same number of shards444 bool dry_run; // calculate and show the final quantization size without performing quantization445 const struct llama_model_imatrix_data * imatrix; // pointer to importance matrix data446 const struct llama_model_kv_override * kv_overrides; // pointer to kv overrides447 const struct llama_model_tensor_override * tt_overrides; // pointer to tensor overrides448 const int32_t * prune_layers; // pointer to layer indices to prune449 size_t max_buf_size; // max bytes of tensor rows kept in memory at once, 0 = default (8 GiB)450 } llama_model_quantize_params;451 452 typedef struct llama_logit_bias {453 llama_token token;454 float bias;455 } llama_logit_bias;456 457 typedef struct llama_sampler_chain_params {458 bool no_perf; // whether to measure performance timings459 } llama_sampler_chain_params;460 461 // used in chat template462 typedef struct llama_chat_message {463 const char * role;464 const char * content;465 } llama_chat_message;466 467 // lora adapter468 struct llama_adapter_lora;469 470 LLAMA_API const char * llama_version(void);471 472 // Helpers for getting default parameters473 // TODO: update API to start accepting pointers to params structs (https://github.com/ggml-org/llama.cpp/discussions/9172)474 LLAMA_API struct llama_model_params llama_model_default_params(void);475 LLAMA_API struct llama_context_params llama_context_default_params(void);476 LLAMA_API struct llama_sampler_chain_params llama_sampler_chain_default_params(void);477 LLAMA_API struct llama_model_quantize_params llama_model_quantize_default_params(void);478 479 // Initialize the llama + ggml backend480 // If numa is true, use NUMA optimizations481 // Call once at the start of the program482 LLAMA_API void llama_backend_init(void);483 484 // Call once at the end of the program - currently only used for MPI485 LLAMA_API void llama_backend_free(void);486 487 //optional:488 LLAMA_API void llama_numa_init(enum ggml_numa_strategy numa);489 490 // Optional: an auto threadpool gets created in ggml if not passed explicitly491 LLAMA_API void llama_attach_threadpool(492 struct llama_context * ctx,493 ggml_threadpool_t threadpool,494 ggml_threadpool_t threadpool_batch);495 496 LLAMA_API void llama_detach_threadpool(struct llama_context * ctx);497 498 typedef void (*llama_model_set_tensor_data_t)(struct ggml_tensor * tensor, void * userdata);499 500 // Create a new model from GGUF metadata as well as a function to set the tensor data501 // - tensors are created as GGML_TYPE_F32 by default,502 // override by adding a tensor with the same name but a different name to the context503 LLAMA_API struct llama_model * llama_model_init_from_user(504 struct gguf_context * metadata,505 llama_model_set_tensor_data_t set_tensor_data, // function to initialize tensor data with506 void * set_tensor_data_ud, // userdata for function507 struct llama_model_params params);508 509 DEPRECATED(LLAMA_API struct llama_model * llama_load_model_from_file(510 const char * path_model,511 struct llama_model_params params),512 "use llama_model_load_from_file instead");513 514 // Load a model from a file515 // If the file is split into multiple parts, the file name must follow this pattern: <name>-%05d-of-%05d.gguf516 // If the split file name does not follow this pattern, use llama_model_load_from_splits517 LLAMA_API struct llama_model * llama_model_load_from_file(518 const char * path_model,519 struct llama_model_params params);520 521 // Load a model from an open FILE pointer522 // The GGUF is read from the current position, so it can be embedded in a larger file523 // mmap needs the GGUF data section at a file offset to be aligned to the CPU tensor alignment (32 bytes)524 LLAMA_API struct llama_model * llama_model_load_from_file_ptr(525 FILE * file,526 struct llama_model_params params);527 528 // Load a model from multiple splits (support custom naming scheme)529 // The paths must be in the correct order530 LLAMA_API struct llama_model * llama_model_load_from_splits(531 const char ** paths,532 size_t n_paths,533 struct llama_model_params params);534 535 LLAMA_API void llama_model_save_to_file(536 const struct llama_model * model,537 const char * path_model);538 539 DEPRECATED(LLAMA_API void llama_free_model(struct llama_model * model),540 "use llama_model_free instead");541 542 LLAMA_API void llama_model_free(struct llama_model * model);543 544 LLAMA_API struct llama_context * llama_init_from_model(545 struct llama_model * model,546 struct llama_context_params params);547 548 DEPRECATED(LLAMA_API struct llama_context * llama_new_context_with_model(549 struct llama_model * model,550 struct llama_context_params params),551 "use llama_init_from_model instead");552 553 // Frees all allocated memory554 LLAMA_API void llama_free(struct llama_context * ctx);555 556 LLAMA_API int64_t llama_time_us(void);557 558 LLAMA_API size_t llama_max_devices(void);559 LLAMA_API size_t llama_max_parallel_sequences(void);560 LLAMA_API size_t llama_max_tensor_buft_overrides(void);561 562 LLAMA_API bool llama_supports_mmap (void);563 LLAMA_API bool llama_supports_mlock (void);564 LLAMA_API bool llama_supports_gpu_offload(void);565 LLAMA_API bool llama_supports_rpc (void);566 567 // NOTE: After creating a llama_context, it is recommended to query the actual values using these functions568 // In some cases the requested values via llama_context_params may differ from the actual values used by the context569 // ref: https://github.com/ggml-org/llama.cpp/pull/17046#discussion_r2503085732570 LLAMA_API uint32_t llama_n_ctx (const struct llama_context * ctx);571 LLAMA_API uint32_t llama_n_ctx_seq (const struct llama_context * ctx);572 LLAMA_API uint32_t llama_n_batch (const struct llama_context * ctx);573 LLAMA_API uint32_t llama_n_ubatch (const struct llama_context * ctx);574 LLAMA_API uint32_t llama_n_seq_max (const struct llama_context * ctx);575 LLAMA_API uint32_t llama_n_rs_seq (const struct llama_context * ctx);576 577 DEPRECATED(LLAMA_API int32_t llama_n_ctx_train(const struct llama_model * model), "use llama_model_n_ctx_train instead");578 DEPRECATED(LLAMA_API int32_t llama_n_embd (const struct llama_model * model), "use llama_model_n_embd instead");579 DEPRECATED(LLAMA_API int32_t llama_n_layer (const struct llama_model * model), "use llama_model_n_layer instead");580 DEPRECATED(LLAMA_API int32_t llama_n_head (const struct llama_model * model), "use llama_model_n_head instead");581 582 DEPRECATED(LLAMA_API int32_t llama_n_vocab (const struct llama_vocab * vocab), "use llama_vocab_n_tokens instead");583 584 LLAMA_API const struct llama_model * llama_get_model (const struct llama_context * ctx);585 LLAMA_API llama_memory_t llama_get_memory (const struct llama_context * ctx);586 LLAMA_API enum llama_pooling_type llama_pooling_type(const struct llama_context * ctx); // TODO: rename to llama_get_pooling_type587 588 LLAMA_API const struct llama_vocab * llama_model_get_vocab(const struct llama_model * model);589 LLAMA_API enum llama_rope_type llama_model_rope_type(const struct llama_model * model);590 591 LLAMA_API int32_t llama_model_n_ctx_train (const struct llama_model * model);592 LLAMA_API int32_t llama_model_n_embd (const struct llama_model * model);593 LLAMA_API int32_t llama_model_n_embd_inp (const struct llama_model * model);594 LLAMA_API int32_t llama_model_n_embd_out (const struct llama_model * model);595 LLAMA_API int32_t llama_model_n_layer (const struct llama_model * model);596 LLAMA_API int32_t llama_model_n_layer_nextn(const struct llama_model * model);597 LLAMA_API int32_t llama_model_n_head (const struct llama_model * model);598 LLAMA_API int32_t llama_model_n_head_kv (const struct llama_model * model);599 LLAMA_API int32_t llama_model_n_swa (const struct llama_model * model);600 601 // Get the model's RoPE frequency scaling factor602 LLAMA_API float llama_model_rope_freq_scale_train(const struct llama_model * model);603 604 // Returns the number of classifier outputs (only valid for classifier models)605 // Undefined behavior for non-classifier models606 LLAMA_API uint32_t llama_model_n_cls_out(const struct llama_model * model);607 608 // Returns label of classifier output by index (<n_cls_out). Returns nullptr if no label provided609 LLAMA_API const char * llama_model_cls_label(const struct llama_model * model, uint32_t i);610 611 LLAMA_API enum llama_vocab_type llama_vocab_type(const struct llama_vocab * vocab);612 613 LLAMA_API int32_t llama_vocab_n_tokens(const struct llama_vocab * vocab);614 615 // Functions to access the model's GGUF metadata scalar values616 // - The functions return the length of the string on success, or -1 on failure617 // - The output string is always null-terminated and cleared on failure618 // - When retrieving a string, an extra byte must be allocated to account for the null terminator619 // - GGUF array values are not supported by these functions620 621 // Get metadata value as a string by key name622 LLAMA_API int32_t llama_model_meta_val_str(const struct llama_model * model, const char * key, char * buf, size_t buf_size);623 624 // Get the number of metadata key/value pairs625 LLAMA_API int32_t llama_model_meta_count(const struct llama_model * model);626 627 // Get sampling metadata key name. Returns nullptr if the key is invalid628 LLAMA_API const char * llama_model_meta_key_str(enum llama_model_meta_key key);629 630 // Get metadata key name by index631 LLAMA_API int32_t llama_model_meta_key_by_index(const struct llama_model * model, int32_t i, char * buf, size_t buf_size);632 633 // Get metadata value as a string by index634 LLAMA_API int32_t llama_model_meta_val_str_by_index(const struct llama_model * model, int32_t i, char * buf, size_t buf_size);635 636 // Get a string describing the model type637 LLAMA_API int32_t llama_model_desc(const struct llama_model * model, char * buf, size_t buf_size);638 639 // Get the model file type (quantization), e.g. LLAMA_FTYPE_MOSTLY_Q8_0640 LLAMA_API enum llama_ftype llama_model_ftype(const struct llama_model * model);641 642 // Returns the total size of all the tensors in the model in bytes643 LLAMA_API uint64_t llama_model_size(const struct llama_model * model);644 645 // Get the default chat template. Returns nullptr if not available646 // If name is NULL, returns the default chat template647 LLAMA_API const char * llama_model_chat_template(const struct llama_model * model, const char * name);648 649 // Returns the total number of parameters in the model650 LLAMA_API uint64_t llama_model_n_params(const struct llama_model * model);651 652 // Returns true if the model contains an encoder that requires llama_encode() call653 LLAMA_API bool llama_model_has_encoder(const struct llama_model * model);654 655 // Returns true if the model contains a decoder that requires llama_decode() call656 LLAMA_API bool llama_model_has_decoder(const struct llama_model * model);657 658 // For encoder-decoder models, this function returns id of the token that must be provided659 // to the decoder to start generating output sequence. For other models, it returns -1.660 LLAMA_API llama_token llama_model_decoder_start_token(const struct llama_model * model);661 662 // Returns true if the model is recurrent (like Mamba, RWKV, etc.)663 LLAMA_API bool llama_model_is_recurrent(const struct llama_model * model);664 665 // Returns true if the model is hybrid (like Jamba, Granite, etc.)666 LLAMA_API bool llama_model_is_hybrid(const struct llama_model * model);667 668 // Returns true if the model is diffusion-based (like LLaDA, Dream, etc.)669 LLAMA_API bool llama_model_is_diffusion(const struct llama_model * model);670 671 // Returns 0 on success672 LLAMA_API uint32_t llama_model_quantize(673 const char * fname_inp,674 const char * fname_out,675 const llama_model_quantize_params * params);676 677 //678 // Adapters679 //680 681 // Load a LoRA adapter from file682 // The adapter is valid as long as the associated model is not freed683 LLAMA_API struct llama_adapter_lora * llama_adapter_lora_init(684 struct llama_model * model,685 const char * path_lora);686 687 // Load a LoRA adapter from an open FILE pointer, reading from its current position688 LLAMA_API struct llama_adapter_lora * llama_adapter_lora_init_from_file_ptr(689 struct llama_model * model,690 FILE * file);691 692 // Functions to access the adapter's GGUF metadata scalar values693 // - The functions return the length of the string on success, or -1 on failure694 // - The output string is always null-terminated and cleared on failure695 // - When retrieving a string, an extra byte must be allocated to account for the null terminator696 // - GGUF array values are not supported by these functions697 698 // Get metadata value as a string by key name699 LLAMA_API int32_t llama_adapter_meta_val_str(const struct llama_adapter_lora * adapter, const char * key, char * buf, size_t buf_size);700 701 // Get the number of metadata key/value pairs702 LLAMA_API int32_t llama_adapter_meta_count(const struct llama_adapter_lora * adapter);703 704 // Get metadata key name by index705 LLAMA_API int32_t llama_adapter_meta_key_by_index(const struct llama_adapter_lora * adapter, int32_t i, char * buf, size_t buf_size);706 707 // Get metadata value as a string by index708 LLAMA_API int32_t llama_adapter_meta_val_str_by_index(const struct llama_adapter_lora * adapter, int32_t i, char * buf, size_t buf_size);709 710 // Manually free a LoRA adapter711 // NOTE: loaded adapters that are not manually freed will be freed when the associated model is deleted712 LLAMA_API void llama_adapter_lora_free(struct llama_adapter_lora * adapter);713 714 // Get the invocation tokens if the current lora is an alora715 LLAMA_API uint64_t llama_adapter_get_alora_n_invocation_tokens(const struct llama_adapter_lora * adapter);716 LLAMA_API const llama_token * llama_adapter_get_alora_invocation_tokens (const struct llama_adapter_lora * adapter);717 718 // The following functions operate on a llama_context, hence the naming: llama_verb_...719 720 // Set LoRa adapters on the context. Will only modify if the adapters currently in context are different.721 LLAMA_API int32_t llama_set_adapters_lora(722 struct llama_context * ctx,723 struct llama_adapter_lora ** adapters,724 size_t n_adapters,725 float * scales);726 727 // Apply a loaded control vector to a llama_context, or if data is NULL, clear728 // the currently loaded vector.729 // n_embd should be the size of a single layer's control, and data should point730 // to an n_embd x n_layers buffer starting from layer 1.731 // il_start and il_end are the layer range the vector should apply to (both inclusive)732 // See llama_control_vector_load in common to load a control vector.733 LLAMA_API int32_t llama_set_adapter_cvec(734 struct llama_context * ctx,735 const float * data,736 size_t len,737 int32_t n_embd,738 int32_t il_start,739 int32_t il_end);740 741 //742 // Memory743 //744 745 // Clear the memory contents746 // If data == true, the data buffers will also be cleared together with the metadata747 LLAMA_API void llama_memory_clear(748 llama_memory_t mem,749 bool data);750 751 // Removes all tokens that belong to the specified sequence and have positions in [p0, p1)752 // Returns false if a partial sequence cannot be removed. Removing a whole sequence never fails753 // seq_id < 0 : match any sequence [TAG_LLAMA_SEQ_ID_NEG]754 // p0 < 0 : [0, p1]755 // p1 < 0 : [p0, inf)756 LLAMA_API bool llama_memory_seq_rm(757 llama_memory_t mem,758 llama_seq_id seq_id,759 llama_pos p0,760 llama_pos p1);761 762 // Copy all tokens that belong to the specified sequence to another sequence763 // p0 < 0 : [0, p1]764 // p1 < 0 : [p0, inf)765 LLAMA_API void llama_memory_seq_cp(766 llama_memory_t mem,767 llama_seq_id seq_id_src,768 llama_seq_id seq_id_dst,769 llama_pos p0,770 llama_pos p1);771 772 // Removes all tokens that do not belong to the specified sequence773 LLAMA_API void llama_memory_seq_keep(774 llama_memory_t mem,775 llama_seq_id seq_id);776 777 // Adds relative position "delta" to all tokens that belong to the specified sequence and have positions in [p0, p1)778 // p0 < 0 : [0, p1]779 // p1 < 0 : [p0, inf)780 LLAMA_API void llama_memory_seq_add(781 llama_memory_t mem,782 llama_seq_id seq_id,783 llama_pos p0,784 llama_pos p1,785 llama_pos delta);786 787 // Integer division of the positions by factor of `d > 1`788 // p0 < 0 : [0, p1]789 // p1 < 0 : [p0, inf)790 LLAMA_API void llama_memory_seq_div(791 llama_memory_t mem,792 llama_seq_id seq_id,793 llama_pos p0,794 llama_pos p1,795 int d);796 797 // Returns the smallest position present in the memory for the specified sequence798 // This is typically non-zero only for SWA caches799 // Note that all positions in the range [pos_min, pos_max] are guaranteed to be present in the memory800 // Return -1 if the sequence is empty801 LLAMA_API llama_pos llama_memory_seq_pos_min(802 llama_memory_t mem,803 llama_seq_id seq_id);804 805 // Returns the largest position present in the memory for the specified sequence806 // Note that all positions in the range [pos_min, pos_max] are guaranteed to be present in the memory807 // Return -1 if the sequence is empty808 LLAMA_API llama_pos llama_memory_seq_pos_max(809 llama_memory_t mem,810 llama_seq_id seq_id);811 812 // Check if the memory supports shifting813 LLAMA_API bool llama_memory_can_shift(llama_memory_t mem);814 815 //816 // State / sessions817 //818 819 // Returns the *actual* size in bytes of the state820 // (logits, embedding and memory)821 // Only use when saving the state, not when restoring it, otherwise the size may be too small.822 LLAMA_API size_t llama_state_get_size(struct llama_context * ctx);823 LLAMA_API DEPRECATED(size_t llama_get_state_size(struct llama_context * ctx),824 "use llama_state_get_size instead");825 826 // Copies the state to the specified destination address.827 // Destination needs to have allocated enough memory.828 // Returns the number of bytes copied829 LLAMA_API size_t llama_state_get_data(830 struct llama_context * ctx,831 uint8_t * dst,832 size_t size);833 LLAMA_API DEPRECATED(size_t llama_copy_state_data(834 struct llama_context * ctx,835 uint8_t * dst),836 "use llama_state_get_data instead");837 838 // Set the state reading from the specified address839 // Returns the number of bytes read840 LLAMA_API size_t llama_state_set_data(841 struct llama_context * ctx,842 const uint8_t * src,843 size_t size);844 LLAMA_API DEPRECATED(size_t llama_set_state_data(845 struct llama_context * ctx,846 const uint8_t * src),847 "use llama_state_set_data instead");848 849 // Save/load session file850 LLAMA_API bool llama_state_load_file(851 struct llama_context * ctx,852 const char * path_session,853 llama_token * tokens_out,854 size_t n_token_capacity,855 size_t * n_token_count_out);856 LLAMA_API DEPRECATED(bool llama_load_session_file(857 struct llama_context * ctx,858 const char * path_session,859 llama_token * tokens_out,860 size_t n_token_capacity,861 size_t * n_token_count_out),862 "use llama_state_load_file instead");863 864 LLAMA_API bool llama_state_save_file(865 struct llama_context * ctx,866 const char * path_session,867 const llama_token * tokens,868 size_t n_token_count);869 LLAMA_API DEPRECATED(bool llama_save_session_file(870 struct llama_context * ctx,871 const char * path_session,872 const llama_token * tokens,873 size_t n_token_count),874 "use llama_state_save_file instead");875 876 // Get the exact size needed to copy the state of a single sequence877 LLAMA_API size_t llama_state_seq_get_size(878 struct llama_context * ctx,879 llama_seq_id seq_id);880 881 // Copy the state of a single sequence into the specified buffer882 LLAMA_API size_t llama_state_seq_get_data(883 struct llama_context * ctx,884 uint8_t * dst,885 size_t size,886 llama_seq_id seq_id);887 888 // Copy the sequence data (originally copied with `llama_state_seq_get_data`) into the specified sequence889 // Returns:890 // - Positive: Ok891 // - Zero: Failed to load892 LLAMA_API size_t llama_state_seq_set_data(893 struct llama_context * ctx,894 const uint8_t * src,895 size_t size,896 llama_seq_id dest_seq_id);897 898 LLAMA_API size_t llama_state_seq_save_file(899 struct llama_context * ctx,900 const char * filepath,901 llama_seq_id seq_id,902 const llama_token * tokens,903 size_t n_token_count);904 905 // If tokens_out is NULL, only the token count is reported through n_token_count_out and no state is loaded906 LLAMA_API size_t llama_state_seq_load_file(907 struct llama_context * ctx,908 const char * filepath,909 llama_seq_id dest_seq_id,910 llama_token * tokens_out,911 size_t n_token_capacity,912 size_t * n_token_count_out);913 914#define LLAMA_STATE_SEQ_FLAGS_NONE 0915 916// for backwards-compat917#define LLAMA_STATE_SEQ_FLAGS_SWA_ONLY 1918 919// work only with partial states, such as SWA KV cache or recurrent cache (e.g. Mamba)920#define LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY 1921 922// Keeps the tensor data on device buffers (i.e. not accessible in host memory, but faster save/load).923// Getting the state for a seq_id with this flag invalidates all prior states gotten for that seq_id with this flag.924#define LLAMA_STATE_SEQ_FLAGS_ON_DEVICE 2925 926 typedef uint32_t llama_state_seq_flags;927 928 LLAMA_API size_t llama_state_seq_get_size_ext(929 struct llama_context * ctx,930 llama_seq_id seq_id,931 llama_state_seq_flags flags);932 933 LLAMA_API size_t llama_state_seq_get_data_ext(934 struct llama_context * ctx,935 uint8_t * dst,936 size_t size,937 llama_seq_id seq_id,938 llama_state_seq_flags flags);939 940 LLAMA_API size_t llama_state_seq_set_data_ext(941 struct llama_context * ctx,942 const uint8_t * src,943 size_t size,944 llama_seq_id dest_seq_id,945 llama_state_seq_flags flags);946 947 //948 // Decoding949 //950 951 // Return batch for single sequence of tokens952 // The sequence ID will be fixed to 0953 // The position of the tokens will be tracked automatically by llama_decode954 //955 // NOTE: this is a helper function to facilitate transition to the new batch API - avoid using it956 //957 LLAMA_API struct llama_batch llama_batch_get_one(958 llama_token * tokens,959 int32_t n_tokens);960 961 // Allocates a batch of tokens on the heap that can hold a maximum of n_tokens962 // Each token can be assigned up to n_seq_max sequence ids963 // The batch has to be freed with llama_batch_free()964 // If embd != 0, llama_batch.embd will be allocated with size of n_tokens * embd * sizeof(float)965 // Otherwise, llama_batch.token will be allocated to store n_tokens llama_token966 // The rest of the llama_batch members are allocated with size n_tokens967 // All members are left uninitialized968 LLAMA_API struct llama_batch llama_batch_init(969 int32_t n_tokens,970 int32_t embd,971 int32_t n_seq_max);972 973 // Frees a batch of tokens allocated with llama_batch_init()974 LLAMA_API void llama_batch_free(struct llama_batch batch);975 976 // Process a batch of tokens.977 // In contrast to llama_decode() - this call does not use KV cache.978 // For encode-decoder contexts, processes the batch using the encoder.979 // Can store the encoder output internally for later use by the decoder's cross-attention layers.980 // 0 - success981 // < 0 - error. the memory state is restored to the state before this call982 LLAMA_API int32_t llama_encode(983 struct llama_context * ctx,984 struct llama_batch batch);985 986 // Process a batch of tokens.987 // Requires the context to have a memory.988 // For encode-decoder contexts, processes the batch using the decoder.989 // Positive return values does not mean a fatal error, but rather a warning.990 // Upon fatal-error or abort, the ubatches that managed to be been processed will remain in the memory state of the context991 // To handle this correctly, query the memory state using llama_memory_seq_pos_min() and llama_memory_seq_pos_max()992 // Upon other return values, the memory state is restored to the state before this call993 // 0 - success994 // 1 - could not find a KV slot for the batch (try reducing the size of the batch or increase the context)995 // 2 - aborted (processed ubatches will remain in the context's memory)996 // -1 - invalid input batch997 // < -1 - fatal error (processed ubatches will remain in the context's memory)998 LLAMA_API int32_t llama_decode(999 struct llama_context * ctx,1000 struct llama_batch batch);1001 1002 // Set the number of threads used for decoding1003 // n_threads is the number of threads used for generation (single token)1004 // n_threads_batch is the number of threads used for prompt and batch processing (multiple tokens)1005 LLAMA_API void llama_set_n_threads(struct llama_context * ctx, int32_t n_threads, int32_t n_threads_batch);1006 1007 // Get the number of threads used for generation of a single token.1008 LLAMA_API int32_t llama_n_threads(struct llama_context * ctx);1009 1010 // Get the number of threads used for prompt and batch processing (multiple token).1011 LLAMA_API int32_t llama_n_threads_batch(struct llama_context * ctx);1012 1013 // Set whether the context outputs embeddings or not1014 // TODO: rename to avoid confusion with llama_get_embeddings()1015 LLAMA_API void llama_set_embeddings(struct llama_context * ctx, bool embeddings);1016 1017 // Set whether to use causal attention or not1018 // If set to true, the model will only attend to the past tokens1019 LLAMA_API void llama_set_causal_attn(struct llama_context * ctx, bool causal_attn);1020 1021 // Set whether the model is in warmup mode or not1022 // If true, all model tensors are activated during llama_decode() to load and cache their weights.1023 //1024 // note: using this can cause extra graph reallocations because it changes the graph topology with MoE models,1025 // so it is generally not recommended to use in practice. will be removed in the future1026 DEPRECATED(LLAMA_API void llama_set_warmup(struct llama_context * ctx, bool warmup),1027 "user code should do warmup runs manually [TAG_LLAMA_GRAPH_NO_WARMUP]");1028 1029 // Set abort callback1030 LLAMA_API void llama_set_abort_callback(struct llama_context * ctx, ggml_abort_callback abort_callback, void * abort_callback_data);1031 1032 // Wait until all computations are finished1033 // This is automatically done when using one of the functions below to obtain the computation results1034 // and is not necessary to call it explicitly in most cases1035 LLAMA_API void llama_synchronize(struct llama_context * ctx);1036 1037 // Token logits obtained from the last call to llama_decode()1038 // The logits for which llama_batch.logits[i] != 0 are stored contiguously1039 // in the order they have appeared in the batch.1040 // Rows: number of tokens for which llama_batch.logits[i] != 01041 // Cols: n_vocab1042 // TODO: deprecate in favor of llama_get_logits_ith() (ref: https://github.com/ggml-org/llama.cpp/pull/14853#issuecomment-3113143522)1043 LLAMA_API float * llama_get_logits(struct llama_context * ctx);1044 1045 // Logits for the ith token. For positive indices, Equivalent to:1046 // llama_get_logits(ctx) + ctx->output_ids[i]*n_vocab1047 // Negative indices can be used to access logits in reverse order, -1 is the last logit.1048 // returns NULL for invalid ids.1049 LLAMA_API float * llama_get_logits_ith(struct llama_context * ctx, int32_t i);1050 1051 // Get all output token embeddings.1052 // when pooling_type == LLAMA_POOLING_TYPE_NONE or when using a generative model,1053 // the embeddings for which llama_batch.logits[i] != 0 are stored contiguously1054 // in the order they have appeared in the batch.1055 // shape: [n_outputs*n_embd]1056 // Otherwise, returns NULL.1057 // TODO: deprecate in favor of llama_get_embeddings_ith() (ref: https://github.com/ggml-org/llama.cpp/pull/14853#issuecomment-3113143522)1058 LLAMA_API float * llama_get_embeddings(struct llama_context * ctx);1059 1060 // Get the embeddings for the ith token. For positive indices, Equivalent to:1061 // llama_get_embeddings(ctx) + ctx->output_ids[i]*n_embd1062 // Negative indices can be used to access embeddings in reverse order, -1 is the last embedding.1063 // shape: [n_embd] (1-dimensional)1064 // returns NULL for invalid ids.1065 LLAMA_API float * llama_get_embeddings_ith(struct llama_context * ctx, int32_t i);1066 1067 // Get the embeddings for a sequence id1068 // Returns NULL if pooling_type is LLAMA_POOLING_TYPE_NONE1069 // when pooling_type == LLAMA_POOLING_TYPE_RANK, returns float[n_cls_out] with the rank(s) of the sequence1070 // otherwise: float[n_embd] (1-dimensional)1071 LLAMA_API float * llama_get_embeddings_seq(struct llama_context * ctx, llama_seq_id seq_id);1072 1073 //1074 // backend sampling API [EXPERIMENTAL]1075 // note: use only if the llama_context was created with at least one llama_sampler_seq_config1076 //1077 1078 // Get the backend sampled token for the ith token.1079 // With multiple outputs, sampler state advances when the token is accepted,1080 // not when it is read through this function.1081 // When accepting multiple outputs, accept a contiguous prefix in output order.1082 // Returns LLAMA_TOKEN_NULL if no token was sampled.1083 LLAMA_API llama_token llama_get_sampled_token_ith(struct llama_context * ctx, int32_t i);1084 1085 // Get the backend sampled probabilities for the ith token1086 // The index matches llama_get_sampled_token_ith().1087 // Returns NULL if no probabilities were generated.1088 LLAMA_API float * llama_get_sampled_probs_ith (struct llama_context * ctx, int32_t i);1089 LLAMA_API uint32_t llama_get_sampled_probs_count_ith(struct llama_context * ctx, int32_t i);1090 1091 // Get the backend sampled logits for the ith token1092 // Returns NULL if no logits were sampled.1093 LLAMA_API float * llama_get_sampled_logits_ith (struct llama_context * ctx, int32_t i);1094 LLAMA_API uint32_t llama_get_sampled_logits_count_ith(struct llama_context * ctx, int32_t i);1095 1096 // Get the backend sampled candidates (token ids) for the ith token1097 // These are needed to map probability/logit indices to vocab token ids.1098 // Returns NULL if no candidates were sampled.1099 LLAMA_API llama_token * llama_get_sampled_candidates_ith (struct llama_context * ctx, int32_t i);1100 LLAMA_API uint32_t llama_get_sampled_candidates_count_ith(struct llama_context * ctx, int32_t i);1101 1102 //1103 // Vocab1104 //1105 1106 LLAMA_API const char * llama_vocab_get_text(const struct llama_vocab * vocab, llama_token token);1107 1108 LLAMA_API float llama_vocab_get_score(const struct llama_vocab * vocab, llama_token token);1109 1110 LLAMA_API enum llama_token_attr llama_vocab_get_attr(const struct llama_vocab * vocab, llama_token token);1111 1112 // Check if the token is supposed to end generation (end-of-generation, eg. EOS, EOT, etc.)1113 LLAMA_API bool llama_vocab_is_eog(const struct llama_vocab * vocab, llama_token token);1114 1115 // Identify if Token Id is a control token or a render-able token1116 LLAMA_API bool llama_vocab_is_control(const struct llama_vocab * vocab, llama_token token);1117 1118 // Special tokens1119 LLAMA_API llama_token llama_vocab_bos(const struct llama_vocab * vocab); // beginning-of-sentence1120 LLAMA_API llama_token llama_vocab_eos(const struct llama_vocab * vocab); // end-of-sentence1121 LLAMA_API llama_token llama_vocab_eot(const struct llama_vocab * vocab); // end-of-turn1122 LLAMA_API llama_token llama_vocab_sep(const struct llama_vocab * vocab); // sentence separator1123 LLAMA_API llama_token llama_vocab_nl (const struct llama_vocab * vocab); // next-line1124 LLAMA_API llama_token llama_vocab_pad(const struct llama_vocab * vocab); // padding1125 LLAMA_API llama_token llama_vocab_mask(const struct llama_vocab * vocab); // mask1126 1127 LLAMA_API bool llama_vocab_get_add_bos(const struct llama_vocab * vocab);1128 LLAMA_API bool llama_vocab_get_add_eos(const struct llama_vocab * vocab);1129 LLAMA_API bool llama_vocab_get_add_sep(const struct llama_vocab * vocab);1130 1131 // model-specific suppress tokens (gguf key: tokenizer.ggml.suppress_tokens)1132 LLAMA_API const llama_token * llama_vocab_get_suppress_tokens(const struct llama_vocab * vocab, int32_t * n_suppress_tokens);1133 1134 LLAMA_API llama_token llama_vocab_fim_pre(const struct llama_vocab * vocab);1135 LLAMA_API llama_token llama_vocab_fim_suf(const struct llama_vocab * vocab);1136 LLAMA_API llama_token llama_vocab_fim_mid(const struct llama_vocab * vocab);1137 LLAMA_API llama_token llama_vocab_fim_pad(const struct llama_vocab * vocab);1138 LLAMA_API llama_token llama_vocab_fim_rep(const struct llama_vocab * vocab);1139 LLAMA_API llama_token llama_vocab_fim_sep(const struct llama_vocab * vocab);1140 1141 DEPRECATED(LLAMA_API const char * llama_token_get_text(const struct llama_vocab * vocab, llama_token token), "use llama_vocab_get_text instead");1142 DEPRECATED(LLAMA_API float llama_token_get_score(const struct llama_vocab * vocab, llama_token token), "use llama_vocab_get_score instead");1143 DEPRECATED(LLAMA_API enum llama_token_attr llama_token_get_attr(const struct llama_vocab * vocab, llama_token token), "use llama_vocab_get_attr instead");1144 DEPRECATED(LLAMA_API bool llama_token_is_eog(const struct llama_vocab * vocab, llama_token token), "use llama_vocab_is_eog instead");1145 DEPRECATED(LLAMA_API bool llama_token_is_control(const struct llama_vocab * vocab, llama_token token), "use llama_vocab_is_control instead");1146 DEPRECATED(LLAMA_API llama_token llama_token_bos(const struct llama_vocab * vocab), "use llama_vocab_bos instead");1147 DEPRECATED(LLAMA_API llama_token llama_token_eos(const struct llama_vocab * vocab), "use llama_vocab_eos instead");1148 DEPRECATED(LLAMA_API llama_token llama_token_eot(const struct llama_vocab * vocab), "use llama_vocab_eot instead");1149 DEPRECATED(LLAMA_API llama_token llama_token_cls(const struct llama_vocab * vocab), "use llama_vocab_cls instead");1150 DEPRECATED(LLAMA_API llama_token llama_token_sep(const struct llama_vocab * vocab), "use llama_vocab_sep instead");1151 DEPRECATED(LLAMA_API llama_token llama_token_nl (const struct llama_vocab * vocab), "use llama_vocab_nl instead");1152 DEPRECATED(LLAMA_API llama_token llama_token_pad(const struct llama_vocab * vocab), "use llama_vocab_pad instead");1153 DEPRECATED(LLAMA_API bool llama_add_bos_token(const struct llama_vocab * vocab), "use llama_vocab_get_add_bos instead");1154 DEPRECATED(LLAMA_API bool llama_add_eos_token(const struct llama_vocab * vocab), "use llama_vocab_get_add_eos instead");1155 DEPRECATED(LLAMA_API llama_token llama_token_fim_pre(const struct llama_vocab * vocab), "use llama_vocab_fim_pre instead");1156 DEPRECATED(LLAMA_API llama_token llama_token_fim_suf(const struct llama_vocab * vocab), "use llama_vocab_fim_suf instead");1157 DEPRECATED(LLAMA_API llama_token llama_token_fim_mid(const struct llama_vocab * vocab), "use llama_vocab_fim_mid instead");1158 DEPRECATED(LLAMA_API llama_token llama_token_fim_pad(const struct llama_vocab * vocab), "use llama_vocab_fim_pad instead");1159 DEPRECATED(LLAMA_API llama_token llama_token_fim_rep(const struct llama_vocab * vocab), "use llama_vocab_fim_rep instead");1160 DEPRECATED(LLAMA_API llama_token llama_token_fim_sep(const struct llama_vocab * vocab), "use llama_vocab_fim_sep instead");1161 1162 // CLS is equivalent to BOS1163 DEPRECATED(LLAMA_API llama_token llama_vocab_cls(const struct llama_vocab * vocab), // classification1164 "use llama_vocab_bos instead");1165 1166 //1167 // Tokenization1168 //1169 // The API is thread-safe.1170 //1171 1172 /// @details Convert the provided text into tokens.1173 /// @param tokens The tokens pointer must be large enough to hold the resulting tokens.1174 /// @return Returns the number of tokens on success, no more than n_tokens_max1175 /// @return Returns a negative number on failure - the number of tokens that would have been returned1176 /// @return Returns INT32_MIN on overflow (e.g., tokenization result size exceeds int32_t limit)1177 /// @param add_special Allow to add BOS and EOS tokens if model is configured to do so.1178 /// @param parse_special Allow tokenizing special and/or control tokens which otherwise are not exposed and treated1179 /// as plaintext. Does not insert a leading space.1180 LLAMA_API int32_t llama_tokenize(1181 const struct llama_vocab * vocab,1182 const char * text,1183 int32_t text_len,1184 llama_token * tokens,1185 int32_t n_tokens_max,1186 bool add_special,1187 bool parse_special);1188 1189 // Token Id -> Piece.1190 // Uses the vocabulary in the provided context.1191 // Does not write null terminator to the buffer.1192 // User can skip up to 'lstrip' leading spaces before copying (useful when encoding/decoding multiple tokens with 'add_space_prefix')1193 // @param special If true, special tokens are rendered in the output.1194 LLAMA_API int32_t llama_token_to_piece(1195 const struct llama_vocab * vocab,1196 llama_token token,1197 char * buf,1198 int32_t length,1199 int32_t lstrip,1200 bool special);