echodict/llama.cpp
version https://git-lfs.github.com/spec/v1 oid sha256:cfc44b7ba25614df70e6b65e3341cae0310163bd32fd31a6b928a542df433faf size 30786
0773
1#ifndef LLAMA_H2#define LLAMA_H3 4#include "ggml.h"5#include "ggml-cpu.h"6#include "ggml-backend.h"7#include "ggml-opt.h"8#include "gguf.h"9 10#include <stddef.h>11#include <stdint.h>12#include <stdio.h>13#include <stdbool.h>14 15#ifdef LLAMA_SHARED16# if defined(_WIN32) && !defined(__MINGW32__)17# ifdef LLAMA_BUILD18# define LLAMA_API __declspec(dllexport)19# else20# define LLAMA_API __declspec(dllimport)21# endif22# else23# define LLAMA_API __attribute__ ((visibility ("default")))24# endif25#else26# define LLAMA_API27#endif28 29#ifdef __GNUC__30# define DEPRECATED(func, hint) func __attribute__((deprecated(hint)))31#elif defined(_MSC_VER)32# define DEPRECATED(func, hint) __declspec(deprecated(hint)) func33#else34# define DEPRECATED(func, hint) func35#endif36 37#define LLAMA_DEFAULT_SEED 0xFFFFFFFF38 39#define LLAMA_TOKEN_NULL -140 41#define LLAMA_FILE_MAGIC_GGLA 0x67676c61u // 'ggla'42#define LLAMA_FILE_MAGIC_GGSN 0x6767736eu // 'ggsn'43#define LLAMA_FILE_MAGIC_GGSQ 0x67677371u // 'ggsq'44 45#define LLAMA_SESSION_MAGIC LLAMA_FILE_MAGIC_GGSN46#define LLAMA_SESSION_VERSION 947 48#define LLAMA_STATE_SEQ_MAGIC LLAMA_FILE_MAGIC_GGSQ49#define LLAMA_STATE_SEQ_VERSION 250 51#ifdef __cplusplus52extern "C" {53#endif54 55 //56 // C interface57 //58 // TODO: show sample usage59 //60 61 struct llama_vocab;62 struct llama_model;63 struct llama_context;64 struct llama_sampler;65 66 typedef struct llama_memory_i * llama_memory_t;67 68 typedef int32_t llama_pos;69 typedef int32_t llama_token;70 typedef int32_t llama_seq_id;71 72 enum llama_vocab_type {73 LLAMA_VOCAB_TYPE_NONE = 0, // For models without vocab74 LLAMA_VOCAB_TYPE_SPM = 1, // LLaMA tokenizer based on byte-level BPE with byte fallback75 LLAMA_VOCAB_TYPE_BPE = 2, // GPT-2 tokenizer based on byte-level BPE76 LLAMA_VOCAB_TYPE_WPM = 3, // BERT tokenizer based on WordPiece77 LLAMA_VOCAB_TYPE_UGM = 4, // T5 tokenizer based on Unigram78 LLAMA_VOCAB_TYPE_RWKV = 5, // RWKV tokenizer based on greedy tokenization79 LLAMA_VOCAB_TYPE_PLAMO2 = 6, // PLaMo-2 tokenizer based on Aho-Corasick with dynamic programming80 };81 82 enum llama_rope_type {83 LLAMA_ROPE_TYPE_NONE = -1,84 LLAMA_ROPE_TYPE_NORM = 0,85 LLAMA_ROPE_TYPE_NEOX = GGML_ROPE_TYPE_NEOX,86 LLAMA_ROPE_TYPE_MROPE = GGML_ROPE_TYPE_MROPE,87 LLAMA_ROPE_TYPE_IMROPE = GGML_ROPE_TYPE_IMROPE,88 LLAMA_ROPE_TYPE_VISION = GGML_ROPE_TYPE_VISION,89 };90 91 enum llama_token_type { //TODO: remove, required until per token attributes are available from GGUF file92 LLAMA_TOKEN_TYPE_UNDEFINED = 0,93 LLAMA_TOKEN_TYPE_NORMAL = 1,94 LLAMA_TOKEN_TYPE_UNKNOWN = 2,95 LLAMA_TOKEN_TYPE_CONTROL = 3,96 LLAMA_TOKEN_TYPE_USER_DEFINED = 4,97 LLAMA_TOKEN_TYPE_UNUSED = 5,98 LLAMA_TOKEN_TYPE_BYTE = 6,99 };100 101 enum llama_token_attr {102 LLAMA_TOKEN_ATTR_UNDEFINED = 0,103 LLAMA_TOKEN_ATTR_UNKNOWN = 1 << 0,104 LLAMA_TOKEN_ATTR_UNUSED = 1 << 1,105 LLAMA_TOKEN_ATTR_NORMAL = 1 << 2,106 LLAMA_TOKEN_ATTR_CONTROL = 1 << 3, // SPECIAL?107 LLAMA_TOKEN_ATTR_USER_DEFINED = 1 << 4,108 LLAMA_TOKEN_ATTR_BYTE = 1 << 5,109 LLAMA_TOKEN_ATTR_NORMALIZED = 1 << 6,110 LLAMA_TOKEN_ATTR_LSTRIP = 1 << 7,111 LLAMA_TOKEN_ATTR_RSTRIP = 1 << 8,112 LLAMA_TOKEN_ATTR_SINGLE_WORD = 1 << 9,113 };114 115 // model file types116 enum llama_ftype {117 LLAMA_FTYPE_ALL_F32 = 0,118 LLAMA_FTYPE_MOSTLY_F16 = 1, // except 1d tensors119 LLAMA_FTYPE_MOSTLY_Q4_0 = 2, // except 1d tensors120 LLAMA_FTYPE_MOSTLY_Q4_1 = 3, // except 1d tensors121 // LLAMA_FTYPE_MOSTLY_Q4_1_SOME_F16 = 4, // tok_embeddings.weight and output.weight are F16122 // LLAMA_FTYPE_MOSTLY_Q4_2 = 5, // support has been removed123 // LLAMA_FTYPE_MOSTLY_Q4_3 = 6, // support has been removed124 LLAMA_FTYPE_MOSTLY_Q8_0 = 7, // except 1d tensors125 LLAMA_FTYPE_MOSTLY_Q5_0 = 8, // except 1d tensors126 LLAMA_FTYPE_MOSTLY_Q5_1 = 9, // except 1d tensors127 LLAMA_FTYPE_MOSTLY_Q2_K = 10, // except 1d tensors128 LLAMA_FTYPE_MOSTLY_Q3_K_S = 11, // except 1d tensors129 LLAMA_FTYPE_MOSTLY_Q3_K_M = 12, // except 1d tensors130 LLAMA_FTYPE_MOSTLY_Q3_K_L = 13, // except 1d tensors131 LLAMA_FTYPE_MOSTLY_Q4_K_S = 14, // except 1d tensors132 LLAMA_FTYPE_MOSTLY_Q4_K_M = 15, // except 1d tensors133 LLAMA_FTYPE_MOSTLY_Q5_K_S = 16, // except 1d tensors134 LLAMA_FTYPE_MOSTLY_Q5_K_M = 17, // except 1d tensors135 LLAMA_FTYPE_MOSTLY_Q6_K = 18, // except 1d tensors136 LLAMA_FTYPE_MOSTLY_IQ2_XXS = 19, // except 1d tensors137 LLAMA_FTYPE_MOSTLY_IQ2_XS = 20, // except 1d tensors138 LLAMA_FTYPE_MOSTLY_Q2_K_S = 21, // except 1d tensors139 LLAMA_FTYPE_MOSTLY_IQ3_XS = 22, // except 1d tensors140 LLAMA_FTYPE_MOSTLY_IQ3_XXS = 23, // except 1d tensors141 LLAMA_FTYPE_MOSTLY_IQ1_S = 24, // except 1d tensors142 LLAMA_FTYPE_MOSTLY_IQ4_NL = 25, // except 1d tensors143 LLAMA_FTYPE_MOSTLY_IQ3_S = 26, // except 1d tensors144 LLAMA_FTYPE_MOSTLY_IQ3_M = 27, // except 1d tensors145 LLAMA_FTYPE_MOSTLY_IQ2_S = 28, // except 1d tensors146 LLAMA_FTYPE_MOSTLY_IQ2_M = 29, // except 1d tensors147 LLAMA_FTYPE_MOSTLY_IQ4_XS = 30, // except 1d tensors148 LLAMA_FTYPE_MOSTLY_IQ1_M = 31, // except 1d tensors149 LLAMA_FTYPE_MOSTLY_BF16 = 32, // except 1d tensors150 //LLAMA_FTYPE_MOSTLY_Q4_0_4_4 = 33, // removed from gguf files, use Q4_0 and runtime repack151 //LLAMA_FTYPE_MOSTLY_Q4_0_4_8 = 34, // removed from gguf files, use Q4_0 and runtime repack152 //LLAMA_FTYPE_MOSTLY_Q4_0_8_8 = 35, // removed from gguf files, use Q4_0 and runtime repack153 LLAMA_FTYPE_MOSTLY_TQ1_0 = 36, // except 1d tensors154 LLAMA_FTYPE_MOSTLY_TQ2_0 = 37, // except 1d tensors155 LLAMA_FTYPE_MOSTLY_MXFP4_MOE = 38, // except 1d tensors156 LLAMA_FTYPE_MOSTLY_NVFP4 = 39, // except 1d tensors157 LLAMA_FTYPE_MOSTLY_Q1_0 = 40, // except 1d tensors158 159 LLAMA_FTYPE_GUESSED = 1024, // not specified in the model file160 };161 162 enum llama_rope_scaling_type {163 LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED = -1,164 LLAMA_ROPE_SCALING_TYPE_NONE = 0,165 LLAMA_ROPE_SCALING_TYPE_LINEAR = 1,166 LLAMA_ROPE_SCALING_TYPE_YARN = 2,167 LLAMA_ROPE_SCALING_TYPE_LONGROPE = 3,168 LLAMA_ROPE_SCALING_TYPE_MAX_VALUE = LLAMA_ROPE_SCALING_TYPE_LONGROPE,169 };170 171 enum llama_pooling_type {172 LLAMA_POOLING_TYPE_UNSPECIFIED = -1,173 LLAMA_POOLING_TYPE_NONE = 0,174 LLAMA_POOLING_TYPE_MEAN = 1,175 LLAMA_POOLING_TYPE_CLS = 2,176 LLAMA_POOLING_TYPE_LAST = 3,177 LLAMA_POOLING_TYPE_RANK = 4, // used by reranking models to attach the classification head to the graph178 };179 180 enum llama_attention_type {181 LLAMA_ATTENTION_TYPE_UNSPECIFIED = -1,182 LLAMA_ATTENTION_TYPE_CAUSAL = 0,183 LLAMA_ATTENTION_TYPE_NON_CAUSAL = 1,184 };185 186 enum llama_flash_attn_type {187 LLAMA_FLASH_ATTN_TYPE_AUTO = -1,188 LLAMA_FLASH_ATTN_TYPE_DISABLED = 0,189 LLAMA_FLASH_ATTN_TYPE_ENABLED = 1,190 };191 192 LLAMA_API const char * llama_flash_attn_type_name(enum llama_flash_attn_type flash_attn_type);193 194 enum llama_split_mode {195 LLAMA_SPLIT_MODE_NONE = 0, // single GPU196 LLAMA_SPLIT_MODE_LAYER = 1, // split layers and KV across GPUs197 LLAMA_SPLIT_MODE_ROW = 2, // split layers and KV across GPUs, use tensor parallelism if supported198 LLAMA_SPLIT_MODE_TENSOR = 3,199 };200 201 // TODO: simplify (https://github.com/ggml-org/llama.cpp/pull/9294#pullrequestreview-2286561979)202 typedef struct llama_token_data {203 llama_token id; // token id204 float logit; // log-odds of the token205 float p; // probability of the token206 } llama_token_data;207 208 typedef struct llama_token_data_array {209 // TODO: consider SoA210 // NOTE: this pointer can be modified by the samplers211 llama_token_data * data;212 size_t size;213 int64_t selected; // this is the index in the data array (i.e. not the token id)214 bool sorted; // note: do not assume the data is sorted - always check this flag215 } llama_token_data_array;216 217 typedef bool (*llama_progress_callback)(float progress, void * user_data);218 219 // Input data for llama_encode/llama_decode220 // A llama_batch object can contain input about one or many sequences221 // The provided arrays (i.e. token, embd, pos, etc.) must have size of n_tokens222 //223 // - token : the token ids of the input (used when embd is NULL)224 // - embd : token embeddings (i.e. float vector of size n_embd) (used when token is NULL)225 // - pos : the positions of the respective token in the sequence226 // (if set to NULL, the token position will be tracked automatically by llama_encode/llama_decode)227 // - seq_id : the sequence to which the respective token belongs228 // (if set to NULL, the sequence ID will be assumed to be 0)229 // - logits : if zero, the logits (and/or the embeddings) for the respective token will not be output230 // (if set to NULL:231 // - if embeddings: all tokens are output232 // - if not: only the last token is output233 // )234 //235 typedef struct llama_batch {236 int32_t n_tokens;237 238 llama_token * token;239 float * embd;240 llama_pos * pos;241 int32_t * n_seq_id;242 llama_seq_id ** seq_id;243 int8_t * logits; // TODO: rename this to "output"244 } llama_batch;245 246 enum llama_model_kv_override_type {247 LLAMA_KV_OVERRIDE_TYPE_INT,248 LLAMA_KV_OVERRIDE_TYPE_FLOAT,249 LLAMA_KV_OVERRIDE_TYPE_BOOL,250 LLAMA_KV_OVERRIDE_TYPE_STR,251 };252 253 enum llama_model_meta_key {254 LLAMA_MODEL_META_KEY_SAMPLING_SEQUENCE,255 LLAMA_MODEL_META_KEY_SAMPLING_TOP_K,256 LLAMA_MODEL_META_KEY_SAMPLING_TOP_P,257 LLAMA_MODEL_META_KEY_SAMPLING_MIN_P,258 LLAMA_MODEL_META_KEY_SAMPLING_XTC_PROBABILITY,259 LLAMA_MODEL_META_KEY_SAMPLING_XTC_THRESHOLD,260 LLAMA_MODEL_META_KEY_SAMPLING_TEMP,261 LLAMA_MODEL_META_KEY_SAMPLING_PENALTY_LAST_N,262 LLAMA_MODEL_META_KEY_SAMPLING_PENALTY_REPEAT,263 LLAMA_MODEL_META_KEY_SAMPLING_MIROSTAT,264 LLAMA_MODEL_META_KEY_SAMPLING_MIROSTAT_TAU,265 LLAMA_MODEL_META_KEY_SAMPLING_MIROSTAT_ETA,266 };267 268 struct llama_model_kv_override {269 enum llama_model_kv_override_type tag;270 271 char key[128];272 273 union {274 int64_t val_i64;275 double val_f64;276 bool val_bool;277 char val_str[128];278 };279 };280 281 struct llama_model_tensor_buft_override {282 const char * pattern;283 ggml_backend_buffer_type_t buft;284 };285 286 struct llama_model_params {287 // NULL-terminated list of devices to use for offloading (if NULL, all available devices are used)288 ggml_backend_dev_t * devices;289 290 // NULL-terminated list of buffer types to use for tensors that match a pattern291 const struct llama_model_tensor_buft_override * tensor_buft_overrides;292 293 int32_t n_gpu_layers; // number of layers to store in VRAM, a negative value means all layers294 enum llama_split_mode split_mode; // how to split the model across multiple GPUs295 296 // the GPU that is used for the entire model when split_mode is LLAMA_SPLIT_MODE_NONE297 int32_t main_gpu;298 299 // proportion of the model (layers or rows) to offload to each GPU, size: llama_max_devices()300 const float * tensor_split;301 302 // Called with a progress value between 0.0 and 1.0. Pass NULL to disable.303 // If the provided progress_callback returns true, model loading continues.304 // If it returns false, model loading is immediately aborted.305 llama_progress_callback progress_callback;306 307 // context pointer passed to the progress callback308 void * progress_callback_user_data;309 310 // override key-value pairs of the model meta data311 const struct llama_model_kv_override * kv_overrides;312 313 // Keep the booleans together to avoid misalignment during copy-by-value.314 bool vocab_only; // only load the vocabulary, no weights315 bool use_mmap; // use mmap if possible316 bool use_direct_io; // use direct io, takes precedence over use_mmap when supported317 bool use_mlock; // force system to keep model in RAM318 bool check_tensors; // validate model tensor data319 bool use_extra_bufts; // use extra buffer types (used for weight repacking)320 bool no_host; // bypass host buffer allowing extra buffers to be used321 bool no_alloc; // only load metadata and simulate memory allocations322 };323 324 struct llama_sampler_seq_config {325 llama_seq_id seq_id;326 struct llama_sampler * sampler;327 };328 329 // NOTE: changing the default values of parameters marked as [EXPERIMENTAL] may cause crashes or incorrect results in certain configurations330 // https://github.com/ggml-org/llama.cpp/pull/7544331 struct llama_context_params {332 uint32_t n_ctx; // text context, 0 = from model333 uint32_t n_batch; // logical maximum batch size that can be submitted to llama_decode334 uint32_t n_ubatch; // physical maximum batch size335 uint32_t n_seq_max; // max number of sequences (i.e. distinct states for recurrent models)336 int32_t n_threads; // number of threads to use for generation337 int32_t n_threads_batch; // number of threads to use for batch processing338 339 enum llama_rope_scaling_type rope_scaling_type; // RoPE scaling type, from `enum llama_rope_scaling_type`340 enum llama_pooling_type pooling_type; // whether to pool (sum) embedding results by sequence id341 enum llama_attention_type attention_type; // attention type to use for embeddings342 enum llama_flash_attn_type flash_attn_type; // when to enable Flash Attention343 344 // ref: https://github.com/ggml-org/llama.cpp/pull/2054345 float rope_freq_base; // RoPE base frequency, 0 = from model346 float rope_freq_scale; // RoPE frequency scaling factor, 0 = from model347 float yarn_ext_factor; // YaRN extrapolation mix factor, negative = from model348 float yarn_attn_factor; // YaRN magnitude scaling factor349 float yarn_beta_fast; // YaRN low correction dim350 float yarn_beta_slow; // YaRN high correction dim351 uint32_t yarn_orig_ctx; // YaRN original context size352 float defrag_thold; // [DEPRECATED] defragment the KV cache if holes/size > thold, <= 0 disabled (default)353 354 ggml_backend_sched_eval_callback cb_eval;355 void * cb_eval_user_data;356 357 enum ggml_type type_k; // data type for K cache [EXPERIMENTAL]358 enum ggml_type type_v; // data type for V cache [EXPERIMENTAL]359 360 // Abort callback361 // if it returns true, execution of llama_decode() will be aborted362 // currently works only with CPU execution363 ggml_abort_callback abort_callback;364 void * abort_callback_data;365 366 // Keep the booleans together and at the end of the struct to avoid misalignment during copy-by-value.367 bool embeddings; // if true, extract embeddings (together with logits)368 bool offload_kqv; // offload the KQV ops (including the KV cache) to GPU369 bool no_perf; // measure performance timings370 bool op_offload; // offload host tensor operations to device371 bool swa_full; // use full-size SWA cache (https://github.com/ggml-org/llama.cpp/pull/13194#issuecomment-2868343055)372 // NOTE: setting to false when n_seq_max > 1 can cause bad performance in some cases373 // ref: https://github.com/ggml-org/llama.cpp/pull/13845#issuecomment-2924800573374 bool kv_unified; // use a unified buffer across the input sequences when computing the attention375 // try to disable when n_seq_max > 1 for improved performance when the sequences do not share a large prefix376 // ref: https://github.com/ggml-org/llama.cpp/pull/14363377 378 // [EXPERIMENTAL]379 // backend sampler chain configuration (make sure the caller keeps the sampler chains alive)380 // note: the samplers must be sampler chains (i.e. use llama_sampler_chain_init)381 struct llama_sampler_seq_config * samplers;382 size_t n_samplers;383 };384 385 struct llama_model_tensor_override {386 const char * pattern;387 enum ggml_type type;388 };389 390 struct llama_model_imatrix_data {391 const char * name;392 const float * data;393 size_t size;394 };395 396 // model quantization parameters397 typedef struct llama_model_quantize_params {398 int32_t nthread; // number of threads to use for quantizing, if <=0 will use std::thread::hardware_concurrency()399 enum llama_ftype ftype; // quantize to this llama_ftype400 enum ggml_type output_tensor_type; // output tensor type401 enum ggml_type token_embedding_type; // token embeddings tensor type402 bool allow_requantize; // allow quantizing non-f32/f16 tensors403 bool quantize_output_tensor; // quantize output.weight404 bool only_copy; // only copy tensors - ftype, allow_requantize and quantize_output_tensor are ignored405 bool pure; // quantize all tensors to the default type406 bool keep_split; // quantize to the same number of shards407 bool dry_run; // calculate and show the final quantization size without performing quantization408 const struct llama_model_imatrix_data * imatrix; // pointer to importance matrix data409 const struct llama_model_kv_override * kv_overrides; // pointer to kv overrides410 const struct llama_model_tensor_override * tt_overrides; // pointer to tensor overrides411 const int32_t * prune_layers; // pointer to layer indices to prune412 } llama_model_quantize_params;413 414 typedef struct llama_logit_bias {415 llama_token token;416 float bias;417 } llama_logit_bias;418 419 typedef struct llama_sampler_chain_params {420 bool no_perf; // whether to measure performance timings421 } llama_sampler_chain_params;422 423 // used in chat template424 typedef struct llama_chat_message {425 const char * role;426 const char * content;427 } llama_chat_message;428 429 // lora adapter430 struct llama_adapter_lora;431 432 // Helpers for getting default parameters433 // TODO: update API to start accepting pointers to params structs (https://github.com/ggml-org/llama.cpp/discussions/9172)434 LLAMA_API struct llama_model_params llama_model_default_params(void);435 LLAMA_API struct llama_context_params llama_context_default_params(void);436 LLAMA_API struct llama_sampler_chain_params llama_sampler_chain_default_params(void);437 LLAMA_API struct llama_model_quantize_params llama_model_quantize_default_params(void);438 439 // Initialize the llama + ggml backend440 // If numa is true, use NUMA optimizations441 // Call once at the start of the program442 LLAMA_API void llama_backend_init(void);443 444 // Call once at the end of the program - currently only used for MPI445 LLAMA_API void llama_backend_free(void);446 447 //optional:448 LLAMA_API void llama_numa_init(enum ggml_numa_strategy numa);449 450 // Optional: an auto threadpool gets created in ggml if not passed explicitly451 LLAMA_API void llama_attach_threadpool(452 struct llama_context * ctx,453 ggml_threadpool_t threadpool,454 ggml_threadpool_t threadpool_batch);455 456 LLAMA_API void llama_detach_threadpool(struct llama_context * ctx);457 458 typedef void (*llama_model_set_tensor_data_t)(struct ggml_tensor * tensor, void * userdata);459 460 // Create a new model from GGUF metadata as well as a function to set the tensor data461 // - tensors are created as GGML_TYPE_F32 by default,462 // override by adding a tensor with the same name but a different name to the context463 LLAMA_API struct llama_model * llama_model_init_from_user(464 struct gguf_context * metadata,465 llama_model_set_tensor_data_t set_tensor_data, // function to initialize tensor data with466 void * set_tensor_data_ud, // userdata for function467 struct llama_model_params params);468 469 DEPRECATED(LLAMA_API struct llama_model * llama_load_model_from_file(470 const char * path_model,471 struct llama_model_params params),472 "use llama_model_load_from_file instead");473 474 // Load a model from a file475 // If the file is split into multiple parts, the file name must follow this pattern: <name>-%05d-of-%05d.gguf476 // If the split file name does not follow this pattern, use llama_model_load_from_splits477 LLAMA_API struct llama_model * llama_model_load_from_file(478 const char * path_model,479 struct llama_model_params params);480 481 // Load a model from an open FILE pointer482 LLAMA_API struct llama_model * llama_model_load_from_file_ptr(483 FILE * file,484 struct llama_model_params params);485 486 // Load a model from multiple splits (support custom naming scheme)487 // The paths must be in the correct order488 LLAMA_API struct llama_model * llama_model_load_from_splits(489 const char ** paths,490 size_t n_paths,491 struct llama_model_params params);492 493 LLAMA_API void llama_model_save_to_file(494 const struct llama_model * model,495 const char * path_model);496 497 DEPRECATED(LLAMA_API void llama_free_model(struct llama_model * model),498 "use llama_model_free instead");499 500 LLAMA_API void llama_model_free(struct llama_model * model);501 502 LLAMA_API struct llama_context * llama_init_from_model(503 struct llama_model * model,504 struct llama_context_params params);505 506 DEPRECATED(LLAMA_API struct llama_context * llama_new_context_with_model(507 struct llama_model * model,508 struct llama_context_params params),509 "use llama_init_from_model instead");510 511 // Frees all allocated memory512 LLAMA_API void llama_free(struct llama_context * ctx);513 514 LLAMA_API int64_t llama_time_us(void);515 516 LLAMA_API size_t llama_max_devices(void);517 LLAMA_API size_t llama_max_parallel_sequences(void);518 LLAMA_API size_t llama_max_tensor_buft_overrides(void);519 520 LLAMA_API bool llama_supports_mmap (void);521 LLAMA_API bool llama_supports_mlock (void);522 LLAMA_API bool llama_supports_gpu_offload(void);523 LLAMA_API bool llama_supports_rpc (void);524 525 // NOTE: After creating a llama_context, it is recommended to query the actual values using these functions526 // In some cases the requested values via llama_context_params may differ from the actual values used by the context527 // ref: https://github.com/ggml-org/llama.cpp/pull/17046#discussion_r2503085732528 LLAMA_API uint32_t llama_n_ctx (const struct llama_context * ctx);529 LLAMA_API uint32_t llama_n_ctx_seq (const struct llama_context * ctx);530 LLAMA_API uint32_t llama_n_batch (const struct llama_context * ctx);531 LLAMA_API uint32_t llama_n_ubatch (const struct llama_context * ctx);532 LLAMA_API uint32_t llama_n_seq_max (const struct llama_context * ctx);533 534 DEPRECATED(LLAMA_API int32_t llama_n_ctx_train(const struct llama_model * model), "use llama_model_n_ctx_train instead");535 DEPRECATED(LLAMA_API int32_t llama_n_embd (const struct llama_model * model), "use llama_model_n_embd instead");536 DEPRECATED(LLAMA_API int32_t llama_n_layer (const struct llama_model * model), "use llama_model_n_layer instead");537 DEPRECATED(LLAMA_API int32_t llama_n_head (const struct llama_model * model), "use llama_model_n_head instead");538 539 DEPRECATED(LLAMA_API int32_t llama_n_vocab (const struct llama_vocab * vocab), "use llama_vocab_n_tokens instead");540 541 LLAMA_API const struct llama_model * llama_get_model (const struct llama_context * ctx);542 LLAMA_API llama_memory_t llama_get_memory (const struct llama_context * ctx);543 LLAMA_API enum llama_pooling_type llama_pooling_type(const struct llama_context * ctx); // TODO: rename to llama_get_pooling_type544 545 LLAMA_API const struct llama_vocab * llama_model_get_vocab(const struct llama_model * model);546 LLAMA_API enum llama_rope_type llama_model_rope_type(const struct llama_model * model);547 548 LLAMA_API int32_t llama_model_n_ctx_train(const struct llama_model * model);549 LLAMA_API int32_t llama_model_n_embd (const struct llama_model * model);550 LLAMA_API int32_t llama_model_n_embd_inp (const struct llama_model * model);551 LLAMA_API int32_t llama_model_n_embd_out (const struct llama_model * model);552 LLAMA_API int32_t llama_model_n_layer (const struct llama_model * model);553 LLAMA_API int32_t llama_model_n_head (const struct llama_model * model);554 LLAMA_API int32_t llama_model_n_head_kv (const struct llama_model * model);555 LLAMA_API int32_t llama_model_n_swa (const struct llama_model * model);556 557 // Get the model's RoPE frequency scaling factor558 LLAMA_API float llama_model_rope_freq_scale_train(const struct llama_model * model);559 560 // Returns the number of classifier outputs (only valid for classifier models)561 // Undefined behavior for non-classifier models562 LLAMA_API uint32_t llama_model_n_cls_out(const struct llama_model * model);563 564 // Returns label of classifier output by index (<n_cls_out). Returns nullptr if no label provided565 LLAMA_API const char * llama_model_cls_label(const struct llama_model * model, uint32_t i);566 567 LLAMA_API enum llama_vocab_type llama_vocab_type(const struct llama_vocab * vocab);568 569 LLAMA_API int32_t llama_vocab_n_tokens(const struct llama_vocab * vocab);570 571 // Functions to access the model's GGUF metadata scalar values572 // - The functions return the length of the string on success, or -1 on failure573 // - The output string is always null-terminated and cleared on failure574 // - When retrieving a string, an extra byte must be allocated to account for the null terminator575 // - GGUF array values are not supported by these functions576 577 // Get metadata value as a string by key name578 LLAMA_API int32_t llama_model_meta_val_str(const struct llama_model * model, const char * key, char * buf, size_t buf_size);579 580 // Get the number of metadata key/value pairs581 LLAMA_API int32_t llama_model_meta_count(const struct llama_model * model);582 583 // Get sampling metadata key name. Returns nullptr if the key is invalid584 LLAMA_API const char * llama_model_meta_key_str(enum llama_model_meta_key key);585 586 // Get metadata key name by index587 LLAMA_API int32_t llama_model_meta_key_by_index(const struct llama_model * model, int32_t i, char * buf, size_t buf_size);588 589 // Get metadata value as a string by index590 LLAMA_API int32_t llama_model_meta_val_str_by_index(const struct llama_model * model, int32_t i, char * buf, size_t buf_size);591 592 // Get a string describing the model type593 LLAMA_API int32_t llama_model_desc(const struct llama_model * model, char * buf, size_t buf_size);594 595 // Returns the total size of all the tensors in the model in bytes596 LLAMA_API uint64_t llama_model_size(const struct llama_model * model);597 598 // Get the default chat template. Returns nullptr if not available599 // If name is NULL, returns the default chat template600 LLAMA_API const char * llama_model_chat_template(const struct llama_model * model, const char * name);601 602 // Returns the total number of parameters in the model603 LLAMA_API uint64_t llama_model_n_params(const struct llama_model * model);604 605 // Returns true if the model contains an encoder that requires llama_encode() call606 LLAMA_API bool llama_model_has_encoder(const struct llama_model * model);607 608 // Returns true if the model contains a decoder that requires llama_decode() call609 LLAMA_API bool llama_model_has_decoder(const struct llama_model * model);610 611 // For encoder-decoder models, this function returns id of the token that must be provided612 // to the decoder to start generating output sequence. For other models, it returns -1.613 LLAMA_API llama_token llama_model_decoder_start_token(const struct llama_model * model);614 615 // Returns true if the model is recurrent (like Mamba, RWKV, etc.)616 LLAMA_API bool llama_model_is_recurrent(const struct llama_model * model);617 618 // Returns true if the model is hybrid (like Jamba, Granite, etc.)619 LLAMA_API bool llama_model_is_hybrid(const struct llama_model * model);620 621 // Returns true if the model is diffusion-based (like LLaDA, Dream, etc.)622 LLAMA_API bool llama_model_is_diffusion(const struct llama_model * model);623 624 // Returns 0 on success625 LLAMA_API uint32_t llama_model_quantize(626 const char * fname_inp,627 const char * fname_out,628 const llama_model_quantize_params * params);629 630 //631 // Adapters632 //633 634 // Load a LoRA adapter from file635 // The adapter is valid as long as the associated model is not freed636 LLAMA_API struct llama_adapter_lora * llama_adapter_lora_init(637 struct llama_model * model,638 const char * path_lora);639 640 // Functions to access the adapter's GGUF metadata scalar values641 // - The functions return the length of the string on success, or -1 on failure642 // - The output string is always null-terminated and cleared on failure643 // - When retrieving a string, an extra byte must be allocated to account for the null terminator644 // - GGUF array values are not supported by these functions645 646 // Get metadata value as a string by key name647 LLAMA_API int32_t llama_adapter_meta_val_str(const struct llama_adapter_lora * adapter, const char * key, char * buf, size_t buf_size);648 649 // Get the number of metadata key/value pairs650 LLAMA_API int32_t llama_adapter_meta_count(const struct llama_adapter_lora * adapter);651 652 // Get metadata key name by index653 LLAMA_API int32_t llama_adapter_meta_key_by_index(const struct llama_adapter_lora * adapter, int32_t i, char * buf, size_t buf_size);654 655 // Get metadata value as a string by index656 LLAMA_API int32_t llama_adapter_meta_val_str_by_index(const struct llama_adapter_lora * adapter, int32_t i, char * buf, size_t buf_size);657 658 // Manually free a LoRA adapter659 // NOTE: loaded adapters that are not manually freed will be freed when the associated model is deleted660 LLAMA_API void llama_adapter_lora_free(struct llama_adapter_lora * adapter);661 662 // Get the invocation tokens if the current lora is an alora663 LLAMA_API uint64_t llama_adapter_get_alora_n_invocation_tokens(const struct llama_adapter_lora * adapter);664 LLAMA_API const llama_token * llama_adapter_get_alora_invocation_tokens (const struct llama_adapter_lora * adapter);665 666 // The following functions operate on a llama_context, hence the naming: llama_verb_...667 668 // Set LoRa adapters on the context. Will only modify if the adapters currently in context are different.669 LLAMA_API int32_t llama_set_adapters_lora(670 struct llama_context * ctx,671 struct llama_adapter_lora ** adapters,672 size_t n_adapters,673 float * scales);674 675 // Apply a loaded control vector to a llama_context, or if data is NULL, clear676 // the currently loaded vector.677 // n_embd should be the size of a single layer's control, and data should point678 // to an n_embd x n_layers buffer starting from layer 1.679 // il_start and il_end are the layer range the vector should apply to (both inclusive)680 // See llama_control_vector_load in common to load a control vector.681 LLAMA_API int32_t llama_set_adapter_cvec(682 struct llama_context * ctx,683 const float * data,684 size_t len,685 int32_t n_embd,686 int32_t il_start,687 int32_t il_end);688 689 //690 // Memory691 //692 693 // Clear the memory contents694 // If data == true, the data buffers will also be cleared together with the metadata695 LLAMA_API void llama_memory_clear(696 llama_memory_t mem,697 bool data);698 699 // Removes all tokens that belong to the specified sequence and have positions in [p0, p1)700 // Returns false if a partial sequence cannot be removed. Removing a whole sequence never fails701 // seq_id < 0 : match any sequence702 // p0 < 0 : [0, p1]703 // p1 < 0 : [p0, inf)704 LLAMA_API bool llama_memory_seq_rm(705 llama_memory_t mem,706 llama_seq_id seq_id,707 llama_pos p0,708 llama_pos p1);709 710 // Copy all tokens that belong to the specified sequence to another sequence711 // p0 < 0 : [0, p1]712 // p1 < 0 : [p0, inf)713 LLAMA_API void llama_memory_seq_cp(714 llama_memory_t mem,715 llama_seq_id seq_id_src,716 llama_seq_id seq_id_dst,717 llama_pos p0,718 llama_pos p1);719 720 // Removes all tokens that do not belong to the specified sequence721 LLAMA_API void llama_memory_seq_keep(722 llama_memory_t mem,723 llama_seq_id seq_id);724 725 // Adds relative position "delta" to all tokens that belong to the specified sequence and have positions in [p0, p1)726 // p0 < 0 : [0, p1]727 // p1 < 0 : [p0, inf)728 LLAMA_API void llama_memory_seq_add(729 llama_memory_t mem,730 llama_seq_id seq_id,731 llama_pos p0,732 llama_pos p1,733 llama_pos delta);734 735 // Integer division of the positions by factor of `d > 1`736 // p0 < 0 : [0, p1]737 // p1 < 0 : [p0, inf)738 LLAMA_API void llama_memory_seq_div(739 llama_memory_t mem,740 llama_seq_id seq_id,741 llama_pos p0,742 llama_pos p1,743 int d);744 745 // Returns the smallest position present in the memory for the specified sequence746 // This is typically non-zero only for SWA caches747 // Note that all positions in the range [pos_min, pos_max] are guaranteed to be present in the memory748 // Return -1 if the sequence is empty749 LLAMA_API llama_pos llama_memory_seq_pos_min(750 llama_memory_t mem,751 llama_seq_id seq_id);752 753 // Returns the largest position present in the memory for the specified sequence754 // Note that all positions in the range [pos_min, pos_max] are guaranteed to be present in the memory755 // Return -1 if the sequence is empty756 LLAMA_API llama_pos llama_memory_seq_pos_max(757 llama_memory_t mem,758 llama_seq_id seq_id);759 760 // Check if the memory supports shifting761 LLAMA_API bool llama_memory_can_shift(llama_memory_t mem);762 763 //764 // State / sessions765 //766 767 // Returns the *actual* size in bytes of the state768 // (logits, embedding and memory)769 // Only use when saving the state, not when restoring it, otherwise the size may be too small.770 LLAMA_API size_t llama_state_get_size(struct llama_context * ctx);771 LLAMA_API DEPRECATED(size_t llama_get_state_size(struct llama_context * ctx),772 "use llama_state_get_size instead");773 774 // Copies the state to the specified destination address.775 // Destination needs to have allocated enough memory.776 // Returns the number of bytes copied777 LLAMA_API size_t llama_state_get_data(778 struct llama_context * ctx,779 uint8_t * dst,780 size_t size);781 LLAMA_API DEPRECATED(size_t llama_copy_state_data(782 struct llama_context * ctx,783 uint8_t * dst),784 "use llama_state_get_data instead");785 786 // Set the state reading from the specified address787 // Returns the number of bytes read788 LLAMA_API size_t llama_state_set_data(789 struct llama_context * ctx,790 const uint8_t * src,791 size_t size);792 LLAMA_API DEPRECATED(size_t llama_set_state_data(793 struct llama_context * ctx,794 const uint8_t * src),795 "use llama_state_set_data instead");796 797 // Save/load session file798 LLAMA_API bool llama_state_load_file(799 struct llama_context * ctx,800 const char * path_session,801 llama_token * tokens_out,802 size_t n_token_capacity,803 size_t * n_token_count_out);804 LLAMA_API DEPRECATED(bool llama_load_session_file(805 struct llama_context * ctx,806 const char * path_session,807 llama_token * tokens_out,808 size_t n_token_capacity,809 size_t * n_token_count_out),810 "use llama_state_load_file instead");811 812 LLAMA_API bool llama_state_save_file(813 struct llama_context * ctx,814 const char * path_session,815 const llama_token * tokens,816 size_t n_token_count);817 LLAMA_API DEPRECATED(bool llama_save_session_file(818 struct llama_context * ctx,819 const char * path_session,820 const llama_token * tokens,821 size_t n_token_count),822 "use llama_state_save_file instead");823 824 // Get the exact size needed to copy the state of a single sequence825 LLAMA_API size_t llama_state_seq_get_size(826 struct llama_context * ctx,827 llama_seq_id seq_id);828 829 // Copy the state of a single sequence into the specified buffer830 LLAMA_API size_t llama_state_seq_get_data(831 struct llama_context * ctx,832 uint8_t * dst,833 size_t size,834 llama_seq_id seq_id);835 836 // Copy the sequence data (originally copied with `llama_state_seq_get_data`) into the specified sequence837 // Returns:838 // - Positive: Ok839 // - Zero: Failed to load840 LLAMA_API size_t llama_state_seq_set_data(841 struct llama_context * ctx,842 const uint8_t * src,843 size_t size,844 llama_seq_id dest_seq_id);845 846 LLAMA_API size_t llama_state_seq_save_file(847 struct llama_context * ctx,848 const char * filepath,849 llama_seq_id seq_id,850 const llama_token * tokens,851 size_t n_token_count);852 853 LLAMA_API size_t llama_state_seq_load_file(854 struct llama_context * ctx,855 const char * filepath,856 llama_seq_id dest_seq_id,857 llama_token * tokens_out,858 size_t n_token_capacity,859 size_t * n_token_count_out);860 861// for backwards-compat862#define LLAMA_STATE_SEQ_FLAGS_SWA_ONLY 1863 864// work only with partial states, such as SWA KV cache or recurrent cache (e.g. Mamba)865#define LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY 1866 867 typedef uint32_t llama_state_seq_flags;868 869 LLAMA_API size_t llama_state_seq_get_size_ext(870 struct llama_context * ctx,871 llama_seq_id seq_id,872 llama_state_seq_flags flags);873 874 LLAMA_API size_t llama_state_seq_get_data_ext(875 struct llama_context * ctx,876 uint8_t * dst,877 size_t size,878 llama_seq_id seq_id,879 llama_state_seq_flags flags);880 881 LLAMA_API size_t llama_state_seq_set_data_ext(882 struct llama_context * ctx,883 const uint8_t * src,884 size_t size,885 llama_seq_id dest_seq_id,886 llama_state_seq_flags flags);887 888 //889 // Decoding890 //891 892 // Return batch for single sequence of tokens893 // The sequence ID will be fixed to 0894 // The position of the tokens will be tracked automatically by llama_decode895 //896 // NOTE: this is a helper function to facilitate transition to the new batch API - avoid using it897 //898 LLAMA_API struct llama_batch llama_batch_get_one(899 llama_token * tokens,900 int32_t n_tokens);901 902 // Allocates a batch of tokens on the heap that can hold a maximum of n_tokens903 // Each token can be assigned up to n_seq_max sequence ids904 // The batch has to be freed with llama_batch_free()905 // If embd != 0, llama_batch.embd will be allocated with size of n_tokens * embd * sizeof(float)906 // Otherwise, llama_batch.token will be allocated to store n_tokens llama_token907 // The rest of the llama_batch members are allocated with size n_tokens908 // All members are left uninitialized909 LLAMA_API struct llama_batch llama_batch_init(910 int32_t n_tokens,911 int32_t embd,912 int32_t n_seq_max);913 914 // Frees a batch of tokens allocated with llama_batch_init()915 LLAMA_API void llama_batch_free(struct llama_batch batch);916 917 // Process a batch of tokens.918 // In contrast to llama_decode() - this call does not use KV cache.919 // For encode-decoder contexts, processes the batch using the encoder.920 // Can store the encoder output internally for later use by the decoder's cross-attention layers.921 // 0 - success922 // < 0 - error. the memory state is restored to the state before this call923 LLAMA_API int32_t llama_encode(924 struct llama_context * ctx,925 struct llama_batch batch);926 927 // Process a batch of tokens.928 // Requires the context to have a memory.929 // For encode-decoder contexts, processes the batch using the decoder.930 // Positive return values does not mean a fatal error, but rather a warning.931 // Upon fatal-error or abort, the ubatches that managed to be been processed will remain in the memory state of the context932 // To handle this correctly, query the memory state using llama_memory_seq_pos_min() and llama_memory_seq_pos_max()933 // Upon other return values, the memory state is restored to the state before this call934 // 0 - success935 // 1 - could not find a KV slot for the batch (try reducing the size of the batch or increase the context)936 // 2 - aborted (processed ubatches will remain in the context's memory)937 // -1 - invalid input batch938 // < -1 - fatal error (processed ubatches will remain in the context's memory)939 LLAMA_API int32_t llama_decode(940 struct llama_context * ctx,941 struct llama_batch batch);942 943 // Set the number of threads used for decoding944 // n_threads is the number of threads used for generation (single token)945 // n_threads_batch is the number of threads used for prompt and batch processing (multiple tokens)946 LLAMA_API void llama_set_n_threads(struct llama_context * ctx, int32_t n_threads, int32_t n_threads_batch);947 948 // Get the number of threads used for generation of a single token.949 LLAMA_API int32_t llama_n_threads(struct llama_context * ctx);950 951 // Get the number of threads used for prompt and batch processing (multiple token).952 LLAMA_API int32_t llama_n_threads_batch(struct llama_context * ctx);953 954 // Set whether the context outputs embeddings or not955 // TODO: rename to avoid confusion with llama_get_embeddings()956 LLAMA_API void llama_set_embeddings(struct llama_context * ctx, bool embeddings);957 958 // Set whether to use causal attention or not959 // If set to true, the model will only attend to the past tokens960 LLAMA_API void llama_set_causal_attn(struct llama_context * ctx, bool causal_attn);961 962 // Set whether the model is in warmup mode or not963 // If true, all model tensors are activated during llama_decode() to load and cache their weights.964 LLAMA_API void llama_set_warmup(struct llama_context * ctx, bool warmup);965 966 // Set abort callback967 LLAMA_API void llama_set_abort_callback(struct llama_context * ctx, ggml_abort_callback abort_callback, void * abort_callback_data);968 969 // Wait until all computations are finished970 // This is automatically done when using one of the functions below to obtain the computation results971 // and is not necessary to call it explicitly in most cases972 LLAMA_API void llama_synchronize(struct llama_context * ctx);973 974 // Token logits obtained from the last call to llama_decode()975 // The logits for which llama_batch.logits[i] != 0 are stored contiguously976 // in the order they have appeared in the batch.977 // Rows: number of tokens for which llama_batch.logits[i] != 0978 // Cols: n_vocab979 // TODO: deprecate in favor of llama_get_logits_ith() (ref: https://github.com/ggml-org/llama.cpp/pull/14853#issuecomment-3113143522)980 LLAMA_API float * llama_get_logits(struct llama_context * ctx);981 982 // Logits for the ith token. For positive indices, Equivalent to:983 // llama_get_logits(ctx) + ctx->output_ids[i]*n_vocab984 // Negative indices can be used to access logits in reverse order, -1 is the last logit.985 // returns NULL for invalid ids.986 LLAMA_API float * llama_get_logits_ith(struct llama_context * ctx, int32_t i);987 988 // Get all output token embeddings.989 // when pooling_type == LLAMA_POOLING_TYPE_NONE or when using a generative model,990 // the embeddings for which llama_batch.logits[i] != 0 are stored contiguously991 // in the order they have appeared in the batch.992 // shape: [n_outputs*n_embd]993 // Otherwise, returns NULL.994 // TODO: deprecate in favor of llama_get_embeddings_ith() (ref: https://github.com/ggml-org/llama.cpp/pull/14853#issuecomment-3113143522)995 LLAMA_API float * llama_get_embeddings(struct llama_context * ctx);996 997 // Get the embeddings for the ith token. For positive indices, Equivalent to:998 // llama_get_embeddings(ctx) + ctx->output_ids[i]*n_embd999 // Negative indices can be used to access embeddings in reverse order, -1 is the last embedding.1000 // shape: [n_embd] (1-dimensional)1001 // returns NULL for invalid ids.1002 LLAMA_API float * llama_get_embeddings_ith(struct llama_context * ctx, int32_t i);1003 1004 // Get the embeddings for a sequence id1005 // Returns NULL if pooling_type is LLAMA_POOLING_TYPE_NONE1006 // when pooling_type == LLAMA_POOLING_TYPE_RANK, returns float[n_cls_out] with the rank(s) of the sequence1007 // otherwise: float[n_embd] (1-dimensional)1008 LLAMA_API float * llama_get_embeddings_seq(struct llama_context * ctx, llama_seq_id seq_id);1009 1010 //1011 // backend sampling API [EXPERIMENTAL]1012 // note: use only if the llama_context was created with at least one llama_sampler_seq_config1013 //1014 1015 // Get the backend sampled token for the ith token.1016 // Returns LLAMA_TOKEN_NULL if no token was sampled.1017 LLAMA_API llama_token llama_get_sampled_token_ith(struct llama_context * ctx, int32_t i);1018 1019 // Get the backend sampled probabilities for the ith token1020 // The index matches llama_get_sampled_token_ith().1021 // Returns NULL if no probabilities were generated.1022 LLAMA_API float * llama_get_sampled_probs_ith (struct llama_context * ctx, int32_t i);1023 LLAMA_API uint32_t llama_get_sampled_probs_count_ith(struct llama_context * ctx, int32_t i);1024 1025 // Get the backend sampled logits for the ith token1026 // Returns NULL if no logits were sampled.1027 LLAMA_API float * llama_get_sampled_logits_ith (struct llama_context * ctx, int32_t i);1028 LLAMA_API uint32_t llama_get_sampled_logits_count_ith(struct llama_context * ctx, int32_t i);1029 1030 // Get the backend sampled candidates (token ids) for the ith token1031 // These are needed to map probability/logit indices to vocab token ids.1032 // Returns NULL if no candidates were sampled.1033 LLAMA_API llama_token * llama_get_sampled_candidates_ith (struct llama_context * ctx, int32_t i);1034 LLAMA_API uint32_t llama_get_sampled_candidates_count_ith(struct llama_context * ctx, int32_t i);1035 1036 //1037 // Vocab1038 //1039 1040 LLAMA_API const char * llama_vocab_get_text(const struct llama_vocab * vocab, llama_token token);1041 1042 LLAMA_API float llama_vocab_get_score(const struct llama_vocab * vocab, llama_token token);1043 1044 LLAMA_API enum llama_token_attr llama_vocab_get_attr(const struct llama_vocab * vocab, llama_token token);1045 1046 // Check if the token is supposed to end generation (end-of-generation, eg. EOS, EOT, etc.)1047 LLAMA_API bool llama_vocab_is_eog(const struct llama_vocab * vocab, llama_token token);1048 1049 // Identify if Token Id is a control token or a render-able token1050 LLAMA_API bool llama_vocab_is_control(const struct llama_vocab * vocab, llama_token token);1051 1052 // Special tokens1053 LLAMA_API llama_token llama_vocab_bos(const struct llama_vocab * vocab); // beginning-of-sentence1054 LLAMA_API llama_token llama_vocab_eos(const struct llama_vocab * vocab); // end-of-sentence1055 LLAMA_API llama_token llama_vocab_eot(const struct llama_vocab * vocab); // end-of-turn1056 LLAMA_API llama_token llama_vocab_sep(const struct llama_vocab * vocab); // sentence separator1057 LLAMA_API llama_token llama_vocab_nl (const struct llama_vocab * vocab); // next-line1058 LLAMA_API llama_token llama_vocab_pad(const struct llama_vocab * vocab); // padding1059 LLAMA_API llama_token llama_vocab_mask(const struct llama_vocab * vocab); // mask1060 1061 LLAMA_API bool llama_vocab_get_add_bos(const struct llama_vocab * vocab);1062 LLAMA_API bool llama_vocab_get_add_eos(const struct llama_vocab * vocab);1063 LLAMA_API bool llama_vocab_get_add_sep(const struct llama_vocab * vocab);1064 1065 LLAMA_API llama_token llama_vocab_fim_pre(const struct llama_vocab * vocab);1066 LLAMA_API llama_token llama_vocab_fim_suf(const struct llama_vocab * vocab);1067 LLAMA_API llama_token llama_vocab_fim_mid(const struct llama_vocab * vocab);1068 LLAMA_API llama_token llama_vocab_fim_pad(const struct llama_vocab * vocab);1069 LLAMA_API llama_token llama_vocab_fim_rep(const struct llama_vocab * vocab);1070 LLAMA_API llama_token llama_vocab_fim_sep(const struct llama_vocab * vocab);1071 1072 DEPRECATED(LLAMA_API const char * llama_token_get_text(const struct llama_vocab * vocab, llama_token token), "use llama_vocab_get_text instead");1073 DEPRECATED(LLAMA_API float llama_token_get_score(const struct llama_vocab * vocab, llama_token token), "use llama_vocab_get_score instead");1074 DEPRECATED(LLAMA_API enum llama_token_attr llama_token_get_attr(const struct llama_vocab * vocab, llama_token token), "use llama_vocab_get_attr instead");1075 DEPRECATED(LLAMA_API bool llama_token_is_eog(const struct llama_vocab * vocab, llama_token token), "use llama_vocab_is_eog instead");1076 DEPRECATED(LLAMA_API bool llama_token_is_control(const struct llama_vocab * vocab, llama_token token), "use llama_vocab_is_control instead");1077 DEPRECATED(LLAMA_API llama_token llama_token_bos(const struct llama_vocab * vocab), "use llama_vocab_bos instead");1078 DEPRECATED(LLAMA_API llama_token llama_token_eos(const struct llama_vocab * vocab), "use llama_vocab_eos instead");1079 DEPRECATED(LLAMA_API llama_token llama_token_eot(const struct llama_vocab * vocab), "use llama_vocab_eot instead");1080 DEPRECATED(LLAMA_API llama_token llama_token_cls(const struct llama_vocab * vocab), "use llama_vocab_cls instead");1081 DEPRECATED(LLAMA_API llama_token llama_token_sep(const struct llama_vocab * vocab), "use llama_vocab_sep instead");1082 DEPRECATED(LLAMA_API llama_token llama_token_nl (const struct llama_vocab * vocab), "use llama_vocab_nl instead");1083 DEPRECATED(LLAMA_API llama_token llama_token_pad(const struct llama_vocab * vocab), "use llama_vocab_pad instead");1084 DEPRECATED(LLAMA_API bool llama_add_bos_token(const struct llama_vocab * vocab), "use llama_vocab_get_add_bos instead");1085 DEPRECATED(LLAMA_API bool llama_add_eos_token(const struct llama_vocab * vocab), "use llama_vocab_get_add_eos instead");1086 DEPRECATED(LLAMA_API llama_token llama_token_fim_pre(const struct llama_vocab * vocab), "use llama_vocab_fim_pre instead");1087 DEPRECATED(LLAMA_API llama_token llama_token_fim_suf(const struct llama_vocab * vocab), "use llama_vocab_fim_suf instead");1088 DEPRECATED(LLAMA_API llama_token llama_token_fim_mid(const struct llama_vocab * vocab), "use llama_vocab_fim_mid instead");1089 DEPRECATED(LLAMA_API llama_token llama_token_fim_pad(const struct llama_vocab * vocab), "use llama_vocab_fim_pad instead");1090 DEPRECATED(LLAMA_API llama_token llama_token_fim_rep(const struct llama_vocab * vocab), "use llama_vocab_fim_rep instead");1091 DEPRECATED(LLAMA_API llama_token llama_token_fim_sep(const struct llama_vocab * vocab), "use llama_vocab_fim_sep instead");1092 1093 // CLS is equivalent to BOS1094 DEPRECATED(LLAMA_API llama_token llama_vocab_cls(const struct llama_vocab * vocab), // classification1095 "use llama_vocab_bos instead");1096 1097 //1098 // Tokenization1099 //1100 // The API is thread-safe.1101 //1102 1103 /// @details Convert the provided text into tokens.1104 /// @param tokens The tokens pointer must be large enough to hold the resulting tokens.1105 /// @return Returns the number of tokens on success, no more than n_tokens_max1106 /// @return Returns a negative number on failure - the number of tokens that would have been returned1107 /// @return Returns INT32_MIN on overflow (e.g., tokenization result size exceeds int32_t limit)1108 /// @param add_special Allow to add BOS and EOS tokens if model is configured to do so.1109 /// @param parse_special Allow tokenizing special and/or control tokens which otherwise are not exposed and treated1110 /// as plaintext. Does not insert a leading space.1111 LLAMA_API int32_t llama_tokenize(1112 const struct llama_vocab * vocab,1113 const char * text,1114 int32_t text_len,1115 llama_token * tokens,1116 int32_t n_tokens_max,1117 bool add_special,1118 bool parse_special);1119 1120 // Token Id -> Piece.1121 // Uses the vocabulary in the provided context.1122 // Does not write null terminator to the buffer.1123 // User can skip up to 'lstrip' leading spaces before copying (useful when encoding/decoding multiple tokens with 'add_space_prefix')1124 // @param special If true, special tokens are rendered in the output.1125 LLAMA_API int32_t llama_token_to_piece(1126 const struct llama_vocab * vocab,1127 llama_token token,1128 char * buf,1129 int32_t length,1130 int32_t lstrip,1131 bool special);1132 1133 /// @details Convert the provided tokens into text (inverse of llama_tokenize()).1134 /// @param text The char pointer must be large enough to hold the resulting text.1135 /// @return Returns the number of chars/bytes on success, no more than text_len_max.1136 /// @return Returns a negative number on failure - the number of chars/bytes that would have been returned.1137 /// @param remove_special Allow to remove BOS and EOS tokens if model is configured to do so.1138 /// @param unparse_special If true, special tokens are rendered in the output.1139 LLAMA_API int32_t llama_detokenize(1140 const struct llama_vocab * vocab,1141 const llama_token * tokens,1142 int32_t n_tokens,1143 char * text,1144 int32_t text_len_max,1145 bool remove_special,1146 bool unparse_special);1147 1148 //1149 // Chat templates1150 //1151 1152 /// Apply chat template. Inspired by hf apply_chat_template() on python.1153 ///1154 /// NOTE: This function does not use a jinja parser. It only support a pre-defined list of template. See more: https://github.com/ggml-org/llama.cpp/wiki/Templates-supported-by-llama_chat_apply_template1155 /// @param tmpl A Jinja template to use for this chat.1156 /// @param chat Pointer to a list of multiple llama_chat_message1157 /// @param n_msg Number of llama_chat_message in this chat1158 /// @param add_ass Whether to end the prompt with the token(s) that indicate the start of an assistant message.1159 /// @param buf A buffer to hold the output formatted prompt. The recommended alloc size is 2 * (total number of characters of all messages)1160 /// @param length The size of the allocated buffer1161 /// @return The total number of bytes of the formatted prompt. If is it larger than the size of buffer, you may need to re-alloc it and then re-apply the template.1162 LLAMA_API int32_t llama_chat_apply_template(1163 const char * tmpl,1164 const struct llama_chat_message * chat,1165 size_t n_msg,1166 bool add_ass,1167 char * buf,1168 int32_t length);1169 1170 // Get list of built-in chat templates1171 LLAMA_API int32_t llama_chat_builtin_templates(const char ** output, size_t len);1172 1173 //1174 // Sampling API1175 //1176 // Sample usage:1177 //1178 // // prepare the sampling chain at the start1179 // auto sparams = llama_sampler_chain_default_params();1180 //1181 // llama_sampler * smpl = llama_sampler_chain_init(sparams);1182 //1183 // llama_sampler_chain_add(smpl, llama_sampler_init_top_k(50));1184 // llama_sampler_chain_add(smpl, llama_sampler_init_top_p(0.9, 1));1185 // llama_sampler_chain_add(smpl, llama_sampler_init_temp (0.8));1186 //1187 // // typically, the chain should end with a sampler such as "greedy", "dist" or "mirostat"1188 // // this sampler will be responsible to select the actual token1189 // llama_sampler_chain_add(smpl, llama_sampler_init_dist(seed));1190 //1191 // ...1192 //1193 // // decoding loop:1194 // while (...) {1195 // ...1196 //1197 // llama_decode(ctx, batch);1198 //1199 // // sample from the logits of the last token in the batch1200 // const llama_token id = llama_sampler_sample(smpl, ctx, -1);