Felipe97/llama-cpp-compiled
01.1k
1// Various helper functions and utilities2 3#pragma once4 5#include "llama-cpp.h"6 7#include "ggml-opt.h"8#include "ggml.h"9#include "llama.h"10 11#include <list>12#include <set>13#include <sstream>14#include <string>15#include <string_view>16#include <vector>17#include <map>18#include <algorithm>19#include <fstream>20 21#if defined(_WIN32) && !defined(_WIN32_WINNT)22#define _WIN32_WINNT 0x0A0023#endif24 25#ifdef _WIN3226#define DIRECTORY_SEPARATOR '\\'27#else28#define DIRECTORY_SEPARATOR '/'29#endif // _WIN3230 31#define COM_DBG(fmt, ...) LOG_DBG("cmn %12.*s: " fmt, 12, __func__, __VA_ARGS__)32#define COM_TRC(fmt, ...) LOG_TRC("cmn %12.*s: " fmt, 12, __func__, __VA_ARGS__)33#define COM_INF(fmt, ...) LOG_INF("cmn %12.*s: " fmt, 12, __func__, __VA_ARGS__)34#define COM_WRN(fmt, ...) LOG_WRN("cmn %12.*s: " fmt, 12, __func__, __VA_ARGS__)35#define COM_ERR(fmt, ...) LOG_ERR("cmn %12.*s: " fmt, 12, __func__, __VA_ARGS__)36#define COM_CNT(fmt, ...) LOG_CNT("" fmt, __VA_ARGS__)37 38#define die(msg) do { fputs("error: " msg "\n", stderr); exit(1); } while (0)39#define die_fmt(fmt, ...) do { fprintf(stderr, "error: " fmt "\n", __VA_ARGS__); exit(1); } while (0)40 41struct common_time_meas {42 common_time_meas(int64_t & t_acc, bool disable = false);43 ~common_time_meas();44 45 const int64_t t_start_us;46 47 int64_t & t_acc;48};49 50struct common_adapter_lora_info {51 std::string path;52 float scale;53 54 std::string task_name;55 std::string prompt_prefix;56 57 struct llama_adapter_lora * ptr;58};59 60using llama_tokens = std::vector<llama_token>;61 62struct common_control_vector_load_info;63 64//65// CPU utils66//67 68struct common_cpu_params {69 int n_threads = -1;70 bool cpumask[GGML_MAX_N_THREADS] = {false}; // CPU affinity mask.71 bool mask_valid = false; // Default: any CPU72 enum ggml_sched_priority priority = GGML_SCHED_PRIO_NORMAL; // Scheduling prio : (0 - normal, 1 - medium, 2 - high, 3 - realtime)73 bool strict_cpu = false; // Use strict CPU placement74 uint32_t poll = 50; // Polling (busywait) level (0 - no polling, 100 - mostly polling)75};76 77int32_t common_cpu_get_num_physical_cores();78int32_t common_cpu_get_num_math();79 80//81// Common params82//83 84enum llama_example {85 LLAMA_EXAMPLE_BATCHED,86 LLAMA_EXAMPLE_DEBUG,87 LLAMA_EXAMPLE_COMMON,88 LLAMA_EXAMPLE_SPECULATIVE,89 LLAMA_EXAMPLE_COMPLETION,90 LLAMA_EXAMPLE_CLI,91 LLAMA_EXAMPLE_EMBEDDING,92 LLAMA_EXAMPLE_PERPLEXITY,93 LLAMA_EXAMPLE_RETRIEVAL,94 LLAMA_EXAMPLE_PASSKEY,95 LLAMA_EXAMPLE_IMATRIX,96 LLAMA_EXAMPLE_BENCH,97 LLAMA_EXAMPLE_SERVER,98 LLAMA_EXAMPLE_CVECTOR_GENERATOR,99 LLAMA_EXAMPLE_EXPORT_LORA,100 LLAMA_EXAMPLE_MTMD,101 LLAMA_EXAMPLE_LOOKUP,102 LLAMA_EXAMPLE_PARALLEL,103 LLAMA_EXAMPLE_TTS,104 LLAMA_EXAMPLE_DIFFUSION,105 LLAMA_EXAMPLE_FINETUNE,106 LLAMA_EXAMPLE_FIT_PARAMS,107 LLAMA_EXAMPLE_RESULTS,108 LLAMA_EXAMPLE_EXPORT_GRAPH_OPS,109 LLAMA_EXAMPLE_DOWNLOAD,110 LLAMA_EXAMPLE_TOKENIZE,111 112 LLAMA_EXAMPLE_COUNT,113};114 115enum common_sampler_type {116 COMMON_SAMPLER_TYPE_NONE = 0,117 COMMON_SAMPLER_TYPE_DRY = 1,118 COMMON_SAMPLER_TYPE_TOP_K = 2,119 COMMON_SAMPLER_TYPE_TOP_P = 3,120 COMMON_SAMPLER_TYPE_MIN_P = 4,121 //COMMON_SAMPLER_TYPE_TFS_Z = 5,122 COMMON_SAMPLER_TYPE_TYPICAL_P = 6,123 COMMON_SAMPLER_TYPE_TEMPERATURE = 7,124 COMMON_SAMPLER_TYPE_XTC = 8,125 COMMON_SAMPLER_TYPE_INFILL = 9,126 COMMON_SAMPLER_TYPE_PENALTIES = 10,127 COMMON_SAMPLER_TYPE_TOP_N_SIGMA = 11,128 COMMON_SAMPLER_TYPE_ADAPTIVE_P = 12,129};130 131// dimensionality reduction methods, used by cvector-generator132enum dimre_method {133 DIMRE_METHOD_PCA,134 DIMRE_METHOD_MEAN,135};136 137enum common_conversation_mode {138 COMMON_CONVERSATION_MODE_DISABLED = 0,139 COMMON_CONVERSATION_MODE_ENABLED = 1,140 COMMON_CONVERSATION_MODE_AUTO = 2,141};142 143enum common_grammar_trigger_type {144 COMMON_GRAMMAR_TRIGGER_TYPE_TOKEN,145 COMMON_GRAMMAR_TRIGGER_TYPE_WORD,146 COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN,147 COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN_FULL,148};149 150struct common_grammar_trigger {151 common_grammar_trigger_type type;152 std::string value;153 llama_token token = LLAMA_TOKEN_NULL;154};155 156enum common_params_sampling_config : uint64_t {157 COMMON_PARAMS_SAMPLING_CONFIG_SAMPLERS = 1 << 0,158 COMMON_PARAMS_SAMPLING_CONFIG_TOP_K = 1 << 1,159 COMMON_PARAMS_SAMPLING_CONFIG_TOP_P = 1 << 2,160 COMMON_PARAMS_SAMPLING_CONFIG_MIN_P = 1 << 3,161 COMMON_PARAMS_SAMPLING_CONFIG_XTC_PROBABILITY = 1 << 4,162 COMMON_PARAMS_SAMPLING_CONFIG_XTC_THRESHOLD = 1 << 5,163 COMMON_PARAMS_SAMPLING_CONFIG_TEMP = 1 << 6,164 COMMON_PARAMS_SAMPLING_CONFIG_PENALTY_LAST_N = 1 << 7,165 COMMON_PARAMS_SAMPLING_CONFIG_PENALTY_REPEAT = 1 << 8,166 COMMON_PARAMS_SAMPLING_CONFIG_MIROSTAT = 1 << 9,167 COMMON_PARAMS_SAMPLING_CONFIG_MIROSTAT_TAU = 1 << 10,168 COMMON_PARAMS_SAMPLING_CONFIG_MIROSTAT_ETA = 1 << 11,169};170 171enum common_speculative_type {172 COMMON_SPECULATIVE_TYPE_NONE, // no speculative decoding173 COMMON_SPECULATIVE_TYPE_DRAFT_SIMPLE, // standalone draft model speculative decoding174 COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3, // Eagle3 speculative decoding175 COMMON_SPECULATIVE_TYPE_DRAFT_MTP, // Multi-token prediction176 COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH, // DFlash speculative decoding177 COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK, // DSpark speculative decoding (DFlash + Markov head)178 COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE, // simple self-speculative decoding based on n-grams179 COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K, // self-speculative decoding with n-gram keys only180 COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K4V, // self-speculative decoding with n-gram keys and 4 m-gram values181 COMMON_SPECULATIVE_TYPE_NGRAM_MOD,182 COMMON_SPECULATIVE_TYPE_NGRAM_CACHE, // self-speculative decoding with 3-level n-gram cache183 COMMON_SPECULATIVE_TYPE_COUNT // number of types, unknown type184};185 186// Grammar type enumeration187enum common_grammar_type {188 COMMON_GRAMMAR_TYPE_NONE, // no grammar set189 COMMON_GRAMMAR_TYPE_USER, // user-provided GBNF (--grammar / "grammar" API field)190 COMMON_GRAMMAR_TYPE_OUTPUT_FORMAT, // auto-generated from JSON schema (--json-schema / "json_schema" API field)191 COMMON_GRAMMAR_TYPE_TOOL_CALLS, // auto-generated by chat template parser for function calling192};193 194// Grammar variant struct with type and grammar string195struct common_grammar {196 common_grammar_type type = COMMON_GRAMMAR_TYPE_NONE;197 std::string grammar;198 199 // Default constructor - no grammar200 common_grammar() = default;201 202 // Constructor with type and grammar string203 common_grammar(common_grammar_type t, std::string g) : type(t), grammar(std::move(g)) {204 GGML_ASSERT(type != COMMON_GRAMMAR_TYPE_NONE || !grammar.empty());205 }206 207 // Check if a grammar is set208 bool empty() const { return type == COMMON_GRAMMAR_TYPE_NONE || grammar.empty(); }209};210 211// Returns the raw grammar string, or empty string if no grammar is set.212inline const std::string & common_grammar_value(const common_grammar & g) {213 return g.grammar;214}215 216// Returns true when the generation_prompt should be prefilled into the grammar sampler.217// Only output-format and tool-call grammars need prefill; user-supplied grammars must not be prefilled.218inline bool common_grammar_needs_prefill(const common_grammar & g) {219 return g.type == COMMON_GRAMMAR_TYPE_OUTPUT_FORMAT220 || g.type == COMMON_GRAMMAR_TYPE_TOOL_CALLS;221}222 223// sampling parameters224struct common_params_sampling {225 uint32_t seed = LLAMA_DEFAULT_SEED; // the seed used to initialize llama_sampler226 227 int32_t n_prev = 64; // number of previous tokens to remember228 int32_t n_probs = 0; // if greater than 0, output the probabilities of top n_probs tokens.229 int32_t min_keep = 0; // 0 = disabled, otherwise samplers should return at least min_keep tokens230 int32_t top_k = 40; // <= 0 to use vocab size231 float top_p = 0.95f; // 1.0 = disabled232 float min_p = 0.05f; // 0.0 = disabled233 float xtc_probability = 0.00f; // 0.0 = disabled234 float xtc_threshold = 0.10f; // > 0.5 disables XTC235 float typ_p = 1.00f; // typical_p, 1.0 = disabled236 float temp = 0.80f; // <= 0.0 to sample greedily, 0.0 to not output probabilities237 float dynatemp_range = 0.00f; // 0.0 = disabled238 float dynatemp_exponent = 1.00f; // controls how entropy maps to temperature in dynamic temperature sampler239 int32_t penalty_last_n = 64; // last n tokens to penalize (0 = disable penalty)240 float penalty_repeat = 1.00f; // 1.0 = disabled241 float penalty_freq = 0.00f; // 0.0 = disabled242 float penalty_present = 0.00f; // 0.0 = disabled243 float dry_multiplier = 0.0f; // 0.0 = disabled; DRY repetition penalty for tokens extending repetition:244 float dry_base = 1.75f; // 0.0 = disabled; multiplier * base ^ (length of sequence before token - allowed length)245 int32_t dry_allowed_length = 2; // tokens extending repetitions beyond this receive penalty246 int32_t dry_penalty_last_n = 64; // how many tokens to scan for repetitions (0 = disable penalty)247 float adaptive_target = -1.0f; // select tokens near this probability (valid range 0.0 to 1.0; negative = disabled)248 float adaptive_decay = 0.90f; // EMA decay for adaptation; history ≈ 1/(1-decay) tokens (0.0 - 0.99)249 int32_t mirostat = 0; // 0 = disabled, 1 = mirostat, 2 = mirostat 2.0250 float top_n_sigma = -1.00f; // -1.0 = disabled251 float mirostat_tau = 5.00f; // target entropy252 float mirostat_eta = 0.10f; // learning rate253 bool ignore_eos = false;254 bool no_perf = false; // disable performance metrics255 bool timing_per_token = false;256 257 uint64_t user_sampling_config = 0; // bitfield to track user-specified samplers258 259 std::vector<std::string> dry_sequence_breakers = {"\n", ":", "\"", "*"}; // default sequence breakers for DRY260 261 std::vector<enum common_sampler_type> samplers = {262 COMMON_SAMPLER_TYPE_PENALTIES,263 COMMON_SAMPLER_TYPE_DRY,264 COMMON_SAMPLER_TYPE_TOP_N_SIGMA,265 COMMON_SAMPLER_TYPE_TOP_K,266 COMMON_SAMPLER_TYPE_TYPICAL_P,267 COMMON_SAMPLER_TYPE_TOP_P,268 COMMON_SAMPLER_TYPE_MIN_P,269 COMMON_SAMPLER_TYPE_XTC,270 COMMON_SAMPLER_TYPE_TEMPERATURE,271 };272 273 common_grammar grammar; // optional grammar constraint (user / output-format / tool-calls)274 bool grammar_lazy = false;275 std::vector<common_grammar_trigger> grammar_triggers; // optional triggers (for lazy grammars)276 std::set<llama_token> preserved_tokens;277 278 std::vector<llama_logit_bias> logit_bias; // logit biases to apply279 std::vector<llama_logit_bias> logit_bias_eog; // pre-calculated logit biases for EOG tokens280 281 // The assistant generation prompt already prefilled into the prompt.282 // Fed to the grammar sampler (to advance past pre-existing tokens) and used283 // to determine the reasoning budget sampler's initial state.284 // Only applied when the grammar is of output-format or tool-calls type.285 std::string generation_prompt;286 287 // reasoning budget sampler parameters288 // these are populated by the server/CLI based on chat template params289 int32_t reasoning_budget_tokens = -1; // -1 = disabled, >= 0 = token budget290 std::vector<llama_token> reasoning_budget_start; // start tag token sequence291 std::vector<llama_tokens> reasoning_budget_end; // end tag token sequences; the first tag is used as the forcing sequence292 std::vector<llama_token> reasoning_budget_forced; // forced sequence (message + first end tag)293 std::string reasoning_budget_message; // message injected before end tag when budget exhausted294 bool reasoning_control = false; // create the budget sampler on demand so reasoning can be ended at runtime295 296 bool backend_sampling = false;297 298 // print the parameters into a string299 std::string print() const;300};301 302struct common_params_model {303 std::string path = ""; // model local path304 std::string url = ""; // model url to download305 std::string hf_repo = ""; // HF repo306 std::string hf_file = ""; // HF file307 std::string docker_repo = ""; // Docker repo308 309 std::string get_name() const {310 if (!hf_repo.empty()) {311 return hf_repo;312 }313 if (!docker_repo.empty()) {314 return docker_repo;315 }316 return path;317 }318 319 bool empty() const {320 return get_name().empty();321 }322};323 324// draft-model-based speculative decoding parameters325struct common_params_speculative_draft {326 int32_t n_max = 3; // maximum number of tokens to draft during speculative decoding327 int32_t n_min = 0; // minimum number of draft tokens to use for speculative decoding328 329 float p_split = 0.1f; // speculative decoding split probability330 float p_min = 0.0f; // minimum speculative decoding probability (greedy)331 332 bool backend_sampling = true; // offload draft sampling to the backend (default: on)333 334 common_params_model mparams;335 336 llama_context * ctx_tgt = nullptr;337 llama_context * ctx_dft = nullptr;338 339 int32_t n_gpu_layers = -1; // number of layers to store in VRAM for the draft model (-1 - use default)340 341 ggml_type cache_type_k = GGML_TYPE_F16; // KV cache data type for the K342 ggml_type cache_type_v = GGML_TYPE_F16; // KV cache data type for the V343 344 common_cpu_params cpuparams;345 common_cpu_params cpuparams_batch;346 347 std::vector<ggml_backend_dev_t> devices; // devices to use for offloading348 349 std::vector<llama_model_tensor_buft_override> tensor_buft_overrides;350};351 352struct common_params_speculative_ngram_mod {353 int32_t n_match = 24;354 355 int32_t n_max = 64;356 int32_t n_min = 48;357};358 359struct common_params_speculative_ngram_map {360 uint16_t size_n = 12; // ngram size for lookup361 uint16_t size_m = 48; // mgram size for speculative tokens362 uint16_t min_hits = 1; // minimum hits at ngram/mgram lookup for mgram to be proposed363};364 365struct common_params_speculative_ngram_cache {366 std::string lookup_cache_static; // path of static ngram cache file for lookup decoding367 std::string lookup_cache_dynamic; // path of dynamic ngram cache file for lookup decoding368};369 370struct common_params_speculative {371 std::vector<enum common_speculative_type> types = { COMMON_SPECULATIVE_TYPE_NONE };372 373 double synth_len = -1.0;374 std::vector<double> synth_rates;375 376 // used by Simple, MTP, Eagle3, etc. - all methods that require some kind of draft model377 common_params_speculative_draft draft;378 379 common_params_speculative_ngram_mod ngram_mod;380 common_params_speculative_ngram_map ngram_simple;381 common_params_speculative_ngram_map ngram_map_k;382 common_params_speculative_ngram_map ngram_map_k4v;383 384 common_params_speculative_ngram_cache ngram_cache;385 386 bool has_dft() const {387 return !draft.mparams.empty();388 }389 390 bool has_synth() const {391 return synth_len != -1.0 || !synth_rates.empty();392 }393 394 uint32_t need_n_rs_seq() const {395 bool needs_rs_seq = std::any_of(types.begin(), types.end(), [&](auto t) {396 return t == COMMON_SPECULATIVE_TYPE_DRAFT_MTP || t == COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3 || t == COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH || t == COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK;397 });398 399 return needs_rs_seq ? draft.n_max : 0u;400 }401};402 403struct common_params_diffusion {404 int32_t steps = 128;405 bool visual_mode = false;406 407 float eps = 0; // epsilon for timesteps408 int32_t block_length = 0; // block length for generation409 410 int32_t algorithm = 4; // default algorithm: low-confidence411 float alg_temp = 0.0f; // algorithm temperature412 413 float cfg_scale = 0; // classifier-free guidance scale414 bool add_gumbel_noise = false; // add gumbel noise to the logits if temp > 0.0415};416 417// reasoning API response format (not to be confused as chat template's reasoning format)418// only used by server419enum common_reasoning_format {420 COMMON_REASONING_FORMAT_NONE,421 COMMON_REASONING_FORMAT_AUTO, // Same as deepseek, using `message.reasoning_content`422 COMMON_REASONING_FORMAT_DEEPSEEK_LEGACY, // Extract thinking tag contents and return as `message.reasoning_content`, or leave inline in <think> tags in stream mode423 COMMON_REASONING_FORMAT_DEEPSEEK, // Extract thinking tag contents and return as `message.reasoning_content`, including in streaming deltas.424 // do not extend this enum unless you absolutely have to425 // in most cases, use COMMON_REASONING_FORMAT_AUTO426 // see: https://github.com/ggml-org/llama.cpp/pull/15408427};428 429 430struct lr_opt {431 float lr0 = 1e-5; // learning rate at first epoch432 float lr_min = -1;433 float decay_epochs = -1; // if >0, the learning rate starts at lr0 and decays to lr_min after this many epochs434 float scale_epoch = 0;435 float wd = 0;436 unsigned epochs = 2;437 438 unsigned epoch; // set by optimizer outer (epochs) loop439 // learning rate decay - constant LR per epoch only for now440 float get_lr(float e) const;441 float get_lr() const { return get_lr(epoch); }442 // must call after arg parse, before get_lr443 void init();444};445 446struct ggml_opt_optimizer_params common_opt_lr_pars(void * userdata);447 448struct common_params {449 int32_t n_predict = -1; // max. number of new tokens to predict, -1 == no limit450 int32_t n_ctx = 0; // context size, 0 == context the model was trained with451 int32_t n_batch = 2048; // logical batch size for prompt processing (must be >=32 to use BLAS)452 int32_t n_ubatch = 512; // physical batch size for prompt processing (must be >=32 to use BLAS)453 int32_t n_keep = 0; // number of tokens to keep from initial prompt454 int32_t n_chunks = -1; // max number of chunks to process (-1 = unlimited)455 int32_t n_parallel = 1; // number of parallel sequences to decode456 int32_t n_sequences = 1; // number of sequences to decode457 int32_t n_outputs_max = 0; // max outputs in a batch (0 = n_batch)458 int32_t n_outputs_max_per_seq = 1; // max outputs per sequence459 int32_t grp_attn_n = 1; // group-attention factor460 int32_t grp_attn_w = 512; // group-attention width461 int32_t n_print = -1; // print token count every n tokens (-1 = disabled)462 float rope_freq_base = 0.0f; // RoPE base frequency463 float rope_freq_scale = 0.0f; // RoPE frequency scaling factor464 float yarn_ext_factor = -1.0f; // YaRN extrapolation mix factor465 float yarn_attn_factor = -1.0f; // YaRN magnitude scaling factor466 float yarn_beta_fast = -1.0f; // YaRN low correction dim467 float yarn_beta_slow = -1.0f; // YaRN high correction dim468 int32_t yarn_orig_ctx = 0; // YaRN original context length469 470 // offload params471 std::vector<ggml_backend_dev_t> devices; // devices to use for offloading472 473 int32_t n_gpu_layers = -1; // number of layers to store in VRAM, -1 is auto, <= -2 is all474 int32_t main_gpu = 0; // the GPU that is used for scratch and small tensors475 float tensor_split[128] = {0}; // how split tensors should be distributed across GPUs476 bool fit_params = true; // whether to fit unset model/context parameters to free device memory477 bool fit_params_print = false; // print the estimated required memory to run the model478 int32_t fit_params_min_ctx = 4096; // minimum context size to set when trying to reduce memory use479 480 // margin per device in bytes for fitting parameters to free memory:481 std::vector<size_t> fit_params_target = std::vector<size_t>(llama_max_devices(), 1024 * 1024*1024);482 483 enum llama_split_mode split_mode = LLAMA_SPLIT_MODE_LAYER; // how to split the model across GPUs484 enum llama_load_mode load_mode = LLAMA_LOAD_MODE_AUTO; // how to load the model485 486 enum llama_lazy_mode lazy_mode = LLAMA_LAZY_MODE_AUTO; // on-demand reading of tensors marked by the arch487 488 common_cpu_params cpuparams;489 common_cpu_params cpuparams_batch;490 491 ggml_backend_sched_eval_callback cb_eval = nullptr;492 void * cb_eval_user_data = nullptr;493 494 ggml_numa_strategy numa = GGML_NUMA_STRATEGY_DISABLED;495 496 enum llama_rope_scaling_type rope_scaling_type = LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED;497 enum llama_pooling_type pooling_type = LLAMA_POOLING_TYPE_UNSPECIFIED; // pooling type for embeddings498 enum llama_attention_type attention_type = LLAMA_ATTENTION_TYPE_UNSPECIFIED; // attention type for embeddings499 enum llama_flash_attn_type flash_attn_type = LLAMA_FLASH_ATTN_TYPE_AUTO; // whether to use Flash Attention500 501 struct common_params_sampling sampling;502 struct common_params_speculative speculative;503 struct common_params_diffusion diffusion;504 505 struct common_params_model model;506 507 std::set<std::string> model_alias; // model aliases // NOLINT508 std::set<std::string> model_tags; // model tags (informational, not used for routing) // NOLINT509 std::string hf_token = ""; // HF token (aka bearer token) // NOLINT510 std::string prompt = ""; // NOLINT511 std::string system_prompt = ""; // NOLINT512 std::string prompt_file = ""; // store the external prompt file name // NOLINT513 std::string path_prompt_cache = ""; // path to file for saving/loading prompt eval state // NOLINT514 std::string input_prefix = ""; // string to prefix user inputs with // NOLINT515 std::string input_suffix = ""; // string to suffix user inputs with // NOLINT516 std::string logits_file = ""; // file for saving *all* logits // NOLINT517 std::string path_prompts_log_dir = ""; // directory with logged prompts // NOLINT518 519 // llama-debug specific options520 std::string logits_output_dir = "data"; // directory for saving logits output files // NOLINT521 bool save_logits = false; // whether to save logits to files // NOLINT522 std::vector<std::string> tensor_filter; // filter tensor names for debug output (regex) // NOLINT523 524 std::vector<std::string> in_files; // all input files525 std::vector<std::string> antiprompt; // strings upon which more user input is prompted (a.k.a. reverse prompts)526 std::vector<llama_model_kv_override> kv_overrides;527 std::vector<llama_model_tensor_buft_override> tensor_buft_overrides;528 529 bool lora_init_without_apply = false; // only load lora to memory, but do not apply it to ctx (user can manually apply lora later using llama_adapter_lora_apply)530 std::vector<common_adapter_lora_info> lora_adapters; // lora adapter path with user defined scale531 532 std::vector<common_control_vector_load_info> control_vectors; // control vector with user defined scale533 534 int32_t verbosity = 3; // LOG_LEVEL_INFO535 int32_t control_vector_layer_start = -1; // layer range for control vector536 int32_t control_vector_layer_end = -1; // layer range for control vector537 bool offline = false;538 539 int32_t ppl_stride = 0; // stride for perplexity calculations. If left at 0, the pre-existing approach will be used.540 int32_t ppl_output_type = 0; // = 0 -> ppl output is as usual, = 1 -> ppl output is num_tokens, ppl, one per line541 // (which is more convenient to use for plotting)542 //543 bool hellaswag = false; // compute HellaSwag score over random tasks from datafile supplied in prompt544 size_t hellaswag_tasks = 400; // number of tasks to use when computing the HellaSwag score545 546 bool winogrande = false; // compute Winogrande score over random tasks from datafile supplied in prompt547 size_t winogrande_tasks = 0; // number of tasks to use when computing the Winogrande score. If 0, all tasks will be computed548 549 bool multiple_choice = false; // compute TruthfulQA score over random tasks from datafile supplied in prompt550 size_t multiple_choice_tasks = 0; // number of tasks to use when computing the TruthfulQA score. If 0, all tasks will be computed551 552 bool kl_divergence = false; // compute KL divergence553 554 bool check = false; // check rather than generate results for llama-results555 556 bool usage = false; // print usage557 bool completion = false; // print source-able completion script558 bool use_color = false; // use color to distinguish generations and inputs559 bool special = false; // enable special token output560 bool interactive = false; // interactive mode561 bool interactive_first = false; // wait for user input immediately562 bool prompt_cache_all = false; // save user input and generations to prompt cache563 bool prompt_cache_ro = false; // open the prompt cache read-only and do not update it564 565 bool escape = true; // escape "\n", "\r", "\t", "\'", "\"", and "\\"566 bool multiline_input = false; // reverse the usage of `\`567 bool simple_io = false; // improves compatibility with subprocesses and limited consoles568 bool cont_batching = true; // insert new sequences for decoding on-the-fly569 bool no_perf = false; // disable performance metrics570 bool show_timings = true; // show timing information on CLI571 bool ctx_shift = false; // context shift on infinite text generation572 bool swa_full = false; // use full-size SWA cache (https://github.com/ggml-org/llama.cpp/pull/13194#issuecomment-2868343055)573 bool kv_unified = false; // enable unified KV cache574 575 bool input_prefix_bos = false; // prefix BOS to user inputs, preceding input_prefix576 bool verbose_prompt = false; // print prompt tokens before generation577 bool display_prompt = true; // print prompt before generation578 bool no_kv_offload = false; // disable KV offloading579 bool warmup = true; // warmup run580 bool check_tensors = false; // validate tensor data581 bool no_op_offload = false; // globally disable offload host tensor operations to device582 bool no_extra_bufts = false; // disable extra buffer types (used for weight repacking)583 bool no_host = false; // bypass host buffer allowing extra buffers to be used584 585 bool single_turn = false; // single turn chat conversation586 587 ggml_type cache_type_k = GGML_TYPE_F16; // KV cache data type for the K588 ggml_type cache_type_v = GGML_TYPE_F16; // KV cache data type for the V589 590 common_conversation_mode conversation_mode = COMMON_CONVERSATION_MODE_AUTO;591 592 // multimodal models (see tools/mtmd)593 struct common_params_model mmproj;594 bool mmproj_use_gpu = true; // use GPU for multimodal model595 ggml_backend_dev_t mmproj_device = nullptr; // GPU device to use for multimodal model596 bool no_mmproj = false; // explicitly disable multimodal model597 std::vector<std::string> image; // path to image file(s) ; TODO: change the name to "media"598 int image_min_tokens = -1;599 int image_max_tokens = -1;600 int mtmd_batch_max_tokens = 1024;601 602 // for video input603 float video_fps = 4.0f;604 int64_t video_timestamp_interval_ms = 5000;605 std::string video_ffmpeg_bin_dir = "";606 607 // finetune608 struct lr_opt lr;609 enum ggml_opt_optimizer_type optimizer = GGML_OPT_OPTIMIZER_TYPE_ADAMW;610 float val_split = 0.05f; // fraction of the data used for the validation set611 612 // embedding613 bool embedding = false; // get only sentence embedding614 int32_t embd_normalize = 2; // normalisation for embeddings (-1=none, 0=max absolute int16, 1=taxicab, 2=euclidean, >2=p-norm)615 std::string embd_out = ""; // empty = default, "array" = [[],[]...], "json" = openai style, "json+" = same "json" + cosine similarity matrix616 std::string embd_sep = "\n"; // separator of embeddings617 std::string cls_sep = "\t"; // separator of classification sequences618 619 // server params620 int32_t port = 8080; // server listens on this network port621 bool reuse_port = false; // allow multiple sockets to bind to the same port622 int32_t timeout_read = 3600; // http read timeout in seconds623 int32_t timeout_write = timeout_read; // http write timeout in seconds624 int32_t sse_ping_interval = 30; // SSE ping interval in seconds625 int32_t n_threads_http = -1; // number of threads to process HTTP requests (TODO: support threadpool)626 int32_t n_cache_reuse = 0; // min chunk size to reuse from the cache via KV shifting627 bool cache_prompt = true; // whether to enable prompt caching628 bool cache_idle_slots = true; // save and clear idle slots upon starting a new task629 int32_t n_ctx_checkpoints = 32; // max number of context checkpoints per slot630 int32_t kv_unified_per_slot = 0; // max context per parallel slot; 0 = unset631 int32_t checkpoint_min_step = 8192; // minimum spacing between context checkpoints632 int32_t cache_ram_mib = 8192; // -1 = no limit, 0 - disable, 1 = 1 MiB, etc.633 634 std::string hostname = "127.0.0.1";635 std::string public_path = ""; // NOLINT636 std::string api_prefix = ""; // NOLINT637 std::string chat_template = ""; // NOLINT638 bool use_jinja = true; // NOLINT639 640 // server CORS params641 std::string cors_origins = "*";642 std::string cors_methods = "GET, POST, DELETE, OPTIONS";643 std::string cors_headers = "*";644 bool cors_credentials = true;645 bool cors_origins_explicit = false; // for --agent option646 647 bool enable_chat_template = true;648 bool force_pure_content_parser = false;649 common_reasoning_format reasoning_format = COMMON_REASONING_FORMAT_DEEPSEEK;650 int enable_reasoning = -1; // -1 = auto, 0 = disable, 1 = enable651 bool prefill_assistant = true; // if true, any trailing assistant message will be prefilled into the response652 int sleep_idle_seconds = -1; // if >0, server will sleep after this many seconds of idle time653 654 std::vector<std::string> api_keys;655 656 std::string ssl_file_key = ""; // NOLINT657 std::string ssl_file_cert = ""; // NOLINT658 659 std::map<std::string, std::string> default_template_kwargs;660 bool preserve_reasoning_specified = false;661 662 // CLI params663 std::string server_base; // if set, connect to this server instead of starting a new one664 665 // UI configs666 bool ui = true;667 bool ui_mcp_proxy = false;668 std::string ui_config_json;669 670 // "advanced" endpoints are disabled by default for better security671 bool endpoint_slots = true;672 bool endpoint_props = false; // only control POST requests, not GET673 bool endpoint_metrics = false;674 675 // enable built-in tools676 std::vector<std::string> server_tools;677 std::string server_tools_runtime;678 679 // MCP server configs (Cursor-compatible JSON)680 std::string mcp_servers_config; // path to JSON file with MCP server definitions681 std::string mcp_servers_json; // inline JSON with MCP server definitions682 683 // router server configs684 std::string models_dir = ""; // directory containing models for the router server685 std::string models_preset = ""; // directory containing model presets for the router server686 int models_max = 4; // maximum number of models to load simultaneously687 bool models_autoload = true; // automatically load models when requested via the router server688 std::string models_preset_hf = ""; // show a warning about remote presets on router loaded (if not empty)689 690 bool log_json = false;691 692 std::string slot_save_path;693 std::string media_path; // path to directory for loading media files694 695 float slot_prompt_similarity = 0.1f;696 697 // batched-bench params698 bool is_pp_shared = false;699 bool is_tg_separate = false;700 701 std::vector<int32_t> n_pp;702 std::vector<int32_t> n_tg;703 std::vector<int32_t> n_pl;704 705 // retrieval params706 std::vector<std::string> context_files; // context files to embed707 708 int32_t chunk_size = 64; // chunk size for context embedding709 710 std::string chunk_separator = "\n"; // chunk separator for context embedding711 712 // passkey params713 int32_t n_junk = 250; // number of times to repeat the junk text714 int32_t i_pos = -1; // position of the passkey in the junk text715 716 // imatrix params717 int32_t n_out_freq = 10; // output the imatrix every n_out_freq iterations718 int32_t n_save_freq = 0; // save the imatrix every n_save_freq iterations719 int32_t i_chunk = 0; // start processing from this chunk720 int8_t imat_dat = 0; // whether the legacy imatrix.dat format should be output (gguf <= 0 < dat)721 722 bool process_output = false; // collect data for the output tensor723 bool compute_ppl = true; // whether to compute perplexity724 bool show_statistics = false; // show imatrix statistics per tensor725 bool parse_special = false; // whether to parse special tokens during imatrix tokenization726 727 // cvector-generator params728 int n_pca_batch = 100;729 int n_pca_iterations = 1000;730 dimre_method cvector_dimre_method = DIMRE_METHOD_PCA;731 std::string cvector_positive_file = "tools/cvector-generator/positive.txt";732 std::string cvector_negative_file = "tools/cvector-generator/negative.txt";733 734 bool spm_infill = false; // suffix/prefix/middle pattern for infill735 736 // batched-bench params737 bool batched_bench_output_jsonl = false;738 739 // tokenize params740 bool tokenize_ids = false; // if true, only print the token IDs741 bool tokenize_stdin = false; // if true, read the prompt from stdin742 bool tokenize_no_bos = false; // if true, do not add the BOS token743 bool tokenize_show_count = false; // if true, print the total token count744 745 // common params746 std::string out_file; // output filename for all example programs747 // optional callback for model loading progress and cancellation:748 // called with a progress value between 0.0 and 1.0.749 // return false from callback to abort model loading or true to continue750 llama_progress_callback load_progress_callback = NULL;751 void * load_progress_callback_user_data = NULL;752 bool no_alloc = false; // Don't allocate model buffers753 754 // TTS params755 std::string tts_lang = "";756 std::string tts_speaker_file = "";757 758 bool is_gen_docs = false; // whether we are running inside llama-gen-docs759};760 761// call once at the start of a program if it uses libcommon762// initializes the logging system and prints info about the build763void common_init();764 765void common_params_print_info(const common_params & params, bool print_devices = true);766std::string common_params_get_system_info(const common_params & params);767 768bool parse_cpu_range(const std::string & range, bool(&boolmask)[GGML_MAX_N_THREADS]);769bool parse_cpu_mask(const std::string & mask, bool(&boolmask)[GGML_MAX_N_THREADS]);770void postprocess_cpu_params(common_cpu_params & cpuparams, const common_cpu_params * role_model = nullptr);771bool set_process_priority(enum ggml_sched_priority prio);772 773//774// String utils775//776 777#ifdef __GNUC__778# if defined(__MINGW32__) && !defined(__clang__)779# define LLAMA_COMMON_ATTRIBUTE_FORMAT(...) __attribute__((format(gnu_printf, __VA_ARGS__)))780# else781# define LLAMA_COMMON_ATTRIBUTE_FORMAT(...) __attribute__((format(printf, __VA_ARGS__)))782# endif783#else784# define LLAMA_COMMON_ATTRIBUTE_FORMAT(...)785#endif786 787LLAMA_COMMON_ATTRIBUTE_FORMAT(1, 2)788std::string string_format(const char * fmt, ...);789 790std::string string_strip(const std::string & str);791std::string string_get_sortable_timestamp();792std::string string_lcs(std::string_view a, std::string_view b);793 794std::string string_join(const std::vector<std::string> & values, const std::string & separator);795std::vector<std::string> string_split(const std::string & str, const std::string & delimiter);796std::string string_repeat(const std::string & str, size_t n);797 798void string_replace_all(std::string & s, const std::string & search, const std::string & replace);799 800std::string regex_escape(const std::string & s);801 802template<class T>803static std::vector<T> string_split(const std::string & str, char delim) {804 static_assert(!std::is_same<T, std::string>::value, "Please use the specialized version for std::string");805 std::vector<T> values;806 std::istringstream str_stream(str);807 std::string token;808 while (std::getline(str_stream, token, delim)) {809 T value;810 std::istringstream token_stream(token);811 token_stream >> value;812 values.push_back(value);813 }814 return values;815}816 817template<>818inline std::vector<std::string> string_split<std::string>(const std::string & str, char delim)819{820 std::vector<std::string> parts;821 size_t begin_pos = 0;822 size_t delim_pos = str.find(delim);823 while (delim_pos != std::string::npos) {824 std::string part = str.substr(begin_pos, delim_pos - begin_pos);825 parts.emplace_back(part);826 begin_pos = delim_pos + 1;827 delim_pos = str.find(delim, begin_pos);828 }829 parts.emplace_back(str.substr(begin_pos));830 return parts;831}832 833// remove when moving to c++20834inline bool string_starts_with(std::string_view str, std::string_view prefix) {835 return str.size() >= prefix.size() &&836 str.compare(0, prefix.size(), prefix) == 0;837}838 839// remove when moving to c++20840inline bool string_starts_with(std::string_view str, char prefix) {841 return !str.empty() && str.front() == prefix;842}843 844// remove when moving to c++20845inline bool string_ends_with(std::string_view str, std::string_view suffix) {846 return str.size() >= suffix.size() &&847 str.compare(str.size() - suffix.size(), suffix.size(), suffix) == 0;848}849 850inline bool string_remove_suffix(std::string & str, std::string_view suffix) {851 if (string_ends_with(str, suffix)) {852 str.resize(str.size() - suffix.size());853 return true;854 }855 return false;856}857 858inline size_t string_find_partial_stop(std::string_view str, std::string_view stop) {859 if (!str.empty() && !stop.empty()) {860 const size_t max_len = std::min(str.size(), stop.size());861 const char last_char = str.back();862 for (size_t len = max_len; len > 0; --len) {863 if (stop[len - 1] == last_char) {864 if (string_ends_with(str, stop.substr(0, len))) {865 return str.size() - len;866 }867 }868 }869 }870 return std::string::npos;871}872 873bool string_parse_kv_override(const char * data, std::vector<llama_model_kv_override> & overrides);874void string_process_escapes(std::string & input);875 876std::string string_from(bool value);877std::string string_from(const std::vector<int> & values);878std::string string_from(const struct llama_context * ctx, const std::vector<llama_token> & tokens);879std::string string_from(const struct llama_context * ctx, const struct llama_batch & batch);880 881bool glob_match(const std::string & pattern, const std::string & str);882 883//884// Environment utils885//886 887// portable environment access, an unset variable reads as an empty string888// and setting an empty value unsets the variable889std::string common_get_env(const std::string & name);890void common_set_env(const std::string & name, const std::string & value);891 892//893// Filesystem utils894//895 896bool fs_validate_filename(const std::string & filename, bool allow_subdirs = false);897bool fs_create_directory_with_parents(const std::string & path);898bool fs_is_directory(const std::string & path);899 900std::string fs_get_cache_directory();901std::string fs_get_cache_file(const std::string & filename);902std::string fs_get_config_directory();903 904struct common_file_info {905 std::string path;906 std::string name;907 size_t size = 0; // in bytes908 bool is_dir = false;909};910std::vector<common_file_info> fs_list(const std::string & path, bool include_directories);911 912// fs open, also handle UTF8 on Windows913std::ifstream fs_open_ifstream(const std::string & fname, std::ios_base::openmode mode);914 915//916// TTY utils917//918 919// Auto-detect if colors can be enabled based on terminal and environment920bool tty_can_use_colors();921 922//923// Model utils924//925 926struct common_sampler;927 928// note: defines the model, context, samplers, ets. lifetimes929struct common_init_result {930 common_init_result(common_params & params, bool model_only = false);931 ~common_init_result();932 933 llama_model * model();934 llama_context * context();935 936 common_sampler * sampler(llama_seq_id seq_id);937 void reset_samplers();938 939 std::vector<llama_adapter_lora_ptr> & lora();940 941private:942 struct impl;943 std::unique_ptr<impl> pimpl;944};945 946using common_init_result_ptr = std::unique_ptr<common_init_result>;947 948common_init_result_ptr common_init_from_params(common_params & params, bool model_only = false);949 950struct llama_model_params common_model_params_to_llama ( common_params & params);951struct llama_context_params common_context_params_to_llama(const common_params & params);952 953// clear LoRA adapters from context, then apply new list of adapters954void common_set_adapter_lora(struct llama_context * ctx, std::vector<common_adapter_lora_info> & lora);955 956// model endpoint from env957std::string common_get_model_endpoint();958 959// for testing purposes960char * common_get_model_or_exit(int, char*[]);961 962//963// Threadpool utils964//965 966struct ggml_threadpool_params ggml_threadpool_params_from_cpu_params(const common_cpu_params & params);967 968struct common_threadpools {969 common_threadpools() = default;970 ~common_threadpools();971 972 common_threadpools(const common_threadpools &) = delete;973 common_threadpools & operator=(const common_threadpools &) = delete;974 975 void init(llama_context * ctx, const common_params & params);976 977private:978 ggml_threadpool * threadpool = nullptr;979 ggml_threadpool * threadpool_batch = nullptr;980 981 decltype(ggml_threadpool_free) * free_fn = nullptr;982};983 984//985// Context utils986//987 988enum common_context_seq_rm_type {989 COMMON_CONTEXT_SEQ_RM_TYPE_NO = 0, // seq_rm not supported (e.g. no memory module)990 COMMON_CONTEXT_SEQ_RM_TYPE_PART = 1, // can seq_rm partial sequences991 COMMON_CONTEXT_SEQ_RM_TYPE_FULL = 2, // can seq_rm full sequences only992 COMMON_CONTEXT_SEQ_RM_TYPE_RS = 3, // can seq_rm partial sequences, bounded by n_rs_seq993};994 995// check if the llama_context can remove sequences996// note: clears the memory of the context997common_context_seq_rm_type common_context_can_seq_rm(llama_context * ctx);998 999struct common_memory {1000 llama_context * ctx_tgt = nullptr;1001 llama_context * ctx_dft = nullptr;1002 1003 void init(llama_context * ctx_tgt, llama_context * ctx_dft = nullptr);1004 1005 // aborts execution on failure1006 void seq_rm (llama_seq_id seq_id, llama_pos p0, llama_pos p1) const;1007 void seq_add(llama_seq_id seq_id, llama_pos p0, llama_pos p1, llama_pos delta) const;1008 void seq_cp (llama_seq_id seq_id_src, llama_seq_id seq_id_dst, llama_pos p0, llama_pos p1) const;1009};1010 1011//1012// Batch utils1013//1014 1015void common_batch_clear(struct llama_batch & batch);1016 1017void common_batch_add(1018 struct llama_batch & batch,1019 llama_token id,1020 llama_pos pos,1021 const std::vector<llama_seq_id> & seq_ids,1022 bool logits);1023 1024// decodes a single batch of tokens for a prompt and manages session tokens1025//1026// Note: We save state before the last token so that we can replay it to ensure1027// compatibility with all memory types. Recurrent/hybrid models cannot remove1028// tokens from memory, so this approach works across all model architectures.1029bool common_prompt_batch_decode(1030 struct llama_context * ctx,1031 const std::vector<llama_token> & all_tokens,1032 int n_new,1033 int & n_past,1034 int n_batch,1035 std::string_view state_path,1036 bool save_state);1037 1038// replays the last token after loading state to regenerate logits1039// used after loading session state to ensure the sampling context has valid logits1040bool common_replay_last_token(struct llama_context * ctx, llama_token last_token, int32_t pos);1041 1042//1043// Vocab utils1044//1045 1046// tokenizes a string into a vector of tokens1047// should work similar to Python's `tokenizer.encode`1048std::vector<llama_token> common_tokenize(1049 const struct llama_context * ctx,1050 const std::string & text,1051 bool add_special,1052 bool parse_special = false);1053 1054std::vector<llama_token> common_tokenize(1055 const struct llama_vocab * vocab,1056 const std::string & text,1057 bool add_special,1058 bool parse_special = false);1059 1060// tokenizes a token into a piece, optionally renders special/control tokens1061// should work similar to Python's `tokenizer.id_to_piece`1062std::string common_token_to_piece(1063 const struct llama_context * ctx,1064 llama_token token,1065 bool special = true);1066 1067std::string common_token_to_piece(1068 const struct llama_vocab * vocab,1069 llama_token token,1070 bool special = true);1071 1072// detokenizes a vector of tokens into a string1073// should work similar to Python's `tokenizer.decode`1074// optionally renders special/control tokens1075std::string common_detokenize(1076 const struct llama_context * ctx,1077 const std::vector<llama_token> & tokens,1078 bool special = true);1079 1080std::string common_detokenize(1081 const struct llama_vocab * vocab,1082 const std::vector<llama_token> & tokens,1083 bool special = true);1084 1085//1086// Embedding utils1087//1088 1089// TODO: replace embd_norm with an enum1090void common_embd_normalize(const float * inp, float * out, int n, int embd_norm);1091 1092float common_embd_similarity_cos(const float * embd1, const float * embd2, int n);1093 1094//1095// Control vector utils1096//1097 1098struct common_control_vector_data {1099 int n_embd;1100 1101 // stores data for layers [1, n_layer] where n_layer = data.size() / n_embd1102 std::vector<float> data;1103};1104 1105struct common_control_vector_load_info {1106 float strength;1107 1108 std::string fname;1109};1110 1111// Load control vectors, scale each by strength, and add them together.1112// On error, returns {-1, empty}1113common_control_vector_data common_control_vector_load(const std::vector<common_control_vector_load_info> & load_infos);1114 1115//1116// Split utils1117//1118 1119namespace {1120 1121const char * const LLM_KV_SPLIT_NO = "split.no";1122const char * const LLM_KV_SPLIT_COUNT = "split.count";1123const char * const LLM_KV_SPLIT_TENSORS_COUNT = "split.tensors.count";1124 1125}1126 1127//1128// FFN offload utils1129//1130 1131const char * const LLM_FFN_EXPS_REGEX = "\\.ffn_(up|down|gate|gate_up)_(ch|)exps";1132 1133const char * const LLM_FFN_DENSE_REGEX = "\\.ffn_(up|down|gate)\\.";1134 1135inline std::string llm_ffn_block_regex(int idx, const char * ffn_regex) {1136 return string_format("blk\\.%d%s", idx, ffn_regex);1137}1138 1139inline llama_model_tensor_buft_override llm_ffn_exps_cpu_override() {1140 return { LLM_FFN_EXPS_REGEX, ggml_backend_cpu_buffer_type() };1141}1142 1143inline void llm_add_n_cpu_ffn_overrides(int n, const char * ffn_regex, std::vector<llama_model_tensor_buft_override> & overrides) {1144 // keep strings alive and avoid leaking memory by storing them in a static list1145 static std::list<std::string> buft_override_strings;1146 for (int i = 0; i < n; ++i) {1147 buft_override_strings.push_back(llm_ffn_block_regex(i, ffn_regex));1148 overrides.push_back({buft_override_strings.back().c_str(), ggml_backend_cpu_buffer_type()});1149 }1150}1151 1152//1153// training utils1154//1155 1156ggml_opt_dataset_t common_opt_dataset_init(struct llama_context * ctx, const std::vector<llama_token> & tokens, int64_t stride);1157 1158// "adamw" or "sgd" (case insensitive)1159enum ggml_opt_optimizer_type common_opt_get_optimizer(const char *);1160 1161//1162// prompt utils1163//1164 1165struct common_prompt_checkpoint {1166 int64_t n_tokens;1167 1168 // (optional) id of the task that created the checkpoint1169 int id_task = -1;1170 1171 llama_pos pos_min;1172 llama_pos pos_max;1173 1174 std::vector<uint8_t> data_tgt;1175 std::vector<uint8_t> data_dft;1176 1177 // (optional) speculative-decoding implementation state stashed with the checkpoint1178 // (e.g. eagle3's deferred-boundary g_embd row)1179 std::vector<uint8_t> data_spec;1180 1181 size_t size() const;1182 1183 bool empty() const;1184 void clear();1185 1186 void update_pos(1187 int64_t n_tokens,1188 llama_pos pos_min,1189 llama_pos pos_max);1190 1191 void update_tgt(1192 llama_context * ctx,1193 llama_seq_id seq_id,1194 llama_state_seq_flags flags);1195 1196 void update_dft(1197 llama_context * ctx,1198 llama_seq_id seq_id,1199 llama_state_seq_flags flags);1200