Felipe97/llama-cpp-compiled
01.1k
1#include "sampling.h"2 3#include "common.h"4#include "fit.h"5#include "log.h"6#include "reasoning-budget.h"7 8#include "ggml.h"9 10#include <algorithm>11#include <cctype>12#include <climits>13#include <cmath>14#include <cstring>15#include <unordered_map>16#include <vector>17 18// the ring buffer works similarly to std::deque, but with a fixed capacity19// TODO: deduplicate with llama-impl.h20template<typename T>21struct ring_buffer {22 ring_buffer(size_t cap) : capacity(cap), data(cap) {}23 24 T & front() {25 if (sz == 0) {26 throw std::runtime_error("ring buffer is empty");27 }28 return data[first];29 }30 31 const T & front() const {32 if (sz == 0) {33 throw std::runtime_error("ring buffer is empty");34 }35 return data[first];36 }37 38 T & back() {39 if (sz == 0) {40 throw std::runtime_error("ring buffer is empty");41 }42 return data[pos];43 }44 45 const T & back() const {46 if (sz == 0) {47 throw std::runtime_error("ring buffer is empty");48 }49 return data[pos];50 }51 52 void push_back(const T & value) {53 if (sz == capacity) {54 // advance the start when buffer is full55 first = (first + 1) % capacity;56 } else {57 sz++;58 }59 data[pos] = value;60 pos = (pos + 1) % capacity;61 }62 63 T pop_front() {64 if (sz == 0) {65 throw std::runtime_error("ring buffer is empty");66 }67 T value = data[first];68 first = (first + 1) % capacity;69 sz--;70 return value;71 }72 73 const T & rat(size_t i) const {74 if (i >= sz) {75 throw std::runtime_error("ring buffer: index out of bounds");76 }77 return data[(first + sz - i - 1) % capacity];78 }79 80 std::vector<T> to_vector() const {81 std::vector<T> result;82 result.reserve(sz);83 for (size_t i = 0; i < sz; i++) {84 result.push_back(data[(first + i) % capacity]);85 }86 return result;87 }88 89 void clear() {90 // here only reset the status of the buffer91 sz = 0;92 first = 0;93 pos = 0;94 }95 96 bool empty() const {97 return sz == 0;98 }99 100 size_t size() const {101 return sz;102 }103 104 size_t capacity = 0;105 size_t sz = 0;106 size_t first = 0;107 size_t pos = 0;108 std::vector<T> data;109};110 111struct common_sampler {112 common_params_sampling params;113 114 struct llama_sampler * grmr;115 struct llama_sampler * rbudget;116 struct llama_sampler * chain;117 118 ring_buffer<llama_token> prev;119 120 std::vector<llama_token_data> cur;121 122 llama_token_data_array cur_p;123 124 void reset() {125 prev.clear();126 127 llama_sampler_reset(chain);128 }129 130 void set_logits(struct llama_context * ctx, int idx) {131 const float * sampled_probs = llama_get_sampled_probs_ith (ctx, idx);132 const float * sampled_logits = llama_get_sampled_logits_ith (ctx, idx);133 const llama_token * sampled_ids = llama_get_sampled_candidates_ith(ctx, idx);134 135 const llama_model * model = llama_get_model(ctx);136 const llama_vocab * vocab = llama_model_get_vocab(model);137 138 const int n_vocab = llama_vocab_n_tokens(vocab);139 140 if (sampled_probs) {141 const uint32_t sampled_probs_count = llama_get_sampled_probs_count_ith(ctx, idx);142 cur.resize(sampled_probs_count);143 for (uint32_t i = 0; i < sampled_probs_count; ++i) {144 cur[i] = llama_token_data{sampled_ids[i], sampled_logits[i], sampled_probs[i]};145 }146 } else if (sampled_logits) {147 const uint32_t sampled_logits_count = llama_get_sampled_logits_count_ith(ctx, idx);148 cur.resize(sampled_logits_count);149 for (uint32_t i = 0; i < sampled_logits_count; i++) {150 cur[i] = llama_token_data{sampled_ids[i], sampled_logits[i], 0.0f};151 }152 } else {153 const auto * logits = llama_get_logits_ith(ctx, idx);154 GGML_ASSERT(logits != nullptr);155 cur.resize(n_vocab);156 for (llama_token token_id = 0; token_id < n_vocab; token_id++) {157 cur[token_id] = llama_token_data{token_id, logits[token_id], 0.0f};158 }159 }160 161 cur_p = { cur.data(), cur.size(), -1, false };162 }163 164 common_time_meas tm() {165 return common_time_meas(t_total_us, params.no_perf);166 }167 168 mutable int64_t t_total_us = 0;169};170 171std::string common_params_sampling::print() const {172 char result[1024];173 174 snprintf(result, sizeof(result),175 "\trepeat_last_n = %d, repeat_penalty = %.3f, frequency_penalty = %.3f, presence_penalty = %.3f\n"176 "\tdry_multiplier = %.3f, dry_base = %.3f, dry_allowed_length = %d, dry_penalty_last_n = %d\n"177 "\ttop_k = %d, top_p = %.3f, min_p = %.3f, xtc_probability = %.3f, xtc_threshold = %.3f, typical_p = %.3f, top_n_sigma = %.3f, temp = %.3f\n"178 "\tmirostat = %d, mirostat_lr = %.3f, mirostat_ent = %.3f, adaptive_target = %.3f, adaptive_decay = %.3f",179 penalty_last_n, penalty_repeat, penalty_freq, penalty_present,180 dry_multiplier, dry_base, dry_allowed_length, dry_penalty_last_n,181 top_k, top_p, min_p, xtc_probability, xtc_threshold, typ_p, top_n_sigma, temp,182 mirostat, mirostat_eta, mirostat_tau, adaptive_target, adaptive_decay);183 184 return std::string(result);185}186 187struct common_sampler * common_sampler_init(188 const struct llama_model * model,189 struct common_params_sampling & params) {190 if (!std::isfinite(params.penalty_repeat) ||191 params.penalty_repeat <= 0.0f ||192 !std::isfinite(1.0f/params.penalty_repeat)) {193 throw std::invalid_argument("penalty_repeat must be finite and greater than 0");194 }195 if (!std::isfinite(params.penalty_freq)) {196 throw std::invalid_argument("penalty_freq must be finite");197 }198 if (!std::isfinite(params.penalty_present)) {199 throw std::invalid_argument("penalty_present must be finite");200 }201 const llama_vocab * vocab = llama_model_get_vocab(model);202 llama_sampler_chain_params lparams = llama_sampler_chain_default_params();203 204 lparams.no_perf = params.no_perf;205 206 llama_sampler * grmr = nullptr;207 llama_sampler * rbudget = nullptr;208 llama_sampler * chain = llama_sampler_chain_init(lparams);209 210 std::vector<llama_sampler *> samplers;211 212 const std::string & grammar_str = common_grammar_value(params.grammar);213 if (grammar_str.compare(0, 11, "%llguidance") == 0) {214#ifdef LLAMA_USE_LLGUIDANCE215 grmr = llama_sampler_init_llg(vocab, "lark", grammar_str.c_str());216#else217 GGML_ABORT("llguidance (cmake -DLLAMA_LLGUIDANCE=ON) is not enabled");218#endif // LLAMA_USE_LLGUIDANCE219 } else {220 std::vector<std::string> trigger_patterns;221 std::vector<llama_token> trigger_tokens;222 for (const auto & trigger : params.grammar_triggers) {223 switch (trigger.type) {224 case COMMON_GRAMMAR_TRIGGER_TYPE_WORD:225 {226 const auto & word = trigger.value;227 trigger_patterns.push_back(regex_escape(word));228 break;229 }230 case COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN:231 {232 trigger_patterns.push_back(trigger.value);233 break;234 }235 case COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN_FULL:236 {237 const auto & pattern = trigger.value;238 std::string anchored = "^$";239 if (!pattern.empty()) {240 anchored = (pattern.front() != '^' ? "^" : "")241 + pattern242 + (pattern.back() != '$' ? "$" : "");243 }244 trigger_patterns.push_back(anchored);245 break;246 }247 case COMMON_GRAMMAR_TRIGGER_TYPE_TOKEN:248 {249 const auto token = trigger.token;250 trigger_tokens.push_back(token);251 break;252 }253 default:254 GGML_ASSERT(false && "unknown trigger type");255 }256 }257 258 std::vector<const char *> trigger_patterns_c;259 trigger_patterns_c.reserve(trigger_patterns.size());260 for (const auto & regex : trigger_patterns) {261 trigger_patterns_c.push_back(regex.c_str());262 }263 264 if (!grammar_str.empty()) {265 if (params.grammar_lazy) {266 grmr = llama_sampler_init_grammar_lazy_patterns(vocab, grammar_str.c_str(), "root",267 trigger_patterns_c.data(), trigger_patterns_c.size(),268 trigger_tokens.data(), trigger_tokens.size());269 } else {270 grmr = llama_sampler_init_grammar(vocab, grammar_str.c_str(), "root");271 }272 }273 }274 if (!grmr && !grammar_str.empty()) {275 throw std::runtime_error("failed to parse grammar");276 }277 278 // Compute prefill tokens from the generation prompt279 std::vector<llama_token> prefill_tokens;280 if (!params.generation_prompt.empty()) {281 GGML_ASSERT(vocab != nullptr);282 auto tokens = common_tokenize(vocab, params.generation_prompt, false, true);283 for (size_t i = 0; i < tokens.size(); i++) {284 std::string piece = common_token_to_piece(vocab, tokens[i], true);285 if (i == 0 && std::isspace(piece[0]) && !std::isspace(params.generation_prompt[0])) {286 // Some tokenizers will add a space before the first special token, need to exclude287 continue;288 }289 LOG_DBG("%s: prefill token: %d = %s\n", __func__, tokens[i], piece.c_str());290 prefill_tokens.push_back(tokens[i]);291 }292 }293 294 // Feed generation prompt tokens to the grammar sampler so it advances past295 // tokens the template already placed in the prompt.296 // Only applies to output-format and tool-call grammars; user-supplied grammars must not be prefilled.297 if (grmr && !params.grammar_lazy && common_grammar_needs_prefill(params.grammar)) {298 try {299 for (const auto & token : prefill_tokens) {300 llama_sampler_accept(grmr, token);301 LOG_DBG("%s: grammar accepted prefill token (%d)\n", __func__, token);302 }303 } catch (std::exception &e) {304 LOG_ERR("%s: error initializing grammar sampler for grammar:\n%s\n\nGeneration prompt:\n'%s'\n", __func__,305 common_grammar_value(params.grammar).c_str(), params.generation_prompt.c_str());306 throw e;307 }308 }309 310 // reasoning budget sampler (skip when budget is unlimited unless a lazy grammar is active, which needs rbudget for thinking-block suppression)311 if (!params.reasoning_budget_start.empty() && !params.reasoning_budget_end.empty() && (params.grammar_lazy || params.reasoning_budget_tokens >= 0 || params.reasoning_control)) {312 rbudget = common_reasoning_budget_init(313 vocab,314 {params.reasoning_budget_start},315 params.reasoning_budget_end,316 params.reasoning_budget_forced,317 params.reasoning_budget_tokens < 0 ? INT_MAX : params.reasoning_budget_tokens);318 319 for (const auto & token : prefill_tokens) {320 llama_sampler_accept(rbudget, token);321 LOG_DBG("%s: reasoning-budget accepted prefill token (%d)\n", __func__, token);322 }323 }324 325 // logit bias: user biases + model suppress tokens (-INFINITY)326 {327 std::vector<llama_logit_bias> merged = params.logit_bias;328 329 int32_t n_suppress = 0;330 const llama_token * suppress = llama_vocab_get_suppress_tokens(vocab, &n_suppress);331 for (int32_t i = 0; i < n_suppress; ++i) {332 merged.push_back({ suppress[i], -INFINITY });333 }334 335 if (!merged.empty()) {336 samplers.push_back(llama_sampler_init_logit_bias(llama_vocab_n_tokens(vocab), merged.size(), merged.data()));337 }338 }339 340 if (params.mirostat == 0) {341 342 bool use_adaptive_p = false; // see below343 344 for (const auto & cnstr : params.samplers) {345 switch (cnstr) {346 case COMMON_SAMPLER_TYPE_DRY:347 {348 std::vector<const char *> c_breakers;349 c_breakers.reserve(params.dry_sequence_breakers.size());350 for (const auto & str : params.dry_sequence_breakers) {351 c_breakers.push_back(str.c_str());352 }353 samplers.push_back(llama_sampler_init_dry(vocab, params.dry_multiplier, params.dry_base, params.dry_allowed_length, params.dry_penalty_last_n, c_breakers.data(), c_breakers.size()));354 }355 break;356 case COMMON_SAMPLER_TYPE_TOP_K:357 samplers.push_back(llama_sampler_init_top_k(params.top_k));358 break;359 case COMMON_SAMPLER_TYPE_TOP_P:360 samplers.push_back(llama_sampler_init_top_p(params.top_p, params.min_keep));361 break;362 case COMMON_SAMPLER_TYPE_TOP_N_SIGMA:363 samplers.push_back(llama_sampler_init_top_n_sigma(params.top_n_sigma));364 break;365 case COMMON_SAMPLER_TYPE_MIN_P:366 samplers.push_back(llama_sampler_init_min_p(params.min_p, params.min_keep));367 break;368 case COMMON_SAMPLER_TYPE_XTC:369 samplers.push_back(llama_sampler_init_xtc(params.xtc_probability, params.xtc_threshold, params.min_keep, params.seed));370 break;371 case COMMON_SAMPLER_TYPE_TYPICAL_P:372 samplers.push_back(llama_sampler_init_typical(params.typ_p, params.min_keep));373 break;374 case COMMON_SAMPLER_TYPE_TEMPERATURE:375 samplers.push_back(llama_sampler_init_temp_ext(params.temp, params.dynatemp_range, params.dynatemp_exponent));376 break;377 case COMMON_SAMPLER_TYPE_INFILL:378 samplers.push_back(llama_sampler_init_infill(vocab));379 break;380 case COMMON_SAMPLER_TYPE_PENALTIES:381 samplers.push_back(llama_sampler_init_penalties(llama_vocab_n_tokens(vocab), params.penalty_last_n, params.penalty_repeat, params.penalty_freq, params.penalty_present));382 break;383 case COMMON_SAMPLER_TYPE_ADAPTIVE_P:384 // the `adaptive-p` sampler is like `dist` and `mirostat` in that it selects385 // a single token, so we will add `dist` at the end of the chain by default,386 // unless the user specifically included `adaptive-p`. we set this flag here387 // so we know to add the sampler at the very end.388 use_adaptive_p = true;389 break;390 default:391 GGML_ASSERT(false && "unknown sampler type");392 }393 }394 if (use_adaptive_p) {395 // only if user explicitly included adaptive-p sampler396 samplers.push_back(llama_sampler_init_adaptive_p(params.adaptive_target, params.adaptive_decay, params.seed));397 } else {398 // default: sample from distribution399 samplers.push_back(llama_sampler_init_dist(params.seed));400 }401 } else if (params.mirostat == 1) {402 samplers.push_back(llama_sampler_init_temp(params.temp));403 samplers.push_back(llama_sampler_init_mirostat(llama_vocab_n_tokens(vocab), params.seed, params.mirostat_tau, params.mirostat_eta, 100));404 } else if (params.mirostat == 2) {405 samplers.push_back(llama_sampler_init_temp(params.temp));406 samplers.push_back(llama_sampler_init_mirostat_v2(params.seed, params.mirostat_tau, params.mirostat_eta));407 } else {408 GGML_ASSERT(false && "unknown mirostat version");409 }410 411 for (auto * smpl : samplers) {412 llama_sampler_chain_add(chain, smpl);413 }414 415 if (grmr && params.backend_sampling) {416 LOG_WRN("%s: backend sampling is not compatible with grammar, disabling\n", __func__);417 418 params.backend_sampling = false;419 }420 421 if (rbudget && params.backend_sampling) {422 LOG_WRN("%s: backend sampling is not compatible with reasoning budget, disabling\n", __func__);423 424 params.backend_sampling = false;425 }426 427 auto * result = new common_sampler {428 /* .params = */ params,429 /* .grmr = */ grmr,430 /* .rbudget = */ rbudget,431 /* .chain = */ chain,432 /* .prev = */ ring_buffer<llama_token>(std::max(32, params.n_prev)),433 /* .cur = */ {},434 /* .cur_p = */ {},435 };436 437 return result;438}439 440void common_sampler_free(struct common_sampler * gsmpl) {441 if (!gsmpl) {442 return;443 }444 445 llama_sampler_free(gsmpl->grmr);446 llama_sampler_free(gsmpl->rbudget);447 llama_sampler_free(gsmpl->chain);448 449 delete gsmpl;450}451 452static bool grammar_should_apply(struct common_sampler * gsmpl) {453 if (!gsmpl->grmr) {454 return false;455 }456 if (!gsmpl->rbudget) {457 return true;458 }459 if (gsmpl->params.grammar_lazy) {460 // if grammar is lazy, only apply when reasoning budget is not active461 const auto state = common_reasoning_budget_get_state(gsmpl->rbudget);462 return state == REASONING_BUDGET_IDLE || state == REASONING_BUDGET_DONE;463 }464 return true;465}466 467void common_sampler_accept(struct common_sampler * gsmpl, llama_token token, bool is_generated) {468 if (!gsmpl) {469 return;470 }471 472 const auto tm = gsmpl->tm();473 474 // grammar_should_apply() checks the reasoning budget state, so calculate this before we accept475 const auto accept_grammar = is_generated && grammar_should_apply(gsmpl);476 477 if (gsmpl->rbudget && is_generated) {478 llama_sampler_accept(gsmpl->rbudget, token);479 480 // if done, replay end sequence which may contain a grammar trigger481 const bool is_done = common_reasoning_budget_get_state(gsmpl->rbudget) == REASONING_BUDGET_DONE;482 if (gsmpl->grmr && !accept_grammar && is_done) {483 const llama_tokens * end_seq = common_reasoning_budget_get_end_match(gsmpl->rbudget);484 if (end_seq) {485 for (const llama_token end_token : *end_seq) {486 llama_sampler_accept(gsmpl->grmr, end_token);487 }488 }489 }490 }491 492 if (gsmpl->grmr && accept_grammar) {493 llama_sampler_accept(gsmpl->grmr, token);494 }495 496 llama_sampler_accept(gsmpl->chain, token);497 498 gsmpl->prev.push_back(token);499}500 501void common_sampler_reset(struct common_sampler * gsmpl) {502 if (!gsmpl) {503 return;504 }505 506 gsmpl->reset();507}508 509struct common_sampler * common_sampler_clone(common_sampler * gsmpl) {510 return new common_sampler {511 /* .params = */ gsmpl->params,512 /* .grmr = */ llama_sampler_clone(gsmpl->grmr),513 /* .rbudget = */ llama_sampler_clone(gsmpl->rbudget),514 /* .chain = */ llama_sampler_clone(gsmpl->chain),515 /* .prev = */ gsmpl->prev,516 /* .cur = */ gsmpl->cur,517 /* .cur_p = */ gsmpl->cur_p,518 };519}520 521void common_sampler_copy(const common_sampler * src, common_sampler * dst) {522 if (!src || !dst || src == dst) {523 return;524 }525 526 GGML_ASSERT((src->grmr == nullptr) == (dst->grmr == nullptr));527 GGML_ASSERT((src->rbudget == nullptr) == (dst->rbudget == nullptr));528 529 llama_sampler_copy(src->grmr, dst->grmr);530 llama_sampler_copy(src->rbudget, dst->rbudget);531 llama_sampler_copy(src->chain, dst->chain);532 533 dst->params = src->params;534 dst->prev = src->prev;535 dst->cur = src->cur;536 dst->cur_p = src->cur_p;537 dst->cur_p.data = src->cur_p.data ? dst->cur.data() : nullptr; // re-point to dst's buffer538 dst->t_total_us = src->t_total_us;539}540 541void common_perf_print(const struct llama_context * ctx, const struct common_sampler * gsmpl) {542 // TODO: measure grammar performance543 544 const double t_sampling_ms = gsmpl ? 1e-3*gsmpl->t_total_us : 0;545 546 llama_perf_sampler_data data_smpl;547 llama_perf_context_data data_ctx;548 549 memset(&data_smpl, 0, sizeof(data_smpl));550 memset(&data_ctx, 0, sizeof(data_ctx));551 552 if (gsmpl) {553 auto & data = data_smpl;554 555 data = llama_perf_sampler(gsmpl->chain);556 557 // note: the sampling time includes the samplers time + extra time spent in common/sampling558 LOG_INF("%s: sampling time = %10.2f ms\n", __func__, t_sampling_ms);559 LOG_INF("%s: samplers time = %10.2f ms / %5d tokens\n", __func__, data.t_sample_ms, data.n_sample);560 }561 562 if (ctx) {563 auto & data = data_ctx;564 565 data = llama_perf_context(ctx);566 567 const double t_end_ms = 1e-3 * ggml_time_us();568 569 const double t_total_ms = t_end_ms - data.t_start_ms;570 const double t_unacc_ms = t_total_ms - (t_sampling_ms + data.t_p_eval_ms + data.t_eval_ms);571 const double t_unacc_pc = 100.0 * t_unacc_ms / t_total_ms;572 573 LOG_INF("%s: load time = %10.2f ms\n", __func__, data.t_load_ms);574 LOG_INF("%s: prompt eval time = %10.2f ms / %5d tokens (%8.2f ms per token, %8.2f tokens per second)\n",575 __func__, data.t_p_eval_ms, data.n_p_eval, data.t_p_eval_ms / data.n_p_eval, 1e3 / data.t_p_eval_ms * data.n_p_eval);576 LOG_INF("%s: eval time = %10.2f ms / %5d runs (%8.2f ms per token, %8.2f tokens per second)\n",577 __func__, data.t_eval_ms, data.n_eval, data.t_eval_ms / data.n_eval, 1e3 / data.t_eval_ms * data.n_eval);578 LOG_INF("%s: total time = %10.2f ms / %5d tokens\n", __func__, (t_end_ms - data.t_start_ms), (data.n_p_eval + data.n_eval));579 LOG_INF("%s: unaccounted time = %10.2f ms / %5.1f %% (total - sampling - prompt eval - eval) / (total)\n", __func__, t_unacc_ms, t_unacc_pc);580 LOG_INF("%s: graphs reused = %10d\n", __func__, data.n_reused);581 582 common_memory_breakdown_print(ctx);583 }584}585 586struct llama_sampler * common_sampler_get(const struct common_sampler * gsmpl) {587 if (!gsmpl) {588 return nullptr;589 }590 591 return gsmpl->chain;592}593 594llama_token common_sampler_sample(struct common_sampler * gsmpl, struct llama_context * ctx, int idx, bool grammar_first) {595 llama_synchronize(ctx);596 597 // start measuring sampling time after the llama_context synchronization in order to not measure any ongoing async operations598 const auto tm = gsmpl->tm();599 600 llama_token id = LLAMA_TOKEN_NULL;601 602 auto & grmr = gsmpl->grmr;603 auto & rbudget = gsmpl->rbudget;604 auto & chain = gsmpl->chain;605 auto & cur_p = gsmpl->cur_p; // initialized by set_logits606 607 gsmpl->set_logits(ctx, idx);608 609 // Check if a backend sampler has already sampled a token in which case we610 // return that token id directly.611 {612 id = llama_get_sampled_token_ith(ctx, idx);613 614 if (id != LLAMA_TOKEN_NULL) {615 LOG_DBG("%s: Backend sampler selected token: '%d'. Will not run any CPU samplers\n", __func__, id);616 617 GGML_ASSERT(!gsmpl->grmr && "using grammar in combination with backend sampling is not supported");618 GGML_ASSERT(!gsmpl->rbudget && "using reasoning budget in combination with backend sampling is not supported");619 620 for (size_t i = 0; i < cur_p.size; ++i) {621 if (cur_p.data[i].id == id) {622 cur_p.selected = i;623 break;624 }625 }626 627 return id;628 }629 }630 631 // apply reasoning budget first632 llama_sampler_apply(rbudget, &cur_p);633 634 if (grammar_first && grammar_should_apply(gsmpl)) {635 llama_sampler_apply(grmr, &cur_p);636 }637 638 llama_sampler_apply(chain, &cur_p);639 640 id = cur_p.data[cur_p.selected].id;641 642 if (grammar_first || !grammar_should_apply(gsmpl)) {643 return id;644 }645 646 // check if it the sampled token fits the grammar (grammar-based rejection sampling)647 {648 llama_token_data single_token_data = { id, 1.0f, 0.0f };649 llama_token_data_array single_token_data_array = { &single_token_data, 1, -1, false };650 651 llama_sampler_apply(grmr, &single_token_data_array);652 653 const bool is_valid = single_token_data_array.data[0].logit != -INFINITY;654 if (is_valid) {655 return id;656 }657 }658 659 // resampling:660 // if the token is not valid, sample again, but first apply the grammar sampler and then the sampling chain661 gsmpl->set_logits(ctx, idx);662 663 llama_sampler_apply(rbudget, &cur_p);664 665 if (grammar_should_apply(gsmpl)) {666 llama_sampler_apply(grmr, &cur_p);667 }668 669 llama_sampler_apply(chain, &cur_p);670 671 GGML_ASSERT(cur_p.selected != -1 && "no selected token during sampling - check your sampling configuration");672 673 id = cur_p.data[cur_p.selected].id;674 675 return id;676}677 678std::vector<llama_token> common_sampler_sample_and_accept_n(struct common_sampler * gsmpl, struct llama_context * ctx, const std::vector<int> & idxs, const llama_tokens & draft, bool grammar_first) {679 GGML_ASSERT(idxs.size() == draft.size() + 1 && "idxs.size() must be draft.size() + 1");680 681 std::vector<llama_token> result;682 result.reserve(idxs.size());683 684 size_t i = 0;685 for (; i < draft.size(); i++) {686 const llama_token id = common_sampler_sample(gsmpl, ctx, idxs[i], grammar_first);687 688 common_sampler_accept(gsmpl, id, true);689 690 result.push_back(id);691 692 if (draft[i] != id) {693 break;694 }695 }696 697 if (i == draft.size()) {698 const llama_token id = common_sampler_sample(gsmpl, ctx, idxs[i], grammar_first);699 700 common_sampler_accept(gsmpl, id, true);701 702 result.push_back(id);703 }704 705 return result;706}707 708std::vector<llama_token> common_sampler_sample_and_accept_n(struct common_sampler * gsmpl, struct llama_context * ctx, const llama_tokens & draft, bool grammar_first) {709 std::vector<int> idxs(draft.size() + 1);710 for (size_t i = 0; i < idxs.size(); ++i) {711 idxs[i] = i;712 }713 714 return common_sampler_sample_and_accept_n(gsmpl, ctx, idxs, draft, grammar_first);715}716 717uint32_t common_sampler_get_seed(const struct common_sampler * gsmpl) {718 return llama_sampler_get_seed(gsmpl->chain);719}720 721bool common_sampler_reasoning_budget_force(struct common_sampler * gsmpl) {722 if (!gsmpl) {723 return false;724 }725 726 return common_reasoning_budget_force(gsmpl->rbudget);727}728 729// helpers730 731llama_token_data_array * common_sampler_get_candidates(struct common_sampler * gsmpl, bool do_sort) {732 const auto tm = gsmpl->tm();733 734 auto * res = &gsmpl->cur_p;735 736 if (do_sort && !res->sorted) {737 // remember the selected token before sorting738 const llama_token id = res->data[res->selected].id;739 740 std::sort(res->data, res->data + res->size, [](const llama_token_data & a, const llama_token_data & b) {741 return a.p > b.p;742 });743 744 // restore the selected token after sorting745 for (size_t i = 0; i < res->size; ++i) {746 if (res->data[i].id == id) {747 res->selected = i;748 break;749 }750 }751 752 res->sorted = true;753 }754 755 return res;756}757 758llama_token common_sampler_last(const struct common_sampler * gsmpl) {759 return gsmpl->prev.rat(0);760}761 762std::string common_sampler_print(const struct common_sampler * gsmpl) {763 std::string result = "logits ";764 765 for (int i = 0; i < llama_sampler_chain_n(gsmpl->chain); i++) {766 const auto * smpl = llama_sampler_chain_get(gsmpl->chain, i);767 result += std::string("-> ");768 result += std::string(llama_sampler_name(smpl)) + " ";769 }770 771 return result;772}773 774std::string common_sampler_prev_str(common_sampler * gsmpl, llama_context * ctx_main, int n) {775 n = std::min(n, (int) gsmpl->prev.size());776 777 if (n <= 0) {778 return "";779 }780 781 std::string result;782 result.reserve(8*n); // 8 is the average length of a token [citation needed], TODO: compute this from the vocab783 784 for (int i = n - 1; i >= 0; i--) {785 const llama_token id = gsmpl->prev.rat(i);786 787 GGML_ASSERT(id != LLAMA_TOKEN_NULL && "null token in the sampling history - should not happen");788 789 result += common_token_to_piece(ctx_main, id);790 }791 792 return result;793}794 795char common_sampler_type_to_chr(enum common_sampler_type cnstr) {796 switch (cnstr) {797 case COMMON_SAMPLER_TYPE_DRY: return 'd';798 case COMMON_SAMPLER_TYPE_TOP_K: return 'k';799 case COMMON_SAMPLER_TYPE_TYPICAL_P: return 'y';800 case COMMON_SAMPLER_TYPE_TOP_P: return 'p';801 case COMMON_SAMPLER_TYPE_TOP_N_SIGMA: return 's';802 case COMMON_SAMPLER_TYPE_MIN_P: return 'm';803 case COMMON_SAMPLER_TYPE_TEMPERATURE: return 't';804 case COMMON_SAMPLER_TYPE_XTC: return 'x';805 case COMMON_SAMPLER_TYPE_INFILL: return 'i';806 case COMMON_SAMPLER_TYPE_PENALTIES: return 'e';807 case COMMON_SAMPLER_TYPE_ADAPTIVE_P: return 'a';808 default : return '?';809 }810}811 812std::string common_sampler_type_to_str(enum common_sampler_type cnstr) {813 switch (cnstr) {814 case COMMON_SAMPLER_TYPE_DRY: return "dry";815 case COMMON_SAMPLER_TYPE_TOP_K: return "top_k";816 case COMMON_SAMPLER_TYPE_TYPICAL_P: return "typ_p";817 case COMMON_SAMPLER_TYPE_TOP_P: return "top_p";818 case COMMON_SAMPLER_TYPE_TOP_N_SIGMA: return "top_n_sigma";819 case COMMON_SAMPLER_TYPE_MIN_P: return "min_p";820 case COMMON_SAMPLER_TYPE_TEMPERATURE: return "temperature";821 case COMMON_SAMPLER_TYPE_XTC: return "xtc";822 case COMMON_SAMPLER_TYPE_INFILL: return "infill";823 case COMMON_SAMPLER_TYPE_PENALTIES: return "penalties";824 case COMMON_SAMPLER_TYPE_ADAPTIVE_P: return "adaptive_p";825 default : return "";826 }827}828 829std::vector<common_sampler_type> common_sampler_types_from_names(const std::vector<std::string> & names) {830 // sampler names can be written multiple ways; generate aliases from canonical names831 static const auto sampler_name_map = []{832 // canonical sampler name mapping833 std::unordered_map<std::string, common_sampler_type> canonical_name_map {834 { "dry", COMMON_SAMPLER_TYPE_DRY },835 { "top_k", COMMON_SAMPLER_TYPE_TOP_K },836 { "top_p", COMMON_SAMPLER_TYPE_TOP_P },837 { "top_n_sigma", COMMON_SAMPLER_TYPE_TOP_N_SIGMA },838 { "typ_p", COMMON_SAMPLER_TYPE_TYPICAL_P },839 { "min_p", COMMON_SAMPLER_TYPE_MIN_P },840 { "temperature", COMMON_SAMPLER_TYPE_TEMPERATURE },841 { "xtc", COMMON_SAMPLER_TYPE_XTC },842 { "infill", COMMON_SAMPLER_TYPE_INFILL },843 { "penalties", COMMON_SAMPLER_TYPE_PENALTIES },844 { "adaptive_p", COMMON_SAMPLER_TYPE_ADAPTIVE_P }845 };846 std::unordered_map<std::string, common_sampler_type> alias_name_map;847 for (const auto & entry : canonical_name_map) {848 const std::string & canonical = entry.first;849 if (canonical.find('_') == std::string::npos) {850 continue;851 }852 // kebab-case: "top-k", "min-p", etc.853 {854 std::string kebab_case = canonical;855 std::replace(kebab_case.begin(), kebab_case.end(), '_', '-');856 alias_name_map.insert({kebab_case, entry.second});857 }858 // no dash: "topk", "minp", etc.859 {860 std::string no_dash = canonical;861 no_dash.erase(std::remove(no_dash.begin(), no_dash.end(), '_'), no_dash.end());862 alias_name_map.insert({no_dash, entry.second});863 }864 }865 // misc. aliases866 alias_name_map.insert({"nucleus", COMMON_SAMPLER_TYPE_TOP_P});867 alias_name_map.insert({"temp", COMMON_SAMPLER_TYPE_TEMPERATURE});868 alias_name_map.insert({"typ", COMMON_SAMPLER_TYPE_TYPICAL_P});869 // include aliases + canonical names in the complete mapping870 alias_name_map.merge(canonical_name_map);871 return alias_name_map;872 }();873 874 std::vector<common_sampler_type> samplers;875 samplers.reserve(names.size());876 877 for (const auto & name : names) {878 std::string name_lower = name;879 std::transform(name_lower.begin(), name_lower.end(), name_lower.begin(), ::tolower);880 auto sampler = sampler_name_map.find(name_lower);881 if (sampler != sampler_name_map.end()) {882 samplers.push_back(sampler->second);883 continue;884 }885 LOG_WRN("%s: unable to match sampler by name '%s'\n", __func__, name_lower.c_str());886 }887 888 return samplers;889}890 891std::vector<common_sampler_type> common_sampler_types_from_chars(const std::string & chars) {892 std::unordered_map<char, common_sampler_type> sampler_name_map = {893 { common_sampler_type_to_chr(COMMON_SAMPLER_TYPE_DRY), COMMON_SAMPLER_TYPE_DRY },894 { common_sampler_type_to_chr(COMMON_SAMPLER_TYPE_TOP_K), COMMON_SAMPLER_TYPE_TOP_K },895 { common_sampler_type_to_chr(COMMON_SAMPLER_TYPE_TYPICAL_P), COMMON_SAMPLER_TYPE_TYPICAL_P },896 { common_sampler_type_to_chr(COMMON_SAMPLER_TYPE_TOP_P), COMMON_SAMPLER_TYPE_TOP_P },897 { common_sampler_type_to_chr(COMMON_SAMPLER_TYPE_TOP_N_SIGMA), COMMON_SAMPLER_TYPE_TOP_N_SIGMA },898 { common_sampler_type_to_chr(COMMON_SAMPLER_TYPE_MIN_P), COMMON_SAMPLER_TYPE_MIN_P },899 { common_sampler_type_to_chr(COMMON_SAMPLER_TYPE_TEMPERATURE), COMMON_SAMPLER_TYPE_TEMPERATURE },900 { common_sampler_type_to_chr(COMMON_SAMPLER_TYPE_XTC), COMMON_SAMPLER_TYPE_XTC },901 { common_sampler_type_to_chr(COMMON_SAMPLER_TYPE_INFILL), COMMON_SAMPLER_TYPE_INFILL },902 { common_sampler_type_to_chr(COMMON_SAMPLER_TYPE_PENALTIES), COMMON_SAMPLER_TYPE_PENALTIES },903 { common_sampler_type_to_chr(COMMON_SAMPLER_TYPE_ADAPTIVE_P), COMMON_SAMPLER_TYPE_ADAPTIVE_P },904 };905 906 std::vector<common_sampler_type> samplers;907 samplers.reserve(chars.size());908 909 for (const auto & c : chars) {910 const auto sampler = sampler_name_map.find(c);911 if (sampler != sampler_name_map.end()) {912 samplers.push_back(sampler->second);913 } else {914 LOG_WRN("%s: unable to match sampler by char '%c'\n", __func__, c);915 }916 }917 918 return samplers;919}920