Felipe97/llama-cpp-compiled
01.1k
1#include "ggml.h"2#include "llama.h"3#include "llama-cpp.h"4#include "common.h"5 6#ifdef NDEBUG7#undef NDEBUG8#endif9 10#include <algorithm>11#include <cmath>12#include <cstdlib>13#include <cstring>14#include <fstream>15#include <functional>16#include <map>17#include <random>18#include <string>19#include <unordered_map>20#include <unordered_set>21#include <vector>22 23struct test_args {24 std::string model;25 std::string test;26 std::string device = "auto";27};28 29struct test_params {30 llama_model_ptr model;31};32 33static llama_model_ptr load_model(const test_args & args) {34 auto mparams = llama_model_default_params();35 36 ggml_backend_dev_t devs[2] = { nullptr, nullptr };37 38 if (args.device != "auto") {39 if (args.device == "gpu") {40 devs[0] = ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_GPU);41 42 if (devs[0] == nullptr) {43 fprintf(stderr, "Error: GPU requested but not available\n");44 return nullptr;45 }46 47 mparams.n_gpu_layers = 999;48 } else if (args.device == "cpu") {49 devs[0] = ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_CPU);50 51 mparams.n_gpu_layers = 0;52 } else {53 fprintf(stderr, "Error: invalid device '%s'\n", args.device.c_str());54 return nullptr;55 }56 57 mparams.devices = devs;58 59 fprintf(stderr, "Using device: %s\n", ggml_backend_dev_name(devs[0]));60 }61 62 llama_model_ptr res;63 64 res.reset(llama_model_load_from_file(args.model.c_str(), mparams));65 66 if (!res) {67 fprintf(stderr, "Warning: failed to load model '%s', skipping test\n", args.model.c_str());68 return nullptr;69 }70 71 return res;72}73 74struct test_context {75 llama_context_ptr ctx;76 77 int n_vocab = 0;78 79 const llama_vocab * vocab = nullptr;80 81 std::unordered_map<llama_seq_id, int32_t> seq_positions;82 std::unordered_map<llama_seq_id, int32_t> last_batch_info;83 84 test_context(85 const test_params & params,86 std::vector<llama_sampler_seq_config> & configs,87 int32_t n_seq_max = -1,88 uint32_t n_outputs_max = 0,89 uint32_t n_ubatch = 0,90 uint32_t n_outputs_max_per_seq = 1) {91 auto * model = params.model.get();92 93 GGML_ASSERT(model);94 GGML_ASSERT(!ctx);95 96 llama_context_params cparams = llama_context_default_params();97 cparams.n_ctx = 512;98 cparams.n_batch = 512;99 if (n_ubatch > 0) {100 cparams.n_ubatch = n_ubatch;101 }102 cparams.n_outputs_max = n_outputs_max;103 cparams.n_outputs_max_per_seq = n_outputs_max_per_seq;104 cparams.samplers = configs.data();105 cparams.n_samplers = configs.size();106 cparams.kv_unified = true;107 108 // If n_seq_max is not specified, calculate it from configs109 if (n_seq_max < 0) {110 int32_t max_seq_id = 0;111 for (const auto & config : configs) {112 max_seq_id = std::max(config.seq_id, max_seq_id);113 }114 cparams.n_seq_max = max_seq_id + 1;115 } else {116 cparams.n_seq_max = n_seq_max;117 }118 119 ctx.reset(llama_init_from_model(model, cparams));120 if (!ctx) {121 throw std::runtime_error("failed to create context");122 }123 124 vocab = llama_model_get_vocab(model);125 n_vocab = llama_vocab_n_tokens(vocab);126 }127 128 bool decode(const std::map<llama_seq_id, std::string> & prompts) {129 GGML_ASSERT(ctx);130 131 last_batch_info.clear();132 llama_batch batch = llama_batch_init(512, 0, prompts.size());133 134 for (const auto & [seq_id, prompt] : prompts) {135 std::vector<llama_token> tokens;136 tokens.push_back(llama_vocab_bos(vocab));137 138 std::vector<llama_token> prompt_tokens(32);139 int n_tokens = llama_tokenize(vocab, prompt.c_str(), prompt.length(),140 prompt_tokens.data(), prompt_tokens.size(),141 false, false);142 if (n_tokens < 0) {143 fprintf(stderr, "Warning: tokenization failed for seq_id %d\n", seq_id);144 llama_batch_free(batch);145 return false;146 }147 148 for (int i = 0; i < n_tokens; i++) {149 tokens.push_back(prompt_tokens[i]);150 }151 152 if (seq_positions.find(seq_id) == seq_positions.end()) {153 seq_positions[seq_id] = 0;154 }155 156 int32_t start_pos = seq_positions[seq_id];157 for (size_t i = 0; i < tokens.size(); i++) {158 common_batch_add(batch, tokens[i], start_pos + i, { seq_id }, i == tokens.size() - 1);159 }160 161 seq_positions[seq_id] = start_pos + tokens.size();162 }163 164 165 printf("Batch contents:\n");166 printf("n_tokens: %d\n", batch.n_tokens);167 for (int i = 0; i < batch.n_tokens; i++) {168 printf("token[%d]: tok=%-5d, pos=%d, n_seq_id=%d, seq_ids=[", i, batch.token[i], batch.pos[i], batch.n_seq_id[i]);169 170 for (int j = 0; j < batch.n_seq_id[i]; j++) {171 printf("%d%s", batch.seq_id[i][j], j < batch.n_seq_id[i]-1 ? ", " : "");172 }173 printf("], logits=%d\n", batch.logits[i]);174 }175 176 if (llama_decode(ctx.get(), batch) != 0) {177 fprintf(stderr, "Warning: llama_decode failed\n");178 llama_batch_free(batch);179 return false;180 }181 182 // Build mapping from seq id to batch token idx183 for (int i = 0; i < batch.n_tokens; i++) {184 if (batch.logits[i]) {185 llama_seq_id seq_id = batch.seq_id[i][0];186 last_batch_info[seq_id] = i;187 }188 }189 190 llama_batch_free(batch);191 return true;192 }193 194 int32_t idx_for_seq(llama_seq_id seq_id) {195 auto it = last_batch_info.find(seq_id);196 if (it == last_batch_info.end()) {197 fprintf(stderr, "Error: no batch index found for seq_id %d\n", seq_id);198 return -1;199 }200 return it->second;201 }202 203 void update_batch_info(const llama_batch & batch) {204 last_batch_info.clear();205 for (int i = 0; i < batch.n_tokens; i++) {206 if (batch.logits[i]) {207 llama_seq_id cur_seq = batch.seq_id[i][0];208 last_batch_info[cur_seq] = i;209 }210 }211 }212 213 bool decode_token(llama_token token, llama_seq_id seq_id = 0) {214 GGML_ASSERT(ctx);215 216 llama_batch batch = llama_batch_init(1, 0, 1);217 int32_t pos = seq_positions[seq_id];218 common_batch_add(batch, token, pos, { seq_id }, true);219 220 if (llama_decode(ctx.get(), batch) != 0) {221 fprintf(stderr, "Warning: llama_decode failed for token %d in seq %d\n", token, seq_id);222 llama_batch_free(batch);223 return false;224 }225 226 update_batch_info(batch);227 228 seq_positions[seq_id]++;229 llama_batch_free(batch);230 231 return true;232 }233 234 bool decode_tokens(const std::map<llama_seq_id, llama_token> & seq_tokens) {235 GGML_ASSERT(ctx);236 237 llama_batch batch = llama_batch_init(seq_tokens.size(), 0, seq_tokens.size());238 239 for (const auto & [seq_id, token] : seq_tokens) {240 int32_t pos = seq_positions[seq_id];241 common_batch_add(batch, token, pos, { seq_id }, true);242 }243 244 if (llama_decode(ctx.get(), batch) != 0) {245 fprintf(stderr, "Warning: llama_decode failed for batch tokens\n");246 llama_batch_free(batch);247 return false;248 }249 250 for (const auto & [seq_id, _] : seq_tokens) {251 seq_positions[seq_id]++;252 }253 254 update_batch_info(batch);255 256 llama_batch_free(batch);257 258 return true;259 }260 261 std::string token_to_piece(llama_token token, bool special) const {262 std::string piece;263 piece.resize(piece.capacity()); // using string internal cache, 15 bytes + '\n'264 const int n_chars = llama_token_to_piece(vocab, token, &piece[0], piece.size(), 0, special);265 if (n_chars < 0) {266 piece.resize(-n_chars);267 int check = llama_token_to_piece(vocab, token, &piece[0], piece.size(), 0, special);268 GGML_ASSERT(check == -n_chars);269 } else {270 piece.resize(n_chars);271 }272 273 return piece;274 }275};276 277struct test_single_output_backend_sampler {278 bool backend_initialized = false;279 uint32_t backend_outputs_max_per_seq = 0;280 int backend_apply_count = 0;281 int apply_count = 0;282};283 284static const char * test_single_output_backend_sampler_name(const llama_sampler * /*smpl*/) {285 return "single-output-backend";286}287 288static void test_single_output_backend_sampler_apply(289 llama_sampler * smpl, llama_token_data_array * /*cur_p*/) {290 auto * ctx = (test_single_output_backend_sampler *) smpl->ctx;291 ctx->apply_count++;292}293 294static void test_single_output_backend_sampler_free(llama_sampler * smpl) {295 delete (test_single_output_backend_sampler *) smpl->ctx;296}297 298static bool test_single_output_backend_sampler_backend_init(299 llama_sampler * smpl, ggml_backend_buffer_type_t /*buft*/, uint32_t n_outputs_max_per_seq) {300 auto * ctx = (test_single_output_backend_sampler *) smpl->ctx;301 ctx->backend_outputs_max_per_seq = n_outputs_max_per_seq;302 if (n_outputs_max_per_seq > 1) {303 return false;304 }305 ctx->backend_initialized = true;306 return true;307}308 309static void test_single_output_backend_sampler_backend_apply(310 llama_sampler * smpl, ggml_context * /*ctx*/, ggml_cgraph * /*gf*/, llama_sampler_data * /*data*/) {311 auto * ctx = (test_single_output_backend_sampler *) smpl->ctx;312 ctx->backend_apply_count++;313}314 315static llama_sampler_i test_single_output_backend_sampler_i = {316 /* .name = */ test_single_output_backend_sampler_name,317 /* .accept = */ nullptr,318 /* .apply = */ test_single_output_backend_sampler_apply,319 /* .reset = */ nullptr,320 /* .clone = */ nullptr,321 /* .free = */ test_single_output_backend_sampler_free,322 /* .backend_init = */ test_single_output_backend_sampler_backend_init,323 /* .backend_accept = */ nullptr,324 /* .backend_apply = */ test_single_output_backend_sampler_backend_apply,325 /* .backend_set_input = */ nullptr,326 /* .backend_reset = */ nullptr,327 /* .copy_state = */ nullptr,328};329 330static llama_sampler * test_single_output_backend_sampler_init(331 test_single_output_backend_sampler ** sampler_ctx) {332 auto * ctx = new test_single_output_backend_sampler;333 *sampler_ctx = ctx;334 return llama_sampler_init(&test_single_output_backend_sampler_i, ctx);335}336 337static void test_backend_greedy_sampling(const test_params & params) {338 const int seq_id = 0;339 340 struct llama_sampler_chain_params backend_sampler_params = llama_sampler_chain_default_params();341 llama_sampler_ptr backend_sampler_chain(llama_sampler_chain_init(backend_sampler_params));342 343 llama_sampler_chain_add(backend_sampler_chain.get(), llama_sampler_init_greedy());344 std::vector<llama_sampler_seq_config> backend_sampler_configs = {{ seq_id, backend_sampler_chain.get() }};345 346 test_context test_ctx(params, backend_sampler_configs);347 348 if (!test_ctx.decode({{seq_id, "Some"}})) {349 GGML_ASSERT(false && "Failed to decode token");350 }351 352 int32_t batch_idx = test_ctx.idx_for_seq(seq_id);353 354 llama_token token = llama_get_sampled_token_ith(test_ctx.ctx.get(), batch_idx);355 printf("greedy sampled id:%d, string:'%s'\n", token, test_ctx.token_to_piece(token, false).c_str());356 GGML_ASSERT(token >= 0 && token < test_ctx.n_vocab);357 358 token = llama_get_sampled_token_ith(test_ctx.ctx.get(), -1);359 printf("greedy sampled id:%d, string:'%s'\n", token, test_ctx.token_to_piece(token, false).c_str());360 GGML_ASSERT(token >= 0 && token < test_ctx.n_vocab);361 362 for (int i = 0; i < 10; i++) {363 int32_t loop_idx = test_ctx.idx_for_seq(seq_id);364 llama_token token = llama_get_sampled_token_ith(test_ctx.ctx.get(), loop_idx);365 printf("Generation step %d: token id:%d, string: %s\n", i, token, test_ctx.token_to_piece(token, false).c_str());366 if (!test_ctx.decode_token(token, 0)) {367 GGML_ASSERT(false && "Failed to decode token");368 }369 }370}371 372static void test_backend_top_k_sampling(const test_params & params) {373 const int seq_id = 0;374 const int32_t k = 8;375 struct llama_sampler_chain_params backend_chain_params = llama_sampler_chain_default_params();376 llama_sampler_ptr backend_sampler_chain(llama_sampler_chain_init(backend_chain_params));377 llama_sampler_chain_add(backend_sampler_chain.get(), llama_sampler_init_top_k(k));378 std::vector<llama_sampler_seq_config> backend_sampler_configs = {{ seq_id, backend_sampler_chain.get() }};379 380 test_context test_ctx(params, backend_sampler_configs);381 382 if (!test_ctx.decode({{seq_id, "Hello"}})) {383 GGML_ASSERT(false && "Failed to decode token");384 }385 386 int32_t batch_idx = test_ctx.idx_for_seq(seq_id);387 388 float * logits = llama_get_sampled_logits_ith(test_ctx.ctx.get(), batch_idx);389 uint32_t n_logits = llama_get_sampled_logits_count_ith(test_ctx.ctx.get(), batch_idx);390 for (size_t i = 0; i < n_logits; ++i) {391 printf("top_k logit[%zu] = %.6f\n", i, logits[i]);392 }393 394 llama_token * candidates = llama_get_sampled_candidates_ith(test_ctx.ctx.get(), batch_idx);395 uint32_t n_candidates = llama_get_sampled_candidates_count_ith(test_ctx.ctx.get(), batch_idx);396 for (size_t i = 0; i < n_candidates; ++i) {397 printf("top_k candidate[%zu] = %d : %s\n", i, candidates[i],398 test_ctx.token_to_piece(candidates[i], false).c_str());399 }400 401 // Sample using CPU sampler for verification that it is possible to do hybrid402 // sampling, first top_k on the backend and then dist on the CPU.403 struct llama_sampler_chain_params chain_params = llama_sampler_chain_default_params();404 llama_sampler_ptr chain(llama_sampler_chain_init(chain_params));405 GGML_ASSERT(chain->iface->backend_apply != nullptr);406 407 llama_sampler_chain_add(chain.get(), llama_sampler_init_dist(18));408 llama_token token = llama_sampler_sample(chain.get(), test_ctx.ctx.get(), batch_idx);409 GGML_ASSERT(token >= 0 && token < test_ctx.n_vocab);410 411 printf("backend top-k hybrid sampling test PASSED\n");412}413 414static void test_backend_temp_sampling(const test_params & params) {415 {416 const float temp_0 = 0.8f;417 struct llama_sampler_chain_params backend_chain_params_0 = llama_sampler_chain_default_params();418 llama_sampler_ptr backend_sampler_chain_0(llama_sampler_chain_init(backend_chain_params_0));419 llama_sampler_chain_add(backend_sampler_chain_0.get(), llama_sampler_init_temp(temp_0));420 421 const float temp_1 = 0.1f;422 struct llama_sampler_chain_params backend_chain_params_1 = llama_sampler_chain_default_params();423 llama_sampler_ptr backend_sampler_chain_1(llama_sampler_chain_init(backend_chain_params_1));424 llama_sampler_chain_add(backend_sampler_chain_1.get(), llama_sampler_init_temp(temp_1));425 426 std::vector<llama_sampler_seq_config> backend_sampler_configs = {427 { 0, backend_sampler_chain_0.get() },428 { 1, backend_sampler_chain_1.get() }429 };430 431 test_context test_ctx(params, backend_sampler_configs);432 433 if (!test_ctx.decode({{0, "Some where over the"}, {1, "Once upon a"}})) {434 GGML_ASSERT(false && "Failed to decode token");435 }436 437 // Verify sequence 0438 {439 int32_t batch_idx = test_ctx.idx_for_seq(0);440 int n_logits = llama_get_sampled_logits_count_ith(test_ctx.ctx.get(), batch_idx);441 GGML_ASSERT(n_logits == test_ctx.n_vocab);442 443 // Sample from sequence 0 using CPU sampler444 struct llama_sampler_chain_params chain_params = llama_sampler_chain_default_params();445 llama_sampler_ptr chain(llama_sampler_chain_init(chain_params));446 llama_sampler_chain_add(chain.get(), llama_sampler_init_dist(18));447 448 llama_token token = llama_sampler_sample(chain.get(), test_ctx.ctx.get(), batch_idx);449 const std::string token_str = test_ctx.token_to_piece(token, false);450 printf("Sequence 0 sampled token id:%d, string: '%s'\n", token, token_str.c_str());451 GGML_ASSERT(token >= 0 && token < test_ctx.n_vocab);452 }453 454 455 // Verify sequence 1456 {457 int32_t batch_idx = test_ctx.idx_for_seq(1);458 459 // Sample from sequence 1 using CPU sampler460 struct llama_sampler_chain_params chain_params = llama_sampler_chain_default_params();461 llama_sampler_ptr chain(llama_sampler_chain_init(chain_params));462 llama_sampler_chain_add(chain.get(), llama_sampler_init_dist(18));463 464 llama_token token = llama_sampler_sample(chain.get(), test_ctx.ctx.get(), batch_idx);465 const std::string token_str = test_ctx.token_to_piece(token, false);466 printf("Sequence 1 sampled token id:%d, string: '%s'\n", token, token_str.c_str());467 GGML_ASSERT(token >= 0 && token < test_ctx.n_vocab);468 }469 }470 471 // lambda for testing non-positive temperature values.472 auto test_argmax_temp = [&](float temp) {473 printf("\nTesting temperature = %.1f\n", temp);474 475 int seq_id = 0;476 struct llama_sampler_chain_params backend_chain_params = llama_sampler_chain_default_params();477 llama_sampler_ptr backend_sampler_chain(llama_sampler_chain_init(backend_chain_params));478 llama_sampler_chain_add(backend_sampler_chain.get(), llama_sampler_init_temp(temp));479 480 std::vector<llama_sampler_seq_config> backend_sampler_configs = {481 { seq_id, backend_sampler_chain.get() },482 };483 484 test_context test_ctx(params, backend_sampler_configs);485 486 if (!test_ctx.decode({{seq_id, "Once"}})) {487 GGML_ASSERT(false && "Failed to decode token");488 }489 490 int32_t batch_idx = test_ctx.idx_for_seq(seq_id);491 492 uint32_t n_logits = llama_get_sampled_logits_count_ith(test_ctx.ctx.get(), batch_idx);493 GGML_ASSERT(n_logits == 1);494 };495 496 test_argmax_temp(0.0f);497 test_argmax_temp(-1.0f);498 499 printf("backend temp sampling test PASSED\n");500}501 502static void test_backend_temp_ext_sampling(const test_params & params) {503 {504 int seq_id = 0;505 const float temp = 0.8f;506 const float delta = 0.5f;507 const float exponent = 1.5f;508 struct llama_sampler_chain_params backend_chain_params = llama_sampler_chain_default_params();509 llama_sampler_ptr backend_sampler_chain(llama_sampler_chain_init(backend_chain_params));510 llama_sampler_chain_add(backend_sampler_chain.get(), llama_sampler_init_temp_ext(temp, delta, exponent));511 512 std::vector<llama_sampler_seq_config> backend_sampler_configs = {513 { seq_id, backend_sampler_chain.get() },514 };515 516 test_context test_ctx(params, backend_sampler_configs);517 518 if (!test_ctx.decode({{seq_id, "Once upon a"}})) {519 GGML_ASSERT(false && "Failed to decode token");520 }521 522 // Verify sequence 0523 {524 int32_t batch_idx = test_ctx.idx_for_seq(seq_id);525 int n_logits = llama_get_sampled_logits_count_ith(test_ctx.ctx.get(), batch_idx);526 GGML_ASSERT(n_logits == test_ctx.n_vocab);527 }528 }529 530 // lambda for testing non-positive temp/delta/exponent values.531 auto test_argmax_temp = [&](float temp, float delta, float exponent) {532 printf("\nTesting temperature = %.1f, delta = %1.f, exponent = %1.f\n", temp, delta, exponent);533 534 int seq_id = 0;535 struct llama_sampler_chain_params backend_chain_params = llama_sampler_chain_default_params();536 llama_sampler_ptr backend_sampler_chain(llama_sampler_chain_init(backend_chain_params));537 llama_sampler_chain_add(backend_sampler_chain.get(), llama_sampler_init_temp_ext(temp, delta, exponent));538 539 std::vector<llama_sampler_seq_config> backend_sampler_configs = {540 { seq_id, backend_sampler_chain.get() },541 };542 543 test_context test_ctx(params, backend_sampler_configs);544 545 if (!test_ctx.decode({{seq_id, "Once"}})) {546 GGML_ASSERT(false && "Failed to decode token");547 }548 549 int32_t batch_idx = test_ctx.idx_for_seq(seq_id);550 551 uint32_t n_logits = llama_get_sampled_logits_count_ith(test_ctx.ctx.get(), batch_idx);552 553 if (temp <= 0.0f && delta >= 0.0f) {554 GGML_ASSERT(n_logits == 1);555 } else {556 GGML_ASSERT(n_logits == (uint32_t) test_ctx.n_vocab);557 }558 };559 560 test_argmax_temp(0.0f, 0.3f, 1.0f); // Greedy (temp=0)561 test_argmax_temp(-1.0f, 0.3f, 2.0f); // Greedy (temp<0)562 test_argmax_temp(0.8f, 0.0f, 2.0f); // Temperature scaling563 564 printf("backend temp_ext sampling test PASSED\n");565}566 567static void test_backend_min_p_sampling(const test_params & params) {568 const int seq_id = 0;569 const float p = 0.1;570 struct llama_sampler_chain_params backend_chain_params = llama_sampler_chain_default_params();571 llama_sampler_ptr backend_sampler_chain(llama_sampler_chain_init(backend_chain_params));572 llama_sampler_chain_add(backend_sampler_chain.get(), llama_sampler_init_min_p(p, 0));573 std::vector<llama_sampler_seq_config> backend_sampler_configs = {{ seq_id, backend_sampler_chain.get() }};574 575 test_context test_ctx(params, backend_sampler_configs);576 577 if (!test_ctx.decode({{seq_id, "Hello"}})) {578 GGML_ASSERT(false && "Failed to decode token");579 }580 581 int32_t batch_idx = test_ctx.idx_for_seq(seq_id);582 583 float * logits = llama_get_sampled_logits_ith(test_ctx.ctx.get(), batch_idx);584 uint32_t n_logits = llama_get_sampled_logits_count_ith(test_ctx.ctx.get(), batch_idx);585 586 // Print the logits that are above the min-p threshold587 std::vector<float> filtered_logits;588 for (size_t i = 0; i < n_logits; ++i) {589 if (logits[i] > -1e9f) {590 filtered_logits.push_back(logits[i]);591 //printf("min_p logit[%zu] = %.6f\n", i, logits[i]);592 }593 }594 GGML_ASSERT(filtered_logits.size() < (size_t) test_ctx.n_vocab);595 596 // Sample using CPU sampler for verification to inspect they are reasonable597 struct llama_sampler_chain_params chain_params = llama_sampler_chain_default_params();598 llama_sampler_ptr chain(llama_sampler_chain_init(chain_params));599 llama_sampler_chain_add(chain.get(), llama_sampler_init_dist(88));600 601 llama_token token = llama_sampler_sample(chain.get(), test_ctx.ctx.get(), batch_idx);602 const std::string token_str = test_ctx.token_to_piece(token, false);603 printf("min-p cpu sampled token id:%d, string: '%s'\n", token, token_str.c_str());604 GGML_ASSERT(token >= 0 && token < test_ctx.n_vocab);605 606 // Decode and sample 10 more tokens607 for (int i = 0; i < 10; i++) {608 int32_t loop_idx = test_ctx.idx_for_seq(seq_id);609 llama_token token = llama_sampler_sample(chain.get(), test_ctx.ctx.get(), loop_idx);610 printf("min-p gen step %d: token id :%5.d, string: %s\n", i, token, test_ctx.token_to_piece(token, false).c_str());611 if (!test_ctx.decode_token(token, 0)) {612 GGML_ASSERT(false && "Failed to decode token");613 }614 }615 616 printf("min-p sampling test PASSED\n");617}618 619static void test_backend_top_p_sampling(const test_params & params) {620 const int seq_id = 0;621 const float p = 0.9;622 struct llama_sampler_chain_params backend_chain_params = llama_sampler_chain_default_params();623 llama_sampler_ptr backend_sampler_chain(llama_sampler_chain_init(backend_chain_params));624 llama_sampler_chain_add(backend_sampler_chain.get(), llama_sampler_init_top_p(p, 0));625 std::vector<llama_sampler_seq_config> backend_sampler_configs = {{ seq_id, backend_sampler_chain.get() }};626 627 test_context test_ctx(params, backend_sampler_configs);628 629 if (!test_ctx.decode({{seq_id, "Hello"}})) {630 return;631 }632 633 int32_t batch_idx = test_ctx.idx_for_seq(seq_id);634 635 float * logits = llama_get_sampled_logits_ith(test_ctx.ctx.get(), batch_idx);636 uint32_t n_logits = llama_get_sampled_logits_count_ith(test_ctx.ctx.get(), batch_idx);637 638 // Print the logits that are above the min-p threshold639 std::vector<float> filtered_logits;640 for (size_t i = 0; i < n_logits; ++i) {641 if (logits[i] > -1e9f) {642 filtered_logits.push_back(logits[i]);643 }644 }645 GGML_ASSERT(filtered_logits.size() < (size_t) test_ctx.n_vocab);646 GGML_ASSERT(filtered_logits.size() > 0);647 648 // Sample using CPU sampler for verification to inspect they are reasonable649 struct llama_sampler_chain_params chain_params = llama_sampler_chain_default_params();650 llama_sampler_ptr chain(llama_sampler_chain_init(chain_params));651 llama_sampler_chain_add(chain.get(), llama_sampler_init_dist(88));652 653 llama_token token = llama_sampler_sample(chain.get(), test_ctx.ctx.get(), batch_idx);654 const std::string token_str = test_ctx.token_to_piece(token, false);655 printf("top-p cpu sampled token id:%d, string: '%s'\n", token, token_str.c_str());656 GGML_ASSERT(token >= 0 && token < test_ctx.n_vocab);657 658 // Decode and sample 10 more tokens659 for (int i = 0; i < 10; i++) {660 int32_t loop_idx = test_ctx.idx_for_seq(seq_id);661 llama_token token = llama_sampler_sample(chain.get(), test_ctx.ctx.get(), loop_idx);662 printf("top-p gen step %d: token id :%5.d, string: %s\n", i, token, test_ctx.token_to_piece(token, false).c_str());663 test_ctx.decode_token(token, 0);664 }665 666 printf("top-p sampling test PASSED\n");667}668 669static void test_backend_multi_sequence_sampling(const test_params & params) {670 struct llama_sampler_chain_params chain_params_0 = llama_sampler_chain_default_params();671 llama_sampler_ptr sampler_chain_0(llama_sampler_chain_init(chain_params_0));672 llama_sampler_chain_add(sampler_chain_0.get(), llama_sampler_init_greedy());673 674 struct llama_sampler_chain_params chain_params_1 = llama_sampler_chain_default_params();675 llama_sampler_ptr sampler_chain_1(llama_sampler_chain_init(chain_params_1));676 llama_sampler_chain_add(sampler_chain_1.get(), llama_sampler_init_temp(0.8f));677 llama_sampler_chain_add(sampler_chain_1.get(), llama_sampler_init_greedy());678 679 std::vector<llama_sampler_seq_config> backend_sampler_configs = {680 { 0, sampler_chain_0.get() },681 { 1, sampler_chain_1.get() }682 };683 684 test_context test_ctx(params, backend_sampler_configs);685 686 std::map<llama_seq_id, std::string> prompts = {687 {0, "Hello"},688 {1, "Some"}689 };690 691 if (!test_ctx.decode(prompts)) {692 GGML_ASSERT(false && "Failed to decode token");693 }694 695 // Verify sequence 0696 {697 int32_t batch_idx = test_ctx.idx_for_seq(0);698 llama_token token = llama_get_sampled_token_ith(test_ctx.ctx.get(), batch_idx);699 const std::string token_str = test_ctx.token_to_piece(token, false);700 printf("Seq 0 sampled token id=%d, string='%s'\n", token, token_str.c_str());701 GGML_ASSERT(token >= 0 && token < test_ctx.n_vocab);702 }703 704 // Verify sequence 1705 {706 int32_t batch_idx= test_ctx.idx_for_seq(1);707 llama_token token = llama_get_sampled_token_ith(test_ctx.ctx.get(), batch_idx);708 const std::string token_str = test_ctx.token_to_piece(token, false);709 printf("Seq 1 sampled token id=%d, string='%s'\n", token, token_str.c_str());710 GGML_ASSERT(token >= 0 && token < test_ctx.n_vocab);711 }712 713 // Generate tokens for each sequence714 printf("\nMulti-sequence generation:\n");715 for (int step = 0; step < 4; step++) {716 std::map<llama_seq_id, llama_token> tokens;717 718 for (llama_seq_id seq_id : {0, 1}) {719 int32_t idx = test_ctx.idx_for_seq(seq_id);720 llama_token token = llama_get_sampled_token_ith(test_ctx.ctx.get(), idx);721 const std::string token_str = test_ctx.token_to_piece(token, false);722 printf(" Seq %d, step %d: token id=%d, string='%s'\n", seq_id, step, token, token_str.c_str());723 tokens[seq_id] = token;724 }725 726 // Decode all tokens in a single batch727 if (!test_ctx.decode_tokens(tokens)) {728 GGML_ASSERT(false && "Failed to decode token");729 }730 }731 732 printf("backend multi-sequence sampling test PASSED\n");733}734 735static void test_backend_dist_sampling(const test_params & params) {736 const int seq_id = 0;737 const int32_t seed = 88;738 739 struct llama_sampler_chain_params backend_chain_params = llama_sampler_chain_default_params();740 llama_sampler_ptr backend_sampler_chain(llama_sampler_chain_init(backend_chain_params));741 llama_sampler_chain_add(backend_sampler_chain.get(), llama_sampler_init_dist(seed));742 std::vector<llama_sampler_seq_config> backend_sampler_configs = {{ seq_id, backend_sampler_chain.get() }};743 744 test_context test_ctx(params, backend_sampler_configs);745 746 if (!test_ctx.decode({{seq_id, "Some"}})) {747 GGML_ASSERT(false && "Failed to decode token");748 }749 750 int32_t batch_idx = test_ctx.idx_for_seq(seq_id);751 llama_token token = llama_get_sampled_token_ith(test_ctx.ctx.get(), batch_idx);752 printf("dist sampled id:%d, string:'%s'\n", token, test_ctx.token_to_piece(token, false).c_str());753 GGML_ASSERT(token >= 0 && token < test_ctx.n_vocab);754 //GGML_ASSERT(llama_get_sampled_logits_ith(test_ctx.ctx.get(), batch_idx) == nullptr);755 756 token = llama_get_sampled_token_ith(test_ctx.ctx.get(), -1);757 printf("dist sampled id:%d, string:'%s'\n", token, test_ctx.token_to_piece(token, false).c_str());758 GGML_ASSERT(token >= 0 && token < test_ctx.n_vocab);759 760 printf("backend dist sampling test PASSED\n");761}762 763static void test_backend_dist_sampling_and_cpu(const test_params & params) {764 const int seq_id = 0;765 const int32_t seed = 88;766 767 struct llama_sampler_chain_params backend_chain_params = llama_sampler_chain_default_params();768 llama_sampler_ptr backend_sampler_chain(llama_sampler_chain_init(backend_chain_params));769 llama_sampler_chain_add(backend_sampler_chain.get(), llama_sampler_init_dist(seed));770 std::vector<llama_sampler_seq_config> backend_sampler_configs = {{ seq_id, backend_sampler_chain.get() }};771 772 test_context test_ctx(params, backend_sampler_configs);773 774 if (!test_ctx.decode({{seq_id, "Some"}})) {775 GGML_ASSERT(false && "Failed to decode token");776 }777 778 int32_t batch_idx = test_ctx.idx_for_seq(seq_id);779 780 // Sample using CPU sampler781 struct llama_sampler_chain_params chain_params = llama_sampler_chain_default_params();782 llama_sampler_ptr chain(llama_sampler_chain_init(chain_params));783 llama_sampler_chain_add(chain.get(), llama_sampler_init_dist(18));784 785 llama_token backend_token = llama_get_sampled_token_ith(test_ctx.ctx.get(), batch_idx);786 llama_token cpu_token = llama_sampler_sample(chain.get(), test_ctx.ctx.get(), batch_idx);787 printf("dist & cpu sampled id:%d, string:'%s'\n", cpu_token, test_ctx.token_to_piece(cpu_token, false).c_str());788 GGML_ASSERT(backend_token == cpu_token);789 790 printf("backend dist & cpu sampling test PASSED\n");791}792 793static void test_backend_logit_bias_sampling(const test_params & params) {794 const auto * model = params.model.get();795 const auto * vocab = llama_model_get_vocab(model);796 797 const int seq_id = 0;798 799 std::vector<llama_logit_bias> logit_bias;800 801 // Get the token for the piece "World".802 const std::string piece = "World";803 std::vector<llama_token> tokens(16);804 llama_tokenize(vocab, piece.c_str(), piece.size(), tokens.data(), tokens.size(), false, false);805 806 llama_token bias_token = tokens[0];807 // TODO: biasing too much here makes the Vulkan sampling fail - should be investigated further808 // https://github.com/ggml-org/llama.cpp/actions/runs/20894267644/job/60030252675?pr=18753#step:3:23350809 //logit_bias.push_back({ bias_token, +100.0f });810 logit_bias.push_back({ bias_token, +10.0f });811 812 printf("biasing token piece '%s' -> token id %d\n", piece.c_str(), bias_token);813 814 struct llama_sampler_chain_params backend_chain_params = llama_sampler_chain_default_params();815 llama_sampler_ptr backend_sampler_chain(llama_sampler_chain_init(backend_chain_params));816 llama_sampler_chain_add(backend_sampler_chain.get(), llama_sampler_init_logit_bias(817 llama_vocab_n_tokens(vocab),818 logit_bias.size(),819 logit_bias.data()));820 llama_sampler_chain_add(backend_sampler_chain.get(), llama_sampler_init_dist(88));821 822 std::vector<llama_sampler_seq_config> backend_sampler_configs = {823 { seq_id, backend_sampler_chain.get() },824 };825 826 test_context test_ctx(params, backend_sampler_configs);827 828 if (!test_ctx.decode({{seq_id, "Hello"}})) {829 GGML_ASSERT(false && "Failed to decode token");830 }831 832 llama_token backend_token = llama_get_sampled_token_ith(test_ctx.ctx.get(), test_ctx.idx_for_seq(seq_id));833 printf("sampled token = %d, expected = %d\n", backend_token, bias_token);834 GGML_ASSERT(backend_token == bias_token);835 836 printf("backend logit bias sampling test PASSED\n");837}838 839static void accept_prompt(llama_sampler * smpl, const llama_vocab * vocab, const std::string & prompt) {840 const llama_token bos = llama_vocab_bos(vocab);841 if (bos != LLAMA_TOKEN_NULL) {842 llama_sampler_accept(smpl, bos);843 }844 845 std::vector<llama_token> tokens(64);846 int32_t n_tokens = llama_tokenize(vocab, prompt.c_str(), (int32_t) prompt.size(),847 tokens.data(), (int32_t) tokens.size(), false, false);848 if (n_tokens < 0) {849 tokens.resize(-n_tokens);850 n_tokens = llama_tokenize(vocab, prompt.c_str(), (int32_t) prompt.size(),851 tokens.data(), (int32_t) tokens.size(), false, false);852 }853 854 for (int32_t i = 0; i < n_tokens; ++i) {855 llama_sampler_accept(smpl, tokens[i]);856 }857}858 859static std::vector<float> decode_raw_logits(const test_params & params, const std::string & prompt) {860 const int seq_id = 0;861 const int n_vocab = llama_vocab_n_tokens(llama_model_get_vocab(params.model.get()));862 std::vector<llama_sampler_seq_config> empty_configs;863 test_context ctx(params, empty_configs);864 865 GGML_ASSERT(ctx.decode({{ seq_id, prompt }}));866 867 float * logits = llama_get_logits_ith(ctx.ctx.get(), ctx.idx_for_seq(seq_id));868 GGML_ASSERT(logits != nullptr);869 return std::vector<float>(logits, logits + n_vocab);870}871 872static std::vector<llama_token_data> apply_cpu_sampler(873 const std::vector<float> & raw_logits,874 llama_sampler * sampler) {875 std::vector<llama_token_data> data;876 data.reserve(raw_logits.size());877 for (llama_token token = 0; token < (llama_token) raw_logits.size(); ++token) {878 data.push_back({ token, raw_logits[token], 0.0f });879 }880 881 llama_token_data_array cur_p = { data.data(), data.size(), -1, false };882 llama_sampler_apply(sampler, &cur_p);883 data.resize(cur_p.size);884 return data;885}886 887using sampler_setup_fn = std::function<void(llama_sampler *)>;888using sampler_init_fn = std::function<llama_sampler *()>;889 890enum class penalties_position {891 before_filter,892 after_filter,893};894 895static void add_filter_and_penalties(896 llama_sampler * chain,897 const sampler_init_fn & init_filter,898 int32_t n_vocab,899 int32_t penalty_last_n,900 float penalty_repeat,901 float penalty_freq,902 float penalty_present,903 penalties_position position) {904 const auto add_penalties = [&]() {905 llama_sampler_chain_add(chain, llama_sampler_init_penalties(906 n_vocab, penalty_last_n, penalty_repeat, penalty_freq, penalty_present));907 };908 909 if (position == penalties_position::before_filter) {910 add_penalties();911 llama_sampler_chain_add(chain, init_filter());912 } else {913 llama_sampler_chain_add(chain, init_filter());914 add_penalties();915 }916}917 918static llama_sampler_ptr make_sampler_chain(919 const sampler_setup_fn & add_samplers,920 const sampler_setup_fn & accept_history) {921 llama_sampler_ptr chain(llama_sampler_chain_init(llama_sampler_chain_default_params()));922 add_samplers(chain.get());923 accept_history(chain.get());924 return chain;925}926 927struct backend_sampler_output {928 std::vector<float> logits;929 std::vector<llama_token> candidates;930};931 932static backend_sampler_output run_backend_sampler(933 const test_params & params,934 const std::string & prompt,935 llama_sampler * sampler) {936 const int seq_id = 0;937 std::vector<llama_sampler_seq_config> configs = {{ seq_id, sampler }};938 test_context ctx(params, configs);939 940 GGML_ASSERT(ctx.decode({{ seq_id, prompt }}));941 llama_synchronize(ctx.ctx.get());942 943 const int32_t idx = ctx.idx_for_seq(seq_id);944 const uint32_t n_logits = llama_get_sampled_logits_count_ith(ctx.ctx.get(), idx);945 const uint32_t n_candidates = llama_get_sampled_candidates_count_ith(ctx.ctx.get(), idx);946 float * logits = llama_get_sampled_logits_ith(ctx.ctx.get(), idx);947 llama_token * candidates = llama_get_sampled_candidates_ith(ctx.ctx.get(), idx);948 GGML_ASSERT(logits != nullptr);949 950 backend_sampler_output result;951 result.logits.assign(logits, logits + n_logits);952 result.candidates.resize(n_logits);953 954 if (n_candidates == 0) {955 for (uint32_t i = 0; i < n_logits; ++i) {956 result.candidates[i] = (llama_token) i;957 }958 } else {959 GGML_ASSERT(candidates != nullptr);960 GGML_ASSERT(n_candidates == n_logits);961 std::memcpy(result.candidates.data(), candidates, n_candidates * sizeof(llama_token));962 }963 964 return result;965}966 967struct sampler_comparison_output {968 std::vector<llama_token_data> expected;969 backend_sampler_output actual;970};971 972static sampler_comparison_output run_sampler_comparison(973 const test_params & params,974 const std::string & prompt,975 const std::vector<float> & raw_logits,976 const sampler_setup_fn & add_samplers,977 const sampler_setup_fn & accept_history) {978 llama_sampler_ptr cpu_chain = make_sampler_chain(add_samplers, accept_history);979 llama_sampler_ptr backend_chain = make_sampler_chain(add_samplers, accept_history);980 return {981 apply_cpu_sampler(raw_logits, cpu_chain.get()),982 run_backend_sampler(params, prompt, backend_chain.get()),983 };984}985 986static std::unordered_map<llama_token, float> map_logits(const std::vector<llama_token_data> & data) {987 std::unordered_map<llama_token, float> result;988 result.reserve(data.size());989 for (const auto & item : data) {990 result[item.id] = item.logit;991 }992 return result;993}994 995struct sampler_comparison_stats {996 int n_mismatch = 0;997 int n_masked = 0;998 float max_diff = 0.0f;999};1000 1001static sampler_comparison_stats compare_sampler_outputs(1002 const char * name,1003 const std::unordered_map<llama_token, float> & expected,1004 const backend_sampler_output & actual,1005 bool allow_extra_candidates = false) {1006 GGML_ASSERT(actual.logits.size() == actual.candidates.size());1007 1008 sampler_comparison_stats result;1009 std::unordered_set<llama_token> seen;1010 seen.reserve(actual.candidates.size());1011 1012 for (size_t i = 0; i < actual.logits.size(); ++i) {1013 const llama_token token = actual.candidates[i];1014 const float logit = actual.logits[i];1015 if (!seen.insert(token).second || std::isnan(logit)) {1016 if (result.n_mismatch < 5) {1017 printf("%s token %d has invalid backend output\n", name, token);1018 }1019 ++result.n_mismatch;1020 continue;1021 }1022 1023 const auto it = expected.find(token);1024 if (it == expected.end()) {1025 if (std::isinf(logit) && logit < 0.0f) {1026 ++result.n_masked;1027 } else if (!allow_extra_candidates) {1028 if (result.n_mismatch < 5) {1029 printf("%s token %d was not masked\n", name, token);1030 }1031 ++result.n_mismatch;1032 }1033 continue;1034 }1035 1036 const float diff = fabsf(it->second - logit);1037 result.max_diff = std::max(result.max_diff, diff);1038 if (!std::isfinite(logit) || diff > 1e-3f) {1039 if (result.n_mismatch < 5) {1040 printf("%s mismatch token %d: cpu=%.6f backend=%.6f diff=%.6f\n",1041 name, token, it->second, logit, diff);1042 }1043 ++result.n_mismatch;1044 }1045 }1046 1047 for (const auto & item : expected) {1048 if (seen.find(item.first) == seen.end()) {1049 if (result.n_mismatch < 5) {1050 printf("%s missing backend token %d\n", name, item.first);1051 }1052 ++result.n_mismatch;1053 }1054 }1055 1056 printf("%s logits: max_diff=%.6f n_masked=%d n_mismatch=%d\n",1057 name, result.max_diff, result.n_masked, result.n_mismatch);1058 return result;1059}1060 1061static float find_backend_logit(const backend_sampler_output & output, llama_token token) {1062 for (size_t i = 0; i < output.candidates.size(); ++i) {1063 if (output.candidates[i] == token) {1064 return output.logits[i];1065 }1066 }1067 GGML_ABORT("backend token not found");1068}1069 1070static sampler_comparison_output run_penalties_comparison(1071 const test_params & params,1072 int32_t penalty_last_n,1073 float penalty_repeat,1074 float penalty_freq,1075 float penalty_present,1076 const std::string & prompt,1077 const std::function<void(llama_sampler *)> & extra_accept = {}) {1078 const auto * vocab = llama_model_get_vocab(params.model.get());1079 const std::vector<float> raw_logits = decode_raw_logits(params, prompt);1080 const auto add_samplers = [&](llama_sampler * chain) {1081 llama_sampler_chain_add(chain, llama_sampler_init_penalties(1082 llama_vocab_n_tokens(vocab), penalty_last_n, penalty_repeat, penalty_freq, penalty_present));1083 };1084 const auto accept_history = [&](llama_sampler * chain) {1085 accept_prompt(chain, vocab, prompt);1086 if (extra_accept) {1087 extra_accept(chain);1088 }1089 };1090 1091 return run_sampler_comparison(1092 params, prompt, raw_logits, add_samplers, accept_history);1093}1094 1095static void compare_penalties_logits(1096 const test_params & params,1097 int32_t penalty_last_n,1098 float penalty_repeat,1099 float penalty_freq,1100 float penalty_present,1101 const std::string & prompt,1102 const std::function<void(llama_sampler *)> & extra_accept = {}) {1103 const sampler_comparison_output output = run_penalties_comparison(1104 params, penalty_last_n, penalty_repeat, penalty_freq, penalty_present, prompt, extra_accept);1105 1106 GGML_ASSERT(output.expected.size() == output.actual.logits.size());1107 1108 const sampler_comparison_stats stats = compare_sampler_outputs(1109 "penalties", map_logits(output.expected), output.actual);1110 GGML_ASSERT(stats.n_masked == 0);1111 GGML_ASSERT(stats.n_mismatch == 0);1112}1113 1114static void test_penalty_parameter_values(const test_params & params) {1115 struct penalty_test_case {1116 const char * name;1117 float repeat;1118 float frequency;1119 float presence;1120 };1121 1122 const penalty_test_case cases[] = {1123 { "frequency -1", 1.0f, -1.0f, 0.0f },1124 { "frequency 0", 1.0f, 0.0f, 0.0f },1125 { "frequency 1", 1.0f, 1.0f, 0.0f },1126 { "presence -1", 1.0f, 0.0f, -1.0f },1127 { "presence 0", 1.0f, 0.0f, 0.0f },1128 { "presence 1", 1.0f, 0.0f, 1.0f },1129 { "repeat 1", 1.0f, 0.0f, 0.0f },1130 };1131 1132 int n_failed = 0;1133 for (const auto & test : cases) {1134 const sampler_comparison_output output = run_penalties_comparison(1135 params, 64, test.repeat, test.frequency, test.presence, "Hello Hello world");1136 GGML_ASSERT(output.expected.size() == output.actual.logits.size());1137 const sampler_comparison_stats stats = compare_sampler_outputs(1138 test.name, map_logits(output.expected), output.actual);1139 n_failed += stats.n_mismatch != 0;1140 }1141 1142 GGML_ASSERT(n_failed == 0);1143}1144 1145static void compare_top_k_penalties_logits(1146 const test_params & params,1147 int32_t k,1148 int32_t penalty_last_n,1149 float penalty_repeat,1150 float penalty_freq,1151 float penalty_present,1152 const std::string & prompt,1153 penalties_position position) {1154 const auto * vocab = llama_model_get_vocab(params.model.get());1155 const std::vector<float> raw_logits = decode_raw_logits(params, prompt);1156 const int n_vocab = (int) raw_logits.size();1157 1158 GGML_ASSERT(n_vocab > k);1159 1160 const sampler_init_fn init_top_k = [k]() {1161 return llama_sampler_init_top_k(k);1162 };1163 llama_sampler_ptr top_k(init_top_k());1164 const std::vector<llama_token_data> top_k_data = apply_cpu_sampler(raw_logits, top_k.get());1165 GGML_ASSERT(top_k_data.size() == (size_t) k);1166 const llama_token retained_history_token = top_k_data[0].id;1167 1168 llama_token excluded_history_token = LLAMA_TOKEN_NULL;1169 for (llama_token token = 0; token < n_vocab; ++token) {1170 const auto it = std::find_if(top_k_data.begin(), top_k_data.end(), [token](const llama_token_data & data) {1171 return data.id == token;1172 });1173 if (it == top_k_data.end()) {1174 excluded_history_token = token;1175 break;1176 }1177 }1178 GGML_ASSERT(excluded_history_token != LLAMA_TOKEN_NULL);1179 1180 const auto add_samplers = [&](llama_sampler * chain) {1181 add_filter_and_penalties(chain, init_top_k, n_vocab,1182 penalty_last_n, penalty_repeat, penalty_freq, penalty_present, position);1183 };1184 1185 auto accept_history = [&](llama_sampler * smpl) {1186 accept_prompt(smpl, vocab, prompt);1187 llama_sampler_accept(smpl, excluded_history_token);1188 llama_sampler_accept(smpl, excluded_history_token);1189 llama_sampler_accept(smpl, retained_history_token);1190 llama_sampler_accept(smpl, retained_history_token);1191 };1192 1193 const sampler_comparison_output output = run_sampler_comparison(1194 params, prompt, raw_logits, add_samplers, accept_history);1195 1196 GGML_ASSERT(output.expected.size() == (size_t) k);1197 GGML_ASSERT(output.actual.logits.size() == (size_t) k);1198 1199 const std::unordered_map<llama_token, float> expected_logits = map_logits(output.expected);1200