Felipe97/llama-cpp-compiled
01.1k
1#include "arg.h"2#include "common.h"3#include "fit.h"4#include "log.h"5#include "llama.h"6 7#include <algorithm>8#include <array>9#include <atomic>10#include <chrono>11#include <clocale>12#include <cmath>13#include <cstdio>14#include <cstring>15#include <ctime>16#include <fstream>17#include <mutex>18#include <random>19#include <sstream>20#include <thread>21#include <vector>22 23#if defined(_MSC_VER)24#pragma warning(disable: 4244 4267) // possible loss of data25#endif26 27struct results_perplexity {28 std::vector<llama_token> tokens;29 double ppl_value;30 std::vector<float> logits;31 std::vector<float> probs;32};33 34struct results_log_softmax {35 double log_softmax;36 float logit;37 float prob;38};39 40static std::vector<float> softmax(const std::vector<float>& logits) {41 std::vector<float> probs(logits.size());42 float max_logit = logits[0];43 for (float v : logits) {44 max_logit = std::max(max_logit, v);45 }46 double sum_exp = 0.0;47 for (size_t i = 0; i < logits.size(); i++) {48 // Subtract the maximum logit value from the current logit value for numerical stability49 const float logit = logits[i] - max_logit;50 const float exp_logit = expf(logit);51 sum_exp += exp_logit;52 probs[i] = exp_logit;53 }54 for (size_t i = 0; i < probs.size(); i++) {55 probs[i] /= sum_exp;56 }57 return probs;58}59 60static results_log_softmax log_softmax(int n_vocab, const float * logits, int tok) {61 float max_logit = logits[0];62 for (int i = 1; i < n_vocab; ++i) {63 max_logit = std::max(max_logit, logits[i]);64 }65 double sum_exp = 0.0;66 for (int i = 0; i < n_vocab; ++i) {67 sum_exp += expf(logits[i] - max_logit);68 }69 return {logits[tok] - max_logit - log(sum_exp), logits[tok], expf(logits[tok] - max_logit) / (float) sum_exp};70}71 72static inline int nearest_int(float fval) {73 //assert(fval <= 4194303.f);74 float val = fval + 12582912.f;75 int i; memcpy(&i, &val, sizeof(int));76 return (i & 0x007fffff) - 0x00400000;77}78 79static double log_softmax(int n_vocab, const float * logits, uint16_t * log_prob, int tok) {80 float max_logit = logits[0];81 float min_logit = logits[0];82 for (int i = 1; i < n_vocab; ++i) {83 max_logit = std::max(max_logit, logits[i]);84 min_logit = std::min(min_logit, logits[i]);85 }86 min_logit = std::max(min_logit, max_logit - 16);87 double sum_exp = 0.0;88 for (int i = 0; i < n_vocab; ++i) {89 sum_exp += expf(logits[i] - max_logit);90 }91 const float log_sum_exp = log(sum_exp);92 const float min_log_prob = min_logit - max_logit - log_sum_exp;93 const float scale = (max_logit - min_logit)/65535.f;94 float * d = (float *)log_prob;95 d[0] = scale;96 d[1] = min_log_prob;97 log_prob += 4;98 if (scale) {99 const float inv_scale = 1/scale;100 for (int i = 0; i < n_vocab; ++i) {101 log_prob[i] = logits[i] > min_logit ? nearest_int(inv_scale*(logits[i] - min_logit)) : 0;102 }103 } else {104 std::memset(log_prob, 0, n_vocab*sizeof(uint16_t));105 }106 return max_logit + log_sum_exp - logits[tok];107}108 109static void process_logits(110 int n_vocab, const float * logits, const int * tokens, int n_token, std::vector<std::thread> & workers,111 double & nll, double & nll2, float * logit_history, float * prob_history112) {113 std::mutex mutex;114 int counter = 0;115 auto compute = [&mutex, &counter, &nll, &nll2, logit_history, prob_history, n_vocab, logits, tokens, n_token] () {116 double local_nll = 0;117 double local_nll2 = 0;118 while (true) {119 std::unique_lock<std::mutex> lock(mutex);120 int i = counter++;121 if (i >= n_token) {122 nll += local_nll; nll2 += local_nll2;123 break;124 }125 lock.unlock();126 const results_log_softmax results = log_softmax(n_vocab, logits + size_t(i)*n_vocab, tokens[i+1]);127 const double v = -results.log_softmax;128 local_nll += v;129 local_nll2 += v*v;130 131 logit_history[i] = results.logit;132 prob_history[i] = results.prob;133 }134 };135 for (auto & w : workers) {136 w = std::thread(compute);137 }138 compute();139 for (auto & w : workers) {140 w.join();141 }142}143 144static void process_logits(std::ostream& out, int n_vocab, const float * logits, const int * tokens, int n_token,145 std::vector<std::thread> & workers, std::vector<uint16_t> & log_probs, double & nll, double & nll2) {146 std::mutex mutex;147 const int nv = 2*((n_vocab + 1)/2) + 4;148 int counter = 0;149 auto compute = [&mutex, &counter, &log_probs, &nll, &nll2, n_vocab, logits, tokens, n_token, nv] () {150 double local_nll = 0;151 double local_nll2 = 0;152 while (true) {153 std::unique_lock<std::mutex> lock(mutex);154 int i = counter++;155 if (i >= n_token) {156 nll += local_nll; nll2 += local_nll2;157 break;158 }159 lock.unlock();160 const double v = log_softmax(n_vocab, logits + size_t(i)*n_vocab, log_probs.data() + size_t(i)*nv, tokens[i+1]);161 local_nll += v;162 local_nll2 += v*v;163 }164 };165 for (auto & w : workers) {166 w = std::thread(compute);167 }168 compute();169 for (auto & w : workers) {170 w.join();171 }172 out.write((const char *)log_probs.data(), size_t(n_token)*nv*sizeof(uint16_t));173}174 175struct kl_divergence_result {176 double sum_nll = 0.0;177 double sum_nll2 = 0.0;178 double sum_nll_base = 0.0;179 double sum_nll_base2 = 0.0;180 double sum_nll_nll_base = 0.0;181 double sum_kld = 0.0;182 double sum_kld2 = 0.0;183 double sum_p_diff = 0.0;184 double sum_p_diff2 = 0.0;185 double sum_p_diff4 = 0.0;186 float max_p_diff = 0.0f;187 size_t n_same_top = 0.0;188 size_t count = 0.0;189};190 191static std::pair<double, float> log_softmax(int n_vocab, const float * logits, const uint16_t * base_log_prob, int tok, kl_divergence_result & kld) {192 float max_logit = logits[0];193 int imax = 0;194 for (int i = 1; i < n_vocab; ++i) {195 if (logits[i] > max_logit) {196 max_logit = logits[i];197 imax = i;198 }199 }200 double sum_exp = 0.0;201 for (int i = 0; i < n_vocab; ++i) {202 sum_exp += expf(logits[i] - max_logit);203 }204 const float log_sum_exp = log(sum_exp);205 const float * d = (const float *)base_log_prob;206 const float scale = d[0];207 const float min_log_prob = d[1];208 base_log_prob += 4;209 210 const float nll = max_logit + log_sum_exp - logits[tok];211 kld.sum_nll += nll;212 kld.sum_nll2 += nll*nll;213 214 const float nll_base = -(scale*base_log_prob[tok] + min_log_prob);215 kld.sum_nll_base += nll_base;216 kld.sum_nll_base2 += nll_base*nll_base;217 218 kld.sum_nll_nll_base += nll*nll_base;219 220 max_logit += log_sum_exp;221 double sum = 0;222 int imax_base = -1;223 float p_log_base_max = 0;224 for (int i = 0; i < n_vocab; ++i) {225 const float p_log_base = scale*base_log_prob[i] + min_log_prob;226 if (i == 0 || p_log_base > p_log_base_max) {227 p_log_base_max = p_log_base;228 imax_base = i;229 }230 if (p_log_base > -16.f) {231 const float p_base = expf(p_log_base);232 sum += p_base * (p_log_base - logits[i] + max_logit);233 }234 }235 kld.sum_kld += sum;236 kld.sum_kld2 += sum*sum;237 ++kld.count;238 if (imax == imax_base) {239 ++kld.n_same_top;240 }241 242 const float p_base = expf(-nll_base);243 const float p = expf(-nll);244 const float p_diff = p - p_base;245 kld.sum_p_diff += p_diff;246 const double p_diff2 = p_diff*p_diff;247 kld.sum_p_diff2 += p_diff2;248 kld.sum_p_diff4 += p_diff2*p_diff2;249 kld.max_p_diff = std::max(kld.max_p_diff, std::fabs(p_diff));250 251 return std::make_pair(sum, p_diff);252}253 254static void process_logits(int n_vocab, const float * logits, const int * tokens, int n_token,255 std::vector<std::thread> & workers, const std::vector<uint16_t> & base_log_probs, kl_divergence_result & kld,256 float * kld_values, float * p_diff_values) {257 std::mutex mutex;258 const int nv = 2*((n_vocab + 1)/2) + 4;259 int counter = 0;260 auto compute = [&mutex, &counter, &base_log_probs, &kld, n_vocab, logits, tokens, n_token, nv, kld_values, p_diff_values] () {261 kl_divergence_result local_kld;262 while (true) {263 std::unique_lock<std::mutex> lock(mutex);264 int i = counter++;265 if (i >= n_token) {266 kld.sum_nll += local_kld.sum_nll;267 kld.sum_nll2 += local_kld.sum_nll2;268 kld.sum_nll_base += local_kld.sum_nll_base;269 kld.sum_nll_base2 += local_kld.sum_nll_base2;270 kld.sum_nll_nll_base += local_kld.sum_nll_nll_base;271 kld.sum_kld += local_kld.sum_kld;272 kld.sum_kld2 += local_kld.sum_kld2;273 kld.sum_p_diff += local_kld.sum_p_diff;274 kld.sum_p_diff2 += local_kld.sum_p_diff2;275 kld.sum_p_diff4 += local_kld.sum_p_diff4;276 kld.n_same_top += local_kld.n_same_top;277 kld.max_p_diff = std::max(kld.max_p_diff, local_kld.max_p_diff);278 kld.count += local_kld.count;279 break;280 }281 lock.unlock();282 std::pair<double, float> v = log_softmax(n_vocab, logits + size_t(i)*n_vocab, base_log_probs.data() + size_t(i)*nv, tokens[i+1], local_kld);283 kld_values[i] = (float)v.first;284 p_diff_values[i] = v.second;285 }286 };287 for (auto & w : workers) {288 w = std::thread(compute);289 }290 compute();291 for (auto & w : workers) {292 w.join();293 }294}295 296static results_perplexity perplexity_v2(llama_context * ctx, const common_params & params) {297 // Download: https://huggingface.co/datasets/ggml-org/ci/resolve/main/wikitext-2-raw-v1.zip298 // Run `./perplexity -m models/7B/ggml-model-q4_0.bin -f wiki.test.raw`299 // Output: `perplexity: 13.5106 [114/114]`300 // BOS tokens will be added for each chunk before eval301 302 const llama_model * model = llama_get_model(ctx);303 const llama_vocab * vocab = llama_model_get_vocab(model);304 305 const bool add_bos = llama_vocab_get_add_bos(vocab);306 GGML_ASSERT(!llama_vocab_get_add_eos(vocab));307 308 LOG_INF("%s: tokenizing the input ..\n", __func__);309 310 std::vector<llama_token> tokens = common_tokenize(ctx, params.prompt, true);311 312 const int n_ctx = llama_n_ctx(ctx);313 314 if (int(tokens.size()) < 2*n_ctx) {315 LOG_ERR("%s: you need at least %d tokens to evaluate perplexity with a context of %d\n",__func__,2*n_ctx,316 n_ctx);317 LOG_ERR("%s: the data file you provided tokenizes to only %zu tokens\n",__func__,tokens.size());318 return {std::move(tokens), 0., {}, {}};319 }320 321 std::vector<float> logit_history;322 std::vector<float> prob_history;323 324 logit_history.resize(tokens.size());325 prob_history.resize(tokens.size());326 327 if (params.ppl_stride <= 0) {328 LOG_ERR("%s: stride is %d but must be greater than zero!\n",__func__,params.ppl_stride);329 return {tokens, -1, logit_history, prob_history};330 }331 332 const int calc_chunk = n_ctx;333 334 LOG_INF("%s: have %zu tokens. Calculation chunk = %d\n", __func__, tokens.size(), calc_chunk);335 336 if (int(tokens.size()) <= calc_chunk) {337 LOG_ERR("%s: there are only %zu tokens, this is not enough for a context size of %d and stride %d\n",__func__,338 tokens.size(), n_ctx, params.ppl_stride);339 return {tokens, -1, logit_history, prob_history};340 }341 342 const int n_chunk_max = (tokens.size() - calc_chunk + params.ppl_stride - 1) / params.ppl_stride;343 344 const int n_chunk = params.n_chunks < 0 ? n_chunk_max : std::min(params.n_chunks, n_chunk_max);345 const int n_batch = params.n_batch;346 347 const int n_vocab = llama_vocab_n_tokens(vocab);348 349 int count = 0;350 double nll = 0.0;351 352 const int n_seq = std::max(1, n_batch / n_ctx);353 LOG_INF("%s: computing over %d chunks, n_ctx=%d, batch_size=%d, n_seq=%d\n", __func__, n_chunk, n_ctx, n_batch, n_seq);354 355 for (int i = 0; i < n_chunk; ++i) {356 const int start = i * params.ppl_stride;357 const int end = start + calc_chunk;358 359 const int num_batches = (calc_chunk + n_batch - 1) / n_batch;360 //LOG_DBG("%s: evaluating %d...%d using %d batches\n", __func__, start, end, num_batches);361 362 std::vector<float> logits;363 364 const auto t_start = std::chrono::high_resolution_clock::now();365 366 // clear the KV cache367 llama_memory_clear(llama_get_memory(ctx), true);368 369 llama_batch batch = llama_batch_init(n_batch, 0, 1);370 371 for (int j = 0; j < num_batches; ++j) {372 const int batch_start = start + j * n_batch;373 const int batch_size = std::min(end - batch_start, n_batch);374 375 common_batch_clear(batch);376 for (int i = 0; i < batch_size; i++) {377 common_batch_add(batch, tokens[batch_start + i], j*n_batch + i, {0}, true);378 }379 380 //LOG_DBG(" Batch %d: starts at %d, size is %d, n_past is %d\n",j,batch_start,batch_size,j * n_batch);381 if (llama_decode(ctx, batch)) {382 //LOG_ERR("%s : failed to eval\n", __func__);383 llama_batch_free(batch);384 return {tokens, -1, logit_history, prob_history};385 }386 387 // save original token and restore it after eval388 const auto token_org = tokens[batch_start];389 390 // add BOS token for the first batch of each chunk391 if (add_bos && j == 0) {392 tokens[batch_start] = llama_vocab_bos(vocab);393 }394 395 const auto * batch_logits = llama_get_logits(ctx);396 logits.insert(logits.end(), batch_logits, batch_logits + size_t(batch_size) * n_vocab);397 398 if (j == 0) {399 tokens[batch_start] = token_org;400 }401 }402 403 llama_batch_free(batch);404 405 const auto t_end = std::chrono::high_resolution_clock::now();406 407 if (i == 0) {408 const float t_total = std::chrono::duration<float>(t_end - t_start).count();409 LOG_INF("%s: %.2f seconds per pass - ETA ", __func__, t_total);410 int total_seconds = (int)(t_total * n_chunk);411 if (total_seconds >= 60*60) {412 LOG("%d hours ", total_seconds / (60*60));413 total_seconds = total_seconds % (60*60);414 }415 LOG("%.2f minutes\n", total_seconds / 60.0);416 }417 418 //LOG_DBG("%s: using tokens %d...%d\n",__func__,params.n_ctx - params.ppl_stride + start, params.n_ctx + start);419 for (int j = n_ctx - params.ppl_stride - 1; j < n_ctx - 1; ++j) {420 // Calculate probability of next token, given the previous ones.421 const std::vector<float> tok_logits(422 logits.begin() + size_t(j + 0) * n_vocab,423 logits.begin() + size_t(j + 1) * n_vocab);424 425 const float prob = softmax(tok_logits)[tokens[start + j + 1]];426 logit_history[start + j + 1] = tok_logits[tokens[start + j + 1]];427 prob_history[start + j + 1] = prob;428 429 nll += -std::log(prob);430 ++count;431 }432 // perplexity is e^(average negative log-likelihood)433 if (params.ppl_output_type == 0) {434 LOG("[%d]%.4lf,", i + 1, std::exp(nll / count));435 } else {436 LOG("%8d %.4lf\n", i*params.ppl_stride, std::exp(nll / count));437 }438 }439 LOG("\n");440 441 return {tokens, std::exp(nll / count), logit_history, prob_history};442}443 444static results_perplexity perplexity(llama_context * ctx, const common_params & params, const int32_t n_ctx) {445 if (params.ppl_stride > 0) {446 return perplexity_v2(ctx, params);447 }448 449 // Download: https://huggingface.co/datasets/ggml-org/ci/resolve/main/wikitext-2-raw-v1.zip450 // Run `./llama-perplexity -m models/7B/ggml-model-q4_0.bin -f wiki.test.raw`451 // Output: `perplexity: 13.5106 [114/114]`452 // BOS tokens will be added for each chunk before eval453 454 const llama_model * model = llama_get_model(ctx);455 const llama_vocab * vocab = llama_model_get_vocab(model);456 457 const bool add_bos = llama_vocab_get_add_bos(vocab);458 GGML_ASSERT(!llama_vocab_get_add_eos(vocab));459 460 std::ofstream logits_stream;461 if (!params.logits_file.empty()) {462 logits_stream.open(params.logits_file.c_str(), std::ios::binary);463 if (!logits_stream.is_open()) {464 LOG_ERR("%s: failed to open %s for writing\n", __func__, params.logits_file.c_str());465 return {};466 }467 LOG_INF("%s: saving all logits to %s\n", __func__, params.logits_file.c_str());468 logits_stream.write("_logits_", 8);469 logits_stream.write(reinterpret_cast<const char *>(&n_ctx), sizeof(n_ctx));470 }471 472 auto tim1 = std::chrono::high_resolution_clock::now();473 LOG_INF("%s: tokenizing the input ..\n", __func__);474 475 std::vector<llama_token> tokens = common_tokenize(ctx, params.prompt, true);476 477 auto tim2 = std::chrono::high_resolution_clock::now();478 LOG_INF("%s: tokenization took %g ms\n",__func__,1e-3*std::chrono::duration_cast<std::chrono::microseconds>(tim2-tim1).count());479 480 if (int(tokens.size()) < 2*n_ctx) {481 LOG_ERR("%s: you need at least %d tokens to evaluate perplexity with a context of %d\n",__func__,2*n_ctx,482 n_ctx);483 LOG_ERR("%s: the data file you provided tokenizes to only %zu tokens\n",__func__,tokens.size());484 return {std::move(tokens), 0., {}, {}};485 }486 487 std::vector<float> logit_history;488 logit_history.resize(tokens.size());489 490 std::vector<float> prob_history;491 prob_history.resize(tokens.size());492 493 const int n_chunk_max = tokens.size() / n_ctx;494 495 const int n_chunk = params.n_chunks < 0 ? n_chunk_max : std::min(params.n_chunks, n_chunk_max);496 const int n_batch = params.n_batch;497 498 const int n_vocab = llama_vocab_n_tokens(vocab);499 500 int count = 0;501 double nll = 0.0;502 double nll2 = 0.0;503 504 const int num_batches = (n_ctx + n_batch - 1) / n_batch;505 const int n_seq = std::max(1, n_batch / n_ctx);506 507 GGML_ASSERT(n_batch < n_ctx || n_batch % n_ctx == 0);508 GGML_ASSERT(params.n_ctx == n_seq * n_ctx);509 510 llama_batch batch = llama_batch_init(std::min(n_batch, n_ctx*n_seq), 0, 1);511 512 std::vector<float> logits;513 if (num_batches > 1) {514 logits.reserve(size_t(n_ctx) * n_vocab);515 }516 517 LOG_INF("%s: calculating perplexity over %d chunks, n_ctx=%d, batch_size=%d, n_seq=%d\n", __func__, n_chunk, n_ctx, n_batch, n_seq);518 519 std::vector<std::thread> workers(std::thread::hardware_concurrency() - 1);520 521 std::vector<uint16_t> log_probs;522 if (!params.logits_file.empty()) {523 logits_stream.write((const char *)&n_vocab, sizeof(n_vocab));524 logits_stream.write((const char *)&n_chunk, sizeof(n_chunk));525 logits_stream.write((const char *)tokens.data(), n_chunk*n_ctx*sizeof(tokens[0]));526 const int nv = 2*((n_vocab + 1)/2) + 4;527 log_probs.resize(size_t(n_ctx) * nv);528 }529 530 // We get the logits for all the tokens in the context window (params.n_ctx)531 // from llama_decode below. Now, based on https://huggingface.co/docs/transformers/perplexity,532 // calculate the perplexity over the last half of the window (so the model always has533 // some context to predict the token).534 //535 // We rely on the fact that attention in the forward pass only looks at previous536 // tokens here, so the logits returned for each token are an accurate representation537 // of what the model would have predicted at that point.538 //539 // Example, we have a context window of 512, we will compute perplexity for each of the540 // last 256 tokens. Then, we split the input up into context window size chunks to541 // process the entire prompt.542 const int first = n_ctx/2;543 544 for (int i = 0; i < n_chunk; i += n_seq) {545 const int start = i * n_ctx;546 const int end = start + n_ctx;547 548 const int n_seq_batch = std::min(n_seq, n_chunk - i);549 550 const auto t_start = std::chrono::high_resolution_clock::now();551 552 // clear the KV cache553 llama_memory_clear(llama_get_memory(ctx), true);554 555 for (int j = 0; j < num_batches; ++j) {556 const int batch_start = start + j * n_batch;557 const int batch_size = std::min(end - batch_start, n_batch);558 559 int n_outputs = 0;560 561 batch.n_tokens = 0;562 for (int seq = 0; seq < n_seq_batch; seq++) {563 int seq_start = batch_start + seq*n_ctx;564 565 // save original token and restore it after decode566 const auto token_org = tokens[seq_start];567 568 // add BOS token for the first batch of each chunk569 if (add_bos && j == 0) {570 tokens[seq_start] = llama_vocab_bos(vocab);571 }572 573 for (int k = 0; k < batch_size; ++k) {574 const int idx = seq*n_ctx + k;575 batch.token [idx] = tokens[seq_start + k];576 batch.pos [idx] = j*n_batch + k;577 batch.n_seq_id[idx] = 1;578 batch.seq_id [idx][0] = seq;579 batch.logits [idx] = batch.pos[idx] >= first ? 1 : 0;580 581 n_outputs += batch.logits[idx] != 0;582 }583 batch.n_tokens += batch_size;584 585 // restore the original token in case it was set to BOS586 tokens[seq_start] = token_org;587 }588 589 if (llama_decode(ctx, batch)) {590 LOG_INF("%s : failed to decode\n", __func__);591 return {tokens, -1, logit_history, prob_history};592 }593 594 if (num_batches > 1 && n_outputs > 0) {595 const auto * batch_logits = llama_get_logits(ctx);596 logits.insert(logits.end(), batch_logits, batch_logits + size_t(n_outputs) * n_vocab);597 }598 }599 600 601 if (i == 0) {602 llama_synchronize(ctx);603 const auto t_end = std::chrono::high_resolution_clock::now();604 const float t_total = std::chrono::duration<float>(t_end - t_start).count();605 LOG_INF("%s: %.2f seconds per pass - ETA ", __func__, t_total);606 int total_seconds = (int)(t_total*n_chunk/n_seq);607 if (total_seconds >= 60*60) {608 LOG("%d hours ", total_seconds / (60*60));609 total_seconds = total_seconds % (60*60);610 }611 LOG("%.2f minutes\n", total_seconds / 60.0);612 }613 614 for (int seq = 0; seq < n_seq_batch; seq++) {615 const float * all_logits = num_batches > 1 ? logits.data() : llama_get_logits_ith(ctx, seq*n_ctx + first);616 617 llama_token * tokens_data = tokens.data() + start + seq*n_ctx + first;618 if (!params.logits_file.empty()) {619 process_logits(logits_stream, n_vocab, all_logits,620 tokens_data, n_ctx - 1 - first,621 workers, log_probs, nll, nll2);622 } else {623 process_logits(n_vocab, all_logits,624 tokens_data, n_ctx - 1 - first,625 workers, nll, nll2,626 logit_history.data() + start + seq*n_ctx + first,627 prob_history.data() + start + seq*n_ctx + first);628 }629 count += n_ctx - first - 1;630 631 // perplexity is e^(average negative log-likelihood)632 if (params.ppl_output_type == 0) {633 LOG("[%d]%.4lf,", i + seq + 1, std::exp(nll / count));634 } else {635 double av = nll/count;636 double av2 = nll2/count - av*av;637 if (av2 > 0) {638 av2 = sqrt(av2/(count-1));639 }640 LOG("%8d %.4lf %4lf %4lf\n", i*n_ctx, std::exp(nll / count), av, av2);641 }642 }643 644 logits.clear();645 }646 LOG("\n");647 648 nll2 /= count;649 nll /= count;650 const double ppl = exp(nll);651 nll2 -= nll * nll;652 if (nll2 > 0) {653 nll2 = sqrt(nll2/(count-1));654 LOG_INF("Final estimate: PPL = %.4lf +/- %.5lf\n", ppl, nll2*ppl);655 } else {656 LOG_ERR("Unexpected negative standard deviation of log(prob)\n");657 }658 659 llama_batch_free(batch);660 661 return {tokens, ppl, logit_history, prob_history};662}663 664static bool decode_helper(llama_context * ctx, llama_batch & batch, std::vector<float> & batch_logits, int n_batch, int n_vocab) {665 int prev_outputs = 0;666 for (int i = 0; i < (int) batch.n_tokens; i += n_batch) {667 const int n_tokens = std::min<int>(n_batch, batch.n_tokens - i);668 669 llama_batch batch_view = {670 n_tokens,671 batch.token + i,672 nullptr,673 batch.pos + i,674 batch.n_seq_id + i,675 batch.seq_id + i,676 batch.logits + i,677 };678 679 const int ret = llama_decode(ctx, batch_view);680 if (ret != 0) {681 LOG_ERR("failed to decode the batch, n_batch = %d, ret = %d\n", n_batch, ret);682 return false;683 }684 685 int n_outputs = 0;686 for (int i = 0; i < n_tokens; ++i) {687 n_outputs += batch_view.logits[i] != 0;688 }689 690 memcpy(batch_logits.data() + size_t(prev_outputs)*n_vocab, llama_get_logits(ctx), size_t(n_outputs)*n_vocab*sizeof(float));691 692 prev_outputs += n_outputs;693 }694 695 return true;696}697 698#define K_TOKEN_CHUNK 4699 700static void compute_logprobs(const float * batch_logits, int n_vocab, std::vector<std::thread>& workers,701 const std::vector<std::pair<size_t, llama_token>>& eval_pairs, std::vector<float>& eval_results) {702 if (eval_results.size() != eval_pairs.size()) {703 eval_results.resize(eval_pairs.size());704 }705 if (eval_pairs.empty()) {706 return;707 }708 709 size_t max_threads = std::min((eval_pairs.size() + K_TOKEN_CHUNK - 1)/K_TOKEN_CHUNK, workers.size());710 711 std::atomic<int> counter(0);712 auto compute = [&counter, &eval_pairs, &eval_results, batch_logits, n_vocab] () {713 float local_logprobs[K_TOKEN_CHUNK];714 while (true) {715 const size_t first = counter.fetch_add(K_TOKEN_CHUNK, std::memory_order_relaxed);716 if (first >= eval_results.size()) {717 break;718 }719 const size_t last = std::min(first + K_TOKEN_CHUNK, eval_results.size());720 for (size_t i = first; i < last; ++i) {721 const auto * logits = batch_logits + eval_pairs[i].first * n_vocab;722 float max_logit = logits[0];723 for (int j = 1; j < n_vocab; ++j) {724 max_logit = std::max(max_logit, logits[j]);725 }726 float sum_p = 0.f;727 for (int j = 0; j < n_vocab; ++j) {728 sum_p += expf(logits[j] - max_logit);729 }730 local_logprobs[i - first] = logits[eval_pairs[i].second] - max_logit - std::log(sum_p);731 }732 std::memcpy(eval_results.data() + first, local_logprobs, (last - first)*sizeof(float));733 }734 };735 736 for (size_t it = 0; it < max_threads; ++it) {737 workers[it] = std::thread(compute);738 }739 for (size_t it = 0; it < max_threads; ++it) {740 workers[it].join();741 }742}743 744static void hellaswag_score(llama_context * ctx, const common_params & params) {745 const llama_model * model = llama_get_model(ctx);746 const llama_vocab * vocab = llama_model_get_vocab(model);747 748 // Calculates hellaswag score (acc_norm) from prompt749 //750 // Data extracted from the HellaSwag validation dataset (MIT license) https://github.com/rowanz/hellaswag/blob/master/data/hellaswag_val.jsonl751 // All used data fields are preprocessed as in https://github.com/EleutherAI/lm-evaluation-harness/blob/df3da98c5405deafd519c2ddca52bb7c3fe36bef/lm_eval/tasks/hellaswag.py#L62-L68752 //753 // All 10042 tasks should be extracted to keep the results standardized like other implementations.754 //755 // Datafile layout:756 // ['??'] denotes json fields757 // 6 lines per task:758 // ['activity_label'] + ": " +['ctx'] - The first part of the query, the context759 // ['label'] - The index the best common sense ending aka gold ending760 // ['endings'][0] - Endings added to the first part of the query761 // ['endings'][1]762 // ['endings'][2]763 // ['endings'][3]764 765 std::vector<std::string> prompt_lines;766 std::istringstream strstream(params.prompt);767 std::string line;768 769 while (std::getline(strstream,line,'\n')) {770 prompt_lines.push_back(line);771 }772 773 if (prompt_lines.size() % 6 != 0) {774 LOG_ERR("%s : number of lines in prompt not a multiple of 6.\n", __func__);775 return;776 }777 778 size_t hs_task_count = prompt_lines.size()/6;779 LOG_INF("%s : loaded %zu tasks from prompt.\n", __func__, hs_task_count);780 781 const bool is_spm = llama_vocab_type(vocab) == LLAMA_VOCAB_TYPE_SPM;782 LOG_INF("================================= is_spm = %d\n", is_spm);783 784 // The tasks should be randomized so the score stabilizes quickly.785 bool randomize_tasks = true;786 787 // Number of tasks to use when computing the score788 if (params.hellaswag_tasks < hs_task_count) {789 hs_task_count = params.hellaswag_tasks;790 }791 792 // The random seed should not impact the final result if the computation is done over enough tasks, so kept hardcoded for now793 std::mt19937 rng(1);794 795 // Dataholder for hellaswag tasks796 struct hs_data_t {797 std::string context;798 size_t gold_ending_idx;799 std::string ending[4];800 size_t ending_logprob_count[4];801 double ending_logprob[4];802 803 size_t i_logits; // starting index of logits in the llama_batch804 size_t common_prefix; // max number of initial tokens that are the same in all sentences805 size_t required_tokens; // needed number of tokens to evaluate all 4 endings806 std::vector<llama_token> seq_tokens[4];807 };808 809 LOG_INF("%s : selecting %zu %s tasks.\n", __func__, hs_task_count, (randomize_tasks?"randomized":"the first") );810 811 // Select and read data from prompt lines812 std::vector<hs_data_t> hs_data(hs_task_count);813 for (size_t i = 0; i < hs_task_count; i++) {814 size_t idx = i;815 816 auto & hs_cur = hs_data[i];817 818 // Select a random example of those left in the prompt819 if (randomize_tasks) {820 std::uniform_int_distribution<size_t> dist(0, prompt_lines.size()/6-1 ) ;821 idx = dist(rng);822 }823 824 hs_cur.context = prompt_lines[idx*6];825 hs_cur.gold_ending_idx = std::stoi( prompt_lines[idx*6+1] );826 for (size_t j = 0; j < 4; j++) {827 hs_cur.ending[j] = prompt_lines[idx*6+2+j];828 hs_cur.seq_tokens[j] = common_tokenize(ctx, hs_cur.context + " " + hs_cur.ending[j], true);829 }830 831 // determine the common prefix of the endings832 hs_cur.common_prefix = 0;833 for (size_t k = 0; k < hs_cur.seq_tokens[0].size(); k++) {834 if (hs_cur.seq_tokens[0][k] != hs_cur.seq_tokens[1][k] ||835 hs_cur.seq_tokens[0][k] != hs_cur.seq_tokens[2][k] ||836 hs_cur.seq_tokens[0][k] != hs_cur.seq_tokens[3][k]) {837 break;838 }839 hs_cur.common_prefix++;840 }841 hs_cur.required_tokens = hs_cur.common_prefix +842 hs_cur.seq_tokens[0].size() - hs_cur.common_prefix +843 hs_cur.seq_tokens[1].size() - hs_cur.common_prefix +844 hs_cur.seq_tokens[2].size() - hs_cur.common_prefix +845 hs_cur.seq_tokens[3].size() - hs_cur.common_prefix;846 847 //GGML_ASSERT(hs_cur.common_prefix >= ::llama_tokenize(ctx, hs_cur.context, true).size());848 849 // Delete the selected random example from the prompt850 if (randomize_tasks) {851 prompt_lines.erase( std::next(prompt_lines.begin(),idx*6) , std::next(prompt_lines.begin(),idx*6+6) );852 }853 }854 855 LOG_INF("%s : calculating hellaswag score over selected tasks.\n", __func__);856 857 LOG("\ntask\tacc_norm\t95%% confidence interval\n");858 859 double acc = 0.0f;860 861 const int n_ctx = llama_n_ctx(ctx);862 const int n_batch = params.n_batch;863 864 const int n_vocab = llama_vocab_n_tokens(vocab);865 866 const int max_tasks_per_batch = 32;867 const int max_seq = std::min(4*max_tasks_per_batch, (int) llama_n_seq_max(ctx));868 869 llama_batch batch = llama_batch_init(n_ctx, 0, 4);870 871 std::vector<float> tok_logits(n_vocab);872 // TODO: this could be made smaller; it's currently the worst-case size873 std::vector<float> batch_logits(size_t(n_ctx)*n_vocab);874 875 std::vector<std::pair<size_t, llama_token>> eval_pairs;876 std::vector<float> eval_results;877 std::vector<std::thread> workers(std::thread::hardware_concurrency());878 879 for (size_t i0 = 0; i0 < hs_task_count; i0++) {880 int n_cur = 0;881 882 size_t i1 = i0;883 size_t i_logits = 0; // this tells us how many logits were needed before this point in the batch884 885 common_batch_clear(batch);886 887 // batch as much tasks as possible into the available context888 // each task has 4 unique sequence ids - one for each ending889 // the common prefix is shared among the 4 sequences to save tokens890 // we extract logits only from the last common token and from all ending tokens of each sequence891 while (n_cur + (int) hs_data[i1].required_tokens <= n_ctx) {892 auto & hs_cur = hs_data[i1];893 int n_logits = 0;894 895 const int s0 = 4*(i1 - i0);896 if (s0 + 4 > max_seq) {897 break;898 }899 900 for (size_t i = 0; i < hs_cur.common_prefix; ++i) {901 common_batch_add(batch, hs_cur.seq_tokens[0][i], i, { s0 + 0, s0 + 1, s0 + 2, s0 + 3 }, false);902 }903 batch.logits[batch.n_tokens - 1] = true; // we need logits for the last token of the common prefix904 n_logits += 1;905 906 for (int s = 0; s < 4; ++s) {907 const size_t seq_tokens_size = hs_cur.seq_tokens[s].size();908 // TODO: don't evaluate the last token of each sequence909 for (size_t i = hs_cur.common_prefix; i < seq_tokens_size; ++i) {910 const bool needs_logits = i < seq_tokens_size - 1;911 common_batch_add(batch, hs_cur.seq_tokens[s][i], i, { s0 + s }, needs_logits);912 n_logits += needs_logits;913 }914 }915 916 hs_cur.i_logits = i_logits;917 i_logits += n_logits;918 919 n_cur += hs_data[i1].required_tokens;920 if (++i1 == hs_task_count) {921 break;922 }923 }924 925 if (i0 == i1) {926 LOG_ERR("%s : task %zu does not fit in the context window (requires %zu tokens)\n", __func__, i0, hs_data[i0].required_tokens);927 return;928 }929 930 llama_memory_clear(llama_get_memory(ctx), true);931 932 // decode all tasks [i0, i1)933 if (!decode_helper(ctx, batch, batch_logits, n_batch, n_vocab)) {934 LOG_ERR("%s: llama_decode() failed\n", __func__);935 return;936 }937 938 // Compute log-probs in parallel939 // First we collect all tasks940 eval_pairs.clear();941 for (size_t i = i0; i < i1; ++i) {942 auto & hs_cur = hs_data[i];943 size_t li = 1; // skip the last logit of the common prefix (computed separately below)944 for (int s = 0; s < 4; ++s) {945 for (size_t j = hs_cur.common_prefix; j < hs_cur.seq_tokens[s].size() - 1; j++) {946 eval_pairs.emplace_back(hs_cur.i_logits + li++, hs_cur.seq_tokens[s][j + 1]);947 }948 }949 }950 // Then we do the actual calculation951 compute_logprobs(batch_logits.data(), n_vocab, workers, eval_pairs, eval_results);952 953 size_t ir = 0;954 955 // compute the logprobs for each ending of the decoded tasks956 for (size_t i = i0; i < i1; ++i) {957 auto & hs_cur = hs_data[i];958 959 // get the logits of the last token of the common prefix960 std::memcpy(tok_logits.data(), batch_logits.data() + hs_cur.i_logits*n_vocab, n_vocab*sizeof(float));961 962 const auto first_probs = softmax(tok_logits);963 964 for (int s = 0; s < 4; ++s) {965 hs_cur.ending_logprob_count[s] = 1;966 hs_cur.ending_logprob[s] = std::log(first_probs[hs_cur.seq_tokens[s][hs_cur.common_prefix]]);967 for (size_t j = hs_cur.common_prefix; j < hs_cur.seq_tokens[s].size() - 1; j++) {968 hs_cur.ending_logprob[s] += eval_results[ir++];969 hs_cur.ending_logprob_count[s]++;970 }971 hs_cur.ending_logprob[s] /= hs_cur.ending_logprob_count[s];972 }973 974 // Find the ending with maximum logprob975 size_t ending_logprob_max_idx = 0;976 double ending_logprob_max_val = hs_cur.ending_logprob[0];977 for (size_t s = 1; s < 4; s++) {978 if (hs_cur.ending_logprob[s] > ending_logprob_max_val) {979 ending_logprob_max_idx = s;980 ending_logprob_max_val = hs_cur.ending_logprob[s];981 }982 }983 984 //LOG("max logprob ending idx %lu, gold ending idx %lu\n", ending_logprob_max_idx, hs_cur.gold_ending_idx);985 986 // If the gold ending got the maximum logprobe add one accuracy point987 if (ending_logprob_max_idx == hs_cur.gold_ending_idx) {988 acc += 1.0;989 }990 991 double freq = acc / double(i + 1);992 993 const double za = 1.95996398454;994 995 // // Wald normal approx996 // double conf =za*sqrt(freq*(1-freq)/double(i + 1));997 // LOG("%zu\t%.8lf +/- %.8lf\n", i + 1, freq*100.0, conf*100.0);998 999 // Wilson score interval, more accurate1000 double z = za * za / double(i + 1);1001 double cnf = z * sqrt(double(i + 1) * (4.0 * freq * (1 - freq) + z)) / (za + za);1002 double a = (freq + z * 0.5 - cnf) / (1.0 + z);1003 double b = (freq + z * 0.5 + cnf) / (1.0 + z);1004 1005 // Print the accumulated accuracy mean x 100 and confidence interval1006 LOG("%zu\t%3.8lf%%\t[%3.4lf%%, %3.4lf%%]\n", i + 1, freq * 100.0, a * 100.0, b * 100.0);1007 }1008 1009 i0 = i1 - 1;1010 }1011 1012 llama_batch_free(batch);1013 1014 LOG("\n");1015}1016 1017struct winogrande_entry {1018 std::string first;1019 std::string second;1020 std::array<std::string, 2> choices;1021 int answer;1022 1023 size_t i_logits;1024 size_t common_prefix;1025 size_t required_tokens;1026 size_t n_base1; // number of tokens for context + choice 11027 size_t n_base2; // number of tokens for context + choice 21028 std::vector<llama_token> seq_tokens[2];1029};1030 1031static std::vector<winogrande_entry> load_winogrande_from_csv(const std::string & prompt) {1032 std::vector<winogrande_entry> result;1033 std::istringstream in(prompt);1034 std::string line;1035 std::array<int, 4> comma_pos;1036 while (true) {1037 std::getline(in, line);1038 if (in.fail() || in.eof()) break;1039 int ipos = 0;1040 bool quote_open = false;1041 for (int i = 0; i < int(line.size()); ++i) {1042 if (!quote_open) {1043 if (line[i] == ',') {1044 comma_pos[ipos++] = i;1045 if (ipos == 4) break;1046 }1047 else if (line[i] == '"') {1048 quote_open = true;1049 }1050 }1051 else {1052 if (line[i] == '"') {1053 quote_open = false;1054 }1055 }1056 }1057 if (ipos != 4) {1058 LOG_ERR("%s: failed to find comma separators in <%s>\n", __func__, line.c_str());1059 continue;1060 }1061 auto sentence = line[comma_pos[0]+1] == '"' ? line.substr(comma_pos[0]+2, comma_pos[1] - comma_pos[0] - 3)1062 : line.substr(comma_pos[0]+1, comma_pos[1] - comma_pos[0] - 1);1063 auto choice1 = line.substr(comma_pos[1]+1, comma_pos[2] - comma_pos[1] - 1);1064 auto choice2 = line.substr(comma_pos[2]+1, comma_pos[3] - comma_pos[2] - 1);1065 auto answer = line.substr(comma_pos[3]+1, line.size() - comma_pos[3] - 1);1066 auto index = line.substr(0, comma_pos[0]);1067 int where = 0;1068 for ( ; where < int(sentence.size()); ++where) {1069 if (sentence[where] == '_') break;1070 }1071 if (where == int(sentence.size())) {1072 LOG_ERR("%s: no _ in <%s>\n", __func__, sentence.c_str());1073 continue;1074 }1075 std::istringstream stream(answer.c_str());1076 int i_answer; stream >> i_answer;1077 if (stream.fail() || i_answer < 1 || i_answer > 2) {1078 LOG_ERR("%s: failed to parse answer <%s>\n", __func__, answer.c_str());1079 continue;1080 }1081 result.emplace_back();1082 auto& wg = result.back();1083 wg.first = sentence.substr(0, where);1084 wg.second = sentence.substr(where + 1, sentence.size() - where - 1);1085 wg.choices[0] = std::move(choice1);1086 wg.choices[1] = std::move(choice2);1087 wg.answer = i_answer;1088 }1089 return result;1090}1091 1092/*1093 * Evaluates the Winogrande score.1094 * Uses a CSV containing task index, dentence, choice 1, choice 2, answer (1 or 2)1095 * You can get one such dataset from e.g. https://huggingface.co/datasets/ikawrakow/winogrande-eval-for-llama.cpp1096 * As an example, the 1st row in the above dataset is1097 *1098 * 0,Sarah was a much better surgeon than Maria so _ always got the easier cases.,Sarah,Maria,21099 *1100 */1101static void winogrande_score(llama_context * ctx, const common_params & params) {1102 const llama_model * model = llama_get_model(ctx);1103 const llama_vocab * vocab = llama_model_get_vocab(model);1104 1105 constexpr int k_min_trailing_ctx = 3;1106 1107 auto data = load_winogrande_from_csv(params.prompt);1108 if (data.empty()) {1109 LOG_ERR("%s: no tasks\n", __func__);1110 return;1111 }1112 1113 LOG_INF("%s : loaded %zu tasks from prompt.\n", __func__, data.size());1114 1115 if (params.winogrande_tasks > 0 && params.winogrande_tasks < data.size()) {1116 LOG_INF("%s : selecting %zu random tasks\n", __func__, params.winogrande_tasks);1117 std::mt19937 rng(1);1118 std::vector<int> aux(data.size());1119 for (int i = 0; i < int(data.size()); ++i) {1120 aux[i] = i;1121 }1122 float scale = 1/(1.f + (float)rng.max());1123 std::vector<winogrande_entry> selected;1124 selected.resize(params.winogrande_tasks);1125 for (int i = 0; i < int(params.winogrande_tasks); ++i) {1126 int j = int(scale*rng()*aux.size());1127 selected[i] = std::move(data[aux[j]]);1128 aux[j] = aux.back();1129 aux.pop_back();1130 }1131 data = std::move(selected);1132 }1133 1134 LOG_INF("%s : tokenizing selected tasks\n", __func__);1135 1136 for (auto & task : data) {1137 task.seq_tokens[0] = common_tokenize(ctx, task.first + task.choices[0] + task.second, true);1138 task.seq_tokens[1] = common_tokenize(ctx, task.first + task.choices[1] + task.second, true);1139 1140 task.common_prefix = 0;1141 for (size_t k = 0; k < task.seq_tokens[0].size(); k++) {1142 if (task.seq_tokens[0][k] != task.seq_tokens[1][k]) {1143 break;1144 }1145 task.common_prefix++;1146 }1147 1148 // TODO: the last token of each of the sequences don't need to be evaluated1149 task.required_tokens = task.common_prefix +1150 task.seq_tokens[0].size() - task.common_prefix +1151 task.seq_tokens[1].size() - task.common_prefix;1152 1153 task.n_base1 = common_tokenize(ctx, task.first + task.choices[0], true).size();1154 task.n_base2 = common_tokenize(ctx, task.first + task.choices[1], true).size();1155 }1156 1157 LOG_INF("%s : calculating winogrande score over selected tasks.\n", __func__);1158 1159 const int n_ctx = llama_n_ctx(ctx);1160 const int n_batch = params.n_batch;1161 1162 const int n_vocab = llama_vocab_n_tokens(vocab);1163 1164 const int max_tasks_per_batch = 128;1165 const int max_seq = std::min(2*max_tasks_per_batch, (int) llama_n_seq_max(ctx));1166 1167 llama_batch batch = llama_batch_init(n_ctx, 0, 2);1168 1169 std::vector<float> tok_logits(n_vocab);1170 // TODO: this could be made smaller; it's currently the worst-case size1171 std::vector<float> batch_logits(size_t(n_ctx)*n_vocab);1172 1173 std::vector<std::pair<size_t, llama_token>> eval_pairs;1174 std::vector<float> eval_results;1175 std::vector<std::thread> workers(std::thread::hardware_concurrency());1176 1177 int n_correct = 0;1178 int n_done = 0;1179 1180 for (size_t i0 = 0; i0 < data.size(); i0++) {1181 int n_cur = 0;1182 1183 size_t i1 = i0;1184 size_t i_logits = 0;1185 1186 common_batch_clear(batch);1187 1188 while (n_cur + (int) data[i1].required_tokens <= n_ctx) {1189 int n_logits = 0;1190 const int s0 = 2*(i1 - i0);1191 if (s0 + 2 > max_seq) {1192 break;1193 }1194 1195 for (size_t i = 0; i < data[i1].common_prefix; ++i) {1196 common_batch_add(batch, data[i1].seq_tokens[0][i], i, { s0 + 0, s0 + 1 }, false);1197 }1198 batch.logits[batch.n_tokens - 1] = true;1199 n_logits += 1;1200