Felipe97/llama-cpp-compiled
01.1k
1// test-fusion: verify the backend fusion logic against a per-device baseline.2//3// for every dummy model generated by test-llama-archs, the tool runs the model on a single4// device with fusion enabled and disabled, and reports:5// - the per-fusion-type counters for each mode (prefill / decode, merged into "any" when the6// per-graph counts match)7// - the NMSE between the fused and unfused logits8// - the NMSE between the device and a CPU reference9//10// the per-fusion-type counters are compared against a per-device baseline file (CSV) so a11// fusion pattern that silently stops matching (or fires when it should not) is caught as a12// regression.13//14// usage:15// test-fusion --models DIR --device MTL0 --record baseline.csv # generate a baseline16// test-fusion --models DIR --device MTL0 --check baseline.csv # validate against it17// test-fusion --model FILE --device MTL0 --check baseline.csv # validate a single model18 19#include "common.h"20#include "log.h"21#include "llama-cpp.h"22 23#include "ggml.h"24#include "gguf.h"25 26#include <algorithm>27#include <array>28#include <cstring>29#include <filesystem>30#include <fstream>31#include <iomanip>32#include <iostream>33#include <map>34#include <random>35#include <string>36#include <vector>37 38// generic fusion debugging API, resolved through the ad-hoc get_proc_address mechanism39// (not part of the official ggml backend interface yet). a backend that adopts fusion debugging40// exports these exact names.41typedef void * ggml_backend_fusion_t;42 43typedef ggml_backend_fusion_t ( * fusion_get_t) (ggml_backend_dev_t);44typedef void ( * fusion_stats_init_t) (ggml_backend_fusion_t);45typedef void ( * fusion_stats_reset_t) (ggml_backend_fusion_t);46typedef int ( * fusion_stats_get_t) (ggml_backend_fusion_t, const char **, uint64_t *, int);47typedef void ( * fusion_set_enabled_t) (ggml_backend_fusion_t, bool);48 49static bool silent_model_load_progress(float, void *) {50 return true;51}52 53struct gguf_context_ptr {54 gguf_context * ctx;55 gguf_context_ptr(gguf_context * c) : ctx(c) {}56 ~gguf_context_ptr() { if (ctx) { gguf_free(ctx); } }57 gguf_context * get() const { return ctx; }58 gguf_context_ptr(const gguf_context_ptr &) = delete;59 gguf_context_ptr & operator=(const gguf_context_ptr &) = delete;60};61 62// NMSE between two vectors (same as tests/test-llama-archs.cpp)63static double nmse(const std::vector<float> & a, const std::vector<float> & b) {64 GGML_ASSERT(a.size() == b.size());65 double mse_a_b = 0.0;66 double mse_a_0 = 0.0;67 68 for (size_t i = 0; i < a.size(); i++) {69 const float a_i = a[i];70 const float b_i = b[i];71 72 mse_a_b += (a_i - b_i) * (a_i - b_i);73 mse_a_0 += a_i * a_i;74 }75 76 return mse_a_b / mse_a_0;77}78 79// deterministic token sequence80static std::vector<llama_token> get_tokens(const uint32_t n_tokens, const uint32_t n_vocab, const size_t seed) {81 std::mt19937 gen(seed);82 std::uniform_int_distribution<> dis(0, n_vocab - 1);83 std::vector<llama_token> ret;84 ret.reserve(n_tokens);85 for (uint32_t i = 0; i < n_tokens; i++) {86 ret.push_back(dis(gen));87 }88 return ret;89}90 91// trim leading/trailing whitespace (used when parsing padded CSV columns)92static std::string trim(const std::string & s) {93 const size_t b = s.find_first_not_of(" \t\r\n");94 if (b == std::string::npos) {95 return "";96 }97 const size_t e = s.find_last_not_of(" \t\r\n");98 return s.substr(b, e - b + 1);99}100 101static std::string get_arch(const std::string & path) {102 gguf_init_params params = { /*no_alloc=*/true, /*ctx=*/nullptr };103 gguf_context_ptr ctx(gguf_init_from_file(path.c_str(), params));104 if (!ctx.get()) {105 throw std::runtime_error("failed to read gguf: " + path);106 }107 const int idx = gguf_find_key(ctx.get(), "general.architecture");108 if (idx < 0) {109 return "unknown";110 }111 const char * val = gguf_get_val_str(ctx.get(), idx);112 return val ? val : "unknown";113}114 115static llama_model_ptr load_model(const std::string & path, ggml_backend_dev_t dev) {116 llama_model_params model_params = llama_model_default_params();117 model_params.progress_callback = silent_model_load_progress;118 std::vector<ggml_backend_dev_t> devs = { dev, nullptr };119 model_params.devices = devs.data();120 model_params.split_mode = LLAMA_SPLIT_MODE_LAYER;121 122 llama_model_ptr model(llama_model_load_from_file(path.c_str(), model_params));123 if (!model) {124 throw std::runtime_error("failed to load model: " + path);125 }126 return model;127}128 129// a fresh context (fresh state) from an already-loaded model130static llama_context_ptr create_ctx(llama_model * model, int n_ubatch) {131 llama_context_params ctx_params = llama_context_default_params();132 ctx_params.n_ctx = 0;133 ctx_params.n_threads = 4;134 ctx_params.n_threads_batch = 4;135 ctx_params.n_ubatch = n_ubatch;136 ctx_params.n_batch = n_ubatch;137 138 llama_context_ptr lctx(llama_init_from_model(model, ctx_params));139 if (!lctx) {140 throw std::runtime_error("failed to init context");141 }142 return lctx;143}144 145// decode all tokens in one batch; returns the logits of every token146static std::vector<float> decode_prefill(llama_model * model, llama_context * lctx, const std::vector<llama_token> & tokens) {147 const uint32_t n_vocab = llama_vocab_n_tokens(llama_model_get_vocab(model));148 llama_batch batch = llama_batch_init(tokens.size(), 0, 1);149 for (size_t i = 0; i < tokens.size(); i++) {150 common_batch_add(batch, tokens[i], i, { 0 }, true);151 }152 batch.n_tokens = tokens.size();153 if (llama_decode(lctx, batch)) {154 llama_batch_free(batch);155 throw std::runtime_error("prefill decode failed");156 }157 158 std::vector<float> ret;159 ret.reserve(tokens.size() * n_vocab);160 for (size_t i = 0; i < tokens.size(); i++) {161 const float * logits_ith = llama_get_logits_ith(lctx, i);162 for (uint32_t j = 0; j < n_vocab; j++) {163 ret.push_back(logits_ith[j]);164 }165 }166 llama_batch_free(batch);167 return ret;168}169 170// decode one token at a time; returns the logits of the last token of each step171static std::vector<float> decode_gen(llama_model * model, llama_context * lctx, const std::vector<llama_token> & tokens) {172 const uint32_t n_vocab = llama_vocab_n_tokens(llama_model_get_vocab(model));173 llama_batch batch = llama_batch_init(1, 0, 1);174 std::vector<float> ret;175 for (size_t i = 0; i < tokens.size(); i++) {176 common_batch_clear(batch);177 common_batch_add(batch, tokens[i], i, { 0 }, true);178 if (llama_decode(lctx, batch)) {179 llama_batch_free(batch);180 throw std::runtime_error("decode failed");181 }182 const float * logits = llama_get_logits_ith(lctx, 0);183 for (uint32_t j = 0; j < n_vocab; j++) {184 ret.push_back(logits[j]);185 }186 }187 llama_batch_free(batch);188 return ret;189}190 191static void read_counts(fusion_stats_get_t api_stats_get, ggml_backend_fusion_t finfo,192 std::vector<const char *> & labels, std::vector<uint64_t> & counts) {193 const int n = api_stats_get(finfo, nullptr, nullptr, 0);194 labels.assign(n, nullptr);195 counts.assign(n, 0);196 api_stats_get(finfo, labels.data(), counts.data(), n);197}198 199// one row of the per-label report200struct fusion_row {201 std::string arch;202 bool moe;203 std::string mode;204 std::string label;205 uint64_t count_fused;206 uint64_t count_unfused;207 uint64_t expected;208 double nmse_fus;209 double nmse_dev;210 bool ok_count; // counts match the baseline211 bool ok_nmse; // nmse within epsilon212};213 214static void usage(const char * argv0) {215 printf("%s: verify fusion counts on a device against a per-device baseline\n\n", argv0);216 printf("usage: %s [options]\n\n", argv0);217 printf("options:\n");218 printf(" --models DIR run over all .gguf models in a directory\n");219 printf(" --model FILE run over a single model file (mutually exclusive with --models)\n");220 printf(" --device NAME device to run on (e.g. MTL0, CPU)\n");221 printf(" --record CSV write the golden baseline\n");222 printf(" --check CSV validate the counters against a baseline (default)\n");223 printf(" -h, --help show this message and exit\n");224}225 226int main(int argc, char ** argv) {227 std::string models_dir;228 std::string model_file;229 std::string device_name;230 std::string record_path;231 std::string check_path;232 233 for (int i = 1; i < argc; i++) {234 const std::string arg = argv[i];235 const auto next = [&](const char * name) -> std::string {236 if (i + 1 >= argc) {237 LOG_ERR("%s: %s requires an argument\n", __func__, name);238 exit(1);239 }240 return argv[++i];241 };242 if (arg == "-h" || arg == "--help") {243 usage(argv[0]);244 exit(0);245 }246 if (arg == "--models") { models_dir = next("--models"); }247 else if (arg == "--model") { model_file = next("--model"); }248 else if (arg == "--device"){ device_name = next("--device"); }249 else if (arg == "--record"){ record_path = next("--record"); }250 else if (arg == "--check") { check_path = next("--check"); }251 else {252 LOG_ERR("%s: unknown argument: %s\n", __func__, arg.c_str());253 return 1;254 }255 }256 257 if (device_name.empty()) {258 LOG_ERR("%s: --device NAME is required\n", __func__);259 return 1;260 }261 if (models_dir.empty() && model_file.empty()) {262 LOG_ERR("%s: --models DIR or --model FILE is required\n", __func__);263 return 1;264 }265 if (!models_dir.empty() && !model_file.empty()) {266 LOG_ERR("%s: --models DIR and --model FILE are mutually exclusive\n", __func__);267 return 1;268 }269 if (!record_path.empty() && !check_path.empty()) {270 LOG_ERR("%s: --record and --check are mutually exclusive\n", __func__);271 return 1;272 }273 274 std::vector<std::string> models;275 if (!model_file.empty()) {276 if (!std::filesystem::is_regular_file(model_file)) {277 LOG_ERR("%s: model file '%s' does not exist\n", __func__, model_file.c_str());278 return 1;279 }280 models.push_back(model_file);281 } else {282 if (!std::filesystem::exists(models_dir) || !std::filesystem::is_directory(models_dir)) {283 LOG_ERR("%s: models directory '%s' does not exist\n", __func__, models_dir.c_str());284 return 1;285 }286 for (const auto & entry : std::filesystem::directory_iterator(models_dir)) {287 if (entry.is_regular_file() && entry.path().extension() == ".gguf") {288 models.push_back(entry.path().string());289 }290 }291 std::sort(models.begin(), models.end());292 293 if (models.empty()) {294 LOG_ERR("%s: no .gguf models found in '%s'\n", __func__, models_dir.c_str());295 return 1;296 }297 }298 299 common_init();300 ggml_backend_load_all();301 302 ggml_backend_dev_t dev = ggml_backend_dev_by_name(device_name.c_str());303 if (!dev) {304 LOG_WRN("%s: device '%s' not found - skipping (baseline is device-specific)\n",305 __func__, device_name.c_str());306 return 0;307 }308 309 // resolve the generic fusion debugging functions through the ad-hoc get_proc_address310 // mechanism; a backend that does not adopt fusion debugging exports none of them311 auto * reg = ggml_backend_dev_backend_reg(dev);312 313 // output naming uses the backend base name (e.g. "MTL") rather than the specific device314 // name (e.g. "MTL0") the test was invoked with315 const std::string base_name = ggml_backend_reg_name(reg);316 317 auto api_get = (fusion_get_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_fusion_get");318 auto api_stats_init = (fusion_stats_init_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_fusion_stats_init");319 auto api_stats_reset = (fusion_stats_reset_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_fusion_stats_reset");320 auto api_stats_get = (fusion_stats_get_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_fusion_stats_get");321 auto api_set_enabled = (fusion_set_enabled_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_fusion_set_enabled");322 323 if (!api_get || !api_stats_init || !api_set_enabled || !api_stats_reset || !api_stats_get) {324 LOG_ERR("%s: device '%s' does not export the generic fusion debugging API "325 "(ggml_backend_fusion_*) - cannot run the fusion regression test\n",326 __func__, device_name.c_str());327 return 1;328 }329 330 ggml_backend_fusion_t finfo = api_get(dev);331 332 // enable fusions stats333 api_stats_init(finfo);334 335 const bool has_counts = true;336 337 // load the baseline (if any): key arch|moe|mode|label -> expected count338 std::map<std::string, uint64_t> baseline;339 if (!check_path.empty()) {340 std::ifstream in(check_path);341 if (!in) {342 LOG_ERR("%s: cannot open baseline '%s'\n", __func__, check_path.c_str());343 return 1;344 }345 std::string line;346 while (std::getline(in, line)) {347 if (line.empty() || line[0] == '#') {348 continue;349 }350 std::vector<std::string> cols;351 size_t pos = 0;352 while ((pos = line.find(',')) != std::string::npos) {353 cols.push_back(trim(line.substr(0, pos)));354 line.erase(0, pos + 1);355 }356 cols.push_back(trim(line));357 if (cols.size() != 5) {358 continue;359 }360 baseline[cols[0] + "|" + cols[1] + "|" + cols[2] + "|" + cols[3]] = std::stoull(cols[4]);361 }362 }363 364 std::vector<fusion_row> rows;365 366 LOG_INF("%s: running fusion test over %zu models on '%s'\n", __func__, models.size(), base_name.c_str());367 368 const size_t seed = 1;369 370 for (const auto & model_path : models) {371 const std::string arch = get_arch(model_path);372 const bool moe = arch.find("moe") != std::string::npos;373 374 llama_model_ptr model;375 llama_model_ptr model_cpu;376 uint32_t n_vocab = 0;377 try {378 model = load_model(model_path, dev);379 model_cpu = load_model(model_path, ggml_backend_dev_by_name("CPU"));380 n_vocab = llama_vocab_n_tokens(llama_model_get_vocab(model.get()));381 } catch (const std::exception & e) {382 LOG_ERR("%s: %s: %s\n", __func__, model_path.c_str(), e.what());383 continue;384 }385 386 struct mode_cfg {387 std::string name;388 std::vector<float> (*decode)(llama_model *, llama_context *, const std::vector<llama_token> &);389 int n_tokens;390 int n_graphs; // graph runs per mode (prefill=1, decode=16)391 };392 const mode_cfg modes[] = {393 { "prefill", decode_prefill, 32, 1 },394 { "decode", decode_gen, 16, 16 },395 };396 397 // per-label, per-mode data for this model; prefill and decode are merged into a single398 // "any" row when their per-graph counts match399 struct mode_data {400 bool present;401 uint64_t count_fused; // per graph402 uint64_t count_unfused; // per graph403 double nmse_fus;404 double nmse_dev;405 bool ok_nmse;406 };407 std::map<std::string, std::array<mode_data, 2>> mdata;408 409 for (int mi = 0; mi < 2; mi++) {410 const mode_cfg & mode = modes[mi];411 const auto tokens = get_tokens(mode.n_tokens, n_vocab, seed);412 413 // CPU reference for this mode (fresh context, fresh state)414 std::vector<float> logits_cpu;415 try {416 llama_context_ptr ctx = create_ctx(model_cpu.get(), 32);417 logits_cpu = mode.decode(model_cpu.get(), ctx.get(), tokens);418 } catch (const std::exception & e) {419 LOG_WRN("%s: %s: cpu reference: %s\n", __func__, model_path.c_str(), e.what());420 }421 422 // fused run on a fresh context (fresh state)423 std::vector<float> logits_fused;424 std::vector<const char *> labels;425 std::vector<uint64_t> counts_fused;426 {427 llama_context_ptr ctx = create_ctx(model.get(), 32);428 if (has_counts) {429 api_set_enabled(finfo, true);430 api_stats_reset(finfo);431 }432 logits_fused = mode.decode(model.get(), ctx.get(), tokens);433 if (has_counts) {434 read_counts(api_stats_get, finfo, labels, counts_fused);435 }436 }437 438 // unfused run on another fresh context (fresh state)439 std::vector<float> logits_unfused;440 std::vector<uint64_t> counts_unfused;441 {442 llama_context_ptr ctx = create_ctx(model.get(), 32);443 if (has_counts) {444 api_set_enabled(finfo, false);445 api_stats_reset(finfo);446 }447 logits_unfused = mode.decode(model.get(), ctx.get(), tokens);448 if (has_counts) {449 read_counts(api_stats_get, finfo, labels, counts_unfused);450 }451 }452 453 const double nmse_fus = nmse(logits_fused, logits_unfused);454 const double nmse_dev = logits_cpu.empty() ? 0.0 : nmse(logits_fused, logits_cpu);455 456 if (has_counts) {457 for (int i = 0; i < (int) labels.size(); i++) {458 const uint64_t fused = counts_fused[i] / mode.n_graphs;459 const uint64_t unfused = counts_unfused[i] / mode.n_graphs;460 if (fused == 0 && unfused == 0) {461 continue;462 }463 auto & d = mdata[labels[i]][mi];464 d.present = true;465 d.count_fused = fused;466 d.count_unfused = unfused;467 d.nmse_fus = nmse_fus;468 d.nmse_dev = nmse_dev;469 d.ok_nmse = nmse_fus <= 1e-4;470 }471 } else {472 rows.push_back({ arch, moe, mode.name, "?", 0, 0, 0, nmse_fus, nmse_dev, true, nmse_fus <= 1e-4 });473 }474 }475 476 // build the per-label rows, merging prefill and decode into "any" when the per-graph477 // counts match (they always do for the deterministic fusion table)478 if (has_counts) {479 for (auto & kv : mdata) {480 const std::string & label = kv.first;481 const auto & d = kv.second;482 const bool both = d[0].present && d[1].present;483 const bool match = both && d[0].count_fused == d[1].count_fused;484 485 if (match) {486 // one "any" row; use the worst NMSE across the two modes487 const std::string any_key = arch + "|" + (moe ? "1" : "0") + "|any|" + label;488 const uint64_t expected = baseline.count(any_key) ? baseline.at(any_key) : 0;489 const bool ok_count = check_path.empty() || d[0].count_fused == expected;490 const bool ok_nmse = d[0].ok_nmse && d[1].ok_nmse;491 const double nmse_fus = std::max(d[0].nmse_fus, d[1].nmse_fus);492 const double nmse_dev = std::max(d[0].nmse_dev, d[1].nmse_dev);493 rows.push_back({ arch, moe, "any", label, d[0].count_fused, d[0].count_unfused,494 expected, nmse_fus, nmse_dev, ok_count, ok_nmse });495 } else {496 // counts differ - keep a separate row per mode497 for (int mi = 0; mi < 2; mi++) {498 if (!d[mi].present) {499 continue;500 }501 const mode_data & a = d[mi];502 const std::string mode_key = arch + "|" + (moe ? "1" : "0") + "|" + modes[mi].name + "|" + label;503 const uint64_t expected = baseline.count(mode_key) ? baseline.at(mode_key) : 0;504 const bool ok_count = check_path.empty() || a.count_fused == expected;505 rows.push_back({ arch, moe, modes[mi].name, label, a.count_fused, a.count_unfused,506 expected, a.nmse_fus, a.nmse_dev, ok_count, a.ok_nmse });507 }508 }509 }510 }511 512 LOG_INF("%s: %-20s (%s) done\n", __func__, arch.c_str(), model_path.c_str());513 }514 515 // print the report516 {517 std::ofstream out(record_path);518 std::ostream & os = record_path.empty() ? std::cout : out;519 if (!record_path.empty()) {520 os << "# test-fusion baseline for device " << base_name << "\n";521 os << "# " << std::left522 << std::setw(18) << "arch" << ','523 << std::setw(4) << "moe" << ','524 << std::setw(8) << "mode" << ','525 << std::setw(28) << "label" << ','526 << std::right << std::setw(7) << "count" << '\n';527 }528 529 LOG_INF("%-20s %-4s %-8s %-22s %7s %7s %7s %10s %10s %s\n",530 "arch", "moe", "mode", "label", "fused", "unfused", "expected", "nmse_fus", "nmse_dev", "status");531 int n_ok = 0;532 int n_bad = 0;533 for (const auto & r : rows) {534 const bool ok = r.ok_count && r.ok_nmse;535 const char * status = ok ? "ok" : "FAIL";536 if (ok) { n_ok++; } else { n_bad++; }537 LOG_INF("%-20s %-4s %-8s %-22s %7llu %7llu %7llu %10.2e %10.2e %s\n",538 r.arch.c_str(), r.moe ? "moe" : "dense", r.mode.c_str(), r.label.c_str(),539 (unsigned long long) r.count_fused, (unsigned long long) r.count_unfused,540 (unsigned long long) r.expected, r.nmse_fus, r.nmse_dev, status);541 if (!record_path.empty()) {542 os << std::left543 << std::setw(20) << r.arch << ','544 << std::setw(4) << (r.moe ? "1" : "0") << ','545 << std::setw(8) << r.mode << ','546 << std::setw(28) << r.label << ','547 << std::right << std::setw(7) << r.count_fused << '\n';548 }549 }550 LOG_INF("summary: %d ok, %d failed\n", n_ok, n_bad);551 if (!record_path.empty()) {552 LOG_INF("%s: baseline written to '%s'\n", __func__, record_path.c_str());553 }554 555 if (n_bad && !models_dir.empty() && !check_path.empty()) {556 LOG_WRN("%s: if the fusion counts are expected to change, run with --record to update the baseline:\n"557 "\n"558 "./bin/test-llama-archs -o %s\n"559 "%s --device %s --models %s --record %s\n",560 __func__, models_dir.c_str(), argv[0], device_name.c_str(), models_dir.c_str(), check_path.c_str());561 }562 563 return n_bad;564 }565}566