Felipe97/llama-cpp-compiled
01.1k
1#include "common.h"2#include "log.h"3#include "ggml-backend.h"4#include "ggml.h"5#include "gguf.h"6#include "ggml-cpp.h"7#include "llama.h"8#include "llama-cpp.h"9 10// TODO: replace with #include "llama-ext.h" in the future11#include "../src/llama-arch.h"12#include "../src/llama-model-saver.h"13 14#include <cinttypes>15#include <cstddef>16#include <cstdio>17#include <cstring>18#include <cstdint>19#include <random>20#include <stdexcept>21#include <string>22#include <utility>23#include <vector>24 25// normalized mean squared error = mse(a, b) / mse(a, 0)26static double nmse(const std::vector<float> & a, const std::vector<float> & b) {27 GGML_ASSERT(a.size() == b.size());28 double mse_a_b = 0.0;29 double mse_a_0 = 0.0;30 31 for (size_t i = 0; i < a.size(); i++) {32 float a_i = a[i];33 float b_i = b[i];34 35 mse_a_b += (a_i - b_i) * (a_i - b_i);36 mse_a_0 += a_i * a_i;37 }38 39 return mse_a_b / mse_a_0;40}41 42static void set_tensor_data(struct ggml_tensor * tensor, void * userdata) {43 size_t seed = *(const size_t *) userdata;44 std::hash<std::string> hasher;45 seed ^= hasher(tensor->name);46 std::mt19937 gen(seed);47 std::normal_distribution<float> dis(0.0f, 1.0e-2f);48 49 const int64_t ne = ggml_nelements(tensor);50 if (tensor->type == GGML_TYPE_F32) {51 std::vector<float> tmp(ne);52 for (int64_t i = 0; i < ne; i++) {53 tmp[i] = dis(gen);54 }55 ggml_backend_tensor_set(tensor, tmp.data(), 0, ggml_nbytes(tensor));56 } else if (tensor->type == GGML_TYPE_F16) {57 std::vector<ggml_fp16_t> tmp(ne);58 for (int64_t i = 0; i < ne; i++) {59 tmp[i] = ggml_fp32_to_fp16(dis(gen));60 }61 ggml_backend_tensor_set(tensor, tmp.data(), 0, ggml_nbytes(tensor));62 } else {63 GGML_ABORT("fatal error");64 }65}66 67static void usage(char ** argv) {68 printf("Usage: %s [-a/--arch arch] [-s/--seed seed] [-o/--out dir] [-v N] [-h/--help]\n", argv[0]);69}70 71static std::vector<llama_token> get_tokens(const uint32_t n_tokens, const uint32_t n_vocab, const size_t seed){72 std::mt19937 gen(seed);73 std::uniform_int_distribution<> dis(0, n_vocab - 1);74 std::vector<llama_token> ret;75 ret.reserve(n_tokens);76 for (uint32_t i = 0; i < n_tokens; i++) {77 ret.push_back(dis(gen));78 }79 return ret;80}81 82static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {83 gguf_context_ptr ret(gguf_init_empty());84 llama_model_saver ms(arch, ret.get());85 const uint32_t n_ctx = 256;86 87 uint32_t n_vocab = 128;88 uint32_t n_embd = 256;89 uint32_t n_head = 2;90 uint32_t n_ff = 384;91 uint32_t n_layer = 2;92 if (arch == LLM_ARCH_LLAMA4) {93 n_layer = 4; // hparams.n_no_rope_layer_step is hard-coded to 494 } else if (arch == LLM_ARCH_GEMMA4) {95 n_embd = 128;96 n_head = 2;97 n_ff = 192;98 n_layer = 5; // need at least 5 for swa_pattern (every 5th is full_attention)99 } else if (arch == LLM_ARCH_GEMMA3N) {100 n_embd = 64;101 n_head = 1;102 n_ff = 96;103 n_layer = 22; // hparams.n_layer_kv_from_start = 20 is hardcoded104 } else if (arch == LLM_ARCH_DEEPSEEK4) {105 // head size 64 so that GPU flash attention kernels support the model106 n_embd = 512;107 n_head = 8;108 n_ff = 1024;109 n_layer = 4;110 } else if (arch == LLM_ARCH_STEP35 || arch == LLM_ARCH_LAGUNA) {111 n_embd = 160; // exercise per-head tensor split granularity with head size 80112 } else if (arch == LLM_ARCH_QWEN3 || arch == LLM_ARCH_MUSE_GLIMMER || arch == LLM_ARCH_AFMOE) {113 n_head = 4;114 } else if (arch == LLM_ARCH_DEEPSEEK2115 || arch == LLM_ARCH_DEEPSEEK32116 || arch == LLM_ARCH_GLM_DSA117 || arch == LLM_ARCH_DOTS3NOTE118 || arch == LLM_ARCH_KIMI_LINEAR119 || arch == LLM_ARCH_BAILINGMOE3120 || arch == LLM_ARCH_KIMI_K3121 || arch == LLM_ARCH_MISTRAL4122 || arch == LLM_ARCH_HY_V4) {123 n_embd = 128;124 n_head = 1;125 n_ff = 192;126 } else if (arch == LLM_ARCH_NEMOTRON_H || arch == LLM_ARCH_NEMOTRON_H_MOE) {127 n_layer = 3;128 } else if (arch == LLM_ARCH_CHAMELEON) {129 n_vocab = 10240;130 } else if (arch == LLM_ARCH_QWEN3TTS) {131 //n_vocab = 4096; // must be >= the hard-coded codec head size (3072)132 n_vocab = 3072; // TODO: should be 4096, but user code cannot get `n_vocab_out` yet [TAG_LLAMA_N_VOCAB_OUT]133 } else if (arch == LLM_ARCH_HRM_TEXT) {134 n_layer = 8; // 1 layer per stack x 2 h-cycles x (3 l-cycles + 1) cache slots135 }136 137 uint32_t n_head_kv = n_head;138 if (arch == LLM_ARCH_QWEN3) {139 n_head_kv = 1; // MQA coverage140 } else if (arch == LLM_ARCH_MUSE_GLIMMER || arch == LLM_ARCH_AFMOE) {141 n_head_kv = 2; // GQA coverage142 }143 const uint32_t n_embd_head = n_embd / n_head;144 145 ms.add_kv(LLM_KV_GENERAL_ARCHITECTURE, llm_arch_name(arch));146 ms.add_kv(LLM_KV_VOCAB_SIZE, n_vocab);147 ms.add_kv(LLM_KV_CONTEXT_LENGTH, n_ctx);148 ms.add_kv(LLM_KV_EMBEDDING_LENGTH, n_embd);149 ms.add_kv(LLM_KV_FEATURES_LENGTH, n_embd);150 ms.add_kv(LLM_KV_BLOCK_COUNT, n_layer);151 ms.add_kv(LLM_KV_LEADING_DENSE_BLOCK_COUNT, uint32_t(1));152 153 if (arch == LLM_ARCH_NEMOTRON_H || arch == LLM_ARCH_NEMOTRON_H_MOE) {154 std::vector<uint32_t> n_ff_per_layer;155 n_ff_per_layer.reserve(n_layer);156 for (uint32_t il = 0; il < n_layer; il++) {157 n_ff_per_layer.push_back(il <= 1 ? 0 : n_ff);158 }159 ms.add_kv(LLM_KV_FEED_FORWARD_LENGTH, n_ff_per_layer);160 } else {161 ms.add_kv(LLM_KV_FEED_FORWARD_LENGTH, n_ff);162 }163 164 ms.add_kv(LLM_KV_USE_PARALLEL_RESIDUAL, false);165 ms.add_kv(LLM_KV_LOGIT_SCALE, 1.0f);166 ms.add_kv(LLM_KV_TIME_MIX_EXTRA_DIM, uint32_t(64));167 ms.add_kv(LLM_KV_TIME_DECAY_EXTRA_DIM, uint32_t(128));168 ms.add_kv(LLM_KV_FULL_ATTENTION_INTERVAL, uint32_t(2));169 170 if (arch == LLM_ARCH_PLAMO2 || arch == LLM_ARCH_JAMBA || arch == LLM_ARCH_NEMOTRON_H || arch == LLM_ARCH_NEMOTRON_H_MOE ||171 arch == LLM_ARCH_GRANITE_HYBRID || arch == LLM_ARCH_LFM2 || arch == LLM_ARCH_LFM2MOE || arch == LLM_ARCH_KIMI_LINEAR ||172 arch == LLM_ARCH_BAILINGMOE3 || arch == LLM_ARCH_KIMI_K3) {173 GGML_ASSERT(n_layer >= 2);174 std::vector<uint32_t> n_head_per_layer;175 n_head_per_layer.reserve(n_layer);176 for (uint32_t il = 0; il < n_layer; il++) {177 n_head_per_layer.push_back(il == 1 ? 0 : n_head);178 }179 ms.add_kv(LLM_KV_ATTENTION_HEAD_COUNT, n_head_per_layer);180 ms.add_kv(LLM_KV_ATTENTION_HEAD_COUNT_KV, n_head_per_layer);181 } else {182 ms.add_kv(LLM_KV_ATTENTION_HEAD_COUNT, n_head);183 ms.add_kv(LLM_KV_ATTENTION_HEAD_COUNT_KV, arch == LLM_ARCH_DEEPSEEK4 ? uint32_t(1) : n_head_kv);184 }185 186 ms.add_kv(LLM_KV_ATTENTION_MAX_ALIBI_BIAS, 8.0f);187 if (arch == LLM_ARCH_DEEPSEEK4) {188 ms.add_kv(LLM_KV_ATTENTION_KEY_LENGTH, n_embd_head);189 ms.add_kv(LLM_KV_ATTENTION_VALUE_LENGTH, n_embd_head);190 ms.add_kv(LLM_KV_ROPE_DIMENSION_COUNT, n_embd_head/2);191 } else if (arch == LLM_ARCH_DEEPSEEK2192 || arch == LLM_ARCH_DEEPSEEK32193 || arch == LLM_ARCH_GLM_DSA194 || arch == LLM_ARCH_DOTS3NOTE195 || arch == LLM_ARCH_KIMI_LINEAR196 || arch == LLM_ARCH_BAILINGMOE3197 || arch == LLM_ARCH_KIMI_K3198 || arch == LLM_ARCH_MISTRAL4199 || arch == LLM_ARCH_HY_V4) {200 ms.add_kv(LLM_KV_ATTENTION_KEY_LENGTH, uint32_t(576));201 ms.add_kv(LLM_KV_ATTENTION_VALUE_LENGTH, uint32_t(512));202 ms.add_kv(LLM_KV_ROPE_DIMENSION_COUNT, uint32_t(64));203 ms.add_kv(LLM_KV_ATTENTION_KEY_LENGTH_MLA, uint32_t(192));204 ms.add_kv(LLM_KV_ATTENTION_VALUE_LENGTH_MLA, uint32_t(128));205 if (arch == LLM_ARCH_DOTS3NOTE) {206 // SWA layers reuse the same MLA geometry as the full layers in this fixture207 ms.add_kv(LLM_KV_ATTENTION_KV_LORA_RANK_SWA, uint32_t(512));208 ms.add_kv(LLM_KV_ATTENTION_KEY_LENGTH_SWA, uint32_t(576));209 ms.add_kv(LLM_KV_ATTENTION_VALUE_LENGTH_SWA, uint32_t(512));210 ms.add_kv(LLM_KV_ATTENTION_KEY_LENGTH_MLA_SWA, uint32_t(192));211 ms.add_kv(LLM_KV_ATTENTION_VALUE_LENGTH_MLA_SWA, uint32_t(128));212 ms.add_kv(LLM_KV_ROPE_FREQ_BASE_SWA, 10000.0f);213 // indexer on the full-attention layers (inverse of the swa pattern)214 std::vector<uint32_t> indexer_types;215 indexer_types.reserve(n_layer);216 for (uint32_t il = 0; il < n_layer; il++) {217 indexer_types.push_back(il % 2 ? 0 : 1);218 }219 ms.add_kv(LLM_KV_ATTENTION_INDEXER_TYPES, indexer_types);220 }221 } else if (arch == LLM_ARCH_MINIMAX_M3) {222 // partial rotary: n_rot must not exceed the indexer key length (64)223 ms.add_kv(LLM_KV_ROPE_DIMENSION_COUNT, uint32_t(64));224 }225 ms.add_kv(LLM_KV_ATTENTION_CLAMP_KQV, 1.0f);226 ms.add_kv(LLM_KV_ATTENTION_LAYERNORM_EPS, 1e-5f);227 ms.add_kv(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, 1e-5f);228 ms.add_kv(LLM_KV_ATTENTION_GROUPNORM_EPS, 1e-5f);229 ms.add_kv(LLM_KV_ATTENTION_GROUPNORM_GROUPS, uint32_t(8));230 ms.add_kv(LLM_KV_ATTENTION_Q_LORA_RANK, arch == LLM_ARCH_DEEPSEEK4 ? uint32_t(64) : uint32_t(512));231 ms.add_kv(LLM_KV_ATTENTION_KV_LORA_RANK, uint32_t(512));232 ms.add_kv(LLM_KV_ATTENTION_RELATIVE_BUCKETS_COUNT, uint32_t(8));233 ms.add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW, n_ctx/8);234 235 if (arch == LLM_ARCH_GEMMA4) {236 ms.add_kv(LLM_KV_EMBEDDING_LENGTH_PER_LAYER, n_embd/2);237 ms.add_kv(LLM_KV_ATTENTION_SHARED_KV_LAYERS, uint32_t(0));238 ms.add_kv(LLM_KV_ATTENTION_KEY_LENGTH_SWA, n_embd_head);239 ms.add_kv(LLM_KV_ATTENTION_VALUE_LENGTH_SWA, n_embd_head);240 ms.add_kv(LLM_KV_ROPE_FREQ_BASE_SWA, 10000.0f);241 // SWA pattern: every 5th layer is full attention (matches E2B layer_types)242 ms.add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, uint32_t(5));243 } else if (arch == LLM_ARCH_COHERE2MOE || arch == LLM_ARCH_MIMO2 || arch == LLM_ARCH_STEP35 || arch == LLM_ARCH_SPARK2_5 ||244 arch == LLM_ARCH_MUSE_GLIMMER || arch == LLM_ARCH_GRANITE_SWA || arch == LLM_ARCH_DOTS3NOTE ||245 arch == LLM_ARCH_MAPLE) {246 std::vector<uint32_t> pattern;247 pattern.reserve(n_layer);248 for (uint32_t il = 0; il < n_layer; il++) {249 pattern.push_back(il % 2);250 }251 ms.add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, pattern);252 } else {253 ms.add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, uint32_t(2));254 }255 256 // MSA requires one indexer head per GQA (KV) head, unlike the DSA archs where the257 // indexer head count is independent of the main attention head count.258 if (arch == LLM_ARCH_QWEN4EXP) {259 ms.add_kv(LLM_KV_HYPER_CONNECTION_COUNT, uint32_t(4));260 ms.add_kv(LLM_KV_HYPER_CONNECTION_LOW_RANK, uint32_t(8));261 // without this the QSA layers fall back to dense and go uncovered262 ms.add_kv(LLM_KV_ATTENTION_COMPRESS_RATIOS, std::vector<uint32_t>(n_layer, 4));263 264 // has_cell_ext() needs ple_n_heads here: the indexer cache serializes no ext without it265 const uint32_t ple_ngram_size = 3;266 const uint32_t ple_heads_per_ngram = 2;267 const uint32_t ple_n_heads = (ple_ngram_size - 1)*ple_heads_per_ngram;268 GGML_ASSERT(n_embd % ple_n_heads == 0);269 const uint32_t ple_head_dim = n_embd/ple_n_heads;270 271 std::vector<uint64_t> ple_head_offsets(ple_n_heads);272 std::vector<uint64_t> ple_head_vocab_sizes(ple_n_heads, n_vocab);273 for (uint32_t h = 0; h < ple_n_heads; h++) {274 ple_head_offsets[h] = uint64_t(h)*n_vocab;275 }276 277 // the PLE history lives in the recurrent cache, so it must sit on a linear attention layer278 ms.add_kv(LLM_KV_PLE_LAYERS, std::vector<uint32_t>({ 0 }));279 ms.add_kv(LLM_KV_PLE_NGRAM_SIZE, ple_ngram_size);280 ms.add_kv(LLM_KV_PLE_HEADS_PER_NGRAM, ple_heads_per_ngram);281 ms.add_kv(LLM_KV_PLE_CONV_KERNEL, uint32_t(4));282 ms.add_kv(LLM_KV_PLE_EOS_TOKEN_ID, uint32_t(0));283 ms.add_kv(LLM_KV_EMBEDDING_LENGTH_PER_LAYER, ple_head_dim);284 ms.add_kv(LLM_KV_PLE_LAYER_MULTIPLIERS, std::vector<uint64_t>({ 1, 3, 5 }));285 ms.add_kv(LLM_KV_PLE_HEAD_OFFSETS, ple_head_offsets);286 ms.add_kv(LLM_KV_PLE_HEAD_VOCAB_SIZES, ple_head_vocab_sizes);287 }288 289 // minimax-m3 keeps one indexer head per GQA head; the rest use a fixed 64 to match the fused290 ms.add_kv(LLM_KV_ATTENTION_INDEXER_HEAD_COUNT, arch == LLM_ARCH_MINIMAX_M3 ? n_head : uint32_t(64));291 // qwen4exp ropes indexer keys with the main rotary width, so its head can't be < n_rot292 ms.add_kv(LLM_KV_ATTENTION_INDEXER_KEY_LENGTH,293 arch == LLM_ARCH_QWEN4EXP ? n_embd_head : uint32_t(128));294 295 ms.add_kv(LLM_KV_ATTENTION_INDEXER_TOP_K, uint32_t(8));296 ms.add_kv(LLM_KV_ATTENTION_INDEXER_BLOCK_SIZE, uint32_t(4));297 ms.add_kv(LLM_KV_ATTENTION_INDEXER_LOCAL_BLOCKS, uint32_t(1));298 ms.add_kv(LLM_KV_ROPE_DIMENSION_SECTIONS, std::vector<uint32_t>({n_embd_head/4, n_embd_head/4, n_embd_head/4, n_embd_head/4}));299 300 if (arch == LLM_ARCH_HY_V4) {301 ms.add_kv(LLM_KV_HYPER_CONNECTION_COUNT, uint32_t(4));302 ms.add_kv(LLM_KV_HYPER_CONNECTION_EPSILON, 1.0e-6f);303 ms.add_kv(LLM_KV_HYPER_CONNECTION_MAGNITUDE, 2.0f);304 ms.add_kv(LLM_KV_SWIGLU_CLAMP_EXP, 10.0f);305 ms.add_kv(LLM_KV_EXPERT_WEIGHTS_SCALE, 1.0f);306 ms.add_kv(LLM_KV_EXPERT_WEIGHTS_NORM, true);307 // layer 0 must own an indexer, the odd layers share it308 std::vector<uint32_t> indexer_types;309 indexer_types.reserve(n_layer);310 for (uint32_t il = 0; il < n_layer; il++) {311 indexer_types.push_back(il % 2 ? 0 : 1);312 }313 ms.add_kv(LLM_KV_ATTENTION_INDEXER_TYPES, indexer_types);314 }315 316 if (arch == LLM_ARCH_DEEPSEEK4) {317 ms.add_kv(LLM_KV_ATTENTION_OUTPUT_GROUP_COUNT, uint32_t(8));318 ms.add_kv(LLM_KV_ATTENTION_OUTPUT_LORA_RANK, uint32_t(32));319 ms.add_kv(LLM_KV_ATTENTION_COMPRESS_RATIOS, std::vector<uint32_t>({0, 0, 4, 128}));320 ms.add_kv(LLM_KV_ATTENTION_COMPRESS_ROPE_FREQ_BASE, 160000.0f);321 ms.add_kv(LLM_KV_HYPER_CONNECTION_COUNT, uint32_t(4));322 ms.add_kv(LLM_KV_HYPER_CONNECTION_SINKHORN_ITERATIONS, uint32_t(2));323 ms.add_kv(LLM_KV_HYPER_CONNECTION_EPSILON, 1.0e-6f);324 ms.add_kv(LLM_KV_HASH_LAYER_COUNT, uint32_t(0));325 ms.add_kv(LLM_KV_SWIGLU_CLAMP_EXP, 10.0f);326 ms.add_kv(LLM_KV_EXPERT_WEIGHTS_SCALE, 1.0f);327 ms.add_kv(LLM_KV_EXPERT_WEIGHTS_NORM, true);328 }329 330 if (arch == LLM_ARCH_HRM_TEXT) {331 // 8 cache slots alias 2 physical blocks: 1 low-stack layer + 1 high-stack layer332 ms.add_kv(LLM_KV_HRM_LAYERS_PER_STACK, uint32_t(1));333 ms.add_kv(LLM_KV_HRM_H_CYCLES, uint32_t(2));334 ms.add_kv(LLM_KV_HRM_L_CYCLES, uint32_t(3));335 }336 337 if (arch == LLM_ARCH_MAPLE) {338 ms.add_kv(LLM_KV_SWIGLU_CLAMP_EXP, 7.0f);339 }340 341 // dummy tokenizer: token ids are derived from fixed-size chunks and detokenized as hex ids342 {343 std::vector<std::string> tokenizer_list(n_vocab);344 std::vector<float> tokenizer_scores(n_vocab, 0.0f);345 346 ms.add_kv(LLM_KV_TOKENIZER_MODEL, "test");347 for (uint32_t i = 0; i < n_vocab; i++) {348 tokenizer_list[i] = "tok_" + std::to_string(i);349 }350 ms.add_kv(LLM_KV_TOKENIZER_LIST, tokenizer_list);351 ms.add_kv(LLM_KV_TOKENIZER_SCORES, tokenizer_scores);352 }353 354 // ms.add_kv(LLM_KV_DENSE_2_FEAT_OUT, n_embd);355 // ms.add_kv(LLM_KV_DENSE_3_FEAT_IN, n_embd);356 357 if (moe) {358 ms.add_kv(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, n_ff);359 ms.add_kv(LLM_KV_EXPERT_SHARED_FEED_FORWARD_LENGTH, n_ff / 2); // distinct from n_ff so a saver key-clobber surfaces on reload360 ms.add_kv(LLM_KV_EXPERT_LATENT_LENGTH, n_ff);361 ms.add_kv(LLM_KV_INTERLEAVE_MOE_LAYER_STEP, uint32_t(2));362 ms.add_kv(LLM_KV_EXPERT_COUNT, uint32_t(2));363 ms.add_kv(LLM_KV_EXPERT_USED_COUNT, uint32_t(2));364 ms.add_kv(LLM_KV_EXPERT_SHARED_COUNT, uint32_t(1));365 ms.add_kv(LLM_KV_EXPERT_GATING_FUNC, arch == LLM_ARCH_DEEPSEEK4 ? uint32_t(4) : uint32_t(2)); // sqrtsoftplus : sigmoid366 ms.add_kv(LLM_KV_EXPERT_GROUP_SCALE, 1.0f);367 ms.add_kv(LLM_KV_EXPERTS_PER_GROUP, uint32_t(1));368 }369 370 ms.add_kv(LLM_KV_POSNET_EMBEDDING_LENGTH, n_embd);371 ms.add_kv(LLM_KV_POSNET_BLOCK_COUNT, n_layer);372 ms.add_kv(LLM_KV_CONVNEXT_EMBEDDING_LENGTH, n_embd);373 ms.add_kv(LLM_KV_CONVNEXT_BLOCK_COUNT, n_layer);374 ms.add_kv(LLM_KV_XIELU_ALPHA_N, 1.0f);375 ms.add_kv(LLM_KV_XIELU_ALPHA_P, 1.0f);376 ms.add_kv(LLM_KV_XIELU_BETA, 1.0f);377 ms.add_kv(LLM_KV_XIELU_EPS, 1.0e-7f);378 ms.add_kv(LLM_KV_SSM_INNER_SIZE, arch == LLM_ARCH_QWEN3NEXT || arch == LLM_ARCH_QWEN35 || arch == LLM_ARCH_QWEN35MOE || arch == LLM_ARCH_QWEN4EXP ? 256 : 2*n_embd);379 ms.add_kv(LLM_KV_SSM_CONV_KERNEL, uint32_t(4));380 ms.add_kv(LLM_KV_SSM_STATE_SIZE, uint32_t(128));381 ms.add_kv(LLM_KV_SSM_TIME_STEP_RANK, n_head);382 ms.add_kv(LLM_KV_SSM_GROUP_COUNT, arch == LLM_ARCH_PLAMO2 ? 0 : uint32_t(2));383 ms.add_kv(LLM_KV_KDA_HEAD_DIM, uint32_t(128));384 ms.add_kv(LLM_KV_KDA_SAFE_GATE, true);385 ms.add_kv(LLM_KV_KDA_GATE_LOWER_BOUND, -5.0f);386 if (arch == LLM_ARCH_BAILINGMOE3) {387 ms.add_kv(LLM_KV_SWIGLU_CLAMP_EXP, std::vector<float>({0.0f, 4.0f}));388 ms.add_kv(LLM_KV_SWIGLU_CLAMP_SHEXP, std::vector<float>({0.0f, 5.0f}));389 }390 ms.add_kv(LLM_KV_WKV_HEAD_SIZE, n_embd/n_head);391 ms.add_kv(LLM_KV_SHORTCONV_L_CACHE, uint32_t(3));392 ms.add_kv(LLM_KV_RESIDUAL_SCALE, 3.5565588200778455f);393 ms.add_kv(LLM_KV_ATTN_RES_BLOCK_SIZE, uint32_t(12));394 ms.add_kv(LLM_KV_ACTIVATION_SITU_BETA, 4.0f);395 ms.add_kv(LLM_KV_ACTIVATION_SITU_LINEAR_BETA, 25.0f);396 ms.add_kv(LLM_KV_KDA_GATE_LOWER_BOUND, -5.0f);397 398 for (uint32_t il = 0; il < n_layer; il++) {399 ggml_tensor t;400 memset(&t, 0, sizeof(ggml_tensor));401 t.type = GGML_TYPE_F16;402 ggml_format_name(&t, "conv%" PRIu32 "d.weight", il);403 gguf_add_tensor(ms.gguf_ctx, &t);404 ggml_format_name(&t, "posnet.%" PRIu32 ".conv1.weight", il);405 gguf_add_tensor(ms.gguf_ctx, &t);406 ggml_format_name(&t, "posnet.%" PRIu32 ".conv2.weight", il);407 gguf_add_tensor(ms.gguf_ctx, &t);408 ggml_format_name(&t, "convnext.%" PRIu32 ".dw.weight", il);409 gguf_add_tensor(ms.gguf_ctx, &t);410 }411 return ret;412}413 414static bool silent_model_load_progress(float /*progress*/, void * /*user_data*/) {415 return true;416}417 418static std::pair<llama_model_ptr, llama_context_ptr> get_model_and_ctx(419 struct gguf_context * gguf_ctx, FILE * file, const size_t seed, const std::vector<ggml_backend_dev_t> & devs,420 const llama_split_mode split_mode = LLAMA_SPLIT_MODE_LAYER, bool encode = false) {421 GGML_ASSERT((gguf_ctx == nullptr) != (file == nullptr));422 llama_model_params model_params = llama_model_default_params();423 model_params.progress_callback = silent_model_load_progress;424 std::vector<ggml_backend_dev_t> devs_copy = devs;425 devs_copy.push_back(nullptr);426 model_params.devices = devs_copy.data();427 model_params.split_mode = split_mode;428 429 llama_context_params ctx_params = llama_context_default_params();430 ctx_params.n_ctx = 0;431 ctx_params.n_threads = 4;432 ctx_params.n_threads_batch = 4;433 if (!encode) {434 ctx_params.n_ubatch = 64;435 }436 437 size_t tmp = seed;438 llama_model_ptr model(gguf_ctx != nullptr ?439 llama_model_init_from_user(gguf_ctx, set_tensor_data, &tmp, model_params) :440 llama_model_load_from_file_ptr(file, model_params));441 if (!model) {442 throw std::runtime_error("failed to create llama model");443 }444 llama_context_ptr lctx(llama_init_from_model(model.get(), ctx_params));445 if (!lctx) {446 throw std::runtime_error("failed to create llama context");447 }448 return std::make_pair(std::move(model), std::move(lctx));449}450 451static std::vector<float> get_logits(452 llama_model * model, llama_context * lctx, const std::vector<llama_token> & tokens, bool encode = false) {453 const uint32_t n_vocab = llama_vocab_n_tokens(llama_model_get_vocab(model));454 const uint32_t n_ctx = llama_n_ctx(lctx);455 const uint32_t n_tokens = tokens.size();456 llama_batch batch = llama_batch_init(n_ctx, 0, 1);457 GGML_ASSERT(n_tokens <= n_ctx);458 for (uint32_t pos = 0; pos < n_tokens; pos++) {459 common_batch_add(batch, tokens[pos], pos, {0}, true);460 }461 batch.n_tokens = n_tokens;462 if (encode) {463 if (llama_encode(lctx, batch)) {464 llama_batch_free(batch);465 throw std::runtime_error("failed to encode batch");466 }467 }468 if (llama_decode(lctx, batch)) {469 llama_batch_free(batch);470 throw std::runtime_error("failed to decode batch");471 }472 473 std::vector<float> ret;474 ret.reserve(n_tokens*n_vocab);475 for (uint32_t i = 0; i < n_tokens; i++) {476 const float * logits_ith = llama_get_logits_ith(lctx, i);477 for (uint32_t j = 0; j < n_vocab; j++) {478 ret.push_back(logits_ith[j]);479 }480 }481 llama_batch_free(batch);482 return ret;483}484 485static bool moe_mandatory(const llm_arch arch) {486 switch (arch) {487 case LLM_ARCH_LLAMA4:488 case LLM_ARCH_COHERE2MOE:489 case LLM_ARCH_GROK:490 case LLM_ARCH_QWEN2MOE:491 case LLM_ARCH_QWEN3MOE:492 case LLM_ARCH_QWEN3NEXT:493 case LLM_ARCH_QWEN3VLMOE:494 case LLM_ARCH_QWEN35MOE:495 case LLM_ARCH_QWEN4EXP:496 case LLM_ARCH_PHIMOE:497 case LLM_ARCH_DBRX:498 case LLM_ARCH_OLMOE:499 case LLM_ARCH_ARCTIC:500 case LLM_ARCH_DEEPSEEK:501 case LLM_ARCH_DEEPSEEK2:502 case LLM_ARCH_DEEPSEEK32:503 case LLM_ARCH_DOTS3NOTE:504 case LLM_ARCH_DEEPSEEK4:505 case LLM_ARCH_GLM4_MOE:506 case LLM_ARCH_GLM_DSA:507 case LLM_ARCH_EXAONE_MOE:508 case LLM_ARCH_BAILINGMOE:509 case LLM_ARCH_BAILINGMOE2:510 case LLM_ARCH_BAILINGMOE3:511 case LLM_ARCH_DOTS1:512 case LLM_ARCH_AFMOE:513 case LLM_ARCH_ERNIE4_5:514 case LLM_ARCH_ERNIE4_5_MOE:515 case LLM_ARCH_HUNYUAN_MOE:516 case LLM_ARCH_HY_V3:517 case LLM_ARCH_HY_V4:518 case LLM_ARCH_OPENAI_MOE:519 case LLM_ARCH_LFM2MOE:520 case LLM_ARCH_SMALLTHINKER:521 case LLM_ARCH_LLADA_MOE:522 case LLM_ARCH_GROVEMOE:523 case LLM_ARCH_MINIMAX_01:524 case LLM_ARCH_MINIMAX_M2:525 case LLM_ARCH_MINIMAX_M3:526 case LLM_ARCH_RND1:527 case LLM_ARCH_PADDLEOCR:528 case LLM_ARCH_MIMO2:529 case LLM_ARCH_KIMI_LINEAR:530 case LLM_ARCH_KIMI_K3:531 case LLM_ARCH_STEP35:532 case LLM_ARCH_MISTRAL4:533 case LLM_ARCH_MELLUM:534 case LLM_ARCH_LAGUNA:535 case LLM_ARCH_MAPLE:536 return true;537 default:538 return false;539 }540}541 542static bool moe_implemented(const llm_arch arch) {543 if (moe_mandatory(arch)) {544 return true;545 }546 switch (arch) {547 case LLM_ARCH_LLAMA:548 case LLM_ARCH_REFACT:549 case LLM_ARCH_MINICPM:550 case LLM_ARCH_GRANITE:551 case LLM_ARCH_GRANITE_MOE:552 case LLM_ARCH_MISTRAL3:553 case LLM_ARCH_LLAMA_EMBED:554 return true;555 default:556 return false;557 }558}559 560static bool arch_supported(const llm_arch arch) {561 if (arch == LLM_ARCH_CLIP || arch == LLM_ARCH_GPTJ || arch == LLM_ARCH_UNKNOWN) {562 return false; // These models don't have usable implementations.563 }564 if (arch == LLM_ARCH_CHAMELEON) {565 return false; // Only half-implemented and to be removed in the future.566 }567 if (arch == LLM_ARCH_WAVTOKENIZER_DEC) {568 return false; // FIXME CUDA backend crashes.569 }570 if (arch == LLM_ARCH_GEMMA4 || arch == LLM_ARCH_GEMMA4_ASSISTANT) {571 return false; // FIXME @ngxson572 }573 if (arch == LLM_ARCH_GRANITE_SWITCH) {574 return false; // FIXME adapter fixture575 }576 if (arch == LLM_ARCH_LLAMA_EMBED || arch == LLM_ARCH_GEMMA_EMBEDDING || arch == LLM_ARCH_T5ENCODER) {577 return false; // FIXME Embedding (?) models produce inconsistent results.578 }579 if (arch == LLM_ARCH_RWKV6 || arch == LLM_ARCH_RWKV6QWEN2 || arch == LLM_ARCH_RWKV7 || arch == LLM_ARCH_ARWKV7) {580 return false; // FIXME RWKV models hang indefinitely.581 }582 if (arch == LLM_ARCH_BERT || arch == LLM_ARCH_MODERN_BERT || arch == LLM_ARCH_NOMIC_BERT || arch == LLM_ARCH_NOMIC_BERT_MOE ||583 arch == LLM_ARCH_NEO_BERT || arch == LLM_ARCH_JINA_BERT_V2 || arch == LLM_ARCH_JINA_BERT_V3 || arch == LLM_ARCH_EUROBERT) {584 return false; // TODO vocab585 }586 if (arch == LLM_ARCH_PLM) {587 return false; // TODO tensor shapes588 }589 if (arch == LLM_ARCH_DEEPSEEK2OCR) {590 return false;591 }592 // FIXME: these hit scheduler/view-backed-output issues with WebGPU on CI.593#ifdef GGML_USE_WEBGPU594 if (arch == LLM_ARCH_DEEPSEEK32 || arch == LLM_ARCH_GLM_DSA || arch == LLM_ARCH_DOTS3NOTE || arch == LLM_ARCH_QWEN4EXP ||595 arch == LLM_ARCH_HY_V4) {596 return false;597 }598#endif // GGML_USE_WEBGPU599 600 // FIXME: jamba produces incorrect output (~0.55 NMSE vs CPU) on the HIP601 // backend on RDNA3.5 (gfx1151); the SSM kernels need investigation.602#ifdef GGML_USE_HIP603 if (arch == LLM_ARCH_JAMBA) {604 return false;605 }606#endif // GGML_USE_HIP607 608 return true;609}610 611static int save_models(const llm_arch target_arch, const size_t seed, const int verbosity, const std::string & dir) {612 struct user_data_t {613 struct {614 ggml_log_callback callback;615 void * user_data;616 } log_old;617 618 int verbosity;619 620 user_data_t(int verbosity) : verbosity(verbosity) {621 llama_log_get(&log_old.callback, &log_old.user_data);622 }623 };624 user_data_t ud(verbosity);625 626 llama_log_set([](ggml_log_level level, const char * text, void * user_data) {627 const user_data_t * ud = (const user_data_t *) user_data;628 int verbosity = common_log_get_verbosity(level);629 if (verbosity <= ud->verbosity) {630 ud->log_old.callback(level, text, ud->log_old.user_data);631 }632 }, &ud);633 634 for (const llm_arch & arch : llm_arch_all()) {635 if (arch == LLM_ARCH_UNKNOWN) {636 continue;637 }638 if (target_arch != LLM_ARCH_UNKNOWN && arch != target_arch) {639 continue;640 }641 if (arch == LLM_ARCH_GEMMA4 || arch == LLM_ARCH_GEMMA4_ASSISTANT) {642 continue; // FIXME: ISWA KV cache initialization needs more fixture params643 }644 if (arch == LLM_ARCH_EAGLE3 || arch == LLM_ARCH_DFLASH) {645 continue;646 }647 for (bool moe : {false, true}) {648 if (moe && !moe_implemented(arch)) {649 continue;650 }651 if (!moe && moe_mandatory(arch)) {652 continue;653 }654 if (!llama_model_saver_supports_arch(arch) || !arch_supported(arch)) {655 LOG_INF("%s: %s model (%s) is unsupported, skipping\n", __func__, llm_arch_name(arch), moe ? "MoE" : "dense");656 continue;657 }658 gguf_context_ptr gguf_ctx = get_gguf_ctx(arch, moe);659 auto model_and_ctx = get_model_and_ctx(gguf_ctx.get(), nullptr, seed, {});660 const std::string path = dir + "/" + llm_arch_name(arch) + (moe ? "-moe.gguf" : "-dense.gguf");661 LOG_INF("%s: Saving %s model (%s) to %s...\n", __func__, llm_arch_name(arch), moe ? "MoE" : "dense", path.c_str());662 llama_model_save_to_file(model_and_ctx.first.get(), path.c_str());663 }664 }665 llama_log_set(ud.log_old.callback, ud.log_old.user_data);666 return 0;667}668 669static int test_backends(const llm_arch target_arch, const size_t seed, const int verbosity) {670 struct user_data_t {671 struct {672 ggml_log_callback callback;673 void * user_data;674 } log_old;675 676 int verbosity;677 678 user_data_t(int verbosity) : verbosity(verbosity) {679 llama_log_get(&log_old.callback, &log_old.user_data);680 }681 };682 user_data_t ud(verbosity);683 684 llama_log_set([](ggml_log_level level, const char * text, void * user_data) {685 const user_data_t * ud = (const user_data_t *) user_data;686 int verbosity = common_log_get_verbosity(level);687 if (verbosity <= ud->verbosity) {688 ud->log_old.callback(level, text, ud->log_old.user_data);689 }690 }, &ud);691 692 const std::vector<llama_token> tokens = get_tokens(128, 128, seed);693 694 struct device_config {695 std::vector<ggml_backend_dev_t> devs;696 std::string label;697 llama_split_mode split_mode;698 699 device_config(std::vector<ggml_backend_dev_t> devs, std::string name, llama_split_mode split_mode)700 : devs(std::move(devs)), label(std::move(name)), split_mode(split_mode) {}701 };702 703 std::vector<device_config> dev_configs;704 size_t max_device_label_length = 4;705 {706 std::vector<ggml_backend_dev_t> devices_meta;707 {708 const size_t device_count = ggml_backend_dev_count();709 for (size_t i = 0; i < device_count; i++) {710 ggml_backend_dev_t dev = ggml_backend_dev_get(i);711 dev_configs.emplace_back(std::vector<ggml_backend_dev_t>{dev}, ggml_backend_dev_description(dev), LLAMA_SPLIT_MODE_LAYER);712 max_device_label_length = std::max(max_device_label_length, dev_configs.back().label.length());713 714 // cpu-based devices cannot be used in tensor split mode715 if (ggml_backend_dev_buffer_type(dev) != ggml_backend_cpu_buffer_type()) {716 devices_meta.push_back(dev);717 }718 }719 }720 721 dev_configs.emplace_back(devices_meta, "Meta", LLAMA_SPLIT_MODE_TENSOR);722 }723 724 size_t max_arch_name_length = 0;725 for (const llm_arch & arch : llm_arch_all()) {726 max_arch_name_length = std::max(max_arch_name_length, strlen(llm_arch_name(arch)));727 }728 729 const std::string template_header = std::string("|%" + std::to_string(max_arch_name_length) + "s|%") + std::to_string(max_device_label_length) + "s|%6s|%15s|%9s|\n";730 const std::string template_row_cfg = std::string("|%" + std::to_string(max_arch_name_length) + "s|%") + std::to_string(max_device_label_length) + "s|%6s|";731 const std::string template_row_res = "%15s %10s|%20s|\n";732 733 bool all_ok = true;734 common_log_flush(common_log_main());735 printf(template_header.c_str(), "Model arch.", "Device", "Config", "NMSE vs. CPU", "Roundtrip");736 printf("|");737 for (size_t i = 0; i < max_arch_name_length; i++) {738 printf("-");739 }740 printf("|");741 for (size_t i = 0; i < max_device_label_length; i++) {742 printf("-");743 }744 printf("|------|---------------|---------|\n");745 for (const llm_arch & arch : llm_arch_all()) {746 if (arch == LLM_ARCH_UNKNOWN) {747 continue;748 }749 if (target_arch != LLM_ARCH_UNKNOWN && arch != target_arch) {750 continue;751 }752 if (arch == LLM_ARCH_GEMMA4 || arch == LLM_ARCH_GEMMA4_ASSISTANT) {753 continue; // FIXME: ISWA KV cache initialization needs more fixture params754 }755 if (arch == LLM_ARCH_EAGLE3 || arch == LLM_ARCH_DFLASH) {756 continue;757 }758 759 const bool encode = arch == LLM_ARCH_T5 || arch == LLM_ARCH_DREAM || arch == LLM_ARCH_LLADA || arch == LLM_ARCH_LLADA_MOE || arch == LLM_ARCH_RND1;760 for (bool moe : {false, true}) {761 if (moe && !moe_implemented(arch)) {762 continue;763 }764 if (!moe && moe_mandatory(arch)) {765 continue;766 }767 const std::string config_name = moe ? "MoE" : "Dense";768 gguf_context_ptr gguf_ctx = get_gguf_ctx(arch, moe);769 if (arch == LLM_ARCH_BAILINGMOE3) {770 GGML_ASSERT(gguf_remove_key(gguf_ctx.get(), "bailingmoe3.kda.safe_gate") >= 0);771 }772 std::pair<llama_model_ptr, llama_context_ptr> model_and_ctx_cpu;773 std::vector<float> logits_cpu;774 for (device_config & dc : dev_configs) {775 // print test config first; should anything fail during model loading or inference, at least we know which test case caused it776 printf(template_row_cfg.c_str(),777 llm_arch_name(arch), dc.label.c_str(), config_name.c_str());778 fflush(stdout);779 780 std::pair<llama_model_ptr, llama_context_ptr> model_and_ctx_dev;781 std::vector<float> logits_dev;782 std::string status_nmse = "\033[1;33mSKIP\033[0m";783 std::string status_roundtrip = "\033[1;33mSKIP\033[0m";784 char nmse_str[12] = {0};785 786 bool skip = !arch_supported(arch) || (dc.split_mode == LLAMA_SPLIT_MODE_TENSOR && dc.devs.empty());787 if (!skip) {788 if (logits_cpu.empty()) {789 model_and_ctx_cpu = get_model_and_ctx(gguf_ctx.get(), nullptr, seed, {}, LLAMA_SPLIT_MODE_LAYER, encode);790 logits_cpu = get_logits(model_and_ctx_cpu.first.get(), model_and_ctx_cpu.second.get(), tokens, encode);791 }792 if (dc.split_mode != LLAMA_SPLIT_MODE_TENSOR || llm_arch_supports_sm_tensor(arch)) {793 model_and_ctx_dev = get_model_and_ctx(gguf_ctx.get(), nullptr, seed, dc.devs, dc.split_mode, encode);794 logits_dev = get_logits(model_and_ctx_dev.first.get(), model_and_ctx_dev.second.get(), tokens, encode);795 const double nmse_val = nmse(logits_cpu, logits_dev);796 snprintf(nmse_str, sizeof(nmse_str), "(%.2e)", nmse_val);797 status_nmse = "\033[1;32mOK\033[0m";798 if (nmse_val > 1e-4) {799 all_ok = false;800 status_nmse = "\033[1;31mFAIL\033[0m";801 }802 }803 804 FILE * file = tmpfile(); // Can be null on Windows without administrator privileges.805 // FIXME: when adding a tensor to a gguf_context a copy is made, this changes the pointer which the meta backend806 // in turn uses to map the tensors to their simple equivalents - this is fundamentally incompatible807 if (file != nullptr && llama_model_saver_supports_arch(arch) && dc.split_mode != LLAMA_SPLIT_MODE_TENSOR) {808 GGML_ASSERT(model_and_ctx_dev.first && model_and_ctx_dev.second);809 llama_model_saver ms = llama_model_saver(model_and_ctx_dev.first.get());810 ms.add_kv_from_model();811 ms.add_tensors_from_model();812 ms.save(file);813 rewind(file);814 815 auto model_and_ctx_roundtrip = get_model_and_ctx(nullptr, file, seed, dc.devs, dc.split_mode, encode);816 const std::vector<float> logits_roundtrip = get_logits(817 model_and_ctx_roundtrip.first.get(), model_and_ctx_roundtrip.second.get(), tokens, encode);818 status_roundtrip = "\033[1;32mOK\033[0m";819 GGML_ASSERT(logits_roundtrip.size() == logits_dev.size());820 for (size_t i = 0; i < logits_roundtrip.size(); i++) {821 if (logits_roundtrip[i] != logits_dev[i]) {822 all_ok = false;823 status_roundtrip = "\033[1;31mFAIL\033[0m";824 break;825 }826 }827 }828 }829 830 // log the results for this test case831 printf(template_row_res.c_str(),832 status_nmse.c_str(), nmse_str, status_roundtrip.c_str());833 }834 }835 }836 llama_log_set(ud.log_old.callback, ud.log_old.user_data);837 return all_ok ? 0 : 1;838}839 840int main(int argc, char ** argv) {841 // init the logger at max verbosity. filter with a custom callback respecting the user-configure verbosity842 common_log_set_verbosity_thold(LOG_LEVEL_DEBUG);843 common_init();844 845 std::random_device rd;846 847 llm_arch arch = LLM_ARCH_UNKNOWN;848 size_t seed = rd();849 std::string out;850 851 int verbosity = LOG_LEVEL_ERROR;852 853 for (int i = 1; i < argc; i++) {854 if (strcmp(argv[i], "-h") == 0 || strcmp(argv[i], "--help") == 0) {855 usage(argv);856 return 0;857 }858 if (strcmp(argv[i], "-a") == 0 || strcmp(argv[i], "--arch") == 0) {859 if (i + 1 < argc) {860 const std::string arch_name = argv[++i];861 arch = llm_arch_from_string(arch_name);862 if (arch == LLM_ARCH_UNKNOWN) {863 LOG_ERR("%s: unkown LLM architecture: %s\n", __func__, arch_name.c_str());864 return 1;865 }866 } else {867 usage(argv);868 return 1;869 }870 }871 if (strcmp(argv[i], "-s") == 0 || strcmp(argv[i], "--seed") == 0) {872 if (i + 1 < argc) {873 seed = std::stoull(argv[++i]);874 } else {875 usage(argv);876 return 1;877 }878 }879 if (strcmp(argv[i], "-v") == 0) {880 if (i + 1 < argc) {881 verbosity = std::stoull(argv[++i]);882 } else {883 usage(argv);884 return 1;885 }886 }887 if (strcmp(argv[i], "-o") == 0 || strcmp(argv[i], "--out") == 0) {888 if (i + 1 < argc) {889 out = argv[++i];890 } else {891 usage(argv);892 return 1;893 }894 }895 }896 printf("%s: using seed %zu\n", __func__, seed);897 898 try {899 if (!out.empty()) {900 return save_models(arch, seed, verbosity, out);901 }902 return test_backends(arch, seed, verbosity);903 } catch (const std::exception & err) {904 fprintf(stderr, "encountered runtime error: %s\n", err.what());905 return -1;906 }907}908