Felipe97/llama-cpp-compiled
01.1k
1#include "models.h"2 3void llama_model_mimo2::load_arch_hparams(llama_model_loader & ml) {4 ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);5 6 hparams.swa_type = LLAMA_SWA_TYPE_STANDARD;7 8 ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all);9 ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa);10 ml.get_key(LLM_KV_ROPE_FREQ_BASE_SWA, hparams.rope_freq_base_train_swa, false);11 12 ml.get_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.is_swa_impl);13 14 float value_scale = 0.0f;15 if (ml.get_key(LLM_KV_ATTENTION_VALUE_SCALE, value_scale, false) && value_scale != 1.0f) {16 hparams.f_attn_value_scale = value_scale;17 }18 19 switch (hparams.n_layer()) {20 case 48: type = LLM_TYPE_310B_A15B; break;21 default: type = LLM_TYPE_UNKNOWN;22 }23}24 25void llama_model_mimo2::load_arch_tensors(llama_model_loader & ml) {26 LLAMA_LOAD_LOCALS;27 28 const std::string mtp_probe = "blk." + std::to_string(n_layer) + ".nextn.eh_proj.weight";29 const bool trunk_only = (hparams.n_layer_nextn > 0) && (ml.get_weight(mtp_probe.c_str()) == nullptr);30 int mtp_flags = trunk_only ? TENSOR_NOT_REQUIRED : 0;31 32 if (!ml.load_mtp) {33 mtp_flags |= TENSOR_SKIP;34 }35 36 tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0);37 38 // output39 output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0);40 output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, 0);41 42 for (int i = 0; i < n_layer_all; ++i) {43 auto & layer = layers[i];44 uint32_t n_embd_k_gqa = hparams.n_embd_k_gqa(i);45 uint32_t n_embd_v_gqa = hparams.n_embd_v_gqa(i);46 uint32_t n_head = hparams.n_head(i);47 48 const bool is_nextn = i >= n_layer;49 const int flags = is_nextn ? mtp_flags : 0;50 51 create_tensor_qkv(layer, i, n_embd, n_embd_head_k * n_head, n_embd_k_gqa, n_embd_v_gqa, flags);52 layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), { n_embd_head_v * n_head, n_embd }, flags);53 54 layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, flags);55 layer.attn_sinks = create_tensor(tn(LLM_TENSOR_ATTN_SINKS, "weight", i), {n_head}, TENSOR_NOT_REQUIRED | flags);56 57 layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, flags);58 59 // non-MoE branch60 layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff}, TENSOR_NOT_REQUIRED | flags);61 layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd}, TENSOR_NOT_REQUIRED | flags);62 layer.ffn_up = create_tensor(tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, TENSOR_NOT_REQUIRED | flags);63 64 // MoE branch65 int64_t n_ff_exp = hparams.n_ff_exp();66 layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd, n_expert}, TENSOR_NOT_REQUIRED | flags);67 layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i), {n_embd, n_ff_exp, n_expert}, TENSOR_NOT_REQUIRED | flags);68 layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {n_ff_exp, n_embd, n_expert}, TENSOR_NOT_REQUIRED | flags);69 layer.ffn_up_exps = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS, "weight", i), {n_embd, n_ff_exp, n_expert}, TENSOR_NOT_REQUIRED | flags);70 layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, "bias", i), {n_expert}, TENSOR_NOT_REQUIRED | flags);71 72 if (is_nextn) {73 layer.nextn.eh_proj = create_tensor(tn(LLM_TENSOR_NEXTN_EH_PROJ, "weight", i), {2 * n_embd, n_embd}, flags);74 layer.nextn.enorm = create_tensor(tn(LLM_TENSOR_NEXTN_ENORM, "weight", i), {n_embd}, flags);75 layer.nextn.hnorm = create_tensor(tn(LLM_TENSOR_NEXTN_HNORM, "weight", i), {n_embd}, flags);76 layer.nextn.embed_tokens = create_tensor(tn(LLM_TENSOR_NEXTN_EMBED_TOKENS, "weight", i), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED | flags);77 layer.nextn.shared_head_head = create_tensor(tn(LLM_TENSOR_NEXTN_SHARED_HEAD_HEAD, "weight", i), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED | flags);78 layer.nextn.shared_head_norm = create_tensor(tn(LLM_TENSOR_NEXTN_SHARED_HEAD_NORM, "weight", i), {n_embd}, TENSOR_NOT_REQUIRED | flags);79 layer.layer_out_norm = create_tensor(tn(LLM_TENSOR_LAYER_OUT_NORM, "weight", i), {n_embd}, TENSOR_NOT_REQUIRED | flags);80 }81 }82}83 84std::unique_ptr<llm_graph_context> llama_model_mimo2::build_arch_graph(const llm_graph_params & params) const {85 if (params.gtype == LLM_GRAPH_TYPE_DECODER_MTP) {86 return std::make_unique<graph_mtp>(*this, params);87 }88 return std::make_unique<graph>(*this, params);89}90 91llama_model_mimo2::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {92 ggml_tensor * cur;93 ggml_tensor * inpL;94 95 inpL = build_inp_embd(model.tok_embd);96 97 ggml_tensor * inp_pos = build_inp_pos();98 auto * inp_attn = build_attn_inp_kv_iswa();99 ggml_tensor * inp_out_ids = build_inp_out_ids();100 101 const float v_scale = hparams.f_attn_value_scale;102 const bool emit_h_nextn = cparams.embeddings_nextn;103 const bool crop_last_layer = inp_out_ids && (!emit_h_nextn || cparams.embeddings_nextn_masked);104 105 for (int il = 0; il < n_layer; ++il) {106 ggml_tensor * inpSA = inpL;107 108 uint32_t n_head_l = hparams.n_head(il);109 uint32_t n_head_kv_l = hparams.n_head_kv(il);110 const float freq_base_l = model.get_rope_freq_base(cparams, il);111 const float freq_scale_l = model.get_rope_freq_scale(cparams, il);112 113 cur = inpL;114 115 // self_attention116 {117 cur = build_norm(inpL, model.layers[il].attn_norm, NULL, LLM_NORM_RMS, il);118 cb(cur, "attn_norm", il);119 120 ggml_tensor * Qcur;121 ggml_tensor * Kcur;122 ggml_tensor * Vcur;123 124 if (model.layers[il].wqkv) {125 // Fused qkv_proj - Q/K share head_dim_k, V uses head_dim_v126 ggml_tensor * qkv = build_lora_mm(model.layers[il].wqkv, cur);127 cb(qkv, "wqkv", il);128 129 const size_t row_k = ggml_row_size(qkv->type, n_embd_head_k);130 const size_t row_v = ggml_row_size(qkv->type, n_embd_head_v);131 const size_t row_full = qkv->nb[1];132 const size_t k_off = row_k * n_head_l;133 const size_t v_off = k_off + row_k * n_head_kv_l;134 135 Qcur = ggml_view_3d(ctx0, qkv, n_embd_head_k, n_head_l, n_tokens, row_k, row_full, 0);136 Kcur = ggml_view_3d(ctx0, qkv, n_embd_head_k, n_head_kv_l, n_tokens, row_k, row_full, k_off);137 Vcur = ggml_view_3d(ctx0, qkv, n_embd_head_v, n_head_kv_l, n_tokens, row_v, row_full, v_off);138 } else {139 // Split path140 Qcur = build_lora_mm(model.layers[il].wq, cur);141 cb(Qcur, "Qcur", il);142 143 Kcur = build_lora_mm(model.layers[il].wk, cur);144 cb(Kcur, "Kcur", il);145 146 Vcur = build_lora_mm(model.layers[il].wv, cur);147 cb(Vcur, "Vcur", il);148 149 Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head_k, n_head_l, n_tokens);150 Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head_k, n_head_kv_l, n_tokens);151 Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head_v, n_head_kv_l, n_tokens);152 }153 154 Qcur = ggml_rope_ext(155 ctx0, Qcur, inp_pos, nullptr,156 n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l,157 ext_factor, attn_factor, beta_fast, beta_slow158 );159 160 Kcur = ggml_rope_ext(161 ctx0, Kcur, inp_pos, nullptr,162 n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l,163 ext_factor, attn_factor, beta_fast, beta_slow164 );165 166 cb(Qcur, "Qcur", il);167 cb(Kcur, "Kcur", il);168 cb(Vcur, "Vcur", il);169 170 ggml_tensor * sinks = model.layers[il].attn_sinks;171 172 cur = build_attn(inp_attn,173 model.layers[il].wo, NULL, model.layers[il].wo_s,174 Qcur, Kcur, Vcur, nullptr, sinks, nullptr, 1.0f/sqrtf(float(n_embd_head_k)), il);175 cb(cur, "attn_out", il);176 177 if (v_scale) {178 cur = ggml_scale(ctx0, cur, v_scale);179 cb(cur, "attn_out_scaled", il);180 }181 }182 183 if (il == n_layer - 1 && crop_last_layer) {184 cur = ggml_get_rows(ctx0, cur, inp_out_ids);185 inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);186 }187 188 ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);189 cb(ffn_inp, "ffn_inp", il);190 191 cur = build_norm(ffn_inp,192 model.layers[il].ffn_norm, NULL,193 LLM_NORM_RMS, il);194 cb(cur, "ffn_norm", il);195 196 // feed-forward network197 if (model.layers[il].ffn_gate_inp == nullptr) {198 // dense branch199 cur = build_ffn(cur,200 model.layers[il].ffn_up, model.layers[il].ffn_up_b, NULL,201 model.layers[il].ffn_gate, model.layers[il].ffn_gate_b, NULL,202 model.layers[il].ffn_down, model.layers[il].ffn_down_b, NULL,203 NULL,204 LLM_FFN_SILU, LLM_FFN_PAR, il);205 cb(cur, "ffn_out", il);206 } else {207 // MoE branch208 cur = build_moe_ffn(cur,209 model.layers[il].ffn_gate_inp,210 model.layers[il].ffn_up_exps,211 model.layers[il].ffn_gate_exps,212 model.layers[il].ffn_down_exps,213 model.layers[il].ffn_exp_probs_b,214 n_expert, n_expert_used,215 LLM_FFN_SILU, true,216 hparams.expert_weights_scale,217 LLAMA_EXPERT_GATING_FUNC_TYPE_SIGMOID,218 il);219 cb(cur, "ffn_moe_out", il);220 }221 222 cur = ggml_add(ctx0, cur, ffn_inp);223 224 cur = build_cvec(cur, il);225 cb(cur, "l_out", il);226 227 // input for next layer228 inpL = cur;229 }230 231 cur = inpL;232 233 if (emit_h_nextn) {234 cb(cur, "h_nextn", -1);235 res->t_h_nextn = cur;236 237 if (!cparams.embeddings_nextn_masked && inp_out_ids) {238 cur = ggml_get_rows(ctx0, cur, inp_out_ids);239 }240 }241 242 cur = build_norm(cur,243 model.output_norm, NULL,244 LLM_NORM_RMS, -1);245 246 cb(cur, "result_norm", -1);247 res->t_embd = cur;248 249 // lm_head250 cur = build_lora_mm(model.output, cur, model.output_s);251 252 cb(cur, "result_output", -1);253 res->t_logits = cur;254 255 ggml_build_forward_expand(gf, cur);256}257 258// Mirrors MiMo's appended NextN block: normalize and fuse token and hidden inputs, run the decoder block,259// expose its pre-head-norm state to the next draft step, then apply the shared output norm and LM head.260// Converted checkpoints may store that shared norm as layer_out_norm, so it remains in the fallback chain.261llama_model_mimo2::graph_mtp::graph_mtp(const llama_model & model, const llm_graph_params & params)262 : llm_graph_context(params) {263 GGML_ASSERT(hparams.n_layer_nextn > 0 && "MIMO2 MTP requires n_layer_nextn > 0");264 265 const int il = hparams.n_layer() + cparams.nextn_layer_offset;266 GGML_ASSERT(cparams.nextn_layer_offset >= 0 &&267 cparams.nextn_layer_offset < (int) hparams.n_layer_nextn &&268 "nextn_layer_offset out of range [0, n_layer_nextn)");269 270 const auto & layer = model.layers[il];271 GGML_ASSERT(layer.nextn.eh_proj && "MIMO2 MTP block missing nextn.eh_proj");272 GGML_ASSERT(layer.nextn.enorm && "MIMO2 MTP block missing nextn.enorm");273 GGML_ASSERT(layer.nextn.hnorm && "MIMO2 MTP block missing nextn.hnorm");274 GGML_ASSERT(layer.wqkv && "MIMO2 MTP requires fused attn_qkv");275 276 const uint32_t n_head_l = hparams.n_head(il);277 const uint32_t n_head_kv_l = hparams.n_head_kv(il);278 279 const float freq_base_l = model.get_rope_freq_base(cparams, il);280 const float freq_scale_l = model.get_rope_freq_scale(cparams, il);281 const float v_scale = hparams.f_attn_value_scale;282 283 auto inp = std::make_unique<llm_graph_input_embd>(hparams.n_embd);284 285 inp->tokens = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_tokens);286 ggml_set_input(inp->tokens);287 288 inp->embd = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, hparams.n_embd, n_tokens);289 ggml_set_input(inp->embd);290 ggml_set_name(inp->embd, "mtp_h_input");291 292 ggml_tensor * tok_embd_w = layer.nextn.embed_tokens ? layer.nextn.embed_tokens : model.tok_embd;293 ggml_tensor * h_input = inp->embd;294 ggml_tensor * tok_embd = ggml_get_rows(ctx0, tok_embd_w, inp->tokens);295 cb(tok_embd, "mtp_tok_embd", il);296 297 res->add_input(std::move(inp));298 299 ggml_tensor * inp_pos = build_inp_pos();300 ggml_tensor * inp_out_ids = build_inp_out_ids();301 auto * inp_attn = build_attn_inp_kv_iswa();302 303 ggml_tensor * h_norm = build_norm(h_input, layer.nextn.hnorm, nullptr, LLM_NORM_RMS, il);304 cb(h_norm, "mtp_hnorm", il);305 306 ggml_tensor * e_norm = build_norm(tok_embd, layer.nextn.enorm, nullptr, LLM_NORM_RMS, il);307 cb(e_norm, "mtp_enorm", il);308 309 ggml_tensor * concat = ggml_concat(ctx0, e_norm, h_norm, /*dim=*/ 0);310 cb(concat, "mtp_concat", il);311 312 ggml_tensor * cur = build_lora_mm(layer.nextn.eh_proj, concat, layer.nextn.eh_proj_s);313 cb(cur, "mtp_eh_proj", il);314 315 ggml_tensor * inpSA = cur;316 317 cur = build_norm(cur, layer.attn_norm, nullptr, LLM_NORM_RMS, il);318 cb(cur, "mtp_attn_norm", il);319 320 ggml_tensor * qkv = build_lora_mm(layer.wqkv, cur, layer.wqkv_s);321 cb(qkv, "mtp_wqkv", il);322 323 const size_t row_k = ggml_row_size(qkv->type, n_embd_head_k);324 const size_t row_v = ggml_row_size(qkv->type, n_embd_head_v);325 const size_t row_full = qkv->nb[1];326 const size_t k_off = row_k * n_head_l;327 const size_t v_off = k_off + row_k * n_head_kv_l;328 329 ggml_tensor * Qcur = ggml_view_3d(ctx0, qkv, n_embd_head_k, n_head_l, n_tokens, row_k, row_full, 0);330 ggml_tensor * Kcur = ggml_view_3d(ctx0, qkv, n_embd_head_k, n_head_kv_l, n_tokens, row_k, row_full, k_off);331 ggml_tensor * Vcur = ggml_view_3d(ctx0, qkv, n_embd_head_v, n_head_kv_l, n_tokens, row_v, row_full, v_off);332 333 Qcur = ggml_rope_ext(334 ctx0, Qcur, inp_pos, nullptr,335 n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l,336 ext_factor, attn_factor, beta_fast, beta_slow);337 338 Kcur = ggml_rope_ext(339 ctx0, Kcur, inp_pos, nullptr,340 n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l,341 ext_factor, attn_factor, beta_fast, beta_slow);342 343 cb(Qcur, "mtp_Qcur", il);344 cb(Kcur, "mtp_Kcur", il);345 cb(Vcur, "mtp_Vcur", il);346 347 cur = build_attn(inp_attn,348 layer.wo, nullptr, layer.wo_s,349 Qcur, Kcur, Vcur, nullptr, layer.attn_sinks, nullptr,350 1.0f / sqrtf(float(n_embd_head_k)), il);351 cb(cur, "mtp_attn_out", il);352 353 if (v_scale) {354 cur = ggml_scale(ctx0, cur, v_scale);355 cb(cur, "mtp_attn_out_scaled", il);356 }357 358 ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);359 cb(ffn_inp, "mtp_ffn_inp", il);360 361 cur = build_norm(ffn_inp, layer.ffn_norm, nullptr, LLM_NORM_RMS, il);362 cb(cur, "mtp_ffn_norm", il);363 364 GGML_ASSERT(layer.ffn_gate && layer.ffn_down && layer.ffn_up && "MIMO2 MTP requires dense FFN tensors");365 cur = build_ffn(cur,366 layer.ffn_up, layer.ffn_up_b, nullptr,367 layer.ffn_gate, layer.ffn_gate_b, nullptr,368 layer.ffn_down, layer.ffn_down_b, nullptr,369 nullptr,370 LLM_FFN_SILU, LLM_FFN_PAR, il);371 cb(cur, "mtp_ffn_out", il);372 373 cur = ggml_add(ctx0, cur, ffn_inp);374 cb(cur, "mtp_post_ffn", il);375 376 cur = ggml_get_rows(ctx0, cur, inp_out_ids);377 378 cb(cur, "h_nextn", -1);379 res->t_h_nextn = cur;380 381 ggml_tensor * head_norm_w = layer.nextn.shared_head_norm382 ? layer.nextn.shared_head_norm383 : (layer.layer_out_norm ? layer.layer_out_norm : model.output_norm);384 GGML_ASSERT(head_norm_w && "MIMO2 MTP missing head norm fallback");385 cur = build_norm(cur, head_norm_w, nullptr, LLM_NORM_RMS, -1);386 cb(cur, "mtp_shared_head_norm", -1);387 388 ggml_tensor * head_w = layer.nextn.shared_head_head ? layer.nextn.shared_head_head : model.output;389 ggml_tensor * head_s = layer.nextn.shared_head_head ? layer.nextn.shared_head_head_s : model.output_s;390 GGML_ASSERT(head_w && "MIMO2 MTP missing LM head fallback");391 cur = build_lora_mm(head_w, cur, head_s);392 cb(cur, "result_output", -1);393 394 res->t_logits = cur;395 ggml_build_forward_expand(gf, cur);396}397