Felipe97/llama-cpp-compiled
01.1k
1#include "models.h"2 3#include <algorithm> // std::max4 5void llama_model_nemotron_h::load_arch_hparams(llama_model_loader & ml) {6 ml.get_key(LLM_KV_SSM_CONV_KERNEL, hparams.ssm_d_conv);7 ml.get_key(LLM_KV_SSM_INNER_SIZE, hparams.ssm_d_inner);8 ml.get_key(LLM_KV_SSM_STATE_SIZE, hparams.ssm_d_state);9 ml.get_key(LLM_KV_SSM_TIME_STEP_RANK, hparams.ssm_dt_rank);10 ml.get_key(LLM_KV_SSM_GROUP_COUNT, hparams.ssm_n_group);11 12 // A layer is recurrent IFF the n_head_kv value is set to 0 and13 // the n_ff value is set to 0. Appended MTP blocks are dense (non-recurrent)14 for (uint32_t i = 0; i < hparams.n_layer_all; ++i) {15 hparams.is_recr_impl[i] = i < hparams.n_layer() && hparams.n_head_kv(i) == 0 && hparams.n_ff(i) == 0;16 }17 18 ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps); // MTP head final_layernorm19 if (!ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps, false)) {20 hparams.f_norm_rms_eps = hparams.f_norm_eps;21 }22 23 // Puzzle models set a different expert FFN size per layer24 ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all, false);25 ml.get_key(LLM_KV_EXPERT_SHARED_FEED_FORWARD_LENGTH, hparams.n_ff_shexp, false);26 ml.get_key(LLM_KV_EXPERT_SHARED_COUNT, hparams.n_expert_shared, false);27 ml.get_key(LLM_KV_EXPERT_WEIGHTS_NORM, hparams.expert_weights_norm, false);28 ml.get_key(LLM_KV_EXPERT_WEIGHTS_SCALE, hparams.expert_weights_scale, false);29 ml.get_key(LLM_KV_MOE_LATENT_SIZE, hparams.moe_latent_size, false);30 31 switch (hparams.n_layer()) {32 case 52: type = LLM_TYPE_31B_A3_5B; break; // Nemotron-H_MOE 31B33 case 56: type = LLM_TYPE_9B; break;34 case 88:35 {36 // Nemotron 3 Super (uniform MoE) and Nemotron 3 Puzzle (per-layer37 // heterogeneous MoE) both have 88 layers; the per-layer top-k array38 // is the discriminator.39 bool heterogeneous = false;40 for (uint32_t i = 1; i < hparams.n_layer(); ++i) {41 heterogeneous |= hparams.n_expert_used_arr[i] != hparams.n_expert_used_arr[0];42 }43 type = heterogeneous ? LLM_TYPE_75B_A9B : LLM_TYPE_120B_A12B;44 } break;45 default: type = LLM_TYPE_UNKNOWN;46 }47}48 49void llama_model_nemotron_h::load_arch_tensors(llama_model_loader & ml) {50 LLAMA_LOAD_LOCALS;51 52 const bool mtp_only = hparams.n_layer_nextn > 0 && ml.get_weight("blk.0.attn_norm.weight") == nullptr;53 const int trunk_flags = mtp_only ? TENSOR_NOT_REQUIRED : 0;54 const int mtp_flags = !ml.load_mtp ? TENSOR_SKIP : 0;55 56 // mamba2 Mixer SSM params57 // NOTE: int64_t for tensor dimensions58 const int64_t d_conv = hparams.ssm_d_conv;59 const int64_t d_inner = hparams.ssm_d_inner;60 const int64_t d_state = hparams.ssm_d_state;61 const int64_t n_ssm_head = hparams.ssm_dt_rank;62 const int64_t n_group = hparams.ssm_n_group;63 const int64_t d_in_proj = 2*d_inner + 2*n_group*d_state + n_ssm_head;64 const int64_t moe_n_embd = hparams.moe_latent_size > 0 ? hparams.moe_latent_size : n_embd;65 66 // embeddings67 tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0);68 69 // output70 {71 output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0);72 output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED);73 // if output is NULL, init from the input tok embed, duplicated to allow offloading74 if (output == NULL) {75 output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, TENSOR_DUPLICATED);76 }77 }78 79 for (int i = 0; i < n_layer; ++i) {80 auto & layer = layers[i];81 82 // all blocks use the attn norm83 layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, trunk_flags);84 85 if (hparams.is_recr(i)) {86 // ssm layers87 layer.ssm_in = create_tensor(tn(LLM_TENSOR_SSM_IN, "weight", i), {n_embd, d_in_proj}, trunk_flags);88 89 layer.ssm_conv1d = create_tensor(tn(LLM_TENSOR_SSM_CONV1D, "weight", i), {d_conv, d_inner + 2*n_group*d_state}, trunk_flags);90 layer.ssm_conv1d_b = create_tensor(tn(LLM_TENSOR_SSM_CONV1D, "bias", i), {d_inner + 2*n_group*d_state}, TENSOR_NOT_REQUIRED);91 92 layer.ssm_dt_b = create_tensor(tn(LLM_TENSOR_SSM_DT, "bias", i), {n_ssm_head}, trunk_flags);93 94 // no "weight" suffix for these95 layer.ssm_a = create_tensor(tn(LLM_TENSOR_SSM_A, i), {1, n_ssm_head}, trunk_flags);96 layer.ssm_d = create_tensor(tn(LLM_TENSOR_SSM_D, i), {1, n_ssm_head}, trunk_flags);97 98 layer.ssm_norm = create_tensor(tn(LLM_TENSOR_SSM_NORM, "weight", i), {d_inner / n_group, n_group}, trunk_flags);99 100 // out_proj101 layer.ssm_out = create_tensor(tn(LLM_TENSOR_SSM_OUT, "weight", i), {d_inner, n_embd}, trunk_flags);102 } else if (hparams.n_ff(i) == 0) {103 // attention layers (with optional bias)104 const int64_t n_head_i = hparams.n_head(i);105 const int64_t n_embd_k_gqa_i = hparams.n_embd_k_gqa(i);106 const int64_t n_embd_v_gqa_i = hparams.n_embd_v_gqa(i);107 create_tensor_qkv(layer, i, n_embd, n_embd_head_k * n_head_i, n_embd_k_gqa_i, n_embd_v_gqa_i, trunk_flags);108 layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd_head_k * n_head_i, n_embd}, trunk_flags);109 layer.wo_b = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd}, TENSOR_NOT_REQUIRED);110 } else {111 if (n_expert != 0) {112 // Use per-layer n_ff_exp; fall back to n_ff/n_expert_used if absent (existing GGUFs).113 const int64_t n_ff_exp_i = hparams.n_ff_exp(i)114 ? (int64_t)hparams.n_ff_exp(i)115 : hparams.n_ff(i) / (int64_t)hparams.n_expert_used(i);116 const int64_t n_ff_shexp = hparams.n_ff_shexp;117 118 layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), { n_embd, n_expert}, trunk_flags);119 layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, "bias", i), {n_expert }, trunk_flags);120 121 // MoE branch122 layer.ffn_latent_down = create_tensor(tn(LLM_TENSOR_FFN_LATENT_DOWN, "weight", i), {n_embd, moe_n_embd}, TENSOR_NOT_REQUIRED);123 layer.ffn_latent_up = create_tensor(tn(LLM_TENSOR_FFN_LATENT_UP, "weight", i), {moe_n_embd, n_embd}, TENSOR_NOT_REQUIRED);124 125 layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {n_ff_exp_i, moe_n_embd, n_expert}, trunk_flags);126 layer.ffn_up_exps = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS, "weight", i), {moe_n_embd, n_ff_exp_i, n_expert}, trunk_flags);127 128 // Shared expert branch129 layer.ffn_down_shexp = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, "weight", i), {n_ff_shexp, n_embd}, trunk_flags);130 layer.ffn_up_shexp = create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP, "weight", i), {n_embd, n_ff_shexp}, trunk_flags);131 132 } else {133 // mlp layers134 layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), { hparams.n_ff(i), n_embd}, trunk_flags);135 layer.ffn_up = create_tensor(tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, hparams.n_ff(i)}, trunk_flags);136 layer.ffn_down_b = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "bias", i), {n_embd}, TENSOR_NOT_REQUIRED);137 layer.ffn_up_b = create_tensor(tn(LLM_TENSOR_FFN_UP, "bias", i), {hparams.n_ff(i)}, TENSOR_NOT_REQUIRED);138 }139 }140 }141 142 // NextN/MTP draft head: each predict layer folds an attention sub-layer and a MoE143 // sub-layer into a single trailing block144 for (int i = n_layer; i < n_layer_all; ++i) {145 auto & layer = layers[i];146 147 const int64_t n_head_i = hparams.n_head(i);148 const int64_t n_embd_k_gqa_i = hparams.n_embd_k_gqa(i);149 const int64_t n_embd_v_gqa_i = hparams.n_embd_v_gqa(i);150 const int64_t n_expert_used_i = hparams.n_expert_used(i);151 const int64_t n_ff_exp_i = hparams.n_ff_exp(i);152 if (n_ff_exp_i == 0 && n_expert_used_i == 0) {153 throw std::runtime_error(format("%s: layer %d declares neither expert_feed_forward_length nor expert_used_count, "154 "cannot determine the expert FFN size", __func__, i));155 }156 const int64_t n_ff_exp = n_ff_exp_i ? n_ff_exp_i : n_ff / n_expert_used_i;157 const int64_t n_ff_shexp = hparams.n_ff_shexp;158 159 // NextN input-fusion tensors160 layer.nextn.enorm = create_tensor(tn(LLM_TENSOR_NEXTN_ENORM, "weight", i), {n_embd}, mtp_flags);161 layer.nextn.hnorm = create_tensor(tn(LLM_TENSOR_NEXTN_HNORM, "weight", i), {n_embd}, mtp_flags);162 layer.nextn.eh_proj = create_tensor(tn(LLM_TENSOR_NEXTN_EH_PROJ, "weight", i), {2*n_embd, n_embd}, mtp_flags);163 layer.nextn.shared_head_norm = create_tensor(tn(LLM_TENSOR_NEXTN_SHARED_HEAD_NORM, "weight", i), {n_embd}, mtp_flags);164 165 // attention sub-layer166 layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, mtp_flags);167 create_tensor_qkv(layer, i, n_embd, n_embd_head_k * n_head_i, n_embd_k_gqa_i, n_embd_v_gqa_i, mtp_flags);168 layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd_head_k * n_head_i, n_embd}, mtp_flags);169 layer.wo_b = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "bias", i), {n_embd}, mtp_flags | TENSOR_NOT_REQUIRED);170 171 // MoE sub-layer172 layer.attn_post_norm = create_tensor(tn(LLM_TENSOR_ATTN_POST_NORM, "weight", i), {n_embd}, mtp_flags);173 layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd, n_expert}, mtp_flags);174 layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, "bias", i), {n_expert}, mtp_flags);175 layer.ffn_latent_down = create_tensor(tn(LLM_TENSOR_FFN_LATENT_DOWN, "weight", i), {n_embd, moe_n_embd}, mtp_flags | TENSOR_NOT_REQUIRED);176 layer.ffn_latent_up = create_tensor(tn(LLM_TENSOR_FFN_LATENT_UP, "weight", i), {moe_n_embd, n_embd}, mtp_flags | TENSOR_NOT_REQUIRED);177 layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {n_ff_exp, moe_n_embd, n_expert}, mtp_flags);178 layer.ffn_up_exps = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS, "weight", i), {moe_n_embd, n_ff_exp, n_expert}, mtp_flags);179 layer.ffn_down_shexp = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, "weight", i), {n_ff_shexp, n_embd}, mtp_flags);180 layer.ffn_up_shexp = create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP, "weight", i), {n_embd, n_ff_shexp}, mtp_flags);181 }182}183 184std::unique_ptr<llm_graph_context> llama_model_nemotron_h::build_arch_graph(const llm_graph_params & params) const {185 return std::make_unique<graph>(*this, params);186}187 188llama_model_nemotron_h::graph::graph(const llama_model & model, const llm_graph_params & params) :189 llm_build_mamba_base(params) {190 const int64_t n_embd_head = hparams.n_embd_head_v();191 GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());192 193 ggml_tensor * cur;194 ggml_tensor * inpL;195 196 inpL = build_inp_embd(model.tok_embd);197 ggml_build_forward_expand(gf, inpL);198 199 auto * inp = build_inp_mem_hybrid();200 201 ggml_tensor * inp_out_ids = build_inp_out_ids();202 const bool extract_final_inp = (size_t) n_layer < cparams.embeddings_layer_inp.size() && cparams.embeddings_layer_inp[n_layer];203 204 for (int il = 0; il < n_layer; ++il) {205 res->t_layer_inp[il] = inpL;206 207 struct ggml_tensor * inpSA = inpL;208 209 // norm210 cur = build_norm(inpL, model.layers[il].attn_norm, NULL, LLM_NORM_RMS, il);211 cb(cur, "attn_norm", il);212 213 if (hparams.is_recr(il)) {214 // ssm layer //215 cur = build_mamba2_layer(inp->get_recr(), cur, model, ubatch, il);216 } else if (hparams.n_ff(il) == 0) {217 // attention layer //218 cur = build_attention_layer(cur, inp->get_attn(), model, n_embd_head, il);219 } else {220 cur = build_ffn_layer(cur, model, il);221 }222 223 if (il == n_layer - 1 && inp_out_ids && cparams.embeddings_nextn_masked && !extract_final_inp) {224 cur = ggml_get_rows(ctx0, cur, inp_out_ids);225 inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);226 }227 228 // add residual229 cur = ggml_add(ctx0, cur, inpSA);230 cb(cur, "nemotron_h_block_out", il);231 232 // input for next layer233 inpL = cur;234 }235 236 cur = inpL;237 if (extract_final_inp) {238 res->t_layer_inp[n_layer] = cur;239 240 if (inp_out_ids && cparams.embeddings_nextn_masked) {241 cur = ggml_get_rows(ctx0, cur, inp_out_ids);242 }243 }244 245 cur = build_norm(cur, model.output_norm, NULL, LLM_NORM_RMS, -1);246 247 // seed for the MTP/NextN draft head248 cb(cur, "h_nextn", -1);249 res->t_h_nextn = cur;250 251 if (!cparams.embeddings_nextn_masked && inp_out_ids) {252 cur = ggml_get_rows(ctx0, cur, inp_out_ids);253 }254 255 cb(cur, "result_norm", -1);256 res->t_embd = cur;257 258 // lm_head259 cur = build_lora_mm(model.output, cur, model.output_s);260 cb(cur, "result_output", -1);261 res->t_logits = cur;262 263 ggml_build_forward_expand(gf, cur);264}265 266ggml_tensor * llama_model_nemotron_h::graph::build_attention_layer(ggml_tensor * cur,267 llm_graph_input_attn_kv * inp_attn,268 const llama_model & model,269 int64_t n_embd_head,270 int il) {271 auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur, n_embd_head, hparams.n_head(il), hparams.n_head_kv(il), il);272 273 const float kq_scale =274 hparams.f_attention_scale == 0.0f ? 1.0f / sqrtf(float(n_embd_head)) : hparams.f_attention_scale;275 cur = build_attn(inp_attn,276 model.layers[il].wo, model.layers[il].wo_b, model.layers[il].wo_s,277 Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il);278 cb(cur, "attn_out", il);279 return cur;280}281 282ggml_tensor * llama_model_nemotron_h::graph::build_ffn_layer(ggml_tensor * cur, const llama_model & model, int il) {283 if (model.layers[il].ffn_gate_inp == nullptr) {284 cur = build_ffn(cur,285 model.layers[il].ffn_up, model.layers[il].ffn_up_b, model.layers[il].ffn_up_s,286 NULL, NULL, NULL,287 model.layers[il].ffn_down, model.layers[il].ffn_down_b, model.layers[il].ffn_down_s,288 NULL,289 LLM_FFN_RELU_SQR, LLM_FFN_PAR, il);290 cb(cur, "ffn_out", il);291 } else {292 ggml_tensor * inp_emb = cur;293 ggml_tensor * inp_latent = cur;294 295 if (model.layers[il].ffn_latent_down) {296 inp_latent = ggml_mul_mat(ctx0, model.layers[il].ffn_latent_down, cur);297 }298 299 ggml_tensor * router_logits = build_lora_mm(model.layers[il].ffn_gate_inp, cur);300 cb(router_logits, "ffn_moe_logits", il);301 302 ggml_tensor * moe_out =303 build_moe_ffn(inp_latent,304 model.layers[il].ffn_gate_inp,305 model.layers[il].ffn_up_exps,306 nullptr, // no gate307 model.layers[il].ffn_down_exps,308 model.layers[il].ffn_exp_probs_b,309 n_expert, (int64_t)hparams.n_expert_used(il),310 LLM_FFN_RELU_SQR, hparams.expert_weights_norm,311 hparams.expert_weights_scale,312 LLAMA_EXPERT_GATING_FUNC_TYPE_SIGMOID,313 il,314 router_logits, nullptr,315 model.layers[il].ffn_up_exps_s,316 nullptr, // no gate317 model.layers[il].ffn_down_exps_s);318 cb(moe_out, "ffn_moe_out", il);319 320 if (model.layers[il].ffn_latent_up) {321 moe_out = ggml_mul_mat(ctx0, model.layers[il].ffn_latent_up, moe_out);322 }323 324 ggml_tensor * ffn_shexp = build_ffn(inp_emb,325 model.layers[il].ffn_up_shexp, NULL, model.layers[il].ffn_up_shexp_s,326 NULL /* no gate */ , NULL, NULL,327 model.layers[il].ffn_down_shexp, NULL, model.layers[il].ffn_down_shexp_s,328 NULL,329 LLM_FFN_RELU_SQR, LLM_FFN_PAR, il);330 cb(ffn_shexp, "ffn_shexp", il);331 332 cur = ggml_add(ctx0, moe_out, ffn_shexp);333 cb(cur, "ffn_out", il);334 }335 336 cur = build_cvec(cur, il);337 cb(cur, "l_out", il);338 339 return cur;340}341