Felipe97/llama-cpp-compiled
01.1k
1#include "models.h"2#include "llama-memory-recurrent.h"3 4void llama_model_kimi_linear::load_arch_hparams(llama_model_loader & ml) {5 ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);6 ml.get_key(LLM_KV_ATTENTION_KEY_LENGTH_MLA, hparams.n_embd_head_k_mla_impl);7 ml.get_key(LLM_KV_ATTENTION_VALUE_LENGTH_MLA, hparams.n_embd_head_v_mla_impl);8 ml.get_key(LLM_KV_ATTENTION_KV_LORA_RANK, hparams.n_lora_kv);9 ml.get_key(LLM_KV_SSM_CONV_KERNEL, hparams.ssm_d_conv);10 ml.get_key(LLM_KV_KDA_HEAD_DIM, hparams.n_embd_head_kda);11 12 // MLA qk_rope_head_dim (for reference)13 // qk_rope_head_dim = 64, qk_nope_head_dim = 128, qk_head_dim = 19214 15 // Mark KDA layers as recurrent using n_head_kv pattern (like Jamba)16 // Set n_head_kv = 0 for KDA layers (recurrent), n_head_kv = n_head for MLA layers (attention)17 for (uint32_t i = 0; i < hparams.n_layer(); ++i) {18 hparams.is_recr_impl[i] = hparams.n_head_kv(i) == 0; // KDA layers are recurrent19 }20 21 // MoE parameters - Kimi uses moe_intermediate_size = 102422 ml.get_key_or_arr(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp_arr, hparams.n_layer_all);23 ml.get_key(LLM_KV_EXPERT_SHARED_COUNT, hparams.n_expert_shared);24 ml.get_key(LLM_KV_LEADING_DENSE_BLOCK_COUNT, hparams.n_layer_dense_lead, false);25 ml.get_key(LLM_KV_EXPERT_WEIGHTS_SCALE, hparams.expert_weights_scale, false);26 ml.get_key(LLM_KV_EXPERT_GATING_FUNC, hparams.expert_gating_func);27 28 switch (hparams.n_layer()) {29 case 27: type = LLM_TYPE_48B_A3B; break; // Kimi-Linear-48B-A3B30 default: type = LLM_TYPE_UNKNOWN;31 }32}33 34void llama_model_kimi_linear::load_arch_tensors(llama_model_loader &) {35 LLAMA_LOAD_LOCALS;36 37 tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0);38 39 // output40 output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0);41 output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, 0);42 43 for (int i = 0; i < n_layer; ++i) {44 auto & layer = layers[i];45 46 layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0);47 48 // Check for KDA specific tensors to determine layer type or if it's a mixed model49 // Assuming KDA layer if KDA tensors are present50 51 // KDA uses head_dim = 128 (from linear_attn_config.head_dim)52 const int64_t n_embd_head_k_kda = hparams.n_embd_head_kda;53 const int64_t n_embd_head_v_kda = hparams.n_embd_head_kda;54 const int64_t ssm_d_conv = hparams.ssm_d_conv;55 56 if (hparams.is_recr(i)) {57 // Conv1d weights: try 4D first, then 3D (quantization may remove trailing 1)58 // 4D: [d_conv, 1, d_inner, 1], 3D: [d_conv, 1, d_inner]59 layer.ssm_q_conv = create_tensor(tn(LLM_TENSOR_SSM_CONV1D_Q, "weight", i), {ssm_d_conv, 1, n_embd_head_k_kda * n_head, 1}, TENSOR_NOT_REQUIRED);60 if (!layer.ssm_q_conv) {61 layer.ssm_q_conv = create_tensor(tn(LLM_TENSOR_SSM_CONV1D_Q, "weight", i), {ssm_d_conv, 1, n_embd_head_k_kda * n_head}, 0);62 }63 64 // KDA Layer - Conv1d weights may be 3D or 4D65 layer.ssm_k_conv = create_tensor(tn(LLM_TENSOR_SSM_CONV1D_K, "weight", i), {ssm_d_conv, 1, n_embd_head_k_kda * n_head, 1}, TENSOR_NOT_REQUIRED);66 if (!layer.ssm_k_conv) {67 layer.ssm_k_conv = create_tensor(tn(LLM_TENSOR_SSM_CONV1D_K, "weight", i), {ssm_d_conv, 1, n_embd_head_k_kda * n_head}, 0);68 }69 layer.ssm_v_conv = create_tensor(tn(LLM_TENSOR_SSM_CONV1D_V, "weight", i), {ssm_d_conv, 1, n_embd_head_v_kda * n_head, 1}, TENSOR_NOT_REQUIRED);70 if (!layer.ssm_v_conv) {71 layer.ssm_v_conv = create_tensor(tn(LLM_TENSOR_SSM_CONV1D_V, "weight", i), {ssm_d_conv, 1, n_embd_head_v_kda * n_head}, 0);72 }73 74 // q, k, v projections75 // Python: q_proj, k_proj, v_proj76 create_tensor_qkv(layer, i, n_embd, n_embd_head_k_kda * n_head, n_embd_head_k_kda * n_head, n_embd_head_v_kda * n_head, 0);77 78 // KDA specific projections79 // f_a_proj, f_b_proj80 layer.ssm_f_a = create_tensor(tn(LLM_TENSOR_SSM_F_A, "weight", i), {n_embd, n_embd_head_k_kda}, 0); // head_dim81 layer.ssm_f_b = create_tensor(tn(LLM_TENSOR_SSM_F_B, "weight", i), {n_embd_head_k_kda, n_embd_head_k_kda * n_head}, 0); // projection_size82 83 // b_proj (beta mixing coefficient)84 layer.ssm_beta = create_tensor(tn(LLM_TENSOR_SSM_BETA, "weight", i), {n_embd, n_head}, 0);85 86 // A_log - Shape in GGUF: [1, num_heads, 1, 1] (4D) or [1, num_heads] (2D after quantization) Note: -exp(A_log) is applied in convert_hf_to_gguf.py87 layer.ssm_a = create_tensor(tn(LLM_TENSOR_SSM_A_NOSCAN, i), {1, n_head, 1, 1}, TENSOR_NOT_REQUIRED);88 if (!layer.ssm_a) {89 layer.ssm_a = create_tensor(tn(LLM_TENSOR_SSM_A_NOSCAN, i), {1, n_head}, 0);90 }91 92 // dt_bias - shape [n_embd_head_k_kda * n_head] = [4096]93 layer.ssm_dt_b = create_tensor(tn(LLM_TENSOR_SSM_DT, "bias", i), {n_embd_head_k_kda * n_head}, 0);94 95 // g_a_proj, g_b_proj (output gate)96 layer.ssm_g_a = create_tensor(tn(LLM_TENSOR_SSM_G_A, "weight", i), {n_embd, n_embd_head_k_kda}, 0);97 layer.ssm_g_b = create_tensor(tn(LLM_TENSOR_SSM_G_B, "weight", i), {n_embd_head_k_kda, n_embd_head_k_kda * n_head}, 0);98 99 // o_norm (reusing SSM_NORM)100 layer.ssm_o_norm = create_tensor(tn(LLM_TENSOR_SSM_NORM, "weight", i), {n_embd_head_k_kda}, 0); // FusedRMSNormGated101 102 // o_proj103 layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd_head_v_kda * n_head, n_embd}, 0);104 105 } else {106 // MLA Layer - use MLA-specific head dimensions107 const int64_t q_lora_rank = hparams.n_lora_q;108 const int64_t kv_lora_rank = hparams.n_lora_kv;109 const int64_t n_embd_head_k_mla = hparams.n_embd_head_k_mla();110 const int64_t n_embd_head_v_mla = hparams.n_embd_head_v_mla();111 112 layer.attn_q_a_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_A_NORM, "weight", i), {q_lora_rank}, TENSOR_NOT_REQUIRED);113 layer.attn_kv_a_norm = create_tensor(tn(LLM_TENSOR_ATTN_KV_A_NORM, "weight", i), {kv_lora_rank}, 0);114 115 if (layer.attn_q_a_norm) {116 layer.wq_a = create_tensor(tn(LLM_TENSOR_ATTN_Q_A, "weight", i), {n_embd, q_lora_rank}, 0);117 layer.wq_b = create_tensor(tn(LLM_TENSOR_ATTN_Q_B, "weight", i), {q_lora_rank, n_head * n_embd_head_k_mla}, 0);118 } else {119 // Kimi MLA without Q compression: wq = [n_embd, n_head * n_embd_head_k_mla]120 layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_head * n_embd_head_k_mla}, 0);121 }122 123 // Kimi: qk_rope_head_dim = 64 (actual RoPE dimension for MLA)124 // Note: hparams.n_rot may be 72 (from conversion) but actual is 64125 const int64_t qk_rope_head_dim = hparams.n_rot(); // From config: qk_rope_head_dim126 layer.wkv_a_mqa = create_tensor(tn(LLM_TENSOR_ATTN_KV_A_MQA, "weight", i), {n_embd, kv_lora_rank + qk_rope_head_dim}, 0);127 // Support Legacy GGUFs that don't split wkv_b (MLA KV cache disabled)128 layer.wkv_b = create_tensor(tn(LLM_TENSOR_ATTN_KV_B, "weight", i),129 {kv_lora_rank, n_head * (n_embd_head_k_mla - qk_rope_head_dim + n_embd_head_v_mla)}, TENSOR_NOT_REQUIRED | TENSOR_SKIP_IF_VIRTUAL);130 if (!layer.wkv_b) { // MLA KV cache enabled131 layer.wk_b = create_tensor(tn(LLM_TENSOR_ATTN_K_B, "weight", i), {n_embd_head_k_mla - qk_rope_head_dim, kv_lora_rank, n_head}, 0);132 layer.wv_b = create_tensor(tn(LLM_TENSOR_ATTN_V_B, "weight", i), {kv_lora_rank, n_embd_head_v_mla, n_head}, 0);133 }134 layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_head * n_embd_head_v_mla, n_embd}, 0);135 }136 137 layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0);138 139 // MoE intermediate size (different from dense FFN)140 const int64_t n_ff_exp = hparams.n_ff_exp();141 142 // Kimi uses n_layer_dense_lead to determine which layers use dense FFN vs MoE143 // first_k_dense_replace = 1 means layer 0 uses dense FFN, layers 1+ use MoE144 if (i < (int) hparams.n_layer_dense_lead) {145 // Dense FFN layer - use normal n_ff146 layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff}, 0);147 layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), {n_ff, n_embd}, 0);148 layer.ffn_up = create_tensor(tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, 0);149 } else {150 // MoE layer - use n_ff_exp (1024) instead of n_ff (9216)151 layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd, n_expert}, 0);152 layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i), {n_embd, n_ff_exp, n_expert}, 0);153 layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {n_ff_exp, n_embd, n_expert}, 0);154 layer.ffn_up_exps = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS, "weight", i), {n_embd, n_ff_exp, n_expert}, 0);155 156 // Shared experts use moe_intermediate_size * num_shared_experts157 // Kimi: shared_expert_intermediate_size = 1024 * 1 = 1024158 // Tensors are 2D: [n_embd, n_ff_shexp] or [n_ff_shexp, n_embd]159 const int64_t n_ff_shexp_actual = n_ff_exp * (hparams.n_expert_shared > 0 ? hparams.n_expert_shared : 1);160 layer.ffn_gate_shexp = create_tensor(tn(LLM_TENSOR_FFN_GATE_SHEXP, "weight", i), {n_embd, n_ff_shexp_actual}, TENSOR_NOT_REQUIRED);161 layer.ffn_down_shexp = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, "weight", i), {n_ff_shexp_actual, n_embd}, TENSOR_NOT_REQUIRED);162 layer.ffn_up_shexp = create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP, "weight", i), {n_embd, n_ff_shexp_actual}, TENSOR_NOT_REQUIRED);163 164 layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, "bias", i), {n_expert}, 0);165 }166 }167}168 169std::unique_ptr<llm_graph_context> llama_model_kimi_linear::build_arch_graph(const llm_graph_params & params) const {170 return std::make_unique<graph>(*this, params);171}172 173// Causal Conv1d function for Q,K,V174// When qkv is 0, it is Q, 1 is K, 2 is V175static ggml_tensor * causal_conv1d(ggml_cgraph * gf, ggml_context * ctx0, ggml_tensor * conv_states_all, ggml_tensor * conv_state_all, int64_t qkv, ggml_tensor * x, ggml_tensor * proj_w, ggml_tensor * conv_w, int64_t d_conv, int64_t head_dim, int64_t n_head, int64_t n_seq_tokens, int64_t n_seqs, int64_t n_tokens, int64_t kv_head) {176 const int64_t d_inner = head_dim * n_head;177 const int64_t conv_state_size = (d_conv - 1) * d_inner;178 const int64_t n_embd_r_total = 3 * conv_state_size; // Q + K + V179 180 // conv_state_all is [n_embd_r_total, n_seqs], split into Q, K, V181 // Each conv state is [(d_conv-1) * d_inner] per sequence, need to reshape to [d_conv-1, d_inner, n_seqs]182 // Memory layout: for each seq, Q state is first conv_state_size elements, then K, then V183 // conv_state_all has stride: nb[0] = element_size, nb[1] = n_embd_r_total * element_size184 // View Q conv state: offset 0, size conv_state_size per seq185 // conv_state_all is [n_embd_r_total, n_seqs] with memory layout:186 // state[i + seq * n_embd_r_total] where i = conv_step + channel * (d_conv-1) + {0, conv_state_size, 2*conv_state_size} for Q/K/V187 // We want [d_conv-1, d_inner, n_seqs] view:188 // nb1 = (d_conv-1) * element_size (stride between channels)189 // nb2 = n_embd_r_total * element_size (stride between seqs)190 ggml_tensor * conv_state_x = ggml_view_3d(ctx0, conv_state_all, d_conv - 1, d_inner, n_seqs,191 (d_conv - 1) * ggml_element_size(conv_state_all), // nb1: stride between channels192 n_embd_r_total * ggml_element_size(conv_state_all), // nb2: stride between seqs193 qkv * conv_state_size * ggml_element_size(conv_state_all));194 195// Causal Conv1d function for Q,K,V196// When qkv is 0, it is Q, 1 is K, 2 is V197 // Step 1: Q, K, V projections -> [d_inner, n_tokens]198 ggml_tensor * x_proj = proj_w ? ggml_mul_mat(ctx0, proj_w, x) : x;199 200 // Reshape input: {d_inner, n_tokens} -> {d_inner, n_seq_tokens, n_seqs}201 ggml_tensor * x_3d = ggml_reshape_3d(ctx0, x_proj, d_inner, n_seq_tokens, n_seqs);202 203 // Concat Q conv state and current input: {d_conv-1 + n_seq_tokens, d_inner, n_seqs}204 ggml_tensor * conv_x = ggml_concat(ctx0, conv_state_x, ggml_transpose(ctx0, x_3d), 0);205 206 // Save last (d_conv-1) columns back to Q conv state207 ggml_tensor * last_conv_x = ggml_view_3d(ctx0, conv_x, d_conv - 1, d_inner, n_seqs,208 conv_x->nb[1], conv_x->nb[2], n_seq_tokens * conv_x->nb[0]);209 ggml_build_forward_expand(gf,210 ggml_cpy(ctx0, last_conv_x,211 ggml_view_3d(ctx0, conv_states_all,212 d_conv - 1, d_inner, n_seqs,213 (d_conv - 1) * ggml_element_size(conv_states_all), // nb1: contiguous within one channel's conv taps214 n_embd_r_total * ggml_element_size(conv_states_all), // nb2: stride between sequences (skip over K,V states)215 (kv_head * n_embd_r_total + qkv * conv_state_size) * ggml_element_size(conv_states_all)))); // offset to first seq's Q/K/V state216 // Reshape conv weight: GGUF [d_conv, 1, d_inner, 1] -> ggml_ssm_conv expects [d_conv, d_inner]217 // GGUF stores as [d_conv, 1, d_inner, 1] with memory layout w[conv_step + channel * d_conv]218 // vLLM stores as [d_inner, d_conv] with memory layout w[channel * d_conv + conv_step]219 // ggml_ssm_conv computes: c[conv_step + channel * d_conv]220 // GGUF layout: [d_conv, 1, d_inner] or [d_conv, 1, d_inner, 1] -> reshape to [d_conv, d_inner]221 // Reshape conv weight from [d_conv, 1, d_inner, 1] to [d_conv, d_inner] for ggml_ssm_conv222 ggml_tensor * conv_weight = ggml_reshape_2d(ctx0, conv_w, d_conv, d_inner);223 224 // Apply conv1d225 // ggml_ssm_conv output: {d_inner, n_seq_tokens, n_seqs}226 ggml_tensor * Xcur = ggml_ssm_conv(ctx0, conv_x, conv_weight);227 // Reshape to 2D for bias add: {d_inner, n_tokens}228 Xcur = ggml_reshape_2d(ctx0, Xcur, d_inner, n_tokens);229 Xcur = ggml_silu(ctx0, Xcur);230 231 return ggml_reshape_4d(ctx0, Xcur, head_dim, n_head, n_seq_tokens, n_seqs);232}233 234llama_model_kimi_linear::graph::graph(const llama_model & model, const llm_graph_params & params) :235 llm_build_delta_net_base(params), model(model) {236 ggml_tensor * cur;237 ggml_tensor * inpL;238 239 inpL = build_inp_embd(model.tok_embd);240 cb(inpL, "model.embed_tokens", -1);241 242 // Note: Kimi MLA does NOT use RoPE (rotary_emb=None in vLLM)243 // So we don't need inp_pos244 245 auto * inp_kv = !hparams.is_mla() ? build_inp_mem_hybrid() : nullptr;246 auto * inp_k = hparams.is_mla() ? build_inp_mem_hybrid_k() : nullptr;247 auto * inp_rs = hparams.is_mla() ? inp_k->get_recr() : inp_kv->get_recr();248 auto * inp_attn_kv = !hparams.is_mla() ? inp_kv->get_attn() : nullptr;249 auto * inp_attn_k = hparams.is_mla() ? inp_k->get_attn() : nullptr;250 251 // Output ids for selecting which tokens to output252 ggml_tensor * inp_out_ids = build_inp_out_ids();253 254 // Kimi dimension constants255 const int64_t n_head = hparams.n_head();256 const int64_t head_dim = hparams.n_embd_head_kda;257 const int64_t d_conv = hparams.ssm_d_conv;258 const int64_t d_inner = n_head * head_dim; // 32 * 128 = 4096259 const int64_t n_seqs = ubatch.n_seqs;260 const int64_t n_seq_tokens = ubatch.n_seq_tokens;261 262 // Verify batch consistency for recurrent layers263 GGML_ASSERT(n_seqs != 0);264 GGML_ASSERT(ubatch.equal_seqs());265 GGML_ASSERT(ubatch.n_tokens == n_seq_tokens * n_seqs);266 267 // MLA params268 const int64_t n_embd_head_k_mla = hparams.n_embd_head_k_mla();269 const int64_t n_embd_head_v_mla = hparams.n_embd_head_v_mla();270 const int64_t kv_lora_rank = hparams.n_lora_kv;271 // qk_rope_head_dim = 64 (from Kimi config) which is hparams.n_rot272 // Confirmed from tensor shape: wkv_a_mqa [2304, 576] = [n_embd, kv_lora_rank + qk_rope_head_dim]273 const int64_t n_embd_head_qk_rope = hparams.n_rot(); // config.qk_rope_head_dim274 const int64_t n_embd_head_qk_nope = n_embd_head_k_mla - n_embd_head_qk_rope; // 192 - 64 = 128275 // Attention scale for MLA276 const float kq_scale_mla = 1.0f / sqrtf((float)n_embd_head_k_mla);277 278 for (int il = 0; il < n_layer; ++il) {279 const auto & layer = model.layers[il];280 ggml_tensor * inpSA = inpL;281 282 // Attention Norm283 cur = build_norm(inpL, layer.attn_norm, NULL, LLM_NORM_RMS, il);284 cb(cur, "attn_norm", il);285 286 ggml_build_forward_expand(gf, cur);287 288 if (hparams.is_recr(il)) {289 // === KDA Layer (Kimi Delta Attention) with Recurrent State ===290 // Reference: vLLM kda.py291 const auto * mctx_cur = inp_rs->mctx;292 const auto kv_head = mctx_cur->get_head();293 294 // Get conv states from r_l tensor (Q, K, V each have separate state)295 ggml_tensor * conv_states_all = mctx_cur->get_r_l(il);296 cb(conv_states_all, "conv_states_all", il);297 ggml_tensor * conv_state_all = build_rs(inp_rs, conv_states_all, hparams.n_embd_r(), n_seqs);298 ggml_tensor * q_in = cur, * k_in = cur, * v_in = cur;299 ggml_tensor * q_w = layer.wq, * k_w = layer.wk, * v_w = layer.wv;300 if (layer.wqkv) {301 ggml_tensor * qkv = ggml_mul_mat(ctx0, layer.wqkv, cur);302 const int64_t d_inner = head_dim * n_head;303 const size_t esize = ggml_element_size(qkv);304 q_in = ggml_cont(ctx0, ggml_view_2d(ctx0, qkv, d_inner, n_tokens, qkv->nb[1], 0));305 k_in = ggml_cont(ctx0, ggml_view_2d(ctx0, qkv, d_inner, n_tokens, qkv->nb[1], d_inner * esize));306 v_in = ggml_cont(ctx0, ggml_view_2d(ctx0, qkv, d_inner, n_tokens, qkv->nb[1], 2 * d_inner * esize));307 q_w = nullptr; k_w = nullptr; v_w = nullptr;308 }309 ggml_tensor * Qcur = causal_conv1d(gf, ctx0, conv_states_all, conv_state_all, 0, q_in, q_w, layer.ssm_q_conv, d_conv, head_dim, n_head, n_seq_tokens, n_seqs, n_tokens, kv_head);310 ggml_tensor * Kcur = causal_conv1d(gf, ctx0, conv_states_all, conv_state_all, 1, k_in, k_w, layer.ssm_k_conv, d_conv, head_dim, n_head, n_seq_tokens, n_seqs, n_tokens, kv_head);311 ggml_tensor * Vcur = causal_conv1d(gf, ctx0, conv_states_all, conv_state_all, 2, v_in, v_w, layer.ssm_v_conv, d_conv, head_dim, n_head, n_seq_tokens, n_seqs, n_tokens, kv_head);312 313 // g1 = -exp(A_log) * softplus(f_b(f_a(x)) + dt_bias)314 ggml_tensor * f_a = ggml_mul_mat(ctx0, layer.ssm_f_a, cur);315 ggml_tensor * g1 = ggml_mul_mat(ctx0, layer.ssm_f_b, f_a);316 cb(g1, "g1 f_b(f_a(cur))", il);317 g1 = ggml_add(ctx0, g1, layer.ssm_dt_b);318 g1 = ggml_softplus(ctx0, g1);319 g1 = ggml_reshape_3d(ctx0, g1, head_dim, n_head, n_tokens);320 321 // A_log shape is [1, n_head] or [1, n_head, 1, 1], need to broadcast to [head_dim, n_head, n_tokens]. No need to -exp(a_log) because it was done in convert_hf_to_gguf.py322 // Reshape to [1, n_head, 1] for broadcasting with g1 [head_dim, n_head, n_tokens]323 ggml_tensor * A = ggml_reshape_3d(ctx0, layer.ssm_a, 1, n_head, 1);324 g1 = ggml_mul(ctx0, g1, A);325 cb(g1, "kda_g1", il);326 327 g1 = ggml_reshape_4d(ctx0, g1, head_dim, n_head, n_seq_tokens, n_seqs);328 329 // Compute beta (mixing coefficient)330 ggml_tensor * beta = ggml_mul_mat(ctx0, layer.ssm_beta, cur);331 beta = ggml_reshape_4d(ctx0, beta, 1, n_head, n_seq_tokens, n_seqs);332 cb(beta, "kda_beta", il);333 334 beta = ggml_sigmoid(ctx0, beta);335 336 // Reshape for KDA recurrence337 // {n_embd, n_tokens} -> {n_embd, n_seq_tokens, n_seqs}338 cur = ggml_reshape_3d(ctx0, cur, cur->ne[0], n_seq_tokens, n_seqs);339 340 // Get SSM state and compute KDA recurrence using ggml_kda_scan341 ggml_tensor * ssm_states_all = mctx_cur->get_s_l(il);342 ggml_tensor * state = build_rs(inp_rs, ssm_states_all, hparams.n_embd_s(), n_seqs);343 state = ggml_reshape_4d(ctx0, state, head_dim, head_dim, n_head, n_seqs);344 345 346 const float eps_norm = hparams.f_norm_rms_eps;347 348 Qcur = build_gdn_l2_norm(ctx0, Qcur, eps_norm);349 Kcur = build_gdn_l2_norm(ctx0, Kcur, eps_norm);350 351 // Choose between build_delta_net_chunking and build_delta_net_recurrent based on n_tokens352 auto attn_out = build_delta_net(Qcur, Kcur, Vcur, g1, beta, state, il);353 354 ggml_tensor * output = ggml_cont(ctx0, attn_out.first);355 ggml_tensor * new_state = attn_out.second;356 cb(output, "attn_output", il);357 cb(new_state, "new_state", il);358 359 // Update the recurrent states360 ggml_build_forward_expand(gf,361 ggml_cpy(ctx0, new_state,362 ggml_view_1d(ctx0, ssm_states_all, hparams.n_embd_s() * n_seqs,363 kv_head * hparams.n_embd_s() * ggml_element_size(ssm_states_all))));364 365 // Output gating g2 = g_b(g_a(x))366 ggml_tensor * cur_2d = ggml_reshape_2d(ctx0, cur, cur->ne[0], n_seq_tokens * n_seqs);367 ggml_tensor * g_a = ggml_mul_mat(ctx0, layer.ssm_g_a, cur_2d);368 ggml_tensor * g2 = ggml_mul_mat(ctx0, layer.ssm_g_b, g_a);369 cb(g2, "g2 g_b(g_a(cur_2d))", il);370 g2 = ggml_reshape_3d(ctx0, g2, head_dim, n_head, n_seq_tokens * n_seqs);371 372 // Apply o_norm with sigmoid gating373 // Note: Kimi model uses sigmoid gating, not SiLU (despite FusedRMSNormGated default being swish)374 // Formula: output = RMSNorm(x) * sigmoid(g)375 ggml_tensor * attn_out_final = ggml_reshape_3d(ctx0, output, head_dim, n_head, n_seq_tokens * n_seqs);376 ggml_tensor * normed = build_norm(attn_out_final, layer.ssm_o_norm, nullptr, LLM_NORM_RMS, il);377 cb(normed, "kda_normed", il);378 ggml_tensor * gate = ggml_sigmoid(ctx0, g2);379 ggml_tensor * gated = ggml_mul(ctx0, normed, gate);380 381 // Output projection382 gated = ggml_cont_2d(ctx0, gated, d_inner, n_tokens);383 cur = ggml_mul_mat(ctx0, layer.wo, gated);384 cb(cur, "kda_out", il);385 386 } else {387 // === MLA Layer (Multi-head Latent Attention) without KV Cache ===388 // Reference: vLLM mla.py389 // Step 1: Q projection and reshape390 // vLLM Kimi: q = q_proj(hidden_states), then view as [n_tokens, n_head, qk_head_dim]391 // Note: Kimi MLA does NOT use RoPE (rotary_emb=None in vLLM)392 ggml_tensor * Qcur = ggml_mul_mat(ctx0, layer.wq, cur);393 394 // Step 2: KV compression395 // kv_cmpr_pe = kv_a_proj_with_mqa(hidden_states) -> [kv_lora_rank + qk_rope_head_dim, n_tokens]396 ggml_tensor * kv_cmpr_pe = ggml_mul_mat(ctx0, layer.wkv_a_mqa, cur);397 398 // Split: kv_cmpr = kv_lora[:kv_lora_rank], k_pe = kv_lora[kv_lora_rank:]399 ggml_tensor * kv_cmpr = ggml_view_2d(ctx0, kv_cmpr_pe, kv_lora_rank, n_tokens,400 ggml_row_size(kv_cmpr_pe->type, kv_lora_rank + n_embd_head_qk_rope), 0);401 ggml_tensor * k_pe = ggml_view_3d(ctx0, kv_cmpr_pe, n_embd_head_qk_rope, 1, n_tokens,402 ggml_row_size(kv_cmpr_pe->type, kv_lora_rank + n_embd_head_qk_rope),403 ggml_row_size(kv_cmpr_pe->type, kv_lora_rank + n_embd_head_qk_rope),404 ggml_row_size(kv_cmpr_pe->type, kv_lora_rank));405 // Note: Kimi MLA does NOT apply RoPE (rotary_emb=None in vLLM)406 // k_pe is used directly without RoPE407 // Normalize kv_c408 kv_cmpr = build_norm(kv_cmpr, layer.attn_kv_a_norm, nullptr, LLM_NORM_RMS, il);409 410 if (layer.wk_b && layer.wv_b) { // MLA KV cache enabled411 // extract q_nope412 ggml_tensor * q_nope =413 ggml_view_3d(ctx0, Qcur, n_embd_head_qk_nope, n_head, n_tokens, ggml_row_size(Qcur->type, n_embd_head_k_mla),414 ggml_row_size(Qcur->type, n_embd_head_k_mla) * n_head, 0);415 cb(q_nope, "q_nope", il);416 417 // and {n_embd_head_qk_rope, n_head, n_tokens}418 ggml_tensor * q_pe = ggml_view_3d(419 ctx0, Qcur, n_embd_head_qk_rope, n_head, n_tokens, ggml_row_size(Qcur->type, n_embd_head_k_mla),420 ggml_row_size(Qcur->type, n_embd_head_k_mla) * n_head, ggml_row_size(Qcur->type, n_embd_head_qk_nope));421 cb(q_pe, "q_pe", il);422 423 // {n_embd_head_qk_nope, n_tokens, n_head}424 q_nope = ggml_permute(ctx0, q_nope, 0, 2, 1, 3);425 cb(q_nope, "q_nope_perm", il);426 427 // {n_embd_head_qk_nope, kv_lora_rank, n_head} x {n_embd_head_qk_nope, n_tokens, n_head}428 ggml_tensor * q_nope_absorbed = ggml_mul_mat(ctx0, layer.wk_b, q_nope);429 cb(q_nope_absorbed, "q_nope_absorbed", il);430 431 // {kv_lora_rank, n_head, n_tokens}432 q_nope_absorbed = ggml_permute(ctx0, q_nope_absorbed, 0, 2, 1, 3);433 cb(q_nope_absorbed, "q_nope_absorbed_perm", il);434 435 // {n_embd_head_qk_rope + kv_lora_rank, n_head, n_tokens}436 // note: rope must go first for in-place context shifting in build_rope_shift()437 Qcur = ggml_concat(ctx0, q_nope_absorbed, q_pe, 0);438 cb(Qcur, "Qcur", il);439 440 kv_cmpr = ggml_reshape_3d(ctx0, kv_cmpr, kv_lora_rank, 1, n_tokens);441 cb(kv_cmpr, "kv_cmpr_reshape", il);442 443 // {n_embd_head_qk_rope + kv_lora_rank, 1, n_tokens}444 ggml_tensor * Kcur = ggml_concat(ctx0, kv_cmpr, k_pe, 0);445 cb(Kcur, "Kcur", il);446 447 // {kv_lora_rank, 1, n_tokens}448 ggml_tensor * Vcur = kv_cmpr;449 cb(Vcur, "Vcur", il);450 451 cur = build_attn(inp_attn_k, layer.wo, NULL, layer.wo_s, Qcur, Kcur, Vcur, nullptr, nullptr, layer.wv_b, kq_scale_mla, il);452 cb(cur, "mla_out", il);453 } else { // MLA KV cache disabled. Fall back to MHA KV cache.454 Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head_k_mla, n_head, n_tokens);455 cb(Qcur, "mla_Q", il);456 // KV decompression: kv = kv_b_proj(kv_c_normed)457 ggml_tensor * kv = ggml_mul_mat(ctx0, layer.wkv_b, kv_cmpr);458 const int64_t kv_per_head = n_embd_head_qk_nope + n_embd_head_v_mla;459 460 // Split kv into k_nope and v461 ggml_tensor * k_nope = ggml_view_3d(ctx0, kv, n_embd_head_qk_nope, n_head, n_tokens,462 ggml_row_size(kv->type, kv_per_head),463 ggml_row_size(kv->type, kv_per_head * n_head), 0);464 ggml_tensor * Vcur = ggml_view_3d(ctx0, kv, n_embd_head_v_mla, n_head, n_tokens,465 ggml_row_size(kv->type, kv_per_head),466 ggml_row_size(kv->type, kv_per_head * n_head),467 ggml_row_size(kv->type, n_embd_head_qk_nope));468 Vcur = ggml_cont(ctx0, Vcur);469 cb(Vcur, "mla_V", il);470 471 // Concatenate k_nope + k_pe (broadcast k_pe to all heads)472 // K = [k_nope, k_pe] where k_nope is [qk_nope_head_dim, n_head, n_tokens]473 // and k_pe is [qk_rope_head_dim, 1, n_tokens] broadcast to all heads474 // Need to broadcast k_pe from [qk_rope, 1, n_tokens] to [qk_rope, n_head, n_tokens]475 ggml_tensor * k_pe_target = ggml_new_tensor_3d(ctx0, k_pe->type, n_embd_head_qk_rope, n_head, n_tokens);476 ggml_tensor * k_pe_repeated = ggml_repeat(ctx0, k_pe, k_pe_target);477 ggml_tensor * Kcur = ggml_concat(ctx0, k_pe_repeated, k_nope, 0);478 cb(Kcur, "mla_K", il);479 480 // Direct softmax attention (with MHA KV cache)481 // Use build_attn with inp_attn for proper mask handling482 cur = build_attn(inp_attn_kv, layer.wo, NULL, layer.wo_s, Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale_mla, il);483 cb(cur, "mla_out", il);484 }485 }486 487 // On last layer, select only the output tokens488 if (il == n_layer - 1 && inp_out_ids) {489 cur = ggml_get_rows(ctx0, cur, inp_out_ids);490 inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);491 }492 493 // Residual494 ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);495 cb(ffn_inp, "ffn_inp", il);496 497 // FFN Norm498 cur = build_norm(ffn_inp, layer.ffn_norm, NULL, LLM_NORM_RMS, il);499 cb(cur, "ffn_norm", il);500 501 if ((uint32_t) il < hparams.n_layer_dense_lead) {502 // Dense FFN layer503 cur = build_ffn(cur,504 layer.ffn_up, NULL, NULL,505 layer.ffn_gate, NULL, NULL,506 layer.ffn_down, NULL, NULL,507 NULL, LLM_FFN_SILU, LLM_FFN_PAR, il);508 cb(cur, "ffn_out", il);509 } else {510 // MoE layer511 // Kimi uses moe_renormalize=True and routed_scaling_factor (stored as expert_weights_scale) = 2.446512 ggml_tensor * moe_out = build_moe_ffn(cur,513 layer.ffn_gate_inp,514 layer.ffn_up_exps,515 layer.ffn_gate_exps,516 layer.ffn_down_exps,517 layer.ffn_exp_probs_b,518 hparams.n_expert,519 hparams.n_expert_used(),520 LLM_FFN_SILU, true,521 hparams.expert_weights_scale,522 (llama_expert_gating_func_type) hparams.expert_gating_func,523 il);524 cb(moe_out, "ffn_moe_out", il);525 526 // Shared expert527 {528 ggml_tensor * ffn_shexp = build_ffn(cur,529 layer.ffn_up_shexp, NULL, NULL,530 layer.ffn_gate_shexp, NULL, NULL,531 layer.ffn_down_shexp, NULL, NULL,532 NULL, LLM_FFN_SILU, LLM_FFN_PAR, il);533 cb(ffn_shexp, "ffn_shexp", il);534 535 cur = ggml_add(ctx0, moe_out, ffn_shexp);536 cb(cur, "ffn_out", il);537 }538 }539 // Residual540 cur = ggml_add(ctx0, cur, ffn_inp);541 542 cur = build_cvec(cur, il);543 cb(cur, "l_out", il);544 545 // input for next layer546 inpL = cur;547 }548 cur = inpL;549 550 // Final Norm551 cur = build_norm(cur, model.output_norm, NULL, LLM_NORM_RMS, -1);552 553 cb(cur, "result_norm", -1);554 res->t_embd = cur;555 556 // Output557 cur = ggml_mul_mat(ctx0, model.output, cur);558 cb(cur, "result_output", -1);559 res->t_logits = cur;560 561 ggml_build_forward_expand(gf, cur);562}563