Felipe97/llama-cpp-compiled
01.1k
1#include "models.h"2#include "llama-memory-recurrent.h"3 4void llama_model_plamo2::load_arch_hparams(llama_model_loader & ml) {5 ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);6 7 // Load Mamba SSM parameters8 ml.get_key(LLM_KV_SSM_CONV_KERNEL, hparams.ssm_d_conv);9 ml.get_key(LLM_KV_SSM_INNER_SIZE, hparams.ssm_d_inner);10 ml.get_key(LLM_KV_SSM_STATE_SIZE, hparams.ssm_d_state);11 ml.get_key(LLM_KV_SSM_TIME_STEP_RANK, hparams.ssm_dt_rank);12 ml.get_key(LLM_KV_SSM_GROUP_COUNT, hparams.ssm_n_group);13 14 // Load attention parameters15 ml.get_key(LLM_KV_ATTENTION_KEY_LENGTH, hparams.n_embd_head_k_full, false);16 ml.get_key(LLM_KV_ATTENTION_VALUE_LENGTH, hparams.n_embd_head_v_full, false);17 18 for (uint32_t i = 0; i < hparams.n_layer(); ++i) {19 hparams.is_recr_impl[i] = hparams.n_head_kv(i) == 0;20 }21 22 switch (hparams.n_layer()) {23 case 16: type = LLM_TYPE_1B; break;24 case 32:25 if (hparams.n_embd == 2048) {26 type = LLM_TYPE_2B;27 } else if (hparams.n_embd == 4096) {28 type = LLM_TYPE_8B;29 }30 break;31 default: type = LLM_TYPE_UNKNOWN;32 }33}34 35void llama_model_plamo2::load_arch_tensors(llama_model_loader &) {36 LLAMA_LOAD_LOCALS;37 38 // mamba parameters39 const uint32_t d_conv = hparams.ssm_d_conv;40 const uint32_t d_state = hparams.ssm_d_state;41 const uint32_t num_heads = hparams.ssm_dt_rank;42 const uint32_t intermediate_size = hparams.ssm_d_inner;43 const int64_t dt_dim = std::max(64, int(hparams.n_embd / 16));44 45 // attention parameters46 const uint32_t qk_dim = hparams.n_embd_head_k();47 const uint32_t v_dim = hparams.n_embd_head_v();48 49 tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0);50 51 // output52 output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0);53 output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED);54 // if output is NULL, init from the input tok embed55 if (output == NULL) {56 output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, TENSOR_DUPLICATED);57 }58 59 for (int i = 0; i < n_layer; ++i) {60 auto & layer = layers[i];61 bool is_mamba_layer = hparams.is_recr(i);62 63 layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0);64 65 if (is_mamba_layer) {66 layer.ssm_in = create_tensor(tn(LLM_TENSOR_SSM_IN, "weight", i), {n_embd, 2 * intermediate_size}, 0);67 layer.ssm_conv1d = create_tensor(tn(LLM_TENSOR_SSM_CONV1D, "weight", i), {d_conv, intermediate_size}, 0);68 69 layer.ssm_x = create_tensor(tn(LLM_TENSOR_SSM_X, "weight", i), {intermediate_size, dt_dim + 2*d_state}, 0);70 layer.ssm_dt = create_tensor(tn(LLM_TENSOR_SSM_DT, "weight", i), {dt_dim, num_heads}, 0);71 layer.ssm_dt_b = create_tensor(tn(LLM_TENSOR_SSM_DT, "bias", i), {num_heads}, 0);72 73 layer.ssm_a = create_tensor(tn(LLM_TENSOR_SSM_A, i), {num_heads}, 0);74 layer.ssm_d = create_tensor(tn(LLM_TENSOR_SSM_D, i), {num_heads}, 0);75 76 layer.ssm_out = create_tensor(tn(LLM_TENSOR_SSM_OUT, "weight", i), {intermediate_size, n_embd}, 0);77 78 layer.ssm_dt_norm = create_tensor(tn(LLM_TENSOR_SSM_DT_NORM, i), {dt_dim}, 0);79 layer.ssm_b_norm = create_tensor(tn(LLM_TENSOR_SSM_B_NORM, i), {d_state}, 0);80 layer.ssm_c_norm = create_tensor(tn(LLM_TENSOR_SSM_C_NORM, i), {d_state}, 0);81 } else {82 const int64_t num_attention_heads = hparams.n_head(i);83 const int64_t q_num_heads = num_attention_heads;84 const int64_t num_key_value_heads = hparams.n_head_kv(i);85 const int64_t k_num_heads = num_key_value_heads;86 const int64_t v_num_heads = num_key_value_heads;87 const int64_t q_proj_dim = q_num_heads * qk_dim;88 const int64_t k_proj_dim = k_num_heads * qk_dim;89 const int64_t v_proj_dim = v_num_heads * v_dim;90 91 layer.wqkv = create_tensor(tn(LLM_TENSOR_ATTN_QKV, "weight", i), {n_embd, q_proj_dim + k_proj_dim + v_proj_dim}, 0);92 layer.attn_q_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_NORM, "weight", i), {qk_dim, num_attention_heads}, 0);93 layer.attn_k_norm = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM, "weight", i), {qk_dim, k_num_heads}, 0);94 layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {q_num_heads * v_dim, n_embd}, 0);95 }96 97 // All layers have post-attention norm, FFN norm, and FFN tensors98 layer.attn_post_norm = create_tensor(tn(LLM_TENSOR_ATTN_POST_NORM, i), {n_embd}, 0);99 layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0);100 layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), {n_ff, n_embd}, 0);101 layer.ffn_up = create_tensor(tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff * 2}, 0);102 layer.ffn_post_norm = create_tensor(tn(LLM_TENSOR_FFN_POST_NORM, i), {n_embd}, 0);103 }104}105 106std::unique_ptr<llm_graph_context> llama_model_plamo2::build_arch_graph(const llm_graph_params & params) const {107 return std::make_unique<graph>(*this, params);108}109 110llama_model_plamo2::graph::graph(const llama_model & model, const llm_graph_params & params) :111 llm_build_mamba_base(params) {112 ggml_tensor * cur;113 ggml_tensor * inpL;114 115 // {n_embd, n_tokens}116 inpL = build_inp_embd(model.tok_embd);117 cb(inpL, "embedding_output", -1);118 119 ggml_tensor * inp_pos = build_inp_pos();120 121 auto * inp_hybrid = build_inp_mem_hybrid();122 123 ggml_tensor * inp_out_ids = build_inp_out_ids();124 125 for (int il = 0; il < n_layer; ++il) {126 ggml_tensor * residual = inpL;127 128 // ggml_graph_add_node(gf, model.layers[il].attn_norm);129 // cb(model.layers[il].attn_norm, "attn_norm", il);130 131 // pre_mixer_norm132 cur = build_norm(inpL, model.layers[il].attn_norm, NULL, LLM_NORM_RMS, il);133 134 // check if this layer is Mamba or Attention135 const bool is_mamba_layer = hparams.is_recr(il);136 137 if (is_mamba_layer) {138 // PLaMo-2 Mamba layer139 cur = build_plamo2_mamba_layer(inp_hybrid->get_recr(), cur, model, ubatch, il);140 } else {141 // PLaMo-2 Attention layer142 cur = build_plamo2_attn_layer(inp_hybrid->get_attn(), inp_pos, cur, model, il);143 }144 145 if (il == n_layer - 1 && inp_out_ids) {146 cur = ggml_get_rows(ctx0, cur, inp_out_ids);147 residual = ggml_get_rows(ctx0, residual, inp_out_ids);148 }149 150 // post_mixer_norm151 cur = build_norm(cur, model.layers[il].attn_post_norm, NULL, LLM_NORM_RMS, il);152 cb(cur, "attn_post_norm", il);153 154 // residual connection155 cur = ggml_add(ctx0, cur, residual);156 cb(cur, "attn_residual", il);157 residual = cur;158 159 // pre-ffn norm160 cur = build_norm(cur, model.layers[il].ffn_norm, NULL, LLM_NORM_RMS, il);161 cb(cur, "ffn_pre_norm", il);162 163 // feed-forward network164 cur = build_ffn(cur,165 model.layers[il].ffn_up, NULL, NULL,166 NULL, NULL, NULL,167 model.layers[il].ffn_down, NULL, NULL,168 NULL, LLM_FFN_SWIGLU, LLM_FFN_SEQ, il);169 cb(cur, "ffn_out", il);170 171 // post ffn norm172 cur = build_norm(cur, model.layers[il].ffn_post_norm, NULL, LLM_NORM_RMS, il);173 cb(cur, "ffn_post_norm", il);174 175 // residual connection176 cur = ggml_add(ctx0, cur, residual);177 cb(cur, "ffn_residual", il);178 179 // input for next layer180 inpL = cur;181 }182 183 cur = inpL;184 185 // final norm186 cur = build_norm(cur, model.output_norm, NULL, LLM_NORM_RMS, -1);187 cb(cur, "result_norm", -1);188 189 res->t_embd = cur;190 191 // lm_head192 cur = build_lora_mm(model.output, cur, model.output_s);193 cb(cur, "result_output", -1);194 195 // Explicitly mark as output tensor to ensure proper backend assignment196 ggml_set_output(cur);197 198 res->t_logits = cur;199 200 ggml_build_forward_expand(gf, cur);201}202 203ggml_tensor * llama_model_plamo2::graph::build_plamo2_attn_layer(llm_graph_input_attn_kv * inp,204 ggml_tensor * inp_pos,205 ggml_tensor * cur,206 const llama_model & model,207 int il) {208 // self-attention209 {210 // PLaMo-2 uses combined QKV tensor211 ggml_tensor * qkv = build_lora_mm(model.layers[il].wqkv, cur);212 cb(qkv, "wqkv", il);213 214 // split QKV tensor into Q, K, V215 const int64_t n_embd_head_q = hparams.n_embd_head_k();216 const int64_t n_embd_head_k = hparams.n_embd_head_k();217 const int64_t n_embd_head_v = hparams.n_embd_head_v();218 int32_t n_head = hparams.n_head(il);219 int32_t n_head_kv = hparams.n_head_kv(il);220 221 const int64_t q_offset = 0;222 const int64_t k_offset = n_embd_head_q * n_head;223 const int64_t v_offset = k_offset + n_embd_head_k * n_head_kv;224 225 ggml_tensor * Qcur = ggml_view_3d(ctx0, qkv, n_embd_head_q, n_head, n_tokens, n_embd_head_q * sizeof(float),226 qkv->nb[1], q_offset * ggml_element_size(qkv));227 ggml_tensor * Kcur = ggml_view_3d(ctx0, qkv, n_embd_head_k, n_head_kv, n_tokens, n_embd_head_k * sizeof(float),228 qkv->nb[1], k_offset * ggml_element_size(qkv));229 ggml_tensor * Vcur = ggml_view_3d(ctx0, qkv, n_embd_head_v, n_head_kv, n_tokens, n_embd_head_v * sizeof(float),230 qkv->nb[1], v_offset * ggml_element_size(qkv));231 232 cb(Qcur, "Qcur", il);233 cb(Kcur, "Kcur", il);234 cb(Vcur, "Vcur", il);235 236 Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, NULL, LLM_NORM_RMS, il);237 cb(Qcur, "Qcur_normed", il);238 239 Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,240 ext_factor, attn_factor, beta_fast, beta_slow);241 242 Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, NULL, LLM_NORM_RMS, il);243 cb(Kcur, "Kcur_normed", il);244 245 Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,246 ext_factor, attn_factor, beta_fast, beta_slow);247 248 cur = build_attn(inp,249 model.layers[il].wo, NULL, model.layers[il].wo_s,250 Qcur, Kcur, Vcur, NULL, NULL, NULL, 1.0f / sqrtf(float(n_embd_head_v)), il);251 }252 253 cb(cur, "attn_out", il);254 255 return cur;256}257 258ggml_tensor * llama_model_plamo2::graph::build_plamo2_mamba_layer(llm_graph_input_rs * inp,259 ggml_tensor * cur,260 const llama_model & model,261 const llama_ubatch & ubatch,262 int il) {263 const auto * mctx_cur = inp->mctx;264 265 const auto kv_head = mctx_cur->get_head();266 267 const int64_t d_conv = hparams.ssm_d_conv;268 const int64_t d_inner = hparams.ssm_d_inner;269 const int64_t d_state = hparams.ssm_d_state;270 const int64_t n_heads = hparams.ssm_dt_rank;271 const int64_t head_dim = d_inner / n_heads;272 const int64_t n_group = hparams.ssm_n_group;273 const int64_t n_seqs = ubatch.n_seqs;274 275 const int64_t n_seq_tokens = ubatch.n_seq_tokens;276 277 GGML_ASSERT(n_seqs != 0);278 GGML_ASSERT(ubatch.equal_seqs());279 GGML_ASSERT(ubatch.n_tokens == n_seq_tokens * n_seqs);280 GGML_ASSERT(d_inner % n_heads == 0);281 GGML_ASSERT(n_group == 0);282 283 ggml_tensor * conv_states_all = mctx_cur->get_r_l(il);284 ggml_tensor * ssm_states_all = mctx_cur->get_s_l(il);285 286 ggml_tensor * conv = build_rs(inp, conv_states_all, hparams.n_embd_r(), n_seqs);287 conv = ggml_reshape_3d(ctx0, conv, d_conv - 1, d_inner + 2 * n_group * d_state, n_seqs);288 289 // {n_embd, n_tokens} => {n_embd, n_seq_tokens, n_seqs}290 cur = ggml_reshape_3d(ctx0, cur, cur->ne[0], n_seq_tokens, n_seqs);291 292 // in_proj: {n_embd, 2*d_inner} @ {n_embd, n_seq_tokens, n_seqs} => {2*d_inner, n_seq_tokens, n_seqs}293 ggml_tensor * zx = build_lora_mm(model.layers[il].ssm_in, cur);294 cb(zx, "mamba_in_proj", il);295 // {8192, 5, 1, 1} -> {8192, 1, 5, 1}296 zx = ggml_permute(ctx0, zx, 0, 2, 1, 3);297 zx = ggml_cont_4d(ctx0, zx, head_dim * 2, n_heads, n_seq_tokens, n_seqs);298 cb(zx, "mamba_in_proj_out", il);299 300 // split into z and x301 // => {head_dim * n_heads, n_seq_tokens, n_seqs}302 ggml_tensor * x = ggml_view_4d(ctx0, zx, head_dim, n_heads, n_seq_tokens, n_seqs, zx->nb[1], zx->nb[2], zx->nb[3],303 head_dim * ggml_element_size(zx));304 x = ggml_cont_3d(ctx0, x, head_dim * n_heads, n_seq_tokens, n_seqs);305 // x = ggml_permute(ctx0, x, 0, 2, 1, 3);306 cb(x, "mamba_x_split", il);307 308 ggml_tensor * z =309 ggml_view_4d(ctx0, zx, head_dim, n_heads, n_seq_tokens, n_seqs, zx->nb[1], zx->nb[2], zx->nb[3], 0);310 cb(z, "mamba_z_split", il);311 312 // conv1d313 {314 // => {d_conv - 1 + n_seq_tokens, d_inner, n_seqs}315 ggml_tensor * conv_x = ggml_concat(ctx0, conv, ggml_transpose(ctx0, x), 0);316 cb(conv_x, "mamba_conv1d_input", il);317 318 // copy last (d_conv - 1) columns back into the state cache319 ggml_tensor * last_conv = ggml_view_3d(ctx0, conv_x, d_conv - 1, d_inner, n_seqs, conv_x->nb[1], conv_x->nb[2],320 n_seq_tokens * (conv_x->nb[0]));321 322 ggml_build_forward_expand(gf, ggml_cpy(ctx0, last_conv,323 ggml_view_1d(ctx0, conv_states_all,324 (d_conv - 1) * (d_inner + 2 * n_group * d_state) * (n_seqs),325 kv_head * (d_conv - 1) * (d_inner + 2 * n_group * d_state) *326 ggml_element_size(conv_states_all))));327 cb(conv_states_all, "mamba_conv1d_state", il);328 329 // 1D convolution330 x = ggml_ssm_conv(ctx0, conv_x, model.layers[il].ssm_conv1d);331 cb(x, "mamba_conv1d", il);332 333 x = ggml_silu(ctx0, x);334 cb(x, "mamba_conv1d_silu", il);335 }336 337 // SSM338 {339 // bcdt_proj: {d_inner, dt_rank + 2*d_state} @ {d_inner, n_seq_tokens, n_seqs} => {dt_rank + 2*d_state, n_seq_tokens, n_seqs}340 ggml_tensor * x_bcdt = build_lora_mm(model.layers[il].ssm_x, x);341 cb(x_bcdt, "mamba_bcdt_proj", il);342 343 // split into dt, B, C344 const int64_t dt_dim = std::max(64, int(hparams.n_embd / 16));345 ggml_tensor * B = ggml_view_3d(ctx0, x_bcdt, d_state, n_seq_tokens, n_seqs, x_bcdt->nb[1], x_bcdt->nb[2], 0);346 ggml_tensor * C = ggml_view_3d(ctx0, x_bcdt, d_state, n_seq_tokens, n_seqs, x_bcdt->nb[1], x_bcdt->nb[2],347 ggml_element_size(x_bcdt) * d_state);348 ggml_tensor * dt = ggml_view_3d(ctx0, x_bcdt, dt_dim, n_seq_tokens, n_seqs, x_bcdt->nb[1], x_bcdt->nb[2],349 ggml_element_size(x_bcdt) * (2 * d_state));350 cb(B, "mamba_B_raw", il);351 cb(C, "mamba_C_raw", il);352 cb(dt, "mamba_dt_raw", il);353 354 // Apply RMS norm to dt, B, C (PLaMo-2 specific)355 B = build_norm(B, model.layers[il].ssm_b_norm, NULL, LLM_NORM_RMS, il);356 C = build_norm(C, model.layers[il].ssm_c_norm, NULL, LLM_NORM_RMS, il);357 dt = build_norm(dt, model.layers[il].ssm_dt_norm, NULL, LLM_NORM_RMS, il);358 cb(B, "mamba_B_normed", il);359 cb(C, "mamba_C_normed", il);360 cb(dt, "mamba_dt_normed", il);361 362 // dt_proj: {dt_rank, d_inner} @ {dt_rank, n_seq_tokens, n_seqs} => {d_inner, n_seq_tokens, n_seqs}363 dt = build_lora_mm(model.layers[il].ssm_dt, dt);364 dt = ggml_add(ctx0, dt, model.layers[il].ssm_dt_b);365 cb(dt, "mamba_dt_proj", il);366 367 ggml_tensor * A = ggml_reshape_2d(ctx0, model.layers[il].ssm_a, 1, n_heads);368 cb(A, "mamba_A", il);369 370 x = ggml_view_4d(ctx0, x, head_dim, n_heads, n_seq_tokens, n_seqs, head_dim * ggml_element_size(x),371 head_dim * n_heads * ggml_element_size(x),372 head_dim * n_heads * n_seq_tokens * ggml_element_size(x), 0);373 B = ggml_view_4d(ctx0, B, d_state, 1, n_seq_tokens, n_seqs, d_state * B->nb[0], B->nb[1], B->nb[2], 0);374 C = ggml_view_4d(ctx0, C, d_state, 1, n_seq_tokens, n_seqs, d_state * C->nb[0], C->nb[1], C->nb[2], 0);375 376 // use the states and the indices provided by build_recurrent_state377 // (this is necessary in order to properly use the states before they are overwritten,378 // while avoiding to make unnecessary copies of the states)379 auto get_ssm_rows = [&](ggml_context * ctx, ggml_tensor * states, ggml_tensor * ids) {380 ggml_tensor * ssm = ggml_reshape_4d(ctx, states, d_state, head_dim, n_heads, mctx_cur->get_size());381 382 // Custom operator to optimize the parallel associative scan383 // as described in the Annex D of the Mamba paper.384 // => {d_inner, n_seq_tokens, n_seqs} and {d_state, d_inner, n_seqs}385 return ggml_ssm_scan(ctx, ssm, x, dt, A, B, C, ids, /*K=*/1);386 };387 388 ggml_tensor * y_ssm = build_rs(inp, ssm_states_all, hparams.n_embd_s(), ubatch.n_seqs, get_ssm_rows);389 cb(y_ssm, "mamba_ssm_scan", il);390 391 // store last states392 ggml_build_forward_expand(393 gf, ggml_cpy(394 ctx0,395 ggml_view_1d(ctx0, y_ssm, n_heads * head_dim * d_state * n_seqs,396 n_heads * head_dim * n_seq_tokens * n_seqs * ggml_element_size(y_ssm)),397 ggml_view_1d(ctx0, ssm_states_all, n_heads * head_dim * d_state * n_seqs,398 kv_head * n_seqs * n_heads * head_dim * d_state * ggml_element_size(ssm_states_all))));399 cb(ssm_states_all, "mamba_ssm_states", il);400 401 ggml_tensor * y = ggml_view_4d(ctx0, y_ssm, head_dim, n_heads, n_seq_tokens, n_seqs,402 head_dim * ggml_element_size(x), head_dim * n_heads * ggml_element_size(x),403 head_dim * n_heads * n_seq_tokens * ggml_element_size(x), 0);404 cb(y, "mamba_y_view", il);405 406 // Add D parameter and apply gating with z407 // {d_inner, n_seq_tokens, n_seqs} * {d_inner} => {d_inner, n_seq_tokens, n_seqs}408 ggml_tensor * D = ggml_reshape_2d(ctx0, model.layers[il].ssm_d, 1, n_heads);409 y = ggml_add(ctx0, y, ggml_mul(ctx0, x, D));410 cb(y, "mamba_y_add_d", il);411 412 y = ggml_swiglu_split(ctx0, ggml_cont(ctx0, z), y);413 cb(y, "mamba_y_swiglu_z", il);414 415 // out_proj: {d_inner, n_embd} @ {d_inner, n_seq_tokens, n_seqs} => {n_embd, n_seq_tokens, n_seqs}416 y = ggml_view_3d(ctx0, y, head_dim * n_heads, n_seq_tokens, n_seqs, y->nb[2], y->nb[3], 0);417 cur = build_lora_mm(model.layers[il].ssm_out, y);418 cb(cur, "mamba_out_proj", il);419 }420 421 // {n_embd, n_seq_tokens, n_seqs} => {n_embd, n_tokens}422 cur = ggml_reshape_2d(ctx0, cur, cur->ne[0], n_seq_tokens * n_seqs);423 cb(cur, "mamba_out", il);424 425 return cur;426}427