Felipe97/llama-cpp-compiled
01.1k
1#include "models.h"2#include "../llama-memory-hybrid-iswa.h"3#include "../llama-memory-hybrid.h"4 5#include <algorithm>6 7void llama_model_lfm2::load_arch_hparams(llama_model_loader & ml) {8 ml.get_key(LLM_KV_SHORTCONV_L_CACHE, hparams.n_shortconv_l_cache);9 ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);10 11 for (uint32_t il = 0; il < hparams.n_layer(); ++il) {12 hparams.is_recr_impl[il] = hparams.n_head_kv(il) == 0;13 }14 15 hparams.n_layer_dense_lead = hparams.n_layer();16 17 switch (hparams.n_ff()) {18 case 2560: type = LLM_TYPE_230M; break;19 case 4608: type = LLM_TYPE_350M; break;20 case 6912: type = LLM_TYPE_700M; break;21 case 8192: type = LLM_TYPE_1_2B; break;22 case 10752: type = LLM_TYPE_2_6B; break;23 default: type = LLM_TYPE_UNKNOWN;24 }25 26 if (const auto is_swa = ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa, false); is_swa && hparams.n_swa > 0) {27 hparams.swa_type = LLAMA_SWA_TYPE_STANDARD;28 for (uint32_t il = 0; il < hparams.n_layer(); ++il) {29 hparams.is_swa_impl[il] = !hparams.is_recr_impl[il];30 }31 }32}33 34void llama_model_lfm2::load_arch_tensors(llama_model_loader &) {35 LLAMA_LOAD_LOCALS;36 37 tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0);38 39 output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM_LFM2, "weight"), {n_embd}, 0);40 output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED);41 42 if (output == NULL) {43 output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, TENSOR_DUPLICATED);44 }45 46 for (int i = 0; i < n_layer; ++i) {47 auto & layer = layers[i];48 49 const bool is_moe_layer = i >= static_cast<int>(hparams.n_layer_dense_lead);50 51 // ffn/moe is same for transformer and conv layers52 layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0);53 if (is_moe_layer) {54 GGML_ASSERT(n_expert && n_expert_used);55 layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd, n_expert}, 0);56 layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i), {n_embd, hparams.n_ff_exp(), n_expert}, 0);57 layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {hparams.n_ff_exp(), n_embd, n_expert}, 0);58 layer.ffn_up_exps = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS, "weight", i), {n_embd, hparams.n_ff_exp(), n_expert}, 0);59 layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, "bias", i), {n_expert}, 0);60 } else { // dense61 layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff}, 0);62 layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd}, 0);63 layer.ffn_up = create_tensor(tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, 0);64 }65 66 // for operator_norm67 layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0);68 69 if (!hparams.is_recr(i)) {70 layer.attn_q_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_NORM, "weight", i), {n_embd_head_k}, 0);71 layer.attn_k_norm = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM, "weight", i), {n_embd_head_k}, 0);72 GGML_ASSERT(n_embd_v_gqa == n_embd_k_gqa);73 74 create_tensor_qkv(layer, i, n_embd, n_embd, hparams.n_embd_k_gqa(i), hparams.n_embd_v_gqa(i), 0);75 76 layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd}, 0);77 } else {78 layer.shortconv.conv = create_tensor(tn(LLM_TENSOR_SHORTCONV_CONV, "weight", i), {hparams.n_shortconv_l_cache, n_embd}, 0);79 layer.shortconv.in_proj = create_tensor(tn(LLM_TENSOR_SHORTCONV_INPROJ, "weight", i), {n_embd, 3 * n_embd}, 0);80 layer.shortconv.out_proj = create_tensor(tn(LLM_TENSOR_SHORTCONV_OUTPROJ, "weight", i), {n_embd, n_embd}, 0);81 }82 }83 84 // for LFM2-ColBert-350M85 dense_2_out_layers = create_tensor(tn(LLM_TENSOR_DENSE_2_OUT, "weight"), {n_embd, hparams.n_embd_out()}, TENSOR_NOT_REQUIRED);86 dense_2_out_layers_b = create_tensor(tn(LLM_TENSOR_DENSE_2_OUT, "bias"), {hparams.n_embd_out() }, TENSOR_NOT_REQUIRED);87}88 89std::unique_ptr<llm_graph_context> llama_model_lfm2::build_arch_graph(const llm_graph_params & params) const {90 if (hparams.swa_type == LLAMA_SWA_TYPE_STANDARD) {91 return std::make_unique<graph<true>>(*this, params);92 } else {93 return std::make_unique<graph<false>>(*this, params);94 }95}96 97template <bool iswa>98llama_model_lfm2::graph<iswa>::graph(const llama_model & model, const llm_graph_params & params) :99 llm_graph_context(params) {100 using inp_hybrid_type = std::conditional_t<iswa, llm_graph_input_mem_hybrid_iswa, llm_graph_input_mem_hybrid>;101 using inp_attn_type = std::conditional_t<iswa, llm_graph_input_attn_kv_iswa, llm_graph_input_attn_kv>;102 using mem_hybrid_ctx = std::conditional_t<iswa, llama_memory_hybrid_iswa_context, llama_memory_hybrid_context>;103 104 // lambda helpers for readability105 auto build_dense_feed_forward = [&model, this](ggml_tensor * cur, int il) -> ggml_tensor * {106 GGML_ASSERT(!model.layers[il].ffn_up_b);107 GGML_ASSERT(!model.layers[il].ffn_gate_b);108 GGML_ASSERT(!model.layers[il].ffn_down_b);109 return build_ffn(cur,110 model.layers[il].ffn_up, NULL, NULL,111 model.layers[il].ffn_gate, NULL, NULL,112 model.layers[il].ffn_down, NULL, NULL,113 NULL, LLM_FFN_SILU, LLM_FFN_PAR, il);114 };115 auto build_moe_feed_forward = [&model, this](ggml_tensor * cur, int il) -> ggml_tensor * {116 return build_moe_ffn(cur,117 model.layers[il].ffn_gate_inp,118 model.layers[il].ffn_up_exps,119 model.layers[il].ffn_gate_exps,120 model.layers[il].ffn_down_exps,121 model.layers[il].ffn_exp_probs_b,122 n_expert, n_expert_used,123 LLM_FFN_SILU, true,124 hparams.expert_weights_scale,125 static_cast<llama_expert_gating_func_type>(hparams.expert_gating_func),126 il);127 };128 auto build_attn_block = [&model, this](ggml_tensor * cur,129 ggml_tensor * inp_pos,130 inp_attn_type * inp_attn,131 int il) -> ggml_tensor * {132 GGML_ASSERT(hparams.n_embd_v_gqa(il) == hparams.n_embd_k_gqa(il));133 const auto n_embd_head = hparams.n_embd_head_v();134 const auto n_head_kv = hparams.n_head_kv(il);135 136 auto [q, k, v] = build_qkv(model.layers[il], cur,137 n_embd_head, n_head, n_head_kv, il);138 139 // qk norm140 q = build_norm(q, model.layers[il].attn_q_norm, NULL, LLM_NORM_RMS, il);141 cb(q, "model.layers.{}.self_attn.q_layernorm", il);142 k = build_norm(k, model.layers[il].attn_k_norm, NULL, LLM_NORM_RMS, il);143 cb(k, "model.layers.{}.self_attn.k_layernorm", il);144 145 // RoPE146 q = ggml_rope_ext(ctx0, q, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor,147 attn_factor, beta_fast, beta_slow);148 k = ggml_rope_ext(ctx0, k, inp_pos, nullptr, n_rot, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor,149 attn_factor, beta_fast, beta_slow);150 151 cur = build_attn(inp_attn,152 model.layers[il].wo, NULL, model.layers[il].wo_s,153 q, k, v, nullptr, nullptr, nullptr, 1.0f / sqrtf(float(n_embd_head)), il);154 155 cb(cur, "model.layers.{}.self_attn.out_proj", il);156 157 return cur;158 };159 auto build_shortconv_block = [&model, this](ggml_tensor * cur,160 llm_graph_input_rs * inp_recr,161 int il) -> ggml_tensor * {162 const auto * mctx_cur = static_cast<const mem_hybrid_ctx *>(mctx)->get_recr();163 const uint32_t kv_head = mctx_cur->get_head();164 const int64_t n_seq_tokens = ubatch.n_seq_tokens;165 const int64_t n_seqs = ubatch.n_seqs;166 GGML_ASSERT(n_seqs != 0);167 GGML_ASSERT(ubatch.equal_seqs());168 GGML_ASSERT(ubatch.n_tokens == n_seq_tokens * n_seqs);169 170 GGML_ASSERT(hparams.n_shortconv_l_cache > 1);171 const uint32_t d_conv = hparams.n_shortconv_l_cache - 1;172 173 // {n_embd, n_tokens} => {n_embd, n_seq_tokens, n_seqs}174 cur = ggml_reshape_3d(ctx0, cur, cur->ne[0], n_seq_tokens, n_seqs);175 176 auto * bcx = build_lora_mm(model.layers[il].shortconv.in_proj, cur);177 cb(bcx, "model.layers.{}.conv.in_proj", il);178 179 constexpr auto n_chunks = 3;180 GGML_ASSERT(bcx->ne[0] % n_chunks == 0);181 const auto chunk_size = bcx->ne[0] / n_chunks;182 auto * b = ggml_view_3d(ctx0, bcx, chunk_size, bcx->ne[1], bcx->ne[2], bcx->nb[1], bcx->nb[2],183 0 * chunk_size * ggml_element_size(bcx));184 auto * c = ggml_view_3d(ctx0, bcx, chunk_size, bcx->ne[1], bcx->ne[2], bcx->nb[1], bcx->nb[2],185 1 * chunk_size * ggml_element_size(bcx));186 auto * x = ggml_view_3d(ctx0, bcx, chunk_size, bcx->ne[1], bcx->ne[2], bcx->nb[1], bcx->nb[2],187 2 * chunk_size * ggml_element_size(bcx));188 189 auto * bx = ggml_transpose(ctx0, ggml_mul(ctx0, b, x));190 191 // read conv state192 auto * conv_state = mctx_cur->get_r_l(il);193 auto * conv_rs = build_rs(inp_recr, conv_state, hparams.n_embd_r(), n_seqs);194 auto * conv = ggml_reshape_3d(ctx0, conv_rs, d_conv, hparams.n_embd, n_seqs);195 196 // causal prepends the state, non-causal pads symmetrically for a centered window197 if (hparams.causal_attn) {198 bx = ggml_concat(ctx0, conv, bx, 0);199 } else {200 const int64_t pad = (hparams.n_shortconv_l_cache - 1) / 2;201 auto * left = ggml_cont(ctx0,202 ggml_view_3d(ctx0, conv, pad, hparams.n_embd, n_seqs, conv->nb[1], conv->nb[2], (d_conv - pad) * conv->nb[0]));203 bx = ggml_pad_ext(ctx0, ggml_concat(ctx0, left, bx, 0), 0, pad, 0, 0, 0, 0, 0, 0);204 }205 GGML_ASSERT(bx->ne[0] > conv->ne[0]);206 207 // write conv states: slot 0 = the final state, slot s = the state s tokens back (partial rollback)208 const int64_t K = hparams.causal_attn && cparams.n_rs_seq > 0 ? (int64_t) cparams.n_rs_seq + 1 : 1;209 const int64_t n_written = std::min<int64_t>(n_seq_tokens, K);210 const auto mem_size = mctx_cur->get_size();211 const size_t row_size = ggml_row_size(conv_state->type, (int64_t) d_conv * n_embd);212 213 for (int64_t slot = 0; slot < n_written; ++slot) {214 auto * conv_snap = ggml_view_3d(ctx0, bx, d_conv, bx->ne[1], bx->ne[2], bx->nb[1], bx->nb[2],215 (bx->ne[0] - d_conv - slot) * ggml_element_size(bx));216 ggml_build_forward_expand(gf, ggml_cpy(ctx0, conv_snap,217 ggml_view_2d(ctx0, conv_state, (int64_t) d_conv * n_embd, n_seqs,218 conv_state->nb[1],219 ((size_t) slot * mem_size + kv_head) * row_size)));220 }221 222 auto * conv_kernel = model.layers[il].shortconv.conv;223 auto * conv_out = ggml_ssm_conv(ctx0, bx, conv_kernel);224 cb(conv_out, "model.layers.{}.conv.conv", il);225 226 auto * y = ggml_mul(ctx0, c, conv_out);227 y = build_lora_mm(model.layers[il].shortconv.out_proj, y);228 cb(y, "model.layers.{}.conv.out_proj", il);229 // {n_embd, n_seq_tokens, n_seqs} => {n_embd, n_tokens}230 y = ggml_reshape_2d(ctx0, y, y->ne[0], n_seq_tokens * n_seqs);231 232 return y;233 };234 235 // actual graph construction starts here236 ggml_tensor * cur = build_inp_embd(model.tok_embd);237 cb(cur, "model.embed_tokens", -1);238 239 ggml_build_forward_expand(gf, cur);240 241 inp_hybrid_type * inp_hybrid = nullptr;242 if constexpr (iswa) {243 inp_hybrid = build_inp_mem_hybrid_iswa();244 } else {245 inp_hybrid = build_inp_mem_hybrid();246 }247 248 ggml_tensor * inp_pos = build_inp_pos();249 ggml_tensor * inp_out_ids = build_inp_out_ids();250 251 for (int il = 0; il < n_layer; ++il) {252 res->t_layer_inp[il] = cur;253 254 const bool is_moe_layer = il >= static_cast<int>(hparams.n_layer_dense_lead);255 256 auto * prev_cur = cur;257 cur = build_norm(cur, model.layers[il].attn_norm, NULL, LLM_NORM_RMS, il);258 cb(cur, "model.layers.{}.operator_norm", il);259 260 cur = hparams.is_recr(il) ? build_shortconv_block(cur, inp_hybrid->get_recr(), il) :261 build_attn_block(cur, inp_pos, inp_hybrid->get_attn(), il);262 263 if (il == n_layer - 1 && inp_out_ids) {264 cur = ggml_get_rows(ctx0, cur, inp_out_ids);265 prev_cur = ggml_get_rows(ctx0, prev_cur, inp_out_ids);266 }267 268 cur = ggml_add(ctx0, prev_cur, cur);269 270 auto * ffn_norm_out = build_norm(cur, model.layers[il].ffn_norm, NULL, LLM_NORM_RMS, il);271 cb(ffn_norm_out, "model.layers.{}.ffn_norm", il);272 273 ggml_tensor * ffn_out =274 is_moe_layer ? build_moe_feed_forward(ffn_norm_out, il) : build_dense_feed_forward(ffn_norm_out, il);275 cb(ffn_norm_out, "model.layers.{}.ffn_out", il);276 277 cur = ggml_add(ctx0, cur, ffn_out);278 279 cur = build_cvec(cur, il);280 cb(cur, "l_out", il);281 }282 283 cur = build_norm(cur, model.output_norm, NULL, LLM_NORM_RMS, -1);284 cb(cur, "result_norm", -1);285 res->t_embd = cur;286 287 if (!cparams.embeddings) {288 cur = build_lora_mm(model.output, cur, model.output_s);289 cb(cur, "result_output", -1);290 291 res->t_logits = cur;292 }293 294 ggml_build_forward_expand(gf, cur);295}296 297// Explicit template instantiations298template struct llama_model_lfm2::graph<true>;299template struct llama_model_lfm2::graph<false>;300