Felipe97/llama-cpp-compiled

1#include "models.h"2 3void llama_model_falcon_h1::load_arch_hparams(llama_model_loader & ml) {4    // Common parameters5    ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);6 7    // SSM parameters8    ml.get_key(LLM_KV_SSM_CONV_KERNEL,    hparams.ssm_d_conv);9    ml.get_key(LLM_KV_SSM_INNER_SIZE,     hparams.ssm_d_inner);10    ml.get_key(LLM_KV_SSM_STATE_SIZE,     hparams.ssm_d_state);11    ml.get_key(LLM_KV_SSM_TIME_STEP_RANK, hparams.ssm_dt_rank);12    ml.get_key(LLM_KV_SSM_GROUP_COUNT,    hparams.ssm_n_group);13 14    std::fill(hparams.is_recr_impl.begin(), hparams.is_recr_impl.end(), true);15 16    switch (hparams.n_layer()) {17        case 36:18            type = LLM_TYPE_0_5B; break;19        case 24:20            type = LLM_TYPE_1_5B; break;21        case 66:22            type = LLM_TYPE_1B; break;23        case 32:24            type = LLM_TYPE_3B; break;25        case 44:26            type = LLM_TYPE_7B; break;27        case 72:28            type = LLM_TYPE_34B; break;29        default:30            type = LLM_TYPE_UNKNOWN;31    }32}33 34void llama_model_falcon_h1::load_arch_tensors(llama_model_loader &) {35    LLAMA_LOAD_LOCALS;36 37    // Common38    const int64_t hidden_size = hparams.n_embd; // hidden_size39 40    // mamba2 Mixer SSM params41    const int64_t ssm_conv_kernel_size  = hparams.ssm_d_conv; // ssm_conv_kernel_size42    const int64_t ssm_n_groups          = hparams.ssm_n_group; // ssm_n_groups43    const int64_t ssm_state_size        = hparams.ssm_d_state; // ssm_state_size44    const int64_t ssm_intermediate_size = hparams.ssm_d_inner; // TODO expand45    const int64_t ssm_num_heads         = hparams.ssm_dt_rank; // ssm_num_heads46    const int64_t ssm_conv_dim          = ssm_intermediate_size + 2 * ssm_n_groups * ssm_state_size;47    const int64_t ssm_projection_size   = ssm_intermediate_size + ssm_conv_dim + ssm_num_heads;48 49    // attn params50    const int64_t attn_num_attention_head = hparams.n_head(0); // rename to: attn_num_attention_head51    const int64_t attn_num_key_value_head = hparams.n_head_kv(0);52 53    // ffn params54    const int64_t ffn_intermediate_size = hparams.n_ff(0);55 56    // embeddings57    tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {hidden_size, n_vocab}, 0);58 59    // output60    output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), {hidden_size, n_vocab}, TENSOR_NOT_REQUIRED);61    output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {hidden_size}, 0);62 63    // if output is NULL, init from the input tok embed64    if (output == NULL) {65        output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {hidden_size, n_vocab}, TENSOR_DUPLICATED);66    }67 68    for (int i = 0; i < n_layer; ++i) {69        auto & layer = layers[i];70 71        /*SSM LAYERS*/72        // ssm in73        layer.ssm_in = create_tensor(tn(LLM_TENSOR_SSM_IN, "weight", i), {hidden_size, ssm_projection_size}, 0);74        // ssm 1d conv75        layer.ssm_conv1d = create_tensor(tn(LLM_TENSOR_SSM_CONV1D, "weight", i), {ssm_conv_kernel_size, ssm_conv_dim}, 0);76        layer.ssm_conv1d_b = create_tensor(tn(LLM_TENSOR_SSM_CONV1D, "bias", i), {ssm_conv_dim}, TENSOR_NOT_REQUIRED);77        // ssm_dt78        layer.ssm_dt_b = create_tensor(tn(LLM_TENSOR_SSM_DT, "bias", i), {ssm_num_heads}, 0);79        // no "weight" suffix for these80        layer.ssm_a = create_tensor(tn(LLM_TENSOR_SSM_A, i), {1, ssm_num_heads}, 0);81        layer.ssm_d = create_tensor(tn(LLM_TENSOR_SSM_D, i), {1, ssm_num_heads}, 0);82        // ssm_norm83        layer.ssm_norm = create_tensor(tn(LLM_TENSOR_SSM_NORM, "weight", i), {ssm_intermediate_size / ssm_n_groups, ssm_n_groups}, TENSOR_NOT_REQUIRED);84        // out_proj85        layer.ssm_out = create_tensor(tn(LLM_TENSOR_SSM_OUT, "weight", i), {ssm_intermediate_size, hidden_size}, 0);86 87        /*ATTENTION LAYERS*/88        // attention layers (with optional bias)89        create_tensor_qkv(layer, i, hidden_size, n_embd_head_k * attn_num_attention_head, attn_num_key_value_head * n_embd_head_k, attn_num_key_value_head * n_embd_head_v, 0);90        layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd_head_k * attn_num_attention_head, hidden_size}, 0);91        layer.wo_b = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "bias", i), {hidden_size}, TENSOR_NOT_REQUIRED);92        layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {hidden_size}, 0);93 94 95        // feed forward (w/ optional biases)96        layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, i), {hidden_size}, 0);97        layer.rope_freqs = create_tensor(tn(LLM_TENSOR_ROPE_FREQS, "weight", i), {n_rot/2}, TENSOR_NOT_REQUIRED | (i != 0 ? TENSOR_DUPLICATED : 0));98        layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", i), {hidden_size,   ffn_intermediate_size}, 0);99        layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), {  ffn_intermediate_size, hidden_size}, 0);100        layer.ffn_up   = create_tensor(tn(LLM_TENSOR_FFN_UP,   "weight", i), {hidden_size,   ffn_intermediate_size}, 0);101 102        layer.ffn_gate_b = create_tensor(tn(LLM_TENSOR_FFN_GATE, "bias", i), {ffn_intermediate_size}, TENSOR_NOT_REQUIRED);103        layer.ffn_down_b = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "bias", i), {hidden_size}, TENSOR_NOT_REQUIRED);104        layer.ffn_up_b   = create_tensor(tn(LLM_TENSOR_FFN_UP,   "bias", i), {ffn_intermediate_size}, TENSOR_NOT_REQUIRED);105    }106}107 108std::unique_ptr<llm_graph_context> llama_model_falcon_h1::build_arch_graph(const llm_graph_params & params) const {109    return std::make_unique<graph>(*this, params);110}111 112llama_model_falcon_h1::graph::graph(const llama_model & model, const llm_graph_params & params) :113    llm_build_mamba_base(params) {114    const int64_t n_embd_head = hparams.n_embd_head_v();115 116    ggml_tensor * cur;117    ggml_tensor * inpL;118 119    inpL = build_inp_embd(model.tok_embd);120 121    // inp_pos - contains the positions122    ggml_tensor * inp_pos = build_inp_pos();123 124    // Build the inputs in the recurrent & kv cache125    auto * inp = build_inp_mem_hybrid();126 127    const float kq_scale =128        hparams.f_attention_scale == 0.0f ? 1.0f / sqrtf(float(n_embd_head)) : hparams.f_attention_scale;129 130    ggml_tensor * inp_out_ids = build_inp_out_ids();131 132    for (int il = 0; il < n_layer; ++il) {133        ggml_tensor * inpSA = inpL;134 135        cur = build_norm(inpL, model.layers[il].attn_norm, NULL, LLM_NORM_RMS, il);136        cb(cur, "attn_norm", il);137 138        // self-attention139        auto [Qcur, Kcur, Vcur] = build_qkv(model.layers[il], cur,140                n_embd_head, n_head, n_head_kv, il);141 142        Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, n_rot, hparams.rope_type, n_ctx_orig, freq_base, freq_scale,143                             ext_factor, attn_factor, beta_fast, beta_slow);144 145        Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, n_rot, hparams.rope_type, n_ctx_orig, freq_base, freq_scale,146                             ext_factor, attn_factor, beta_fast, beta_slow);147 148        cb(Qcur, "Qcur-post-rope", il);149        cb(Kcur, "Kcur-post-rope", il);150        cb(Vcur, "Vcur-post-rope", il);151 152        ggml_tensor * attn_out = build_attn(inp->get_attn(),153                                    model.layers[il].wo, NULL, model.layers[il].wo_s,154                                    Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il);155        cb(attn_out, "attn_out", il);156 157        cur = build_norm(inpL, model.layers[il].attn_norm, NULL, LLM_NORM_RMS, il);158        // Mamba2 layer159        cb(cur, "ssm_in", il);160 161        ggml_tensor * ssm_out = build_mamba2_layer(inp->get_recr(), cur, model, ubatch, il);162        cb(ssm_out, "ssm_out", il);163 164        // // Aggregation165        cur   = ggml_add(ctx0, attn_out, ssm_out);166        inpSA = ggml_add(ctx0, cur, inpSA);167        cb(cur, "layer_out", il);168 169        if (il == n_layer - 1 && inp_out_ids) {170            cur   = ggml_get_rows(ctx0, cur, inp_out_ids);171            inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);172        }173        ggml_tensor * ffn_inp = inpSA;174        cb(ffn_inp, "ffn_inp", il);175 176        // feed-forward network177        cur = build_norm(ffn_inp, model.layers[il].ffn_norm, NULL, LLM_NORM_RMS, il);178        cb(cur, "ffn_norm", il);179 180        cur = build_ffn(cur,181                model.layers[il].ffn_up, model.layers[il].ffn_up_b, NULL,182                model.layers[il].ffn_gate, model.layers[il].ffn_gate_b, NULL,183                model.layers[il].ffn_down, model.layers[il].ffn_down_b, NULL,184                NULL, LLM_FFN_SILU, LLM_FFN_PAR, il);185        cb(cur, "ffn_out", il);186 187        cur = ggml_add(ctx0, cur, inpSA);188 189        cur = build_cvec(cur, il);190        cb(cur, "l_out", il);191 192        // input for next layer193        inpL = cur;194    }195    cur = inpL;196 197    cur = build_norm(cur, model.output_norm, NULL, LLM_NORM_RMS, -1);198 199    cb(cur, "result_norm", -1);200    res->t_embd = cur;201 202    // lm_head203    cur = build_lora_mm(model.output, cur, model.output_s);204 205    cb(cur, "result_output", -1);206    res->t_logits = cur;207 208    ggml_build_forward_expand(gf, cur);209}210