Felipe97/llama-cpp-compiled

1#include "models.h"2 3void llama_model_wavtokenizer_dec::load_arch_hparams(llama_model_loader & ml) {4    ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS,    hparams.f_norm_eps);5    ml.get_key(LLM_KV_ATTENTION_GROUPNORM_EPS,    hparams.f_norm_group_eps);6    ml.get_key(LLM_KV_ATTENTION_GROUPNORM_GROUPS, hparams.n_norm_groups);7}8 9void llama_model_wavtokenizer_dec::load_arch_tensors(llama_model_loader &) {10    LLAMA_LOAD_LOCALS;11 12    tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {hparams.n_embd, n_vocab}, 0);13 14    conv1d   = create_tensor(tn(LLM_TENSOR_CONV1D, "weight", 0), {7, hparams.n_embd, hparams.posnet.n_embd}, 0);15    conv1d_b = create_tensor(tn(LLM_TENSOR_CONV1D, "bias",   0), {1, hparams.posnet.n_embd}, 0);16 17    // posnet18    {19        const int64_t n_embd = hparams.posnet.n_embd;20 21        for (uint32_t i = 0; i < hparams.posnet.n_layer; ++i) {22            auto & layer = layers[i].posnet;23 24            // posnet:25            //26            //  - resnet27            //  - resnet28            //  - attn29            //  - resnet30            //  - resnet31            //  - norm32            //33            switch (i) {34                case 0:35                case 1:36                case 3:37                case 4:38                    {39                        layer.norm1   = create_tensor(tn(LLM_TENSOR_POS_NET_NORM1, "weight", i), {1, n_embd}, 0);40                        layer.norm1_b = create_tensor(tn(LLM_TENSOR_POS_NET_NORM1, "bias",   i), {1, n_embd}, 0);41 42                        layer.conv1   = create_tensor(tn(LLM_TENSOR_POS_NET_CONV1, "weight", i), {3, n_embd, n_embd}, 0);43                        layer.conv1_b = create_tensor(tn(LLM_TENSOR_POS_NET_CONV1, "bias",   i), {1, n_embd}, 0);44 45                        layer.norm2   = create_tensor(tn(LLM_TENSOR_POS_NET_NORM2, "weight", i), {1, n_embd}, 0);46                        layer.norm2_b = create_tensor(tn(LLM_TENSOR_POS_NET_NORM2, "bias",   i), {1, n_embd}, 0);47 48                        layer.conv2   = create_tensor(tn(LLM_TENSOR_POS_NET_CONV2, "weight", i), {3, n_embd, n_embd}, 0);49                        layer.conv2_b = create_tensor(tn(LLM_TENSOR_POS_NET_CONV2, "bias",   i), {1, n_embd}, 0);50                    } break;51                case 2:52                    {53                        layer.attn_norm   = create_tensor(tn(LLM_TENSOR_POS_NET_ATTN_NORM, "weight", i), {1, n_embd}, 0);54                        layer.attn_norm_b = create_tensor(tn(LLM_TENSOR_POS_NET_ATTN_NORM, "bias",   i), {1, n_embd}, 0);55 56                        layer.attn_q      = create_tensor(tn(LLM_TENSOR_POS_NET_ATTN_Q,    "weight", i), {1, n_embd, n_embd}, 0);57                        layer.attn_q_b    = create_tensor(tn(LLM_TENSOR_POS_NET_ATTN_Q,    "bias",   i), {1, n_embd}, 0);58 59                        layer.attn_k      = create_tensor(tn(LLM_TENSOR_POS_NET_ATTN_K,    "weight", i), {1, n_embd, n_embd}, 0);60                        layer.attn_k_b    = create_tensor(tn(LLM_TENSOR_POS_NET_ATTN_K,    "bias",   i), {1, n_embd}, 0);61 62                        layer.attn_v      = create_tensor(tn(LLM_TENSOR_POS_NET_ATTN_V,    "weight", i), {1, n_embd, n_embd}, 0);63                        layer.attn_v_b    = create_tensor(tn(LLM_TENSOR_POS_NET_ATTN_V,    "bias",   i), {1, n_embd}, 0);64 65                        layer.attn_o      = create_tensor(tn(LLM_TENSOR_POS_NET_ATTN_OUT,  "weight", i), {1, n_embd, n_embd}, 0);66                        layer.attn_o_b    = create_tensor(tn(LLM_TENSOR_POS_NET_ATTN_OUT,  "bias",   i), {1, n_embd}, 0);67                    } break;68                case 5:69                    {70                        layer.norm   = create_tensor(tn(LLM_TENSOR_POS_NET_ATTN_NORM, "weight", i), {1, n_embd}, 0);71                        layer.norm_b = create_tensor(tn(LLM_TENSOR_POS_NET_ATTN_NORM, "bias",   i), {1, n_embd}, 0);72                    } break;73                default: GGML_ABORT("unknown posnet layer");74            };75        }76    }77 78    GGML_ASSERT(hparams.posnet.n_embd == hparams.convnext.n_embd);79 80    tok_norm   = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD_NORM, "weight", 0), {hparams.posnet.n_embd}, 0);81    tok_norm_b = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD_NORM, "bias",   0), {hparams.posnet.n_embd}, 0);82 83    // convnext84    {85        const int64_t n_embd = hparams.convnext.n_embd;86 87        for (uint32_t i = 0; i < hparams.convnext.n_layer; ++i) {88            auto & layer = layers[i].convnext;89 90            layer.dw     = create_tensor(tn(LLM_TENSOR_CONVNEXT_DW,    "weight", i), {7, 1, n_embd}, 0);91            layer.dw_b   = create_tensor(tn(LLM_TENSOR_CONVNEXT_DW,    "bias",   i), {1, n_embd}, 0);92 93            layer.norm   = create_tensor(tn(LLM_TENSOR_CONVNEXT_NORM,  "weight", i), {n_embd}, 0);94            layer.norm_b = create_tensor(tn(LLM_TENSOR_CONVNEXT_NORM,  "bias",   i), {n_embd}, 0);95 96            layer.pw1    = create_tensor(tn(LLM_TENSOR_CONVNEXT_PW1,   "weight", i), {n_embd, n_ff}, 0);97            layer.pw1_b  = create_tensor(tn(LLM_TENSOR_CONVNEXT_PW1,   "bias",   i), {n_ff}, 0);98 99            layer.pw2    = create_tensor(tn(LLM_TENSOR_CONVNEXT_PW2,   "weight", i), {n_ff, n_embd}, 0);100            layer.pw2_b  = create_tensor(tn(LLM_TENSOR_CONVNEXT_PW2,   "bias",   i), {n_embd}, 0);101 102            layer.gamma  = create_tensor(tn(LLM_TENSOR_CONVNEXT_GAMMA, "weight", i), {n_embd}, 0);103        }104 105        // output106        output_norm   = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0);107        output_norm_b = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "bias"),   {n_embd}, 0);108    }109 110    output   = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), {hparams.convnext.n_embd, hparams.n_embd_out()}, 0);111    output_b = create_tensor(tn(LLM_TENSOR_OUTPUT, "bias"),   {hparams.n_embd_out()}, 0);112}113 114std::unique_ptr<llm_graph_context> llama_model_wavtokenizer_dec::build_arch_graph(const llm_graph_params & params) const {115    return std::make_unique<graph>(*this, params);116}117 118llama_model_wavtokenizer_dec::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {119    ggml_tensor * cur;120    ggml_tensor * inpL;121 122    inpL = build_inp_embd(model.tok_embd);123 124    cur = ggml_cont(ctx0, ggml_transpose(ctx0, inpL));125 126    cur = ggml_conv_1d_ph(ctx0, model.conv1d, cur, 1, 1);127    cur = ggml_add(ctx0, cur, model.conv1d_b);128 129    // posnet130    for (uint32_t il = 0; il < hparams.posnet.n_layer; ++il) {131        const auto & layer = model.layers[il].posnet;132 133        inpL = cur;134 135        switch (il) {136            case 0:137            case 1:138            case 3:139            case 4:140                {141                    cur = build_norm(cur,142                            layer.norm1,143                            layer.norm1_b,144                            LLM_NORM_GROUP, 0);145 146                    cur = ggml_mul(ctx0, ggml_sigmoid(ctx0, cur), cur);147 148                    cur = ggml_conv_1d_ph(ctx0, layer.conv1, cur, 1, 1);149                    cur = ggml_add(ctx0, cur, layer.conv1_b);150 151                    cur = build_norm(cur,152                            layer.norm2,153                            layer.norm2_b,154                            LLM_NORM_GROUP, 0);155 156                    cur = ggml_mul(ctx0, ggml_sigmoid(ctx0, cur), cur);157 158                    cur = ggml_conv_1d_ph(ctx0, layer.conv2, cur, 1, 1);159                    cur = ggml_add(ctx0, cur, layer.conv2_b);160 161                    cur = ggml_add(ctx0, cur, inpL);162                } break;163            case 2:164                {165                    cur = build_norm(cur,166                            layer.attn_norm,167                            layer.attn_norm_b,168                            LLM_NORM_GROUP, 0);169 170                    ggml_tensor * q;171                    ggml_tensor * k;172                    ggml_tensor * v;173 174                    q = ggml_conv_1d_ph(ctx0, layer.attn_q, cur, 1, 1);175                    k = ggml_conv_1d_ph(ctx0, layer.attn_k, cur, 1, 1);176                    v = ggml_conv_1d_ph(ctx0, layer.attn_v, cur, 1, 1);177 178                    q = ggml_add(ctx0, q, layer.attn_q_b);179                    k = ggml_add(ctx0, k, layer.attn_k_b);180                    v = ggml_add(ctx0, v, layer.attn_v_b);181 182                    q = ggml_cont(ctx0, ggml_transpose(ctx0, q));183                    k = ggml_cont(ctx0, ggml_transpose(ctx0, k));184 185                    ggml_tensor * kq = ggml_mul_mat(ctx0, k, q);186 187                    kq = ggml_soft_max_ext(ctx0, kq, nullptr, 1.0f/sqrtf(float(hparams.posnet.n_embd)), 0.0f);188 189                    cur = ggml_mul_mat(ctx0, kq, v);190 191                    cur = ggml_conv_1d_ph(ctx0, layer.attn_o, cur, 1, 1);192                    cur = ggml_add(ctx0, cur, layer.attn_o_b);193 194                    cur = ggml_add(ctx0, cur, inpL);195                } break;196            case 5:197                {198                    cur = build_norm(cur,199                            layer.norm,200                            layer.norm_b,201                            LLM_NORM_GROUP, 0);202                } break;203            default: GGML_ABORT("unknown posnet layer");204        };205    }206    cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur));207 208    cur = build_norm(cur,209            model.tok_norm,210            model.tok_norm_b,211            LLM_NORM, 0);212 213    cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur));214 215    inpL = cur;216 217    // convnext218    for (uint32_t il = 0; il < hparams.convnext.n_layer; ++il) {219        const auto & layer = model.layers[il].convnext;220 221        cur = inpL;222 223        cur = ggml_conv_1d_dw_ph(ctx0, layer.dw, cur, 1, 1);224        cur = ggml_add(ctx0, cur, layer.dw_b);225 226        cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur));227 228        cur = build_norm(cur,229                layer.norm,230                layer.norm_b,231                LLM_NORM, -1);232 233        cur = build_ffn(cur,234                layer.pw1, layer.pw1_b, NULL,235                NULL,      NULL,        NULL,236                layer.pw2, layer.pw2_b, NULL,237                NULL,238                LLM_FFN_GELU, LLM_FFN_SEQ, il);239 240        cur = ggml_mul(ctx0, cur, layer.gamma);241 242        cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur));243 244        inpL = ggml_add(ctx0, cur, inpL);245    }246    cur = inpL;247 248    cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur));249 250    cur = build_norm(cur,251            model.output_norm,252            model.output_norm_b,253            LLM_NORM, -1);254 255    // lm_head256    cur = build_lora_mm(model.output, cur, model.output_s);257 258    cur = ggml_add(ctx0, cur, model.output_b);259 260    cb(cur, "result_embd", -1);261    res->t_embd = cur;262 263    ggml_build_forward_expand(gf, cur);264}265