echodict/llama.cpp

1#include "models.h"2 3ggml_cgraph * clip_graph_cogvlm::build() {4    GGML_ASSERT(model.class_embedding != nullptr);5    GGML_ASSERT(model.position_embeddings != nullptr);6 7    const int n_pos = n_patches + 1; // +1 for [CLS]8 9    // build input and concatenate class embedding10    ggml_tensor * inp = build_inp();11    inp = ggml_concat(ctx0, inp, model.class_embedding, 1);12 13    inp = ggml_add(ctx0, inp, model.position_embeddings);14    cb(inp, "inp_pos", -1);15 16    ggml_tensor * inpL = inp;17 18    for (int il = 0; il < n_layer; il++) {19        auto & layer = model.layers[il];20        ggml_tensor * cur = inpL;21 22        cur = build_mm(layer.qkv_w, cur);23 24        cur = ggml_add(ctx0, cur, layer.qkv_b);25 26        ggml_tensor * Qcur = ggml_view_3d(ctx0, cur, d_head, n_head, n_pos, d_head*sizeof(float),27            cur->nb[1], 0);28        ggml_tensor * Kcur = ggml_view_3d(ctx0, cur, d_head, n_head, n_pos, d_head*sizeof(float),29            cur->nb[1], n_embd * sizeof(float));30        ggml_tensor * Vcur = ggml_view_3d(ctx0, cur, d_head, n_head, n_pos, d_head*sizeof(float),31            cur->nb[1], 2 * n_embd * sizeof(float));32 33        cb(Qcur, "Qcur", il);34        cb(Kcur, "Kcur", il);35        cb(Vcur, "Vcur", il);36 37        cur = build_attn(layer.o_w, layer.o_b,38            Qcur, Kcur, Vcur, nullptr, kq_scale, il);39        cb(cur, "attn_out", il);40 41        cur = build_norm(cur, layer.ln_1_w, layer.ln_1_b, NORM_TYPE_NORMAL, eps, il);42        cb(cur, "attn_post_norm", il);43 44        cur = ggml_add(ctx0, cur, inpL);45        inpL = cur;46 47        cur = build_ffn(cur,48            layer.ff_up_w, layer.ff_up_b,49            layer.ff_gate_w, layer.ff_gate_b,50            layer.ff_down_w, layer.ff_down_b,51            hparams.ffn_op, il);52 53        cb(cur, "ffn_out", il);54 55        cur = build_norm(cur, layer.ln_2_w, layer.ln_2_b, NORM_TYPE_NORMAL, eps, il);56        cb(cur, "ffn_post_norm", il);57 58        cur = ggml_add(ctx0, cur, inpL);59        cb(cur, "layer_out", il);60        inpL = cur;61 62    }63 64    // remove CLS token (like build_llama4 does)65    ggml_tensor * cur = ggml_view_2d(ctx0, inpL,66        n_embd, n_patches,67        ggml_row_size(inpL->type, n_embd), 0);68 69    // Multiply with mm_model_proj70    cur = build_mm(model.mm_model_proj, cur);71 72    // Apply layernorm, weight, bias73    cur = build_norm(cur, model.mm_post_fc_norm_w, model.mm_post_fc_norm_b, NORM_TYPE_NORMAL, 1e-5, -1);74 75    // Apply GELU76    cur = ggml_gelu_inplace(ctx0, cur);77 78    // Branch 1: multiply with mm_h_to_4h_w79    ggml_tensor * h_to_4h = build_mm(model.mm_h_to_4h_w, cur);80 81    // Branch 2: multiply with mm_gate_w82    ggml_tensor * gate = build_mm(model.mm_gate_w, cur);83 84    // Apply silu85    gate = ggml_swiglu_split(ctx0, gate, h_to_4h);86 87    // Apply mm_4h_to_h_w88    cur = build_mm(model.mm_4h_to_h_w, gate);89 90    // Concatenate with boi and eoi91    cur = ggml_concat(ctx0, model.mm_boi, cur, 1);92    cur = ggml_concat(ctx0, cur, model.mm_eoi, 1);93 94    // build the graph95    ggml_build_forward_expand(gf, cur);96 97    return gf;98}99