echodict/llama.cpp

1#include "models.h"2 3ggml_cgraph * clip_graph_qwen2vl::build() {4    GGML_ASSERT(model.patch_bias == nullptr);5    GGML_ASSERT(model.class_embedding == nullptr);6 7    const int batch_size       = 1;8    const bool use_window_attn = hparams.n_wa_pattern > 0;9    const int n_wa_pattern     = hparams.n_wa_pattern;10    const int n_pos            = n_patches;11    const int num_position_ids = n_pos * 4; // m-rope requires 4 dim per position12 13    norm_type norm_t = proj_type == PROJECTOR_TYPE_QWEN25VL14        ? NORM_TYPE_RMS // qwen 2.5 vl15        : NORM_TYPE_NORMAL; // qwen 2 vl16 17    int mrope_sections[4] = {d_head/4, d_head/4, d_head/4, d_head/4};18 19    ggml_tensor * inp_raw = build_inp_raw();20    ggml_tensor * inp = ggml_conv_2d(ctx0, model.patch_embeddings_0, inp_raw, patch_size, patch_size, 0, 0, 1, 1);21 22    GGML_ASSERT(img.nx % (patch_size * 2) == 0);23    GGML_ASSERT(img.ny % (patch_size * 2) == 0);24 25    // second conv dimension26    {27        auto inp_1 = ggml_conv_2d(ctx0, model.patch_embeddings_1, inp_raw, patch_size, patch_size, 0, 0, 1, 1);28        inp = ggml_add(ctx0, inp, inp_1);29 30        inp = ggml_permute(ctx0, inp, 1, 2, 0, 3);  // [w, h, c, b] -> [c, w, h, b]31        inp = ggml_cont_4d(32            ctx0, inp,33            n_embd * 2, n_patches_x / 2, n_patches_y, batch_size);34        inp = ggml_reshape_4d(35            ctx0, inp,36            n_embd * 2, n_patches_x / 2, 2, batch_size * (n_patches_y / 2));37        inp = ggml_permute(ctx0, inp, 0, 2, 1, 3);38        inp = ggml_cont_3d(39            ctx0, inp,40            n_embd, n_patches_x * n_patches_y, batch_size);41    }42 43    ggml_tensor * inpL           = inp;44    ggml_tensor * window_mask    = nullptr;45    ggml_tensor * window_idx     = nullptr;46    ggml_tensor * inv_window_idx = nullptr;47 48    ggml_tensor * positions = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, num_position_ids);49    ggml_set_name(positions, "positions");50    ggml_set_input(positions);51 52    // pre-layernorm53    if (model.pre_ln_w) {54        inpL = build_norm(inpL, model.pre_ln_w, model.pre_ln_b, norm_t, eps, -1);55    }56 57    if (use_window_attn) {58        // handle window attention inputs59        inv_window_idx = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_pos / 4);60        ggml_set_name(inv_window_idx, "inv_window_idx");61        ggml_set_input(inv_window_idx);62        // mask for window attention63        window_mask = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_pos, n_pos);64        ggml_set_name(window_mask, "window_mask");65        ggml_set_input(window_mask);66 67        // if flash attn is used, we need to pad the mask and cast to f1668        if (flash_attn_type == CLIP_FLASH_ATTN_TYPE_ENABLED) {69            window_mask = ggml_cast(ctx0, window_mask, GGML_TYPE_F16);70        }71 72        // inpL shape: [n_embd, n_patches_x * n_patches_y, batch_size]73        GGML_ASSERT(batch_size == 1);74        inpL = ggml_reshape_2d(ctx0, inpL, n_embd * 4, n_patches_x * n_patches_y * batch_size / 4);75        inpL = ggml_get_rows(ctx0, inpL, inv_window_idx);76        inpL = ggml_reshape_3d(ctx0, inpL, n_embd, n_patches_x * n_patches_y, batch_size);77    }78 79    // loop over layers80    for (int il = 0; il < n_layer; il++) {81        const auto & layer = model.layers[il];82        const bool full_attn = use_window_attn ? (il + 1) % n_wa_pattern == 0 : true;83 84        ggml_tensor * cur = inpL; // inpL = residual, cur = hidden_states85 86        // layernorm187        cur = build_norm(cur, layer.ln_1_w, layer.ln_1_b, norm_t, eps, il);88        cb(cur, "ln1", il);89 90        // self-attention91        {92            ggml_tensor * Qcur = ggml_add(ctx0,93                build_mm(layer.q_w, cur), layer.q_b);94            ggml_tensor * Kcur = ggml_add(ctx0,95                build_mm(layer.k_w, cur), layer.k_b);96            ggml_tensor * Vcur = ggml_add(ctx0,97                build_mm(layer.v_w, cur), layer.v_b);98 99            Qcur = ggml_reshape_3d(ctx0, Qcur, d_head, n_head, n_patches);100            Kcur = ggml_reshape_3d(ctx0, Kcur, d_head, n_head, n_patches);101            Vcur = ggml_reshape_3d(ctx0, Vcur, d_head, n_head, n_patches);102 103            cb(Qcur, "Qcur", il);104            cb(Kcur, "Kcur", il);105            cb(Vcur, "Vcur", il);106 107            // apply M-RoPE108            Qcur = ggml_rope_multi(109                ctx0, Qcur, positions, nullptr,110                d_head/2, mrope_sections, GGML_ROPE_TYPE_VISION, 32768, 10000, 1, 0, 1, 32, 1);111            Kcur = ggml_rope_multi(112                ctx0, Kcur, positions, nullptr,113                d_head/2, mrope_sections, GGML_ROPE_TYPE_VISION, 32768, 10000, 1, 0, 1, 32, 1);114 115            cb(Qcur, "Qcur_rope", il);116            cb(Kcur, "Kcur_rope", il);117 118            ggml_tensor * attn_mask = full_attn ? nullptr : window_mask;119 120            cur = build_attn(layer.o_w, layer.o_b,121                Qcur, Kcur, Vcur, attn_mask, kq_scale, il);122            cb(cur, "attn_out", il);123        }124 125        // re-add the layer input, e.g., residual126        cur = ggml_add(ctx0, cur, inpL);127 128        inpL = cur; // inpL = residual, cur = hidden_states129 130        cb(cur, "ffn_inp", il);131 132        // layernorm2133        cur = build_norm(cur, layer.ln_2_w, layer.ln_2_b, norm_t, eps, il);134        cb(cur, "ffn_inp_normed", il);135 136        // ffn137        cur = build_ffn(cur,138            layer.ff_up_w, layer.ff_up_b,139            layer.ff_gate_w, layer.ff_gate_b,140            layer.ff_down_w, layer.ff_down_b,141            hparams.ffn_op, il);142 143        cb(cur, "ffn_out", il);144 145        // residual 2146        cur = ggml_add(ctx0, inpL, cur);147        cb(cur, "layer_out", il);148 149        inpL = cur;150    }151 152    // post-layernorm153    if (model.post_ln_w) {154        inpL = build_norm(inpL, model.post_ln_w, model.post_ln_b, norm_t, eps, n_layer);155    }156 157    // multimodal projection158    ggml_tensor * embeddings = inpL;159    embeddings = ggml_reshape_3d(ctx0, embeddings, n_embd * 4, n_pos / 4, batch_size);160    embeddings = build_ffn(embeddings,161                        model.mm_0_w, model.mm_0_b,162                        nullptr, nullptr,163                        model.mm_1_w, model.mm_1_b,164                        FFN_GELU,165                        -1);166 167    if (use_window_attn) {168        window_idx = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_pos / 4);169        ggml_set_name(window_idx, "window_idx");170        ggml_set_input(window_idx);171 172        // embeddings shape: [n_embd, n_patches_x * n_patches_y, batch_size]173        GGML_ASSERT(batch_size == 1);174        embeddings = ggml_reshape_2d(ctx0, embeddings, hparams.projection_dim, n_patches_x * n_patches_y / 4);175        embeddings = ggml_get_rows(ctx0, embeddings, window_idx);176        embeddings = ggml_reshape_3d(ctx0, embeddings, hparams.projection_dim, n_patches_x * n_patches_y / 4, batch_size);177    }178 179    // build the graph180    ggml_build_forward_expand(gf, embeddings);181 182    return gf;183}184