echodict/llama.cpp

1#include "models.h"2 3ggml_cgraph * clip_graph_paddleocr::build() {4    const int n_pos            = n_patches;5    const int num_position_ids = n_pos * 4; // m-rope requires 4 dim per position6 7    int mrope_sections[4] = {d_head/4, d_head/4, d_head/4, d_head/4};8 9    ggml_tensor * positions = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, num_position_ids);10    ggml_set_name(positions, "positions");11    ggml_set_input(positions);12 13    auto add_pos = [&](ggml_tensor * cur, const clip_layer &) {14        return ggml_rope_multi(15                    ctx0, cur, positions, nullptr,16                    d_head/2, mrope_sections, GGML_ROPE_TYPE_VISION,17                    32768, 10000, 1, 0, 1, 32, 1);18    };19 20    ggml_tensor * learned_pos_embd = resize_position_embeddings();21    ggml_tensor * inp = build_inp();22    ggml_tensor * cur = build_vit(23                            inp, n_patches,24                            NORM_TYPE_NORMAL,25                            hparams.ffn_op,26                            learned_pos_embd,27                            add_pos);28 29    cb(cur, "vit_out", -1);30 31    {32        // mlp_AR paddleocr projector33        float proj_norm_eps = 1e-5;34        cur = build_norm(cur,35                    model.mm_input_norm_w, model.mm_input_norm_b,36                    NORM_TYPE_NORMAL, proj_norm_eps, -1);37 38        const int scale_factor = model.hparams.n_merge;39        cur = build_patch_merge_permute(cur, scale_factor);40        cur = build_ffn(cur,41                    model.mm_1_w, model.mm_1_b,42                    nullptr, nullptr,43                    model.mm_2_w, model.mm_2_b,44                    hparams.ffn_op, -1);45        cb(cur, "mlp_out", -1);46    }47 48    // build the graph49    ggml_build_forward_expand(gf, cur);50 51    return gf;52}53