echodict/llama.cpp

1#pragma once2 3#include "ggml.h"4#include "ggml-cpp.h"5#include "clip.h"6#include "clip-impl.h"7#include "clip-model.h"8 9#include <vector>10#include <functional>11 12#define DEFAULT_INTERPOLATION_MODE (GGML_SCALE_MODE_BILINEAR | GGML_SCALE_FLAG_ANTIALIAS)13 14struct clip_graph {15    const clip_model & model;16    const clip_hparams & hparams;17    projector_type proj_type;18 19    // we only support single image per batch20    const clip_image_f32 & img;21 22    const int patch_size;23    const int n_patches_x;24    const int n_patches_y;25    const int n_patches;26    const int n_embd;27    const int n_head;28    const int d_head;29    const int n_layer;30    const int n_mmproj_embd;31    const float eps;32    float kq_scale; // TODO: maybe move this to hparams33    const clip_flash_attn_type flash_attn_type;34 35    ggml_context_ptr ctx0_ptr;36    ggml_context * ctx0;37    ggml_cgraph * gf;38 39    clip_graph(clip_ctx * ctx, const clip_image_f32 & img);40 41    virtual ~clip_graph() = default;42    virtual ggml_cgraph * build() = 0;43 44    // wrapper around ggml_mul_mat, allow hooking (e.g. LoRA, clamping) depending on the model45    // tensor w should be the weight matrix, and tensor x should be the input46    virtual ggml_tensor * build_mm(ggml_tensor * w, ggml_tensor * x) const;47    // TODO: build_mm(w, b, x) to support bias48 49    //50    // utility functions51    //52    void cb(ggml_tensor * cur0, const char * name, int il) const;53 54    // siglip2 naflex55    ggml_tensor * resize_position_embeddings(uint32_t interpolation_mode = DEFAULT_INTERPOLATION_MODE);56 57    // build vision transformer (ViT) cgraph58    // this function should cover most of the models59    // if your model has specific features, you should probably duplicate this function60    ggml_tensor * build_vit(61                ggml_tensor * inp,62                int64_t n_pos,63                norm_type norm_t,64                ffn_op_type ffn_t,65                ggml_tensor * learned_pos_embd,66                std::function<ggml_tensor *(ggml_tensor *, const clip_layer &)> add_pos);67 68    // build the input after conv2d (inp_raw --> patches)69    // returns tensor with shape [n_embd, n_patches]70    ggml_tensor * build_inp();71 72    ggml_tensor * build_inp_raw(int channels = 3);73 74    ggml_tensor * build_norm(75            ggml_tensor * cur,76            ggml_tensor * mw,77            ggml_tensor * mb,78            norm_type type,79            float norm_eps,80            int il) const;81 82    ggml_tensor * build_ffn(83            ggml_tensor * cur,84            ggml_tensor * up,85            ggml_tensor * up_b,86            ggml_tensor * gate,87            ggml_tensor * gate_b,88            ggml_tensor * down,89            ggml_tensor * down_b,90            ffn_op_type type_op,91            int il) const;92 93    ggml_tensor * build_attn(94            ggml_tensor * wo,95            ggml_tensor * wo_b,96            ggml_tensor * q_cur,97            ggml_tensor * k_cur,98            ggml_tensor * v_cur,99            ggml_tensor * kq_mask,100            float kq_scale,101            int il) const;102 103    // implementation of the 2D RoPE without adding a new op in ggml104    // this is not efficient (use double the memory), but works on all backends105    // TODO: there was a more efficient which relies on ggml_view and ggml_rope_ext_inplace, but the rope inplace does not work well with non-contiguous tensors ; we should fix that and revert back to the original implementation in https://github.com/ggml-org/llama.cpp/pull/13065106    ggml_tensor * build_rope_2d(107        ggml_context * ctx0,108        ggml_tensor * cur,109        ggml_tensor * pos_a, // first half110        ggml_tensor * pos_b, // second half111        const float freq_base,112        const bool interleave_freq113    );114 115    // aka pixel_shuffle / pixel_unshuffle / patch_merger (Kimi-VL)116    // support dynamic resolution117    ggml_tensor * build_patch_merge_permute(ggml_tensor * cur, int scale_factor);118 119    // Generic function to stack frames for audio processing120    // Abstracts out the StackAudioFrames logic used by ultravox121    ggml_tensor * build_stack(ggml_tensor * cur, int32_t stack_factor, int32_t n_embed);122};123