echodict/llama.cpp

1#pragma once2 3#include "ggml.h"4#include "gguf.h"5#include "clip.h"6 7#include <climits>8#include <cstdarg>9#include <cinttypes>10#include <string>11#include <map>12#include <sstream>13#include <vector>14#include <memory>15 16// Internal header for clip.cpp17 18#define MTMD_INTERNAL_HEADER19 20#define KEY_FTYPE               "general.file_type"21#define KEY_NAME                "general.name"22#define KEY_DESCRIPTION         "general.description"23#define KEY_PROJ_TYPE           "clip.projector_type"24#define KEY_HAS_AUDIO_ENC       "clip.has_audio_encoder"25#define KEY_HAS_VISION_ENC      "clip.has_vision_encoder"26#define KEY_USE_GELU            "clip.use_gelu"27#define KEY_USE_SILU            "clip.use_silu"28 29#define KEY_N_EMBD              "clip.%s.embedding_length"30#define KEY_N_FF                "clip.%s.feed_forward_length"31#define KEY_N_BLOCK             "clip.%s.block_count"32#define KEY_PROJ_DIM            "clip.%s.projection_dim"33#define KEY_N_HEAD              "clip.%s.attention.head_count"34#define KEY_LAYER_NORM_EPS      "clip.%s.attention.layer_norm_epsilon"35 36// vision-specific37#define KEY_VISION_PROJ_TYPE    "clip.vision.projector_type" // for models with mixed modalities38#define KEY_IMAGE_SIZE          "clip.vision.image_size"39#define KEY_IMAGE_MIN_PIXELS    "clip.vision.image_min_pixels"40#define KEY_IMAGE_MAX_PIXELS    "clip.vision.image_max_pixels"41#define KEY_PREPROC_MIN_TILES   "clip.vision.preproc_min_tiles"42#define KEY_PREPROC_MAX_TILES   "clip.vision.preproc_max_tiles"43#define KEY_PREPROC_IMAGE_SIZE  "clip.vision.preproc_image_size"44#define KEY_PATCH_SIZE          "clip.vision.patch_size"45#define KEY_IMAGE_MEAN          "clip.vision.image_mean"46#define KEY_IMAGE_STD           "clip.vision.image_std"47#define KEY_FEATURE_LAYER       "clip.vision.feature_layer"48#define KEY_PROJ_SCALE_FACTOR   "clip.vision.projector.scale_factor"49#define KEY_SPATIAL_MERGE_SIZE  "clip.vision.spatial_merge_size"50#define KEY_IS_DEEPSTACK_LAYERS "clip.vision.is_deepstack_layers"51 52#define KEY_MM_PATCH_MERGE_TYPE    "clip.vision.mm_patch_merge_type"53#define KEY_IMAGE_GRID_PINPOINTS   "clip.vision.image_grid_pinpoints"54#define KEY_WIN_ATTN_PATTERN       "clip.vision.n_wa_pattern"55#define KEY_WIN_ATTN_LAYER_INDEXES "clip.vision.wa_layer_indexes"56#define KEY_ATTN_WINDOW_SIZE       "clip.vision.window_size"57#define KEY_MINICPMV_VERSION       "clip.minicpmv_version"58#define KEY_MINICPMV_QUERY_NUM     "clip.minicpmv_query_num"59#define KEY_SAM_N_HEAD             "clip.vision.sam.head_count"60#define KEY_SAM_N_BLOCK            "clip.vision.sam.block_count"61#define KEY_SAM_N_EMBD             "clip.vision.sam.embedding_length"62// audio-specific63#define KEY_AUDIO_PROJ_TYPE     "clip.audio.projector_type" // for models with mixed modalities64#define KEY_A_NUM_MEL_BINS      "clip.audio.num_mel_bins"65#define KEY_A_PROJ_STACK_FACTOR "clip.audio.projector.stack_factor"66 67 68//69// tensor name constants70//71 72#define TN_POS_EMBD        "%s.position_embd.weight"73#define TN_CLASS_EMBD      "v.class_embd"74#define TN_PATCH_EMBD      "v.patch_embd.weight"  // not rename tensor with ".0" postfix for backward compat75#define TN_PATCH_EMBD_1    "v.patch_embd.weight.1"76#define TN_PATCH_BIAS      "v.patch_embd.bias"77#define TN_NORM_EMBD       "v.norm_embd.%s"78#define TN_ATTN_QKV        "%s.blk.%d.attn_qkv.%s"79#define TN_ATTN_K          "%s.blk.%d.attn_k.%s"80#define TN_ATTN_Q          "%s.blk.%d.attn_q.%s"81#define TN_ATTN_V          "%s.blk.%d.attn_v.%s"82#define TN_ATTN_OUTPUT     "%s.blk.%d.attn_out.%s"83#define TN_ATTN_K_NORM     "%s.blk.%d.attn_k_norm.%s"84#define TN_ATTN_Q_NORM     "%s.blk.%d.attn_q_norm.%s"85#define TN_FFN_DOWN        "%s.blk.%d.ffn_down.%s"86#define TN_FFN_GATE        "%s.blk.%d.ffn_gate.%s"87#define TN_FFN_UP          "%s.blk.%d.ffn_up.%s"88#define TN_FFN_GATE        "%s.blk.%d.ffn_gate.%s"89#define TN_LN_1            "%s.blk.%d.ln1.%s" // layer norm90#define TN_LN_2            "%s.blk.%d.ln2.%s" // layer norm91#define TN_LS_1            "%s.blk.%d.ls1.%s"         // layer scale92#define TN_LS_2            "%s.blk.%d.ls2.%s"         // layer scale93#define TN_LS_OUT          "%s.blk.%d.out_scale.%s"      // layer out scale (gemma4)94#define TN_ATTN_POST_NORM  "%s.blk.%d.attn_post_norm.%s" // post-attn norm (gemma4)95#define TN_FFN_POST_NORM   "%s.blk.%d.ffn_post_norm.%s"  // post-FFN norm (gemma4)96#define TN_LN_PRE          "%s.pre_ln.%s"97#define TN_LN_POST         "%s.post_ln.%s"98#define TN_LLAVA_PROJ      "mm.%d.%s"99#define TN_MM_UP           "mm.up.%s"100#define TN_MM_GATE         "mm.gate.%s"101#define TN_MM_DOWN         "mm.down.%s"102#define TN_MM_POST_NORM    "mm.post_norm.%s"103#define TN_MVLM_PROJ_MLP   "mm.model.mlp.%d.%s"104#define TN_MVLM_PROJ_BLOCK "mm.model.mb_block.%d.block.%d.%s"105#define TN_MVLM_PROJ_PEG   "mm.model.peg.%d.%s"106#define TN_IMAGE_NEWLINE   "v.image_newline"107#define TN_IMAGE_SEPERATOR "v.view_seperator"108#define TN_MM_INP_NORM     "mm.input_norm.weight"109#define TN_MM_INP_NORM_B   "mm.input_norm.bias"110#define TN_MM_INP_PROJ     "mm.input_projection.weight" // gemma3111#define TN_MM_SOFT_EMB_N   "mm.soft_emb_norm.weight"    // gemma3112#define TN_MM_PROJECTOR    "mm.model.fc.%s"             // idefics3, deepseekocr113#define TN_MM_PATCH_MERGER "mm.patch_merger.%s"         // mistral small 3.1, glm4v114#define TN_TOK_IMG_BREAK   "v.token_embd.img_break"     // pixtral115#define TN_TOK_GLM_BOI     "adapter.boi"                // glm-edge (these embeddings are not in text model)116#define TN_TOK_GLM_EOI     "adapter.eoi"                // glm-edge (these embeddings are not in text model)117#define TN_DEEPSTACK_NORM  "v.deepstack.%d.norm.%s"     // qwen3vl deepstack118#define TN_DEEPSTACK_FC1   "v.deepstack.%d.fc1.%s"      // qwen3vl deepstack119#define TN_DEEPSTACK_FC2   "v.deepstack.%d.fc2.%s"      // qwen3vl deepstack120 121// mimicpmv122#define TN_MINICPMV_POS_EMBD_K "resampler.pos_embed_k"123#define TN_MINICPMV_QUERY      "resampler.query"124#define TN_MINICPMV_PROJ       "resampler.proj.weight"125#define TN_MINICPMV_KV_PROJ    "resampler.kv.weight"126#define TN_MINICPMV_ATTN       "resampler.attn.%s.%s"127#define TN_MINICPMV_LN         "resampler.ln_%s.%s"128 129#define TN_GLM_ADAPER_CONV      "adapter.conv.%s"130#define TN_GLM_ADAPTER_LINEAR   "adapter.linear.linear.%s"131#define TN_GLM_ADAPTER_NORM_1   "adapter.linear.norm1.%s"132#define TN_GLM_ADAPTER_D_H_2_4H "adapter.linear.dense_h_to_4h.%s"133#define TN_GLM_ADAPTER_GATE     "adapter.linear.gate.%s"134#define TN_GLM_ADAPTER_D_4H_2_H "adapter.linear.dense_4h_to_h.%s"135 136// ultravox137#define TN_CONV1D       "a.conv1d.%d.%s"138#define TN_CONV2D       "a.conv2d.%d.%s"139#define TN_CONV_OUT     "a.conv_out.%s"140#define TN_MM_AUDIO_MLP "mm.a.mlp.%d.%s"141#define TN_MM_AUDIO_FC  "mm.a.fc.%s" // fully connected layer142#define TN_MM_NORM_PRE  "mm.a.norm_pre.%s"143#define TN_MM_NORM_MID  "mm.a.norm_mid.%s"144 145// cogvlm146#define TN_MM_POST_FC_NORM "mm.post_fc_norm.%s"147#define TN_MM_H_TO_4H      "mm.up.%s"148#define TN_MM_GATE         "mm.gate.%s"149#define TN_MM_4H_TO_H      "mm.down.%s"150#define TN_TOK_BOI         "v.boi"151#define TN_TOK_EOI         "v.eoi"152 153// hunyuanocr154#define TN_MM_PRE_NORM     "mm.pre_norm.%s"155#define TN_TOK_IMG_BEGIN   "mm.image_begin"156#define TN_TOK_IMG_END     "mm.image_end"157 158// deepseek-ocr159#define TN_SAM_POS_EMBD   "v.sam.pos_embd.%s"160#define TN_SAM_PATCH_EMBD "v.sam.patch_embd.%s"161#define TN_SAM_PRE_NORM   "v.sam.blk.%d.pre_ln.%s"162#define TN_SAM_POST_NORM  "v.sam.blk.%d.post_ln.%s"163#define TN_SAM_ATTN_POS_H "v.sam.blk.%d.attn.pos_h.%s"164#define TN_SAM_ATTN_POS_W "v.sam.blk.%d.attn.pos_w.%s"165#define TN_SAM_ATTN_QKV   "v.sam.blk.%d.attn.qkv.%s"166#define TN_SAM_ATTN_OUT   "v.sam.blk.%d.attn.out.%s"167#define TN_SAM_FFN_UP     "v.sam.blk.%d.mlp.lin1.%s"168#define TN_SAM_FFN_DOWN   "v.sam.blk.%d.mlp.lin2.%s"169#define TN_SAM_NECK       "v.sam.neck.%d.%s"170#define TN_SAM_NET        "v.sam.net_%d.%s"171// (conformer) lfm2172#define TN_PRE_ENCODE_OUT  "a.pre_encode.out.%s"173#define TN_FFN_NORM        "%s.blk.%d.ffn_norm.%s"174#define TN_FFN_NORM_1      "%s.blk.%d.ffn_norm_1.%s"175#define TN_FFN_UP_1        "%s.blk.%d.ffn_up_1.%s"176#define TN_FFN_DOWN_1      "%s.blk.%d.ffn_down_1.%s"177#define TN_POS_BIAS_U      "%s.blk.%d.pos_bias_u"178#define TN_POS_BIAS_V      "%s.blk.%d.pos_bias_v"179#define TN_NORM_CONV       "%s.blk.%d.norm_conv.%s"180#define TN_LINEAR_POS      "%s.blk.%d.linear_pos.%s"181#define TN_CONV_DW         "%s.blk.%d.conv_dw.%s"182#define TN_CONV_NORM       "%s.blk.%d.conv_norm.%s"183#define TN_CONV_PW1        "%s.blk.%d.conv_pw1.%s"184#define TN_CONV_PW2        "%s.blk.%d.conv_pw2.%s"185 186// gemma4 audio conformer187#define TN_A_MM_INP_PROJ     "mm.a.input_projection.%s"188#define TN_A_MM_SOFT_EMB_N   "mm.a.soft_emb_norm.%s"189#define TN_A_INP_PROJ        "a.input_projection.%s"190#define TN_A_CONV1D          "a.conv1d.%d.%s"191#define TN_A_CONV1D_NORM     "a.conv1d.%d.norm.%s"192#define TN_A_OUT_PROJ        "a.pre_encode.out.%s"193#define TN_A_ATTN_PRE_NORM   "%s.blk.%d.attn_pre_norm.%s"194#define TN_A_ATTN_POST_NORM  "%s.blk.%d.attn_post_norm.%s"195#define TN_A_ATTN_K_REL      "%s.blk.%d.attn_k_rel.%s"196#define TN_A_PER_DIM_SCALE   "%s.blk.%d.per_dim_scale.%s"197#define TN_A_PER_DIM_K_SCALE "%s.blk.%d.per_dim_k_scale.%s"198#define TN_A_FFN_POST_NORM   "%s.blk.%d.ffn_post_norm.%s"199#define TN_A_FFN_POST_NORM_1 "%s.blk.%d.ffn_post_norm_1.%s"200 201// mobilenetv5 (gemma3n) definitions202#define TN_MNV5_STEM_CONV        "v.conv_stem.conv.weight"203#define TN_MNV5_STEM_BIAS        "v.conv_stem.conv.bias"204#define TN_MNV5_STEM_BN          "v.conv_stem.bn.weight"205 206// Stage 0 Block (Edge Residual)207#define TN_MNV5_BLK_S0_EXP_W     "v.blk.%d.%d.conv_exp.weight"208#define TN_MNV5_BLK_S0_BN1_W     "v.blk.%d.%d.bn1.weight"209#define TN_MNV5_BLK_S0_PWL_W     "v.blk.%d.%d.conv_pwl.weight"210#define TN_MNV5_BLK_S0_BN2_W     "v.blk.%d.%d.bn2.weight"211 212// Stage 1+ Block (Universal Inverted Residual)213#define TN_MNV5_BLK_DW_START_W   "v.blk.%d.%d.dw_start.conv.weight"214#define TN_MNV5_BLK_DW_START_BN  "v.blk.%d.%d.dw_start.bn.weight"215#define TN_MNV5_BLK_DW_MID_W     "v.blk.%d.%d.dw_mid.conv.weight"216#define TN_MNV5_BLK_DW_MID_BN    "v.blk.%d.%d.dw_mid.bn.weight"217#define TN_MNV5_BLK_PW_EXP_W     "v.blk.%d.%d.pw_exp.conv.weight"218#define TN_MNV5_BLK_PW_EXP_BN    "v.blk.%d.%d.pw_exp.bn.weight"219#define TN_MNV5_BLK_PW_PROJ_W    "v.blk.%d.%d.pw_proj.conv.weight"220#define TN_MNV5_BLK_PW_PROJ_BN   "v.blk.%d.%d.pw_proj.bn.weight"221#define TN_MNV5_BLK_LAYER_SCALE  "v.blk.%d.%d.layer_scale.gamma"222 223// Attention Components224#define TN_MNV5_ATTN_Q_W         "v.blk.%d.%d.attn.query.proj.weight"225#define TN_MNV5_ATTN_K_W         "v.blk.%d.%d.attn.key.proj.weight"226#define TN_MNV5_ATTN_V_W         "v.blk.%d.%d.attn.value.proj.weight"227#define TN_MNV5_ATTN_O_W         "v.blk.%d.%d.attn.output.proj.weight"228#define TN_MNV5_ATTN_K_DW        "v.blk.%d.%d.attn.key.down_conv.weight"229#define TN_MNV5_ATTN_K_NORM      "v.blk.%d.%d.attn.key.norm.weight"230#define TN_MNV5_ATTN_V_DW        "v.blk.%d.%d.attn.value.down_conv.weight"231#define TN_MNV5_ATTN_V_NORM      "v.blk.%d.%d.attn.value.norm.weight"232#define TN_MNV5_ATTN_NORM        "v.blk.%d.%d.norm.weight" // Block norm used in attn blocks233 234// MSFA235#define TN_MNV5_MSFA_FFN_EXP_W   "v.msfa.ffn.pw_exp.conv.weight"236#define TN_MNV5_MSFA_FFN_EXP_BN  "v.msfa.ffn.pw_exp.bn.weight"237#define TN_MNV5_MSFA_FFN_PROJ_W  "v.msfa.ffn.pw_proj.conv.weight"238#define TN_MNV5_MSFA_FFN_PROJ_BN "v.msfa.ffn.pw_proj.bn.weight"239#define TN_MNV5_MSFA_NORM        "v.msfa.norm.weight"240 241// gemma4242#define TN_STD_BIAS              "v.std_bias"243#define TN_STD_SCALE             "v.std_scale"244 245 246// align x to upper multiple of n247#define CLIP_ALIGN(x, n) ((((x) + (n) - 1) / (n)) * (n))248 249// forward declaration250// TODO: improve this later251struct clip_ctx;252 253enum projector_type {254    PROJECTOR_TYPE_MLP,255    PROJECTOR_TYPE_MLP_NORM,256    PROJECTOR_TYPE_LDP,257    PROJECTOR_TYPE_LDPV2,258    PROJECTOR_TYPE_MINICPMV,259    PROJECTOR_TYPE_GLM_EDGE,260    PROJECTOR_TYPE_QWEN2VL,261    PROJECTOR_TYPE_QWEN3VL,262    PROJECTOR_TYPE_STEP3VL,263    PROJECTOR_TYPE_GEMMA3,264    PROJECTOR_TYPE_GEMMA3NV,265    PROJECTOR_TYPE_GEMMA3NA,266    PROJECTOR_TYPE_GEMMA4V,267    PROJECTOR_TYPE_GEMMA4A,268    PROJECTOR_TYPE_PHI4,269    PROJECTOR_TYPE_IDEFICS3,270    PROJECTOR_TYPE_PIXTRAL,271    PROJECTOR_TYPE_QWEN25VL,272    PROJECTOR_TYPE_ULTRAVOX,273    PROJECTOR_TYPE_INTERNVL,274    PROJECTOR_TYPE_LLAMA4,275    PROJECTOR_TYPE_QWEN2A,276    PROJECTOR_TYPE_QWEN3A,277    PROJECTOR_TYPE_GLMA,278    PROJECTOR_TYPE_QWEN25O, // will be replaced by QWEN2A or QWEN25VL depending on clip_ctx279    PROJECTOR_TYPE_VOXTRAL,280    PROJECTOR_TYPE_MERALION,281    PROJECTOR_TYPE_MUSIC_FLAMINGO,282    PROJECTOR_TYPE_LFM2,283    PROJECTOR_TYPE_KIMIVL,284    PROJECTOR_TYPE_PADDLEOCR,285    PROJECTOR_TYPE_LIGHTONOCR,286    PROJECTOR_TYPE_COGVLM,287    PROJECTOR_TYPE_JANUS_PRO,288    PROJECTOR_TYPE_DOTS_OCR,289    PROJECTOR_TYPE_DEEPSEEKOCR,290    PROJECTOR_TYPE_LFM2A,291    PROJECTOR_TYPE_GLM4V,292    PROJECTOR_TYPE_YOUTUVL,293    PROJECTOR_TYPE_KIMIK25,294    PROJECTOR_TYPE_NEMOTRON_V2_VL,295    PROJECTOR_TYPE_HUNYUANOCR,296    PROJECTOR_TYPE_UNKNOWN,297};298 299static std::map<projector_type, std::string> PROJECTOR_TYPE_NAMES = {300    { PROJECTOR_TYPE_MLP,       "mlp" },301    { PROJECTOR_TYPE_LDP,       "ldp" },302    { PROJECTOR_TYPE_LDPV2,     "ldpv2"},303    { PROJECTOR_TYPE_MINICPMV,  "resampler"},304    { PROJECTOR_TYPE_GLM_EDGE,  "adapter"},305    { PROJECTOR_TYPE_QWEN2VL,   "qwen2vl_merger"},306    { PROJECTOR_TYPE_QWEN25VL,  "qwen2.5vl_merger"},307    { PROJECTOR_TYPE_QWEN3VL,   "qwen3vl_merger"},308    { PROJECTOR_TYPE_STEP3VL,   "step3vl"},309    { PROJECTOR_TYPE_GEMMA3,    "gemma3"},310    { PROJECTOR_TYPE_GEMMA3NV,  "gemma3nv"},311    { PROJECTOR_TYPE_GEMMA3NA,  "gemma3na"},312    { PROJECTOR_TYPE_GEMMA4V,   "gemma4v"},313    { PROJECTOR_TYPE_GEMMA4A,   "gemma4a"},314    { PROJECTOR_TYPE_PHI4,      "phi4"},315    { PROJECTOR_TYPE_IDEFICS3,  "idefics3"},316    { PROJECTOR_TYPE_PIXTRAL,   "pixtral"},317    { PROJECTOR_TYPE_ULTRAVOX,  "ultravox"},318    { PROJECTOR_TYPE_INTERNVL,  "internvl"},319    { PROJECTOR_TYPE_LLAMA4,    "llama4"},320    { PROJECTOR_TYPE_QWEN2A,    "qwen2a"},321    { PROJECTOR_TYPE_QWEN3A,    "qwen3a"},322    { PROJECTOR_TYPE_GLMA,      "glma"},323    { PROJECTOR_TYPE_QWEN25O,   "qwen2.5o"},324    { PROJECTOR_TYPE_VOXTRAL,   "voxtral"},325    { PROJECTOR_TYPE_MERALION,  "meralion"},326    { PROJECTOR_TYPE_MUSIC_FLAMINGO, "musicflamingo"},327    { PROJECTOR_TYPE_LFM2,      "lfm2"},328    { PROJECTOR_TYPE_KIMIVL,    "kimivl"},329    { PROJECTOR_TYPE_PADDLEOCR, "paddleocr"},330    { PROJECTOR_TYPE_LIGHTONOCR,"lightonocr"},331    { PROJECTOR_TYPE_COGVLM,    "cogvlm"},332    { PROJECTOR_TYPE_JANUS_PRO, "janus_pro"},333    { PROJECTOR_TYPE_DOTS_OCR,  "dots_ocr"},334    { PROJECTOR_TYPE_DEEPSEEKOCR,"deepseekocr"},335    { PROJECTOR_TYPE_LFM2A,     "lfm2a"},336    { PROJECTOR_TYPE_GLM4V,     "glm4v"},337    { PROJECTOR_TYPE_YOUTUVL,   "youtuvl"},338    { PROJECTOR_TYPE_KIMIK25,   "kimik25"},339    { PROJECTOR_TYPE_NEMOTRON_V2_VL, "nemotron_v2_vl"},340    { PROJECTOR_TYPE_HUNYUANOCR, "hunyuanocr"},341};342 343static projector_type clip_projector_type_from_string(const std::string & str) {344    for (const auto & pair : PROJECTOR_TYPE_NAMES) {345        if (pair.second == str) {346            return pair.first;347        }348    }349    return PROJECTOR_TYPE_UNKNOWN;350}351 352// RGB uint8 image353struct clip_image_u8 {354    int nx;355    int ny;356 357    std::vector<uint8_t> buf;358};359 360// For images, buf.size() == nx*ny*3361//     Memory layout: RGBRGBRGB...362// For audio, only one channel is used, buf.size() == nx*ny363//     nx will be n_frames and ny will be n_mel364struct clip_image_f32 {365    int nx;366    int ny;367 368    std::vector<float> buf;369};370 371//372// logging373//374 375static void clip_log_callback_default(enum ggml_log_level level, const char * text, void * user_data) {376    (void) level;377    (void) user_data;378    fputs(text, stderr);379    fflush(stderr);380}381 382struct clip_logger_state {383    ggml_log_callback log_callback;384    void * log_callback_user_data;385};386 387extern struct clip_logger_state g_logger_state;388 389static void clip_log_internal_v(enum ggml_log_level level, const char * format, va_list args) {390    if (format == NULL) {391        return;392    }393    va_list args_copy;394    va_copy(args_copy, args);395    char buffer[128];396    int len = vsnprintf(buffer, 128, format, args);397    if (len < 128) {398        g_logger_state.log_callback(level, buffer, g_logger_state.log_callback_user_data);399    } else {400        char * buffer2 = (char *) calloc(len + 1, sizeof(char));401        vsnprintf(buffer2, len + 1, format, args_copy);402        buffer2[len] = 0;403        g_logger_state.log_callback(level, buffer2, g_logger_state.log_callback_user_data);404        free(buffer2);405    }406    va_end(args_copy);407}408 409static void clip_log_internal(enum ggml_log_level level, const char * format, ...) {410    va_list args;411    va_start(args, format);412    clip_log_internal_v(level, format, args);413    va_end(args);414}415 416#define LOG_INF(...) clip_log_internal(GGML_LOG_LEVEL_INFO,  __VA_ARGS__)417#define LOG_WRN(...) clip_log_internal(GGML_LOG_LEVEL_WARN,  __VA_ARGS__)418#define LOG_ERR(...) clip_log_internal(GGML_LOG_LEVEL_ERROR, __VA_ARGS__)419#define LOG_DBG(...) clip_log_internal(GGML_LOG_LEVEL_DEBUG, __VA_ARGS__)420#define LOG_CNT(...) clip_log_internal(GGML_LOG_LEVEL_CONT,  __VA_ARGS__)421 422//423// cpp wrappers424//425 426// wrapper for clip_image_size427struct clip_image_size_deleter {428    void operator()(clip_image_size * val) { clip_image_size_free(val); }429};430typedef std::unique_ptr<clip_image_size, clip_image_size_deleter> clip_image_size_ptr;431 432// wrapper for clip_image_u8433struct clip_image_u8_deleter {434    void operator()(clip_image_u8 * val) { clip_image_u8_free(val); }435};436typedef std::unique_ptr<clip_image_u8, clip_image_u8_deleter> clip_image_u8_ptr;437 438// wrapper for clip_image_f32439struct clip_image_f32_deleter {440    void operator()(clip_image_f32 * val) { clip_image_f32_free(val); }441};442typedef std::unique_ptr<clip_image_f32, clip_image_f32_deleter> clip_image_f32_ptr;443 444struct clip_image_u8_batch {445    std::vector<clip_image_u8_ptr> entries;446};447 448struct clip_image_f32_batch {449    std::vector<clip_image_f32_ptr> entries;450    bool is_audio = false;451 452    // for llava-uhd style models, we need to know the grid size453    // note: entries.size() == grid_x * grid_y + 1 (one overview image)454    int grid_x = 0;455    int grid_y = 0;456 457    clip_image_f32_batch clone() const {458        clip_image_f32_batch new_batch{459            /* entries  */ {},460            /* is_audio */ is_audio,461            /* grid_x   */ grid_x,462            /* grid_y   */ grid_y,463        };464        new_batch.entries.reserve(entries.size());465        for (const auto & entry : entries) {466            new_batch.entries.emplace_back(new clip_image_f32(*entry));467        }468        return new_batch;469    }470};471 472//473// common utils474//475 476static std::string string_format(const char * fmt, ...) {477    va_list ap;478    va_list ap2;479    va_start(ap, fmt);480    va_copy(ap2, ap);481    int size = vsnprintf(NULL, 0, fmt, ap);482    GGML_ASSERT(size >= 0 && size < INT_MAX); // NOLINT483    std::vector<char> buf(size + 1);484    int size2 = vsnprintf(buf.data(), size + 1, fmt, ap2);485    GGML_ASSERT(size2 == size);486    va_end(ap2);487    va_end(ap);488    return std::string(buf.data(), buf.size());489}490 491static void string_replace_all(std::string & s, const std::string & search, const std::string & replace) {492    if (search.empty()) {493        return;494    }495    std::string builder;496    builder.reserve(s.length());497    size_t pos = 0;498    size_t last_pos = 0;499    while ((pos = s.find(search, last_pos)) != std::string::npos) {500        builder.append(s, last_pos, pos - last_pos);501        builder.append(replace);502        last_pos = pos + search.length();503    }504    builder.append(s, last_pos, std::string::npos);505    s = std::move(builder);506}507 508// split string by a `std::string delim` instead of `char delim`509static std::vector<std::string> string_split_str(std::string s, const std::string & delimiter) {510    std::vector<std::string> tokens;511    size_t pos = 0;512    std::string token;513    while ((pos = s.find(delimiter)) != std::string::npos) {514        token = s.substr(0, pos);515        tokens.push_back(token);516        s.erase(0, pos + delimiter.length());517    }518    tokens.push_back(s);519    return tokens;520}521 522// remove when moving to c++20523inline bool string_starts_with(std::string_view str, std::string_view prefix) {524    return str.size() >= prefix.size() &&525           str.compare(0, prefix.size(), prefix) == 0;526}527 528// remove when moving to c++20529inline bool string_ends_with(std::string_view str, std::string_view suffix) {530    return str.size() >= suffix.size() &&531           str.compare(str.size() - suffix.size(), suffix.size(), suffix) == 0;532}533 534//535// gguf utils536//537 538static std::string gguf_data_to_str(enum gguf_type type, const void * data, int i) {539    switch (type) {540        case GGUF_TYPE_UINT8:   return std::to_string(((const uint8_t  *)data)[i]);541        case GGUF_TYPE_INT8:    return std::to_string(((const int8_t   *)data)[i]);542        case GGUF_TYPE_UINT16:  return std::to_string(((const uint16_t *)data)[i]);543        case GGUF_TYPE_INT16:   return std::to_string(((const int16_t  *)data)[i]);544        case GGUF_TYPE_UINT32:  return std::to_string(((const uint32_t *)data)[i]);545        case GGUF_TYPE_INT32:   return std::to_string(((const int32_t  *)data)[i]);546        case GGUF_TYPE_UINT64:  return std::to_string(((const uint64_t *)data)[i]);547        case GGUF_TYPE_INT64:   return std::to_string(((const int64_t  *)data)[i]);548        case GGUF_TYPE_FLOAT32: return std::to_string(((const float    *)data)[i]);549        case GGUF_TYPE_FLOAT64: return std::to_string(((const double   *)data)[i]);550        case GGUF_TYPE_BOOL:    return ((const int8_t *)data)[i] != 0 ? "true" : "false";551        default:                return string_format("unknown type %d", type);552    }553}554 555static std::string gguf_kv_to_str(const struct gguf_context * ctx_gguf, int i) {556    const enum gguf_type type = gguf_get_kv_type(ctx_gguf, i);557 558    switch (type) {559        case GGUF_TYPE_STRING:560            return gguf_get_val_str(ctx_gguf, i);561        case GGUF_TYPE_ARRAY:562            {563                const enum gguf_type arr_type = gguf_get_arr_type(ctx_gguf, i);564                int arr_n = gguf_get_arr_n(ctx_gguf, i);565                const void * data = arr_type == GGUF_TYPE_STRING ? nullptr : gguf_get_arr_data(ctx_gguf, i);566                std::stringstream ss;567                ss << "[";568                for (int j = 0; j < arr_n; j++) {569                    if (arr_type == GGUF_TYPE_STRING) {570                        std::string val = gguf_get_arr_str(ctx_gguf, i, j);571                        // escape quotes572                        string_replace_all(val, "\\", "\\\\");573                        string_replace_all(val, "\"", "\\\"");574                        ss << '"' << val << '"';575                    } else if (arr_type == GGUF_TYPE_ARRAY) {576                        ss << "???";577                    } else {578                        ss << gguf_data_to_str(arr_type, data, j);579                    }580                    if (j < arr_n - 1) {581                        ss << ", ";582                    }583                }584                ss << "]";585                return ss.str();586            }587        default:588            return gguf_data_to_str(type, gguf_get_val_data(ctx_gguf, i), 0);589    }590}591 592//593// debugging594//595 596static void print_tensor_shape(ggml_tensor * t) {597    printf("%s.shape = [", t->name);598    for (int i = 0; i < ggml_n_dims(t); ++i) {599        printf("%" PRId64, t->ne[i]);600        if (i < ggml_n_dims(t) - 1) {601            printf(", ");602        }603    }604    printf("]\n");605}606 607static void print_tensor_data(ggml_tensor * t, uint8_t * data, int64_t n) {608    ggml_type type = t->type;609    int64_t * ne = t->ne;610    size_t * nb = t->nb;611    for (int64_t i3 = 0; i3 < ne[3]; i3++) {612        printf("%s.data: [\n", t->name);613        for (int64_t i2 = 0; i2 < ne[2]; i2++) {614            if (i2 == n && ne[2] > 2*n) {615                printf("     ..., \n");616                i2 = ne[2] - n;617            }618            printf("     [\n");619            for (int64_t i1 = 0; i1 < ne[1]; i1++) {620                if (i1 == n && ne[1] > 2*n) {621                    printf("      ..., \n");622                    i1 = ne[1] - n;623                }624                printf("      [");625                for (int64_t i0 = 0; i0 < ne[0]; i0++) {626                    if (i0 == n && ne[0] > 2*n) {627                        printf("..., ");628                        i0 = ne[0] - n;629                    }630                    size_t i = i3 * nb[3] + i2 * nb[2] + i1 * nb[1] + i0 * nb[0];631                    float v;632                    if (type == GGML_TYPE_F16) {633                        v = ggml_fp16_to_fp32(*(ggml_fp16_t *) &data[i]);634                    } else if (type == GGML_TYPE_F32) {635                        v = *(float *) &data[i];636                    } else if (type == GGML_TYPE_I32) {637                        v = (float) *(int32_t *) &data[i];638                    } else if (type == GGML_TYPE_I16) {639                        v = (float) *(int16_t *) &data[i];640                    } else if (type == GGML_TYPE_I8) {641                        v = (float) *(int8_t *) &data[i];642                    } else {643                        GGML_ABORT("fatal error");644                    }645                    printf("%8.4f", v);646                    if (i0 < ne[0] - 1) printf(", ");647                }648                printf("],\n");649            }650            printf("     ],\n");651        }652        printf("    ]\n");653    }654}655 656//657// API used internally with mtmd658//659 660projector_type clip_get_projector_type(const struct clip_ctx * ctx);661void clip_set_debug_output_embeddings(struct clip_ctx * ctx, bool debug);662