echodict/llama.cpp
version https://git-lfs.github.com/spec/v1 oid sha256:cfc44b7ba25614df70e6b65e3341cae0310163bd32fd31a6b928a542df433faf size 30786
0773
1#include "clip.h"2#include "clip-impl.h"3#include "mtmd.h"4#include "mtmd-audio.h"5#include "mtmd-image.h"6#include "debug/mtmd-debug.h"7 8#include "llama.h"9 10// fix problem with std::min and std::max11#if defined(_WIN32)12#define WIN32_LEAN_AND_MEAN13#ifndef NOMINMAX14# define NOMINMAX15#endif16#include <windows.h>17#endif18 19#include <algorithm>20#include <cerrno>21#include <cstdio>22#include <cstdlib>23#include <cstring>24#include <vector>25 26// represents raw image data, layout is RGBRGBRGB...27// length of data must be nx * ny * 328struct mtmd_bitmap {29 uint32_t nx;30 uint32_t ny;31 std::vector<unsigned char> data;32 std::string id; // optional user-defined id, for ex: can be set to image hash, useful for KV cache tracking33 bool is_audio = false; // true if the bitmap is audio34};35 36// position indexing for decoder model37enum mtmd_pos_type {38 MTMD_POS_TYPE_NORMAL, // number of positions equals to number of tokens39 MTMD_POS_TYPE_MROPE, // qwen-vl mrope style, each image takes max(t,h,w) position indexes40};41 42struct mtmd_image_tokens {43 uint32_t nx; // number of tokens in x direction44 uint32_t ny; // number of tokens in y direction45 mtmd_pos_type pos = MTMD_POS_TYPE_NORMAL;46 uint32_t n_tokens() const { return nx * ny; }47 clip_image_f32_batch batch_f32; // preprocessed image patches48 std::string id; // optional user-defined ID, useful for KV cache tracking49 50 mtmd_image_tokens clone() {51 return mtmd_image_tokens{52 nx,53 ny,54 pos,55 batch_f32.clone(),56 id57 };58 }59};60using mtmd_image_tokens_ptr = std::unique_ptr<mtmd_image_tokens>;61 62struct mtmd_audio_tokens {63 uint32_t n_tokens; // number of tokens64 clip_image_f32_batch batch_f32; // preprocessed image patches65 std::string id; // optional user-defined ID, useful for KV cache tracking66 67 mtmd_audio_tokens clone() {68 return mtmd_audio_tokens{69 n_tokens,70 batch_f32.clone(),71 id72 };73 }74};75using mtmd_audio_tokens_ptr = std::unique_ptr<mtmd_audio_tokens>;76 77struct mtmd_input_chunk {78 mtmd_input_chunk_type type;79 std::vector<llama_token> tokens_text;80 mtmd_image_tokens_ptr tokens_image;81 mtmd_audio_tokens_ptr tokens_audio;82};83 84struct mtmd_input_chunks {85 std::vector<mtmd_input_chunk> entries;86};87 88// slice template, used by some llava-uhd models to correctly place the special tokens around image embeddings89// models not having it (llava-1.6) will process embeddings without any special tokens in-between90enum mtmd_slice_tmpl {91 MTMD_SLICE_TMPL_NONE,92 MTMD_SLICE_TMPL_MINICPMV_2_5,93 MTMD_SLICE_TMPL_MINICPMV_2_6,94 MTMD_SLICE_TMPL_LLAMA4,95 MTMD_SLICE_TMPL_IDEFICS3,96 MTMD_SLICE_TMPL_LFM2,97 MTMD_SLICE_TMPL_STEP3VL,98};99 100const char * mtmd_default_marker() {101 return "<__media__>";102}103 104static clip_flash_attn_type mtmd_get_clip_flash_attn_type(enum llama_flash_attn_type flash_attn_type) {105 switch (flash_attn_type) {106 case LLAMA_FLASH_ATTN_TYPE_AUTO: return CLIP_FLASH_ATTN_TYPE_AUTO;107 case LLAMA_FLASH_ATTN_TYPE_DISABLED: return CLIP_FLASH_ATTN_TYPE_DISABLED;108 case LLAMA_FLASH_ATTN_TYPE_ENABLED: return CLIP_FLASH_ATTN_TYPE_ENABLED;109 }110 return CLIP_FLASH_ATTN_TYPE_AUTO;111}112 113mtmd_context_params mtmd_context_params_default() {114 mtmd_context_params params {115 /* use_gpu */ true,116 /* print_timings */ true,117 /* n_threads */ 4,118 /* image_marker */ nullptr,119 /* media_marker */ mtmd_default_marker(),120 /* flash_attn_type */ LLAMA_FLASH_ATTN_TYPE_AUTO,121 /* warmup */ true,122 /* image_min_tokens */ -1,123 /* image_max_tokens */ -1,124 /* cb_eval */ nullptr,125 /* cb_eval_user_data */ nullptr,126 };127 return params;128}129 130struct mtmd_context {131 struct clip_ctx * ctx_v; // vision132 struct clip_ctx * ctx_a; // audio133 const struct llama_model * text_model;134 std::vector<float> image_embd_v; // image embedding vector135 136 bool print_timings;137 int n_threads;138 std::string media_marker;139 const int n_embd_text;140 mtmd_pos_type pos_type;141 142 // these are not token, but strings used to mark the beginning and end of image/audio embeddings143 std::string img_beg;144 std::string img_end;145 std::string aud_beg;146 std::string aud_end;147 148 // for llava-uhd style models, we need special tokens in-between slices149 // minicpmv calls them "slices", llama 4 calls them "tiles"150 mtmd_slice_tmpl slice_tmpl = MTMD_SLICE_TMPL_NONE;151 std::vector<llama_token> tok_ov_img_start; // overview image152 std::vector<llama_token> tok_ov_img_end; // overview image153 std::vector<llama_token> tok_slices_start; // start of all slices154 std::vector<llama_token> tok_slices_end; // end of all slices155 std::vector<llama_token> tok_sli_img_start; // single slice start156 std::vector<llama_token> tok_sli_img_end; // single slice end157 std::vector<llama_token> tok_sli_img_mid; // between 2 slices158 std::vector<llama_token> tok_row_end; // end of row159 bool tok_row_end_trail = false;160 bool ov_img_first = false;161 162 // string template for slice image delimiters with row/col (idefics3)163 std::string sli_img_start_tmpl;164 165 std::unique_ptr<mtmd_audio_preprocessor> audio_preproc;166 std::unique_ptr<mtmd_image_preprocessor> image_preproc;167 168 // TODO @ngxson : add timings169 170 mtmd_context(const char * mmproj_fname,171 const llama_model * text_model,172 const mtmd_context_params & ctx_params) :173 text_model (text_model),174 print_timings(ctx_params.print_timings),175 n_threads (ctx_params.n_threads),176 media_marker (ctx_params.media_marker),177 n_embd_text (llama_model_n_embd_inp(text_model))178 {179 if (ctx_params.image_marker != nullptr) {180 throw std::runtime_error("custom image_marker is not supported anymore, use media_marker instead");181 }182 183 if (media_marker.empty()) {184 throw std::runtime_error("media_marker must not be empty");185 }186 187 auto decoder_rope_type = llama_model_rope_type(text_model);188 switch (decoder_rope_type) {189 case LLAMA_ROPE_TYPE_NORM:190 case LLAMA_ROPE_TYPE_NEOX:191 {192 pos_type = MTMD_POS_TYPE_NORMAL;193 } break;194 case LLAMA_ROPE_TYPE_MROPE:195 case LLAMA_ROPE_TYPE_IMROPE:196 {197 pos_type = MTMD_POS_TYPE_MROPE;198 } break;199 default:200 throw std::runtime_error(string_format("unsupported decoder rope type: %d\n", decoder_rope_type));201 }202 203 clip_context_params ctx_clip_params {204 /* use_gpu */ ctx_params.use_gpu,205 /* flash_attn_type */ mtmd_get_clip_flash_attn_type(ctx_params.flash_attn_type),206 /* image_min_tokens */ ctx_params.image_min_tokens,207 /* image_max_tokens */ ctx_params.image_max_tokens,208 /* warmup */ ctx_params.warmup,209 /* cb_eval */ ctx_params.cb_eval,210 /* cb_eval_user_data */ ctx_params.cb_eval_user_data,211 };212 213 auto res = clip_init(mmproj_fname, ctx_clip_params);214 ctx_v = res.ctx_v;215 ctx_a = res.ctx_a;216 if (!ctx_v && !ctx_a) {217 throw std::runtime_error(string_format("Failed to load CLIP model from %s\n", mmproj_fname));218 }219 220 // if both vision and audio mmproj are present, we need to validate their n_embd221 if (ctx_v && ctx_a) {222 int n_embd_v = clip_n_mmproj_embd(ctx_v);223 int n_embd_a = clip_n_mmproj_embd(ctx_a);224 if (n_embd_v != n_embd_a) {225 throw std::runtime_error(string_format(226 "mismatch between vision and audio mmproj (n_embd_v = %d, n_embd_a = %d)\n",227 n_embd_v, n_embd_a));228 }229 }230 231 // since we already validate n_embd of vision and audio mmproj,232 // we can safely assume that they are the same233 int n_embd_clip = clip_n_mmproj_embd(ctx_v ? ctx_v : ctx_a);234 if (n_embd_text != n_embd_clip) {235 throw std::runtime_error(string_format(236 "mismatch between text model (n_embd = %d) and mmproj (n_embd = %d)\n"237 "hint: you may be using wrong mmproj\n",238 n_embd_text, n_embd_clip));239 }240 if (ctx_v) {241 init_vision();242 }243 if (ctx_a) {244 init_audio();245 }246 }247 248 void init_vision() {249 GGML_ASSERT(ctx_v != nullptr);250 image_preproc.reset();251 252 projector_type proj = clip_get_projector_type(ctx_v);253 254 switch (proj) {255 case PROJECTOR_TYPE_MLP:256 case PROJECTOR_TYPE_MLP_NORM:257 case PROJECTOR_TYPE_LDP:258 case PROJECTOR_TYPE_LDPV2:259 case PROJECTOR_TYPE_COGVLM:260 case PROJECTOR_TYPE_JANUS_PRO:261 case PROJECTOR_TYPE_GLM_EDGE:262 {263 bool has_pinpoints = !clip_get_hparams(ctx_v)->image_res_candidates.empty();264 if (has_pinpoints) {265 image_preproc = std::make_unique<mtmd_image_preprocessor_llava_uhd>(ctx_v);266 } else {267 image_preproc = std::make_unique<mtmd_image_preprocessor_fixed_size>(ctx_v);268 }269 } break;270 case PROJECTOR_TYPE_MINICPMV:271 {272 int minicpmv_version = clip_is_minicpmv(ctx_v);273 if (minicpmv_version == 2) {274 // minicpmv 2.5 format:275 // <image> (overview) </image><slice><image> (slice) </image><image> (slice) </image>\n ... </slice>276 slice_tmpl = MTMD_SLICE_TMPL_MINICPMV_2_5;277 tok_ov_img_start = {lookup_token("<image>")};278 tok_ov_img_end = {lookup_token("</image>")};279 tok_slices_start = {lookup_token("<slice>")};280 tok_slices_end = {lookup_token("</slice>")};281 tok_sli_img_start = tok_ov_img_start;282 tok_sli_img_end = tok_ov_img_end;283 tok_row_end = {lookup_token("\n")};284 tok_row_end_trail = false; // no trailing end-of-row token285 ov_img_first = true;286 } else if (minicpmv_version == 3 || minicpmv_version == 4 || minicpmv_version == 5 || minicpmv_version == 6 || minicpmv_version == 100045) {287 // minicpmv 2.6 format:288 // <image> (overview) </image><slice> (slice) </slice><slice> (slice) </slice>\n ...289 slice_tmpl = MTMD_SLICE_TMPL_MINICPMV_2_6;290 tok_ov_img_start = {lookup_token("<image>")};291 tok_ov_img_end = {lookup_token("</image>")};292 tok_sli_img_start = {lookup_token("<slice>")};293 tok_sli_img_end = {lookup_token("</slice>")};294 tok_row_end = {lookup_token("\n")};295 tok_row_end_trail = false; // no trailing end-of-row token296 ov_img_first = true;297 298 } else if (minicpmv_version != 0) {299 throw std::runtime_error(string_format("unsupported minicpmv version: %d\n", minicpmv_version));300 }301 image_preproc = std::make_unique<mtmd_image_preprocessor_llava_uhd>(ctx_v);302 } break;303 case PROJECTOR_TYPE_QWEN2VL:304 case PROJECTOR_TYPE_QWEN25VL:305 case PROJECTOR_TYPE_QWEN3VL:306 {307 // <|vision_start|> ... (image embeddings) ... <|vision_end|>308 img_beg = "<|vision_start|>";309 img_end = "<|vision_end|>";310 image_preproc = std::make_unique<mtmd_image_preprocessor_dyn_size>(ctx_v);311 } break;312 case PROJECTOR_TYPE_YOUTUVL:313 {314 // <|vision_start|> ... (image embeddings) ... <|vision_end|>315 img_beg = "<|vision_start|>";316 img_end = "<|vision_end|>";317 image_preproc = std::make_unique<mtmd_image_preprocessor_youtuvl>(ctx_v);318 } break;319 case PROJECTOR_TYPE_GEMMA3:320 case PROJECTOR_TYPE_GEMMA3NV:321 {322 // <start_of_image> ... (image embeddings) ... <end_of_image>323 img_beg = "<start_of_image>";324 img_end = "<end_of_image>";325 image_preproc = std::make_unique<mtmd_image_preprocessor_fixed_size>(ctx_v);326 } break;327 case PROJECTOR_TYPE_IDEFICS3:328 {329 // https://github.com/huggingface/transformers/blob/a42ba80fa520c784c8f11a973ca9034e5f859b79/src/transformers/models/idefics3/processing_idefics3.py#L192-L215330 slice_tmpl = MTMD_SLICE_TMPL_IDEFICS3;331 tok_ov_img_start = {lookup_token("\n\n"), lookup_token("<fake_token_around_image>"), lookup_token("<global-img>")};332 tok_ov_img_end = {lookup_token("<fake_token_around_image>")};333 tok_row_end = {lookup_token("\n")};334 sli_img_start_tmpl = "<fake_token_around_image><row_%d_col_%d>";335 image_preproc = std::make_unique<mtmd_image_preprocessor_idefics3>(ctx_v);336 } break;337 case PROJECTOR_TYPE_PIXTRAL:338 {339 // https://github.com/huggingface/transformers/blob/1cd110c6cb6a6237614130c470e9a902dbc1a4bd/docs/source/en/model_doc/pixtral.md340 img_end = "[IMG_END]";341 image_preproc = std::make_unique<mtmd_image_preprocessor_dyn_size>(ctx_v);342 } break;343 case PROJECTOR_TYPE_PHI4:344 {345 // Phi-4 uses media marker insertion only. Keep image boundary text empty.346 image_preproc = std::make_unique<mtmd_image_preprocessor_dyn_size>(ctx_v);347 } break;348 case PROJECTOR_TYPE_LLAMA4:349 {350 // (more details in mtmd_context constructor)351 img_beg = "<|image_start|>";352 img_end = "<|image_end|>";353 LOG_WRN("%s: llama 4 vision is known to have degraded quality:\n"354 " https://github.com/ggml-org/llama.cpp/pull/13282\n", __func__);355 image_preproc = std::make_unique<mtmd_image_preprocessor_llava_uhd>(ctx_v);356 } break;357 case PROJECTOR_TYPE_STEP3VL:358 {359 // Step3 format:360 // <patch_start> (patch) <patch_end> [<patch_newline>]361 // ... (all patch rows)362 // <im_start> (overview) <im_end>363 slice_tmpl = MTMD_SLICE_TMPL_STEP3VL;364 tok_ov_img_start = {lookup_token("<im_start>")};365 tok_ov_img_end = {lookup_token("<im_end>")};366 tok_sli_img_start = {lookup_token("<patch_start>")};367 tok_sli_img_end = {lookup_token("<patch_end>")};368 tok_row_end = {lookup_token("<patch_newline>")};369 tok_row_end_trail = false;370 ov_img_first = false; // patches first, overview last371 image_preproc = std::make_unique<mtmd_image_preprocessor_step3vl>(ctx_v);372 } break;373 case PROJECTOR_TYPE_INTERNVL:374 {375 // <img> ... (image embeddings) ... </img>376 img_beg = "<img>";377 img_end = "</img>";378 image_preproc = std::make_unique<mtmd_image_preprocessor_internvl>(ctx_v);379 } break;380 case PROJECTOR_TYPE_KIMIVL:381 {382 // <|media_start|> ... (image embeddings) ... <|media_end|>383 img_beg = "<|media_start|>";384 img_end = "<|media_end|>";385 image_preproc = std::make_unique<mtmd_image_preprocessor_dyn_size>(ctx_v);386 } break;387 case PROJECTOR_TYPE_KIMIK25:388 {389 // <|media_begin|> ... (image embeddings) ... <|media_end|>390 img_beg = "<|media_begin|>";391 img_end = "<|media_end|>";392 image_preproc = std::make_unique<mtmd_image_preprocessor_dyn_size>(ctx_v);393 } break;394 case PROJECTOR_TYPE_LIGHTONOCR:395 {396 // <|im_start|> ... (image embeddings) ... <|im_end|>397 img_beg = "<|im_start|>";398 img_end = "<|im_end|>";399 image_preproc = std::make_unique<mtmd_image_preprocessor_longest_edge>(ctx_v);400 } break;401 case PROJECTOR_TYPE_DOTS_OCR:402 {403 // <|img|> ... (image embeddings) ... <|endofimg|>404 img_beg = "<|img|>";405 img_end = "<|endofimg|>";406 image_preproc = std::make_unique<mtmd_image_preprocessor_dyn_size>(ctx_v);407 } break;408 case PROJECTOR_TYPE_NEMOTRON_V2_VL:409 {410 image_preproc = std::make_unique<mtmd_image_preprocessor_fixed_size>(ctx_v);411 } break;412 case PROJECTOR_TYPE_LFM2:413 {414 // multi-tile:415 // <|image_start|>416 // <|img_row_1_col_1|> (tile) <|img_row_1_col_2|> (tile) ...417 // <|img_thumbnail|> (thumbnail)418 // <|image_end|>419 // single-tile:420 // <|image_start|> (image) <|image_end|>421 img_beg = "<|image_start|>";422 img_end = "<|image_end|>";423 slice_tmpl = MTMD_SLICE_TMPL_LFM2;424 sli_img_start_tmpl = "<|img_row_%d_col_%d|>";425 tok_ov_img_start = {lookup_token("<|img_thumbnail|>")};426 ov_img_first = false;427 image_preproc = std::make_unique<mtmd_image_preprocessor_lfm2>(ctx_v);428 } break;429 case PROJECTOR_TYPE_GLM4V:430 {431 // <|begin_of_image|> ... (image embeddings) ... <|end_of_image|>432 img_beg = "<|begin_of_image|>";433 img_end = "<|end_of_image|>";434 image_preproc = std::make_unique<mtmd_image_preprocessor_dyn_size>(ctx_v);435 } break;436 case PROJECTOR_TYPE_PADDLEOCR:437 {438 // <|IMAGE_START|> ... (image embeddings) ... <|IMAGE_END|>439 img_beg = "<|IMAGE_START|>";440 img_end = "<|IMAGE_END|>";441 image_preproc = std::make_unique<mtmd_image_preprocessor_dyn_size>(ctx_v);442 } break;443 case PROJECTOR_TYPE_GEMMA4V:444 {445 // <|image> ... (image embeddings) ... <image|>446 img_beg = "<|image>";447 img_end = "<image|>";448 image_preproc = std::make_unique<mtmd_image_preprocessor_dyn_size>(ctx_v);449 } break;450 case PROJECTOR_TYPE_DEEPSEEKOCR:451 {452 img_end = "\n"; // prevent empty batch on llama-server453 image_preproc = std::make_unique<mtmd_image_preprocessor_deepseekocr>(ctx_v);454 } break;455 case PROJECTOR_TYPE_HUNYUANOCR:456 {457 // note: these use fullwidth ｜ (U+FF5C) and ▁ (U+2581) to match the tokenizer vocabulary458 img_beg = "<｜hy_place▁holder▁no▁100｜>";459 img_end = "<｜hy_place▁holder▁no▁101｜>";460 image_preproc = std::make_unique<mtmd_image_preprocessor_dyn_size>(ctx_v);461 } break;462 default:463 throw std::runtime_error(string_format("%s: unexpected vision projector type %d\n", __func__, proj));464 }465 466 GGML_ASSERT(image_preproc != nullptr);467 }468 469 void init_audio() {470 GGML_ASSERT(ctx_a != nullptr);471 audio_preproc.reset();472 473 projector_type proj = clip_get_projector_type(ctx_a);474 475 LOG_WRN("%s: audio input is in experimental stage and may have reduced quality:\n"476 " https://github.com/ggml-org/llama.cpp/discussions/13759\n", __func__);477 478 // set preprocessor479 switch (proj) {480 case PROJECTOR_TYPE_QWEN2A:481 case PROJECTOR_TYPE_QWEN3A:482 case PROJECTOR_TYPE_QWEN25O:483 {484 // <|audio_bos|> ... (embeddings) ... <|audio_eos|>485 aud_beg = "<|audio_bos|>";486 aud_end = "<|audio_eos|>";487 audio_preproc = std::make_unique<mtmd_audio_preprocessor_whisper>(ctx_a);488 } break;489 case PROJECTOR_TYPE_VOXTRAL:490 {491 // [BEGIN_AUDIO] ... (embeddings) ...492 aud_beg = "[BEGIN_AUDIO]";493 audio_preproc = std::make_unique<mtmd_audio_preprocessor_whisper>(ctx_a);494 } break;495 case PROJECTOR_TYPE_MUSIC_FLAMINGO:496 {497 // <sound> ... (embeddings) ...498 aud_beg = "<sound>";499 audio_preproc = std::make_unique<mtmd_audio_preprocessor_whisper>(ctx_a);500 } break;501 case PROJECTOR_TYPE_ULTRAVOX:502 case PROJECTOR_TYPE_GLMA:503 case PROJECTOR_TYPE_MERALION:504 {505 audio_preproc = std::make_unique<mtmd_audio_preprocessor_whisper>(ctx_a);506 } break;507 case PROJECTOR_TYPE_LFM2A:508 {509 audio_preproc = std::make_unique<mtmd_audio_preprocessor_conformer>(ctx_a);510 } break;511 case PROJECTOR_TYPE_GEMMA4A:512 {513 aud_beg = "<|audio>";514 aud_end = "<audio|>";515 audio_preproc = std::make_unique<mtmd_audio_preprocessor_gemma4a>(ctx_a);516 } break;517 default:518 throw std::runtime_error(string_format("%s: unexpected audio projector type %d\n", __func__, proj));519 }520 521 // initialize audio preprocessor522 GGML_ASSERT(audio_preproc != nullptr);523 audio_preproc->initialize();524 }525 526 // get clip ctx based on chunk type527 clip_ctx * get_clip_ctx(const mtmd_input_chunk * chunk) const {528 if (chunk->type == MTMD_INPUT_CHUNK_TYPE_IMAGE) {529 return ctx_v;530 } else if (chunk->type == MTMD_INPUT_CHUNK_TYPE_AUDIO) {531 return ctx_a;532 }533 GGML_ABORT("unknown chunk type");534 }535 536 projector_type proj_type_v() const {537 return ctx_v ? clip_get_projector_type(ctx_v) : PROJECTOR_TYPE_UNKNOWN;538 }539 540 projector_type proj_type_a() const {541 return ctx_a ? clip_get_projector_type(ctx_a) : PROJECTOR_TYPE_UNKNOWN;542 }543 544 ~mtmd_context() {545 clip_free(ctx_a);546 clip_free(ctx_v);547 }548 549private:550 llama_token lookup_token(const std::string & token_text) {551 const llama_vocab * vocab = llama_model_get_vocab(text_model);552 const int n_vocab = llama_vocab_n_tokens(vocab);553 for (int i = 0; i < n_vocab; i++) {554 if (token_to_piece(vocab, i, true) == token_text) {555 return i;556 }557 }558 return LLAMA_TOKEN_NULL;559 }560 561 std::string token_to_piece(const llama_vocab * vocab, llama_token token, bool special) {562 std::string piece;563 piece.resize(piece.capacity()); // using string internal cache, 15 bytes + '\n'564 const int n_chars = llama_token_to_piece(vocab, token, &piece[0], piece.size(), 0, special);565 if (n_chars < 0) {566 piece.resize(-n_chars);567 int check = llama_token_to_piece(vocab, token, &piece[0], piece.size(), 0, special);568 GGML_ASSERT(check == -n_chars);569 } else {570 piece.resize(n_chars);571 }572 return piece;573 }574};575 576mtmd_context * mtmd_init_from_file(const char * mmproj_fname,577 const struct llama_model * text_model,578 const struct mtmd_context_params ctx_params) {579 try {580 return new mtmd_context(mmproj_fname, text_model, ctx_params);581 } catch (const std::exception & e) {582 LOG_ERR("%s: error: %s\n", __func__, e.what());583 return nullptr;584 }585}586 587void mtmd_free(mtmd_context * ctx) {588 delete ctx;589}590 591struct mtmd_tokenizer {592 mtmd_context * ctx;593 std::vector<const mtmd_bitmap *> bitmaps;594 595 std::string input_text;596 bool add_special;597 bool parse_special;598 const llama_vocab * vocab;599 600 mtmd_input_chunks cur;601 602 mtmd_tokenizer(mtmd_context * ctx,603 const mtmd_input_text * text,604 const mtmd_bitmap ** bitmaps,605 size_t n_bitmaps) : ctx(ctx), bitmaps(bitmaps, bitmaps + n_bitmaps) {606 add_special = text->add_special;607 parse_special = text->parse_special;608 input_text = text->text;609 vocab = llama_model_get_vocab(ctx->text_model);610 }611 612 int32_t tokenize(mtmd_input_chunks * output) {613 cur.entries.clear();614 std::vector<std::string> parts = split_text(input_text, ctx->media_marker);615 size_t i_bm = 0; // index of the current bitmap616 for (auto & part : parts) {617 if (part == ctx->media_marker) {618 // this is a marker, we should add the next bitmap619 if (i_bm >= bitmaps.size()) {620 LOG_ERR("%s: error: number of bitmaps (%zu) does not match number of markers (%zu)\n",621 __func__, bitmaps.size(), parts.size() - 1);622 return 1;623 }624 const mtmd_bitmap * bitmap = bitmaps[i_bm++];625 int32_t res = add_media(bitmap);626 if (res != 0) {627 return res;628 }629 } else {630 // this is a text part, we should add it as text631 add_text(part, parse_special);632 }633 }634 635 if (add_special && llama_vocab_get_add_bos(vocab)) {636 // if first chunk is text, we add BOS token to first text chunk637 // otherwise, create a new text chunk with BOS token638 if (!cur.entries.empty() && cur.entries[0].type == MTMD_INPUT_CHUNK_TYPE_TEXT) {639 // add BOS token to the beginning of first text chunk640 cur.entries[0].tokens_text.insert(cur.entries[0].tokens_text.begin(), llama_vocab_bos(vocab));641 } else {642 // create a new text chunk with BOS token at the beginning643 mtmd_input_chunk bos_chunk{644 MTMD_INPUT_CHUNK_TYPE_TEXT,645 {llama_vocab_bos(vocab)},646 nullptr, // image tokens647 nullptr, // audio tokens648 };649 cur.entries.insert(cur.entries.begin(), std::move(bos_chunk));650 }651 }652 653 if (add_special && llama_vocab_get_add_eos(vocab)) {654 // if last chunk is text, we add EOS token to it655 add_text({llama_vocab_eos(vocab)});656 }657 658 if (i_bm != bitmaps.size()) {659 LOG_ERR("%s: error: number of bitmaps (%zu) does not match number of markers (%zu)\n",660 __func__, bitmaps.size(), parts.size() - 1);661 return 1;662 }663 664 *output = std::move(cur);665 666 return 0;667 }668 669 void add_text(const std::string & txt, bool parse_special) {670 LOG_DBG("%s: %s\n", __func__, txt.c_str());671 auto tokens = mtmd_tokenize_text_internal(vocab, txt, /* add_special */ false, parse_special);672 add_text(tokens);673 }674 675 void add_text(const std::vector<llama_token> & tokens) {676 if (tokens.empty()) {677 return;678 }679 // if last entry is also a text chunk, add tokens to it instead of creating new chunk680 if (!cur.entries.empty() && cur.entries.back().type == MTMD_INPUT_CHUNK_TYPE_TEXT) {681 cur.entries.back().tokens_text.insert(682 cur.entries.back().tokens_text.end(),683 tokens.begin(),684 tokens.end());685 } else {686 mtmd_input_chunk chunk{687 MTMD_INPUT_CHUNK_TYPE_TEXT,688 tokens,689 nullptr, // image tokens690 nullptr, // audio tokens691 };692 cur.entries.emplace_back(std::move(chunk));693 }694 }695 696 int32_t add_media(const mtmd_bitmap * bitmap) {697 if (!bitmap->is_audio) {698 // handle image699 700 if (!ctx->ctx_v) {701 LOG_ERR("%s: error: model does not support vision input\n", __func__);702 return 2;703 }704 705 if (!ctx->img_beg.empty()) {706 add_text(ctx->img_beg, true); // add image begin token707 }708 709 // sanity check710 GGML_ASSERT(bitmap->nx > 0 && bitmap->ny > 0);711 GGML_ASSERT(bitmap->data.size() == (size_t)bitmap->nx * bitmap->ny * 3);712 GGML_ASSERT(ctx->image_preproc != nullptr);713 714 // convert mtmd_bitmap to clip_image_u8715 clip_image_u8_ptr img_u8(clip_image_u8_init());716 img_u8->nx = bitmap->nx;717 img_u8->ny = bitmap->ny;718 img_u8->buf.resize(bitmap->data.size());719 std::memcpy(img_u8->buf.data(), bitmap->data.data(), img_u8->nx * img_u8->ny * 3);720 721 // preprocess image722 clip_image_f32_batch batch_f32;723 bool ok = ctx->image_preproc->preprocess(*img_u8, batch_f32);724 if (!ok) {725 LOG_ERR("Unable to preprocess image\n");726 return 2;727 }728 729 // handle llava-uhd style preprocessing730 const bool has_tiling_grid = batch_f32.grid_x > 0 && batch_f32.grid_y > 0;731 if (732 ctx->slice_tmpl == MTMD_SLICE_TMPL_MINICPMV_2_5733 || ctx->slice_tmpl == MTMD_SLICE_TMPL_MINICPMV_2_6734 || ctx->slice_tmpl == MTMD_SLICE_TMPL_LLAMA4735 || ctx->slice_tmpl == MTMD_SLICE_TMPL_IDEFICS3736 || ctx->slice_tmpl == MTMD_SLICE_TMPL_STEP3VL737 || (ctx->slice_tmpl == MTMD_SLICE_TMPL_LFM2 && has_tiling_grid)738 ) {739 const int n_col = batch_f32.grid_x;740 const int n_row = batch_f32.grid_y;741 // split batch into chunks of single images742 // NOTE: batch_f32 will be invalidated after this call743 auto chunks = split_batch_to_chunk(std::move(batch_f32), bitmap->id);744 GGML_ASSERT(chunks.size() > 0);745 746 auto ov_chunk = std::move(chunks.front());747 chunks.erase(chunks.begin());748 749 // add overview image (first)750 if (ctx->ov_img_first) {751 add_text(ctx->tok_ov_img_start);752 cur.entries.emplace_back(std::move(ov_chunk));753 add_text(ctx->tok_ov_img_end);754 }755 756 // add slices (or tiles)757 if (!chunks.empty()) {758 GGML_ASSERT((int)chunks.size() == n_row * n_col);759 add_text(ctx->tok_slices_start);760 for (int y = 0; y < n_row; y++) {761 for (int x = 0; x < n_col; x++) {762 const bool is_last_in_row = (x == n_col - 1);763 if (!ctx->tok_sli_img_start.empty()) {764 add_text(ctx->tok_sli_img_start);765 } else if (!ctx->sli_img_start_tmpl.empty()) {766 // If using a template to preceed a slice image767 const size_t sz = std::snprintf(nullptr, 0, ctx->sli_img_start_tmpl.c_str(), y+1, x+1) + 1;768 std::unique_ptr<char[]> buf(new char[sz]);769 std::snprintf(buf.get(), sz, ctx->sli_img_start_tmpl.c_str(), y+1, x+1);770 add_text(std::string(buf.get(), buf.get() + sz - 1), true);771 }772 cur.entries.emplace_back(std::move(chunks[y * n_col + x]));773 add_text(ctx->tok_sli_img_end);774 if (!is_last_in_row) {775 add_text(ctx->tok_sli_img_mid);776 }777 }778 if ((y != n_row - 1 || ctx->tok_row_end_trail)) {779 add_text(ctx->tok_row_end);780 }781 }782 add_text(ctx->tok_slices_end);783 }784 785 // add overview image (last)786 if (!ctx->ov_img_first) {787 add_text(ctx->tok_ov_img_start);788 cur.entries.emplace_back(std::move(ov_chunk));789 add_text(ctx->tok_ov_img_end);790 }791 792 } else {793 size_t n_tokens = 0;794 for (const auto & entry : batch_f32.entries) {795 n_tokens += clip_n_output_tokens(ctx->ctx_v, entry.get());796 }797 798 mtmd_image_tokens_ptr image_tokens(new mtmd_image_tokens);799 if (mtmd_decode_use_mrope(ctx)) {800 // for Qwen2VL, we need this information for M-RoPE decoding positions801 image_tokens->nx = clip_n_output_tokens_x(ctx->ctx_v, batch_f32.entries[0].get());802 image_tokens->ny = clip_n_output_tokens_y(ctx->ctx_v, batch_f32.entries[0].get());803 } else {804 // other models, we only need the total number of tokens805 image_tokens->nx = n_tokens;806 image_tokens->ny = 1;807 }808 image_tokens->pos = ctx->pos_type;809 image_tokens->batch_f32 = std::move(batch_f32);810 image_tokens->id = bitmap->id; // optional811 812 LOG_DBG("image_tokens->nx = %d\n", image_tokens->nx);813 LOG_DBG("image_tokens->ny = %d\n", image_tokens->ny);814 LOG_DBG("batch_f32 size = %d\n", (int)image_tokens->batch_f32.entries.size());815 816 mtmd_input_chunk chunk{817 MTMD_INPUT_CHUNK_TYPE_IMAGE,818 {}, // text tokens819 std::move(image_tokens),820 nullptr, // audio tokens821 };822 cur.entries.emplace_back(std::move(chunk));823 }824 825 if (!ctx->img_end.empty()) {826 add_text(ctx->img_end, true); // add image end token827 }828 829 } else {830 // handle audio831 832 if (!ctx->ctx_a) {833 LOG_ERR("%s: error: model does not support audio input\n", __func__);834 return 2;835 }836 837 if (bitmap->data.size() == 0) {838 LOG_ERR("%s: error: empty audio data\n", __func__);839 return 2;840 }841 842 if (!ctx->aud_beg.empty()) {843 add_text(ctx->aud_beg, true); // add audio begin token844 }845 846 // sanity check847 GGML_ASSERT(ctx->audio_preproc != nullptr);848 GGML_ASSERT(bitmap->data.size() > sizeof(float));849 GGML_ASSERT(bitmap->data.size() % sizeof(float) == 0);850 851 // preprocess audio852 std::vector<mtmd_audio_mel> mel_spec_chunks;853 const float * samples = (const float *)bitmap->data.data();854 size_t n_samples = bitmap->data.size() / sizeof(float);855 bool ok = ctx->audio_preproc->preprocess(samples, n_samples, mel_spec_chunks);856 if (!ok) {857 LOG_ERR("Unable to preprocess audio\n");858 return 2;859 }860 861 // consider each mel_spec as a separate audio chunk862 // TODO: maybe support batching, but this may come with memory cost863 for (auto & mel_spec : mel_spec_chunks) {864 clip_image_f32_ptr mel_f32(clip_image_f32_init());865 mel_f32->nx = mel_spec.n_len;866 mel_f32->ny = mel_spec.n_mel;867 mel_f32->buf = std::move(mel_spec.data);868 size_t n_tokens = clip_n_output_tokens(ctx->ctx_a, mel_f32.get());869 870 clip_image_f32_batch batch_f32;871 batch_f32.is_audio = true;872 batch_f32.entries.push_back(std::move(mel_f32));873 874 mtmd_audio_tokens_ptr audio_tokens(new mtmd_audio_tokens);875 audio_tokens->n_tokens = n_tokens;876 audio_tokens->batch_f32 = std::move(batch_f32);877 audio_tokens->id = bitmap->id; // optional878 879 LOG_DBG("audio_tokens->n_tokens = %d\n", audio_tokens->n_tokens);880 881 mtmd_input_chunk chunk{882 MTMD_INPUT_CHUNK_TYPE_AUDIO,883 {}, // text tokens884 nullptr, // image tokens885 std::move(audio_tokens),886 };887 cur.entries.emplace_back(std::move(chunk));888 }889 890 if (!ctx->aud_end.empty()) {891 add_text(ctx->aud_end, true); // add audio end token892 }893 }894 895 return 0;896 }897 898 std::vector<mtmd_input_chunk> split_batch_to_chunk(clip_image_f32_batch && batch_f32, const std::string & id) {899 std::vector<mtmd_input_chunk> chunks;900 901 for (auto & entry : batch_f32.entries) {902 mtmd_image_tokens_ptr image_tokens(new mtmd_image_tokens);903 image_tokens->nx = clip_n_output_tokens(ctx->ctx_v, entry.get());904 image_tokens->ny = 1;905 image_tokens->batch_f32.entries.push_back(std::move(entry));906 image_tokens->id = id;907 908 mtmd_input_chunk chunk{909 MTMD_INPUT_CHUNK_TYPE_IMAGE,910 {}, // text tokens911 std::move(image_tokens),912 nullptr, // audio tokens913 };914 chunks.emplace_back(std::move(chunk));915 }916 917 return chunks;918 }919 920 // for example: "a <__media__> b <__media__> c" --> "a", "<__media__>", "b", "<__media__>", "c"921 static std::vector<std::string> split_text(const std::string & input, const std::string & delimiter) {922 std::vector<std::string> result;923 if (input.empty()) {924 return result;925 }926 size_t start = 0;927 size_t pos = 0;928 while ((pos = input.find(delimiter, start)) != std::string::npos) {929 if (pos > start) {930 result.push_back(input.substr(start, pos - start));931 }932 result.push_back(delimiter);933 start = pos + delimiter.length();934 }935 if (start < input.length()) {936 result.push_back(input.substr(start));937 }938 return result;939 }940 941 // copied from common_tokenize942 static std::vector<llama_token> mtmd_tokenize_text_internal(943 const struct llama_vocab * vocab,944 const std::string & text,945 bool add_special,946 bool parse_special) {947 // upper limit for the number of tokens948 int n_tokens = text.length() + 2 * add_special;949 std::vector<llama_token> result(n_tokens);950 n_tokens = llama_tokenize(vocab, text.data(), text.length(), result.data(), result.size(), add_special, parse_special);951 if (n_tokens < 0) {952 result.resize(-n_tokens);953 int check = llama_tokenize(vocab, text.data(), text.length(), result.data(), result.size(), add_special, parse_special);954 GGML_ASSERT(check == -n_tokens);955 } else {956 result.resize(n_tokens);957 }958 return result;959 }960};961 962int32_t mtmd_tokenize(mtmd_context * ctx,963 mtmd_input_chunks * output,964 const mtmd_input_text * text,965 const mtmd_bitmap ** bitmaps,966 size_t n_bitmaps) {967 mtmd_tokenizer tokenizer(ctx, text, bitmaps, n_bitmaps);968 return tokenizer.tokenize(output);969}970 971int32_t mtmd_encode_chunk(mtmd_context * ctx, const mtmd_input_chunk * chunk) {972 if (chunk->type == MTMD_INPUT_CHUNK_TYPE_TEXT) {973 LOG_WRN("mtmd_encode_chunk has no effect for text chunks\n");974 return 0;975 } else if (chunk->type == MTMD_INPUT_CHUNK_TYPE_IMAGE) {976 if (!ctx->ctx_v) {977 LOG_ERR("%s: model does not support vision input\n", __func__);978 return 1;979 }980 return mtmd_encode(ctx, chunk->tokens_image.get());981 } else if (chunk->type == MTMD_INPUT_CHUNK_TYPE_AUDIO) {982 if (!ctx->ctx_a) {983 LOG_ERR("%s: model does not support audio input\n", __func__);984 return 1;985 }986 int n_mmproj_embd = ctx->n_embd_text;987 ctx->image_embd_v.resize(chunk->tokens_audio->n_tokens * n_mmproj_embd);988 bool ok = clip_image_batch_encode(989 ctx->ctx_a,990 ctx->n_threads,991 &chunk->tokens_audio->batch_f32,992 ctx->image_embd_v.data());993 return ok ? 0 : 1;994 }995 996 LOG_ERR("%s: unknown chunk type %d\n", __func__, (int)chunk->type);997 return 1;998}999 1000int32_t mtmd_encode(mtmd_context * ctx, const mtmd_image_tokens * image_tokens) {1001 clip_ctx * ctx_clip = ctx->ctx_v;1002 if (!ctx_clip) {1003 LOG_ERR("%s: this API does not support non-vision input, please use mtmd_encode_chunk instead\n", __func__);1004 return 1;1005 }1006 auto proj_type = clip_get_projector_type(ctx_clip);1007 int n_mmproj_embd = clip_n_mmproj_embd(ctx_clip);1008 ctx->image_embd_v.resize(image_tokens->n_tokens() * n_mmproj_embd);1009 bool ok = false;1010 1011 if (clip_is_llava(ctx_clip)1012 || clip_is_minicpmv(ctx_clip)1013 || clip_is_glm(ctx_clip)1014 || proj_type == PROJECTOR_TYPE_INTERNVL) {1015 // TODO @ngxson : llava does not support batched encoding ; this should be fixed inside clip_image_batch_encode()1016 const auto & entries = image_tokens->batch_f32.entries;1017 for (size_t i = 0; i < entries.size(); i++) {1018 int n_tokens_per_image = clip_n_output_tokens(ctx_clip, entries[i].get());1019 ok = clip_image_encode(1020 ctx_clip,1021 ctx->n_threads,1022 entries[i].get(),1023 ctx->image_embd_v.data() + i*n_mmproj_embd*n_tokens_per_image);1024 }1025 } else {1026 ok = clip_image_batch_encode(1027 ctx_clip,1028 ctx->n_threads,1029 &image_tokens->batch_f32,1030 ctx->image_embd_v.data());1031 }1032 1033 return ok ? 0 : 1;1034}1035 1036float * mtmd_get_output_embd(mtmd_context * ctx) {1037 return ctx->image_embd_v.data();1038}1039 1040bool mtmd_decode_use_non_causal(const mtmd_context * ctx, const mtmd_input_chunk * chunk) {1041 auto proj_type = ctx->proj_type_v();1042 if (chunk && chunk->type == MTMD_INPUT_CHUNK_TYPE_AUDIO) {1043 proj_type = ctx->proj_type_a();1044 }1045 switch (proj_type) {1046 case PROJECTOR_TYPE_GEMMA3:1047 case PROJECTOR_TYPE_GEMMA4V:1048 return true;1049 default:1050 return false;1051 }1052}1053 1054bool mtmd_decode_use_mrope(const mtmd_context * ctx) {1055 return ctx->pos_type == MTMD_POS_TYPE_MROPE;1056}1057 1058bool mtmd_support_vision(const mtmd_context * ctx) {1059 return ctx->ctx_v != nullptr;1060}1061 1062bool mtmd_support_audio(const mtmd_context * ctx) {1063 return ctx->ctx_a != nullptr;1064}1065 1066int mtmd_get_audio_sample_rate(const mtmd_context * ctx) {1067 if (!ctx->ctx_a) {1068 return -1;1069 }1070 return clip_get_hparams(ctx->ctx_a)->audio_sample_rate;1071}1072 1073//1074// public API functions1075//1076 1077// mtmd_bitmap1078 1079mtmd_bitmap * mtmd_bitmap_init(uint32_t nx,1080 uint32_t ny,1081 const unsigned char * data) {1082 mtmd_bitmap * bitmap = new mtmd_bitmap;1083 bitmap->nx = nx;1084 bitmap->ny = ny;1085 size_t data_size = (size_t)nx * ny * 3;1086 bitmap->data.resize(data_size);1087 std::memcpy(bitmap->data.data(), data, data_size);1088 return bitmap;1089}1090 1091mtmd_bitmap * mtmd_bitmap_init_from_audio(size_t n_samples,1092 const float * data) {1093 mtmd_bitmap * bitmap = new mtmd_bitmap;1094 bitmap->nx = n_samples;1095 bitmap->ny = 1;1096 bitmap->is_audio = true;1097 size_t data_size = n_samples * sizeof(float);1098 bitmap->data.resize(data_size);1099 std::memcpy(bitmap->data.data(), data, data_size);1100 return bitmap;1101}1102 1103uint32_t mtmd_bitmap_get_nx(const mtmd_bitmap * bitmap) {1104 return bitmap->nx;1105}1106 1107uint32_t mtmd_bitmap_get_ny(const mtmd_bitmap * bitmap) {1108 return bitmap->ny;1109}1110 1111const unsigned char * mtmd_bitmap_get_data(const mtmd_bitmap * bitmap) {1112 return bitmap->data.data();1113}1114 1115size_t mtmd_bitmap_get_n_bytes(const mtmd_bitmap * bitmap) {1116 return bitmap->data.size();1117}1118 1119bool mtmd_bitmap_is_audio(const mtmd_bitmap * bitmap) {1120 return bitmap->is_audio;1121}1122 1123const char * mtmd_bitmap_get_id(const mtmd_bitmap * bitmap) {1124 return bitmap->id.c_str();1125}1126 1127void mtmd_bitmap_set_id(mtmd_bitmap * bitmap, const char * id) {1128 if (id) {1129 bitmap->id = std::string(id);1130 } else {1131 bitmap->id.clear();1132 }1133}1134 1135void mtmd_bitmap_free(mtmd_bitmap * bitmap) {1136 if (bitmap) {1137 delete bitmap;1138 }1139}1140 1141// mtmd_input_chunks1142 1143mtmd_input_chunks * mtmd_input_chunks_init() {1144 return new mtmd_input_chunks;1145}1146 1147size_t mtmd_input_chunks_size(const mtmd_input_chunks * chunks) {1148 return chunks->entries.size();1149}1150 1151const mtmd_input_chunk * mtmd_input_chunks_get(const mtmd_input_chunks * chunks, size_t idx) {1152 if (idx >= chunks->entries.size()) {1153 return nullptr;1154 }1155 return &chunks->entries[idx];1156}1157 1158void mtmd_input_chunks_free(mtmd_input_chunks * chunks) {1159 if (chunks) {1160 delete chunks;1161 }1162}1163 1164// mtmd_input_chunk1165 1166enum mtmd_input_chunk_type mtmd_input_chunk_get_type(const mtmd_input_chunk * chunk) {1167 return chunk->type;1168}1169 1170const llama_token * mtmd_input_chunk_get_tokens_text(const mtmd_input_chunk * chunk, size_t * n_tokens_output) {1171 if (chunk->type == MTMD_INPUT_CHUNK_TYPE_TEXT) {1172 *n_tokens_output = chunk->tokens_text.size();1173 return chunk->tokens_text.data();1174 }1175 *n_tokens_output = 0;1176 return nullptr;1177}1178 1179const mtmd_image_tokens * mtmd_input_chunk_get_tokens_image(const mtmd_input_chunk * chunk) {1180 if (chunk->type == MTMD_INPUT_CHUNK_TYPE_IMAGE) {1181 return chunk->tokens_image.get();1182 }1183 return nullptr;1184}1185 1186size_t mtmd_input_chunk_get_n_tokens(const mtmd_input_chunk * chunk) {1187 if (chunk->type == MTMD_INPUT_CHUNK_TYPE_TEXT) {1188 return chunk->tokens_text.size();1189 } else if (chunk->type == MTMD_INPUT_CHUNK_TYPE_IMAGE) {1190 return mtmd_image_tokens_get_n_tokens(chunk->tokens_image.get());1191 } else if (chunk->type == MTMD_INPUT_CHUNK_TYPE_AUDIO) {1192 return chunk->tokens_audio->n_tokens;1193 } else {1194 GGML_ABORT("invalid chunk type");1195 }1196}1197 1198llama_pos mtmd_input_chunk_get_n_pos(const mtmd_input_chunk * chunk) {1199 if (chunk->type == MTMD_INPUT_CHUNK_TYPE_TEXT) {1200 return chunk->tokens_text.size();