echodict/llama.cpp
version https://git-lfs.github.com/spec/v1 oid sha256:cfc44b7ba25614df70e6b65e3341cae0310163bd32fd31a6b928a542df433faf size 30786
0773
1#include "clip.h"2#include "clip-impl.h"3#include "clip-model.h"4#include "clip-graph.h"5#include "models/models.h"6 7#include "ggml.h"8#include "ggml-cpp.h"9#include "ggml-alloc.h"10#include "ggml-backend.h"11#include "gguf.h"12 13#include <algorithm>14#include <cassert>15#include <cmath>16#include <cstdlib>17#include <cstring>18#include <fstream>19#include <map>20#include <stdexcept>21#include <unordered_set>22#include <vector>23#include <cinttypes>24#include <limits>25#include <array>26#include <functional>27#include <float.h>28 29struct clip_logger_state g_logger_state = {clip_log_callback_default, NULL};30 31//#define CLIP_DEBUG_FUNCTIONS32 33#ifdef CLIP_DEBUG_FUNCTIONS34static void clip_image_write_image_to_ppm(const clip_image_u8& img, const std::string& filename) {35 std::ofstream file(filename, std::ios::binary);36 if (!file.is_open()) {37 LOG_ERR("Failed to open file for writing: %s\n", filename.c_str());38 return;39 }40 41 // PPM header: P6 format, width, height, and max color value42 file << "P6\n" << img.nx << " " << img.ny << "\n255\n";43 44 // Write pixel data45 for (size_t i = 0; i < img.buf.size(); i += 3) {46 // PPM expects binary data in RGB format, which matches our image buffer47 file.write(reinterpret_cast<const char*>(&img.buf[i]), 3);48 }49 50 file.close();51}52 53static void clip_image_save_to_bmp(const clip_image_u8& img, const std::string& filename) {54 std::ofstream file(filename, std::ios::binary);55 if (!file.is_open()) {56 LOG_ERR("Failed to open file for writing: %s\n", filename.c_str());57 return;58 }59 60 int fileSize = 54 + 3 * img.nx * img.ny; // File header + info header + pixel data61 int bytesPerPixel = 3;62 int widthInBytes = img.nx * bytesPerPixel;63 int paddingAmount = (4 - (widthInBytes % 4)) % 4;64 int stride = widthInBytes + paddingAmount;65 66 // Bitmap file header67 unsigned char fileHeader[14] = {68 'B','M', // Signature69 0,0,0,0, // Image file size in bytes70 0,0,0,0, // Reserved71 54,0,0,0 // Start of pixel array72 };73 74 // Total file size75 fileSize = 54 + (stride * img.ny);76 fileHeader[2] = (unsigned char)(fileSize);77 fileHeader[3] = (unsigned char)(fileSize >> 8);78 fileHeader[4] = (unsigned char)(fileSize >> 16);79 fileHeader[5] = (unsigned char)(fileSize >> 24);80 81 // Bitmap information header (BITMAPINFOHEADER)82 unsigned char infoHeader[40] = {83 40,0,0,0, // Size of this header (40 bytes)84 0,0,0,0, // Image width85 0,0,0,0, // Image height86 1,0, // Number of color planes87 24,0, // Bits per pixel88 0,0,0,0, // No compression89 0,0,0,0, // Image size (can be 0 for no compression)90 0,0,0,0, // X pixels per meter (not specified)91 0,0,0,0, // Y pixels per meter (not specified)92 0,0,0,0, // Total colors (color table not used)93 0,0,0,0 // Important colors (all are important)94 };95 96 // Width and height in the information header97 infoHeader[4] = (unsigned char)(img.nx);98 infoHeader[5] = (unsigned char)(img.nx >> 8);99 infoHeader[6] = (unsigned char)(img.nx >> 16);100 infoHeader[7] = (unsigned char)(img.nx >> 24);101 infoHeader[8] = (unsigned char)(img.ny);102 infoHeader[9] = (unsigned char)(img.ny >> 8);103 infoHeader[10] = (unsigned char)(img.ny >> 16);104 infoHeader[11] = (unsigned char)(img.ny >> 24);105 106 // Write file headers107 file.write(reinterpret_cast<char*>(fileHeader), sizeof(fileHeader));108 file.write(reinterpret_cast<char*>(infoHeader), sizeof(infoHeader));109 110 // Pixel data111 std::vector<unsigned char> padding(3, 0); // Max padding size to be added to each row112 for (int y = img.ny - 1; y >= 0; --y) { // BMP files are stored bottom-to-top113 for (int x = 0; x < img.nx; ++x) {114 // Each pixel115 size_t pixelIndex = (y * img.nx + x) * 3;116 unsigned char pixel[3] = {117 img.buf[pixelIndex + 2], // BMP stores pixels in BGR format118 img.buf[pixelIndex + 1],119 img.buf[pixelIndex]120 };121 file.write(reinterpret_cast<char*>(pixel), 3);122 }123 // Write padding for the row124 file.write(reinterpret_cast<char*>(padding.data()), paddingAmount);125 }126 127 file.close();128}129 130// debug function to convert f32 to u8131static void clip_image_convert_f32_to_u8(const clip_image_f32& src, clip_image_u8& dst) {132 dst.nx = src.nx;133 dst.ny = src.ny;134 dst.buf.resize(3 * src.nx * src.ny);135 for (size_t i = 0; i < src.buf.size(); ++i) {136 dst.buf[i] = static_cast<uint8_t>(std::min(std::max(int(src.buf[i] * 255.0f), 0), 255));137 }138}139#endif140 141 142struct clip_ctx {143 clip_model model;144 145 gguf_context_ptr ctx_gguf;146 ggml_context_ptr ctx_data;147 148 std::vector<uint8_t> buf_compute_meta;149 150 std::vector<ggml_backend_t> backend_ptrs;151 std::vector<ggml_backend_buffer_type_t> backend_buft;152 153 ggml_backend_t backend = nullptr;154 ggml_backend_t backend_cpu = nullptr;155 ggml_backend_buffer_ptr buf;156 157 158 int max_nodes = 8192;159 ggml_backend_sched_ptr sched;160 clip_flash_attn_type flash_attn_type = CLIP_FLASH_ATTN_TYPE_AUTO;161 bool is_allocated = false;162 163 bool debug_output_embeddings = false;164 165 clip_ctx(clip_context_params & ctx_params) {166 flash_attn_type = ctx_params.flash_attn_type;167 backend_cpu = ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_CPU, nullptr);168 if (!backend_cpu) {169 throw std::runtime_error("failed to initialize CPU backend");170 }171 if (ctx_params.use_gpu) {172 auto backend_name = std::getenv("MTMD_BACKEND_DEVICE");173 if (backend_name != nullptr) {174 backend = ggml_backend_init_by_name(backend_name, nullptr);175 if (!backend) {176 LOG_WRN("%s: Warning: Failed to initialize \"%s\" backend, falling back to default GPU backend\n", __func__, backend_name);177 }178 }179 if (!backend) {180 backend = ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_GPU, nullptr);181 backend = backend ? backend : ggml_backend_init_by_type(GGML_BACKEND_DEVICE_TYPE_IGPU, nullptr);182 }183 }184 185 if (backend) {186 LOG_INF("%s: CLIP using %s backend\n", __func__, ggml_backend_name(backend));187 backend_ptrs.push_back(backend);188 backend_buft.push_back(ggml_backend_get_default_buffer_type(backend));189 } else {190 backend = backend_cpu;191 LOG_INF("%s: CLIP using CPU backend\n", __func__);192 }193 194 if (ctx_params.image_min_tokens > 0) {195 model.hparams.custom_image_min_tokens = ctx_params.image_min_tokens;196 }197 if (ctx_params.image_max_tokens > 0) {198 model.hparams.custom_image_max_tokens = ctx_params.image_max_tokens;199 }200 201 backend_ptrs.push_back(backend_cpu);202 backend_buft.push_back(ggml_backend_get_default_buffer_type(backend_cpu));203 204 sched.reset(205 ggml_backend_sched_new(backend_ptrs.data(), backend_buft.data(), backend_ptrs.size(), 8192, false, true)206 );207 208 if (ctx_params.cb_eval != nullptr) {209 ggml_backend_sched_set_eval_callback(sched.get(), ctx_params.cb_eval, ctx_params.cb_eval_user_data);210 }211 212 debug_output_embeddings = std::getenv("MTMD_DEBUG_EMBEDDINGS") != nullptr;213 }214 215 ~clip_ctx() {216 ggml_backend_free(backend);217 if (backend != backend_cpu) {218 ggml_backend_free(backend_cpu);219 }220 }221 222 // this function is added so that we don't change too much of the existing code223 projector_type proj_type() const {224 return model.proj_type;225 }226};227 228//229// clip_graph230//231 232clip_graph::clip_graph(clip_ctx * ctx, const clip_image_f32 & img) :233 model(ctx->model),234 hparams(model.hparams),235 proj_type(ctx->proj_type()),236 img(img),237 patch_size(hparams.patch_size),238 n_patches_x(img.nx / patch_size),239 n_patches_y(img.ny / patch_size),240 n_patches(n_patches_x * n_patches_y),241 n_embd(hparams.n_embd),242 n_head(hparams.n_head),243 d_head(n_embd / n_head),244 n_layer(hparams.n_layer),245 n_mmproj_embd(clip_n_mmproj_embd(ctx)),246 eps(hparams.eps),247 kq_scale(1.0f / sqrtf((float)d_head)),248 flash_attn_type(ctx->flash_attn_type) {249 struct ggml_init_params params = {250 /*.mem_size =*/ ctx->buf_compute_meta.size(),251 /*.mem_buffer =*/ ctx->buf_compute_meta.data(),252 /*.no_alloc =*/ true,253 };254 ctx0_ptr.reset(ggml_init(params));255 ctx0 = ctx0_ptr.get();256 gf = ggml_new_graph_custom(ctx0, ctx->max_nodes, false);257}258 259ggml_tensor * clip_graph::build_mm(ggml_tensor * w, ggml_tensor * x) const {260 return ggml_mul_mat(ctx0, w, x);261}262 263void clip_graph::cb(ggml_tensor * cur, const char * name, int il) const {264 if (il >= 0) {265 ggml_format_name(cur, "%s-%d", name, il);266 } else {267 ggml_set_name(cur, name);268 }269}270 271// siglip2 naflex272ggml_tensor * clip_graph::resize_position_embeddings(uint32_t interpolation_mode) {273 ggml_tensor * pos_embd = model.position_embeddings;274 const int height = img.ny / patch_size;275 const int width = img.nx / patch_size;276 const uint32_t mode = interpolation_mode;277 const int n_per_side = (int)std::sqrt(pos_embd->ne[1]);278 279 GGML_ASSERT(pos_embd);280 281 if (height == n_per_side && width == n_per_side) {282 return pos_embd;283 }284 285 pos_embd = ggml_reshape_3d(ctx0, pos_embd, n_embd, n_per_side, n_per_side); // -> (n_embd, n_per_side, n_per_side)286 pos_embd = ggml_permute(ctx0, pos_embd, 2, 0, 1, 3); // -> (n_per_side, n_per_side, n_embd)287 pos_embd = ggml_interpolate(ctx0, pos_embd, width, height, n_embd, 1, mode); // -> (width, height, n_embd)288 pos_embd = ggml_permute(ctx0, pos_embd, 1, 2, 0, 3); // -> (n_embd, width, height)289 pos_embd = ggml_cont_2d(ctx0, pos_embd, n_embd, width * height); // -> (n_embd, width * height)290 291 return pos_embd;292}293 294// build vision transformer (ViT) cgraph295// this function should cover most of the models296// if your model has specific features, you should probably duplicate this function297ggml_tensor * clip_graph::build_vit(298 ggml_tensor * inp,299 int64_t n_pos,300 norm_type norm_t,301 ffn_op_type ffn_t,302 ggml_tensor * learned_pos_embd,303 std::function<ggml_tensor *(ggml_tensor *, const clip_layer &)> add_pos304 ) {305 if (learned_pos_embd) {306 inp = ggml_add(ctx0, inp, learned_pos_embd);307 cb(inp, "pos_embed", -1);308 }309 310 ggml_tensor * inpL = inp;311 312 // pre-layernorm313 if (model.pre_ln_w) {314 inpL = build_norm(inpL, model.pre_ln_w, model.pre_ln_b, norm_t, eps, -1);315 cb(inpL, "pre_ln", -1);316 }317 318 // loop over layers319 for (int il = 0; il < n_layer; il++) {320 auto & layer = model.layers[il];321 ggml_tensor * cur = inpL; // inpL = residual, cur = hidden_states322 323 // layernorm1324 cur = build_norm(cur, layer.ln_1_w, layer.ln_1_b, norm_t, eps, il);325 cb(cur, "layer_inp_normed", il);326 327 // self-attention328 {329 ggml_tensor * Qcur = nullptr;330 ggml_tensor * Kcur = nullptr;331 ggml_tensor * Vcur = nullptr;332 if (layer.qkv_w != nullptr) {333 // fused qkv334 cur = build_mm(layer.qkv_w, cur);335 if (layer.qkv_b != nullptr) {336 cur = ggml_add(ctx0, cur, layer.qkv_b);337 }338 339 Qcur = ggml_view_3d(ctx0, cur, d_head, n_head, n_pos,340 /* nb1 */ ggml_row_size(cur->type, d_head),341 /* nb2 */ cur->nb[1],342 /* offset */ 0);343 344 Kcur = ggml_view_3d(ctx0, cur, d_head, n_head, n_pos,345 /* nb1 */ ggml_row_size(cur->type, d_head),346 /* nb2 */ cur->nb[1],347 /* offset */ ggml_row_size(cur->type, n_embd));348 349 Vcur = ggml_view_3d(ctx0, cur, d_head, n_head, n_pos,350 /* nb1 */ ggml_row_size(cur->type, d_head),351 /* nb2 */ cur->nb[1],352 /* offset */ ggml_row_size(cur->type, 2 * n_embd));353 354 if (layer.q_norm) {355 GGML_ASSERT(layer.q_norm->ne[0] == Qcur->ne[0]);356 Qcur = build_norm(Qcur, layer.q_norm, NULL, norm_t, eps, il);357 cb(Qcur, "Qcur_norm", il);358 }359 360 if (layer.k_norm) {361 GGML_ASSERT(layer.k_norm->ne[0] == Kcur->ne[0]);362 Kcur = build_norm(Kcur, layer.k_norm, NULL, norm_t, eps, il);363 cb(Kcur, "Kcur_norm", il);364 }365 366 } else {367 // separate q, k, v368 Qcur = build_mm(layer.q_w, cur);369 if (layer.q_b) {370 Qcur = ggml_add(ctx0, Qcur, layer.q_b);371 }372 373 Kcur = build_mm(layer.k_w, cur);374 if (layer.k_b) {375 Kcur = ggml_add(ctx0, Kcur, layer.k_b);376 }377 378 Vcur = build_mm(layer.v_w, cur);379 if (layer.v_b) {380 Vcur = ggml_add(ctx0, Vcur, layer.v_b);381 }382 383 // if true, norm must be applied after reshaping to (d_head, n_head, n_pos)384 bool norm_per_head = layer.q_norm && layer.q_norm->ne[0] == d_head;385 386 if (!norm_per_head) {387 if (layer.q_norm) {388 Qcur = build_norm(Qcur, layer.q_norm, NULL, norm_t, eps, il);389 cb(Qcur, "Qcur_norm", il);390 }391 if (layer.k_norm) {392 Kcur = build_norm(Kcur, layer.k_norm, NULL, norm_t, eps, il);393 cb(Kcur, "Kcur_norm", il);394 }395 }396 397 Qcur = ggml_reshape_3d(ctx0, Qcur, d_head, n_head, n_pos);398 Kcur = ggml_reshape_3d(ctx0, Kcur, d_head, n_head, n_pos);399 Vcur = ggml_reshape_3d(ctx0, Vcur, d_head, n_head, n_pos);400 401 if (norm_per_head) {402 if (layer.q_norm) {403 Qcur = build_norm(Qcur, layer.q_norm, NULL, norm_t, eps, il);404 cb(Qcur, "Qcur_norm_per_head", il);405 }406 if (layer.k_norm) {407 Kcur = build_norm(Kcur, layer.k_norm, NULL, norm_t, eps, il);408 cb(Kcur, "Kcur_norm_per_head", il);409 }410 }411 }412 413 cb(Qcur, "Qcur", il);414 cb(Kcur, "Kcur", il);415 cb(Vcur, "Vcur", il);416 417 if (add_pos) {418 Qcur = add_pos(Qcur, layer);419 Kcur = add_pos(Kcur, layer);420 cb(Qcur, "Qcur_pos", il);421 cb(Kcur, "Kcur_pos", il);422 }423 424 if (proj_type == PROJECTOR_TYPE_GEMMA4V) {425 Vcur = ggml_rms_norm(ctx0, Vcur, eps);426 cb(Vcur, "Vcur_normed", il);427 }428 429 cur = build_attn(layer.o_w, layer.o_b,430 Qcur, Kcur, Vcur, nullptr, kq_scale, il);431 cb(cur, "attn_out", il);432 }433 434 if (layer.ls_1_w) {435 cur = ggml_mul(ctx0, cur, layer.ls_1_w);436 cb(cur, "attn_out_scaled", il);437 }438 439 if (layer.attn_post_norm_w) {440 cur = build_norm(cur, layer.attn_post_norm_w, nullptr, norm_t, eps, il);441 cb(cur, "attn_post_normed", il);442 }443 444 // re-add the layer input, e.g., residual445 cur = ggml_add(ctx0, cur, inpL);446 447 inpL = cur; // inpL = residual, cur = hidden_states448 449 cb(cur, "ffn_inp", il);450 451 // layernorm2 (pre-ffn norm)452 cur = build_norm(cur, layer.ln_2_w, layer.ln_2_b, norm_t, eps, il);453 cb(cur, "ffn_inp_normed", il);454 455 // ffn456 cur = build_ffn(cur,457 layer.ff_up_w, layer.ff_up_b,458 layer.ff_gate_w, layer.ff_gate_b,459 layer.ff_down_w, layer.ff_down_b,460 ffn_t, il);461 462 cb(cur, "ffn_out", il);463 464 if (layer.ff_post_norm_w) {465 cur = build_norm(cur, layer.ff_post_norm_w, nullptr, norm_t, eps, il);466 cb(cur, "ffn_post_normed", il);467 }468 469 if (layer.ls_2_w) {470 cur = ggml_mul(ctx0, cur, layer.ls_2_w);471 cb(cur, "ffn_out_scaled", il);472 }473 474 // residual 2475 cur = ggml_add(ctx0, inpL, cur);476 cb(cur, "layer_out", il);477 478 if (layer.ls_out_w) {479 cur = ggml_mul(ctx0, cur, layer.ls_out_w);480 cb(cur, "layer_out_scaled", il);481 }482 483 inpL = cur;484 }485 486 if (model.audio_has_avgpool()) {487 ggml_tensor * cur = inpL;488 cur = ggml_transpose(ctx0, cur);489 cur = ggml_cont(ctx0, cur);490 cur = ggml_pool_1d(ctx0, cur, GGML_OP_POOL_AVG, 2, 2, 0);491 cur = ggml_transpose(ctx0, cur);492 cur = ggml_cont(ctx0, cur);493 inpL = cur;494 }495 496 // post-layernorm497 if (model.post_ln_w) {498 inpL = build_norm(inpL, model.post_ln_w, model.post_ln_b, norm_t, eps, -1);499 }500 return inpL;501}502 503// build the input after conv2d (inp_raw --> patches)504// returns tensor with shape [n_embd, n_patches]505ggml_tensor * clip_graph::build_inp() {506 ggml_tensor * inp_raw = build_inp_raw();507 ggml_tensor * inp = ggml_conv_2d(ctx0, model.patch_embeddings_0, inp_raw, patch_size, patch_size, 0, 0, 1, 1);508 inp = ggml_reshape_2d(ctx0, inp, n_patches, n_embd);509 inp = ggml_cont(ctx0, ggml_transpose(ctx0, inp));510 if (model.patch_bias) {511 inp = ggml_add(ctx0, inp, model.patch_bias);512 cb(inp, "patch_bias", -1);513 }514 return inp;515}516 517ggml_tensor * clip_graph::build_inp_raw(int channels) {518 ggml_tensor * inp_raw = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, img.nx, img.ny, channels);519 ggml_set_name(inp_raw, "inp_raw");520 ggml_set_input(inp_raw);521 return inp_raw;522}523 524ggml_tensor * clip_graph::build_norm(525 ggml_tensor * cur,526 ggml_tensor * mw,527 ggml_tensor * mb,528 norm_type type,529 float norm_eps,530 int il) const {531 532 cur = type == NORM_TYPE_RMS533 ? ggml_rms_norm(ctx0, cur, norm_eps)534 : ggml_norm(ctx0, cur, norm_eps);535 536 if (mw) {537 cur = ggml_mul(ctx0, cur, mw);538 cb(cur, "norm_w", il);539 }540 541 if (mb) {542 cur = ggml_add(ctx0, cur, mb);543 cb(cur, "norm_b", il);544 }545 546 return cur;547}548 549ggml_tensor * clip_graph::build_ffn(550 ggml_tensor * cur,551 ggml_tensor * up,552 ggml_tensor * up_b,553 ggml_tensor * gate,554 ggml_tensor * gate_b,555 ggml_tensor * down,556 ggml_tensor * down_b,557 ffn_op_type type_op,558 int il) const {559 560 ggml_tensor * tmp = up ? build_mm(up, cur) : cur;561 cb(tmp, "ffn_up", il);562 563 if (up_b) {564 tmp = ggml_add(ctx0, tmp, up_b);565 cb(tmp, "ffn_up_b", il);566 }567 568 if (gate) {569 cur = build_mm(gate, cur);570 cb(cur, "ffn_gate", il);571 572 if (gate_b) {573 cur = ggml_add(ctx0, cur, gate_b);574 cb(cur, "ffn_gate_b", il);575 }576 } else {577 cur = tmp;578 }579 580 // we only support parallel ffn for now581 switch (type_op) {582 case FFN_SILU:583 if (gate) {584 cur = ggml_swiglu_split(ctx0, cur, tmp);585 cb(cur, "ffn_swiglu", il);586 } else {587 cur = ggml_silu(ctx0, cur);588 cb(cur, "ffn_silu", il);589 } break;590 case FFN_GELU:591 if (gate) {592 cur = ggml_geglu_split(ctx0, cur, tmp);593 cb(cur, "ffn_geglu", il);594 } else {595 cur = ggml_gelu(ctx0, cur);596 cb(cur, "ffn_gelu", il);597 } break;598 case FFN_GELU_ERF:599 if (gate) {600 cur = ggml_geglu_erf_split(ctx0, cur, tmp);601 cb(cur, "ffn_geglu_erf", il);602 } else {603 cur = ggml_gelu_erf(ctx0, cur);604 cb(cur, "ffn_gelu_erf", il);605 } break;606 case FFN_GELU_QUICK:607 if (gate) {608 cur = ggml_geglu_quick_split(ctx0, cur, tmp);609 cb(cur, "ffn_geglu_quick", il);610 } else {611 cur = ggml_gelu_quick(ctx0, cur);612 cb(cur, "ffn_gelu_quick", il);613 } break;614 case FFN_RELU_SQR:615 {616 cur = ggml_relu(ctx0, cur);617 cur = ggml_sqr(ctx0, cur);618 cb(cur, "ffn_relu_sqr", il);619 } break;620 }621 622 if (down) {623 cur = build_mm(down, cur);624 }625 626 if (down_b) {627 cb(cur, "ffn_down", il);628 }629 630 if (down_b) {631 cur = ggml_add(ctx0, cur, down_b);632 }633 634 return cur;635}636 637ggml_tensor * clip_graph::build_attn(638 ggml_tensor * wo,639 ggml_tensor * wo_b,640 ggml_tensor * q_cur,641 ggml_tensor * k_cur,642 ggml_tensor * v_cur,643 ggml_tensor * kq_mask,644 float kq_scale,645 int il) const {646 // these nodes are added to the graph together so that they are not reordered647 // by doing so, the number of splits in the graph is reduced648 ggml_build_forward_expand(gf, q_cur);649 ggml_build_forward_expand(gf, k_cur);650 ggml_build_forward_expand(gf, v_cur);651 652 ggml_tensor * q = ggml_permute(ctx0, q_cur, 0, 2, 1, 3);653 //cb(q, "q", il);654 655 ggml_tensor * k = ggml_permute(ctx0, k_cur, 0, 2, 1, 3);656 //cb(k, "k", il);657 658 ggml_tensor * cur;659 660 if (flash_attn_type == CLIP_FLASH_ATTN_TYPE_ENABLED) {661 ggml_tensor * v = ggml_permute(ctx0, v_cur, 0, 2, 1, 3);662 663 k = ggml_cast(ctx0, k, GGML_TYPE_F16);664 v = ggml_cast(ctx0, v, GGML_TYPE_F16);665 666 cur = ggml_flash_attn_ext(ctx0, q, k, v, kq_mask, kq_scale, 0.0f, 0.0f);667 ggml_flash_attn_ext_set_prec(cur, GGML_PREC_F32);668 669 cur = ggml_reshape_2d(ctx0, cur, cur->ne[0]*cur->ne[1], cur->ne[2]*cur->ne[3]);670 671 } else {672 ggml_tensor * v = ggml_permute(ctx0, v_cur, 1, 2, 0, 3);673 v = ggml_cont(ctx0, v);674 675 ggml_tensor * kq = ggml_mul_mat(ctx0, k, q);676 // F32 may not needed for vision encoders?677 // ggml_mul_mat_set_prec(kq, GGML_PREC_F32);678 679 kq = ggml_soft_max_ext(ctx0, kq, kq_mask, kq_scale, 0.0f);680 681 ggml_tensor * kqv = ggml_mul_mat(ctx0, v, kq);682 cur = ggml_permute(ctx0, kqv, 0, 2, 1, 3);683 cur = ggml_cont_2d(ctx0, cur, cur->ne[0] * cur->ne[1], cur->ne[2] * cur->ne[3]);684 }685 686 cb(cur, "kqv_out", il);687 688 if (wo) {689 cur = build_mm(wo, cur);690 }691 692 if (wo_b) {693 cur = ggml_add(ctx0, cur, wo_b);694 }695 696 return cur;697}698 699// implementation of the 2D RoPE without adding a new op in ggml700// this is not efficient (use double the memory), but works on all backends701// TODO: there was a more efficient which relies on ggml_view and ggml_rope_ext_inplace, but the rope inplace does not work well with non-contiguous tensors ; we should fix that and revert back to the original implementation in https://github.com/ggml-org/llama.cpp/pull/13065702ggml_tensor * clip_graph::build_rope_2d(703 ggml_context * ctx0,704 ggml_tensor * cur,705 ggml_tensor * pos_a, // first half706 ggml_tensor * pos_b, // second half707 const float freq_base,708 const bool interleave_freq709) {710 const int64_t n_dim = cur->ne[0];711 const int64_t n_head = cur->ne[1];712 const int64_t n_pos = cur->ne[2];713 714 // for example, if we have cur tensor of shape (n_dim=8, n_head, n_pos)715 // we will have a list of 4 inv_freq: 1e-0, 1e-1, 1e-2, 1e-3716 // first half of cur will use 1e-0, 1e-2 (even)717 // second half of cur will use 1e-1, 1e-3 (odd)718 // the trick here is to rotate just half of n_dim, so inv_freq will automatically be even719 // ^ don't ask me why, it's math! -2(2i) / n_dim == -2i / (n_dim/2)720 // then for the second half, we use freq_scale to shift the inv_freq721 // ^ why? replace (2i) with (2i+1) in the above equation722 const float freq_scale_odd = interleave_freq723 ? std::pow(freq_base, (float)-2/n_dim)724 : 1.0;725 726 // first half727 ggml_tensor * first;728 {729 first = ggml_view_3d(ctx0, cur,730 n_dim/2, n_head, n_pos,731 cur->nb[1],732 cur->nb[2],733 0);734 first = ggml_rope_ext(735 ctx0,736 first,737 pos_a, // positions738 nullptr, // freq factors739 n_dim/2, // n_dims740 0, 0, freq_base,741 1.0f, 0.0f, 1.0f, 0.0f, 0.0f742 );743 }744 745 // second half746 ggml_tensor * second;747 {748 second = ggml_view_3d(ctx0, cur,749 n_dim/2, n_head, n_pos,750 cur->nb[1],751 cur->nb[2],752 n_dim/2 * ggml_element_size(cur));753 second = ggml_rope_ext(754 ctx0,755 second,756 pos_b, // positions757 nullptr, // freq factors758 n_dim/2, // n_dims759 0, 0, freq_base,760 freq_scale_odd,761 0.0f, 1.0f, 0.0f, 0.0f762 );763 }764 765 cur = ggml_concat(ctx0, first, second, 0);766 return cur;767}768 769// Generic function to stack frames for audio processing770// Abstracts out the StackAudioFrames logic used by ultravox771ggml_tensor * clip_graph::build_stack(ggml_tensor * cur, int32_t stack_factor, int32_t n_embed) {772 if (stack_factor <= 1) {773 return cur;774 }775 776 int64_t total_elements = ggml_nelements(cur);777 int64_t stride = n_embed * stack_factor;778 779 // Calculate padded length780 int64_t padded_len = GGML_PAD(total_elements, stride);781 int64_t pad = padded_len - total_elements;782 783 if (pad > 0) {784 // Pad the tensor to make it divisible by stride785 cur = ggml_view_1d(ctx0, cur, total_elements, 0);786 cur = ggml_pad(ctx0, cur, pad, 0, 0, 0);787 }788 789 // Reshape to [stride, padded_len / stride]790 cur = ggml_view_2d(ctx0, cur, stride, padded_len / stride,791 ggml_row_size(cur->type, stride), 0);792 return cur;793}794 795// aka pixel_shuffle / pixel_unshuffle / patch_merger (Kimi-VL)796// support dynamic resolution797ggml_tensor * clip_graph::build_patch_merge_permute(ggml_tensor * cur, int scale_factor) {798 GGML_ASSERT(scale_factor > 1);799 800 const int n_embd = cur->ne[0];801 int width = img.nx / patch_size;802 int height = img.ny / patch_size;803 804 // pad width and height to factor805 const int64_t pad_width = CLIP_ALIGN(width, scale_factor) - width;806 const int64_t pad_height = CLIP_ALIGN(height, scale_factor) - height;807 cur = ggml_reshape_3d(ctx0, cur, n_embd, width, height);808 if (pad_width || pad_height) {809 cur = ggml_pad(ctx0, cur, 0, pad_width, pad_height, 0);810 width += pad_width;811 height += pad_height;812 }813 814 // unshuffle h815 cur = ggml_reshape_3d(ctx0, cur, n_embd * scale_factor, width / scale_factor, height);816 cur = ggml_permute(ctx0, cur, 0, 2, 1, 3);817 818 // unshuffle w819 cur = ggml_cont_3d(ctx0, cur, n_embd * scale_factor * scale_factor, height / scale_factor, width / scale_factor);820 cur = ggml_permute(ctx0, cur, 0, 2, 1, 3);821 822 cur = ggml_cont_2d(ctx0, cur, cur->ne[0], cur->ne[1] * cur->ne[2]);823 cb(cur, "pixel_shuffle", -1);824 825 return cur;826}827 828static ggml_cgraph * clip_image_build_graph(clip_ctx * ctx, const clip_image_f32_batch & imgs) {829 GGML_ASSERT(imgs.entries.size() == 1 && "n_batch > 1 is not supported");830 831 const clip_image_f32 & img = *imgs.entries[0];832 std::unique_ptr<clip_graph> builder;833 834 switch (ctx->proj_type()) {835 case PROJECTOR_TYPE_GEMMA3:836 case PROJECTOR_TYPE_IDEFICS3:837 case PROJECTOR_TYPE_LFM2:838 case PROJECTOR_TYPE_JANUS_PRO:839 case PROJECTOR_TYPE_PHI4:840 {841 builder = std::make_unique<clip_graph_siglip>(ctx, img);842 } break;843 case PROJECTOR_TYPE_GEMMA3NV:844 {845 builder = std::make_unique<clip_graph_mobilenetv5>(ctx, img);846 } break;847 case PROJECTOR_TYPE_GEMMA4V:848 {849 builder = std::make_unique<clip_graph_gemma4v>(ctx, img);850 } break;851 case PROJECTOR_TYPE_PIXTRAL:852 case PROJECTOR_TYPE_LIGHTONOCR:853 {854 builder = std::make_unique<clip_graph_pixtral>(ctx, img);855 } break;856 case PROJECTOR_TYPE_DOTS_OCR:857 {858 builder = std::make_unique<clip_graph_dotsocr>(ctx, img);859 } break;860 case PROJECTOR_TYPE_QWEN2VL:861 case PROJECTOR_TYPE_QWEN25VL:862 {863 builder = std::make_unique<clip_graph_qwen2vl>(ctx, img);864 } break;865 case PROJECTOR_TYPE_QWEN3VL:866 {867 builder = std::make_unique<clip_graph_qwen3vl>(ctx, img);868 } break;869 case PROJECTOR_TYPE_STEP3VL:870 {871 builder = std::make_unique<clip_graph_step3vl>(ctx, img);872 } break;873 case PROJECTOR_TYPE_MINICPMV:874 {875 builder = std::make_unique<clip_graph_minicpmv>(ctx, img);876 } break;877 case PROJECTOR_TYPE_INTERNVL:878 {879 builder = std::make_unique<clip_graph_internvl>(ctx, img);880 } break;881 case PROJECTOR_TYPE_NEMOTRON_V2_VL:882 {883 builder = std::make_unique<clip_graph_nemotron_v2_vl>(ctx, img);884 } break;885 case PROJECTOR_TYPE_LLAMA4:886 {887 builder = std::make_unique<clip_graph_llama4>(ctx, img);888 } break;889 case PROJECTOR_TYPE_ULTRAVOX:890 case PROJECTOR_TYPE_VOXTRAL:891 case PROJECTOR_TYPE_QWEN2A:892 case PROJECTOR_TYPE_GLMA:893 case PROJECTOR_TYPE_MERALION:894 case PROJECTOR_TYPE_MUSIC_FLAMINGO:895 {896 builder = std::make_unique<clip_graph_whisper_enc>(ctx, img);897 } break;898 case PROJECTOR_TYPE_KIMIVL:899 {900 builder = std::make_unique<clip_graph_kimivl>(ctx, img);901 } break;902 case PROJECTOR_TYPE_PADDLEOCR:903 {904 builder = std::make_unique<clip_graph_paddleocr>(ctx, img);905 } break;906 case PROJECTOR_TYPE_KIMIK25:907 {908 builder = std::make_unique<clip_graph_kimik25>(ctx, img);909 } break;910 case PROJECTOR_TYPE_COGVLM:911 {912 builder = std::make_unique<clip_graph_cogvlm>(ctx, img);913 } break;914 case PROJECTOR_TYPE_HUNYUANOCR:915 {916 builder = std::make_unique<clip_graph_hunyuanocr>(ctx, img);917 } break;918 case PROJECTOR_TYPE_MLP:919 case PROJECTOR_TYPE_MLP_NORM:920 case PROJECTOR_TYPE_LDP:921 case PROJECTOR_TYPE_LDPV2:922 case PROJECTOR_TYPE_GLM_EDGE:923 {924 builder = std::make_unique<clip_graph_llava>(ctx, img);925 } break;926 case PROJECTOR_TYPE_DEEPSEEKOCR:927 {928 builder = std::make_unique<clip_graph_deepseekocr>(ctx, img);929 } break;930 case PROJECTOR_TYPE_LFM2A:931 {932 builder = std::make_unique<clip_graph_conformer>(ctx, img);933 } break;934 case PROJECTOR_TYPE_GEMMA4A:935 {936 builder = std::make_unique<clip_graph_gemma4a>(ctx, img);937 } break;938 case PROJECTOR_TYPE_GLM4V:939 {940 builder = std::make_unique<clip_graph_glm4v>(ctx, img);941 } break;942 case PROJECTOR_TYPE_QWEN3A:943 {944 builder = std::make_unique<clip_graph_qwen3a>(ctx, img);945 } break;946 case PROJECTOR_TYPE_YOUTUVL:947 {948 builder = std::make_unique<clip_graph_youtuvl>(ctx, img);949 } break;950 default:951 GGML_ABORT("missing cgraph builder");952 }953 954 return builder->build();955}956 957//958// clip_model_loader959//960 961struct clip_model_loader {962 ggml_context_ptr ctx_meta;963 gguf_context_ptr ctx_gguf;964 965 std::string fname;966 967 size_t model_size = 0; // in bytes968 969 bool has_vision = false;970 bool has_audio = false;971 972 // TODO @ngxson : we should not pass clip_ctx here, it should be clip_model973 clip_model_loader(const char * fname) : fname(fname) {974 struct ggml_context * meta = nullptr;975 976 struct gguf_init_params params = {977 /*.no_alloc = */ true,978 /*.ctx = */ &meta,979 };980 981 ctx_gguf = gguf_context_ptr(gguf_init_from_file(fname, params));982 if (!ctx_gguf.get()) {983 throw std::runtime_error(string_format("%s: failed to load CLIP model from %s. Does this file exist?\n", __func__, fname));984 }985 986 ctx_meta.reset(meta);987 988 const int n_tensors = gguf_get_n_tensors(ctx_gguf.get());989 990 // print gguf info991 {992 std::string name;993 get_string(KEY_NAME, name, false);994 std::string description;995 get_string(KEY_DESCRIPTION, description, false);996 LOG_INF("%s: model name: %s\n", __func__, name.c_str());997 LOG_INF("%s: description: %s\n", __func__, description.c_str());998 LOG_INF("%s: GGUF version: %d\n", __func__, gguf_get_version(ctx_gguf.get()));999 LOG_INF("%s: alignment: %zu\n", __func__, gguf_get_alignment(ctx_gguf.get()));1000 LOG_INF("%s: n_tensors: %d\n", __func__, n_tensors);1001 LOG_INF("%s: n_kv: %d\n", __func__, (int)gguf_get_n_kv(ctx_gguf.get()));1002 LOG_INF("\n");1003 }1004 1005 // modalities1006 {1007 get_bool(KEY_HAS_VISION_ENC, has_vision, false);1008 get_bool(KEY_HAS_AUDIO_ENC, has_audio, false);1009 1010 if (has_vision) {1011 LOG_INF("%s: has vision encoder\n", __func__);1012 }1013 if (has_audio) {1014 LOG_INF("%s: has audio encoder\n", __func__);1015 }1016 }1017 1018 // tensors1019 {1020 for (int i = 0; i < n_tensors; ++i) {1021 const char * name = gguf_get_tensor_name(ctx_gguf.get(), i);1022 const size_t offset = gguf_get_tensor_offset(ctx_gguf.get(), i);1023 enum ggml_type type = gguf_get_tensor_type(ctx_gguf.get(), i);1024 ggml_tensor * cur = ggml_get_tensor(meta, name);1025 size_t tensor_size = ggml_nbytes(cur);1026 model_size += tensor_size;1027 LOG_DBG("%s: tensor[%d]: n_dims = %d, name = %s, tensor_size=%zu, offset=%zu, shape:[%" PRIu64 ", %" PRIu64 ", %" PRIu64 ", %" PRIu64 "], type = %s\n",1028 __func__, i, ggml_n_dims(cur), cur->name, tensor_size, offset, cur->ne[0], cur->ne[1], cur->ne[2], cur->ne[3], ggml_type_name(type));1029 }1030 }1031 }1032 1033 void load_hparams(clip_model & model, clip_modality modality) {1034 auto & hparams = model.hparams;1035 std::string log_ffn_op; // for logging1036 1037 // sanity check1038 if (modality == CLIP_MODALITY_VISION) {1039 GGML_ASSERT(has_vision);1040 } else if (modality == CLIP_MODALITY_AUDIO) {1041 GGML_ASSERT(has_audio);1042 }1043 model.modality = modality;1044 1045 1046 // projector type1047 std::string proj_type;1048 {1049 // default key1050 get_string(KEY_PROJ_TYPE, proj_type, false);1051 1052 // for models with mixed modalities1053 if (proj_type.empty()) {1054 if (modality == CLIP_MODALITY_VISION) {1055 get_string(KEY_VISION_PROJ_TYPE, proj_type, false);1056 } else if (modality == CLIP_MODALITY_AUDIO) {1057 get_string(KEY_AUDIO_PROJ_TYPE, proj_type, false);1058 } else {1059 GGML_ABORT("unknown modality");1060 }1061 }1062 1063 model.proj_type = clip_projector_type_from_string(proj_type);1064 1065 if (model.proj_type == PROJECTOR_TYPE_UNKNOWN) {1066 throw std::runtime_error(string_format("%s: unknown projector type: %s\n", __func__, proj_type.c_str()));1067 }1068 1069 // correct arch for multimodal models (legacy method)1070 if (model.proj_type == PROJECTOR_TYPE_QWEN25O) {1071 model.proj_type = modality == CLIP_MODALITY_VISION1072 ? PROJECTOR_TYPE_QWEN25VL1073 : PROJECTOR_TYPE_QWEN2A;1074 }1075 }1076 1077 const bool is_vision = model.modality == CLIP_MODALITY_VISION;1078 const bool is_audio = model.modality == CLIP_MODALITY_AUDIO;1079 1080 // other hparams1081 {1082 const char * prefix = is_vision ? "vision" : "audio";1083 get_u32(string_format(KEY_N_EMBD, prefix), hparams.n_embd);1084 get_u32(string_format(KEY_N_HEAD, prefix), hparams.n_head);1085 get_u32(string_format(KEY_N_FF, prefix), hparams.n_ff);1086 get_u32(string_format(KEY_N_BLOCK, prefix), hparams.n_layer);1087 get_u32(string_format(KEY_PROJ_DIM, prefix), hparams.projection_dim);1088 get_f32(string_format(KEY_LAYER_NORM_EPS, prefix), hparams.eps);1089 1090 if (is_vision) {1091 get_u32(KEY_IMAGE_SIZE, hparams.image_size);1092 get_u32(KEY_PATCH_SIZE, hparams.patch_size);1093 get_i32(KEY_MINICPMV_VERSION, hparams.minicpmv_version, false); // legacy1094 get_u32(KEY_MINICPMV_QUERY_NUM, hparams.minicpmv_query_num, false);1095 if (hparams.minicpmv_query_num == 0) {1096 // Fallback to hardcoded values for legacy models1097 if (hparams.minicpmv_version == 3) {1098 hparams.minicpmv_query_num = 64;1099 } else if (hparams.minicpmv_version == 4) {1100 hparams.minicpmv_query_num = 64;1101 } else if (hparams.minicpmv_version == 5) {1102 hparams.minicpmv_query_num = 64;1103 } else if (hparams.minicpmv_version == 6) {1104 hparams.minicpmv_query_num = 64;1105 } else if (hparams.minicpmv_version == 100045) {1106 hparams.minicpmv_query_num = 64;1107 } else {1108 hparams.minicpmv_query_num = 96;1109 }1110 }1111 } else if (is_audio) {1112 get_u32(KEY_A_NUM_MEL_BINS, hparams.n_mel_bins);1113 // some hparams are unused, but still need to set to avoid issues1114 hparams.image_size = 0;1115 hparams.patch_size = 1;1116 1117 } else {1118 GGML_ASSERT(false && "unknown modality");1119 }1120 1121 // for pinpoints, we need to convert it into a list of resolution candidates1122 {1123 std::vector<int> pinpoints;1124 get_arr_int(KEY_IMAGE_GRID_PINPOINTS, pinpoints, false);1125 if (!pinpoints.empty()) {1126 for (size_t i = 0; i < pinpoints.size(); i += 2) {1127 hparams.image_res_candidates.push_back({1128 pinpoints[i],1129 pinpoints[i+1],1130 });1131 }1132 }1133 }1134 1135 // default warmup value1136 hparams.warmup_image_size = hparams.image_size;1137 1138 {1139 bool use_gelu = false;1140 bool use_silu = false;1141 get_bool(KEY_USE_GELU, use_gelu, false);1142 get_bool(KEY_USE_SILU, use_silu, false);1143 if (use_gelu && use_silu) {1144 throw std::runtime_error(string_format("%s: both use_gelu and use_silu are set to true\n", __func__));1145 }1146 if (use_gelu) {1147 hparams.ffn_op = FFN_GELU;1148 log_ffn_op = "gelu";1149 } else if (use_silu) {1150 hparams.ffn_op = FFN_SILU;1151 log_ffn_op = "silu";1152 } else {1153 hparams.ffn_op = FFN_GELU_QUICK;1154 log_ffn_op = "gelu_quick";1155 }1156 }1157 1158 {1159 std::string mm_patch_merge_type;1160 get_string(KEY_MM_PATCH_MERGE_TYPE, mm_patch_merge_type, false);1161 if (mm_patch_merge_type == "spatial_unpad") {1162 hparams.mm_patch_merge_type = PATCH_MERGE_SPATIAL_UNPAD;1163 }1164 }1165 1166 if (is_vision) {1167 int idx_mean = gguf_find_key(ctx_gguf.get(), KEY_IMAGE_MEAN);1168 int idx_std = gguf_find_key(ctx_gguf.get(), KEY_IMAGE_STD);1169 GGML_ASSERT(idx_mean >= 0 && "image_mean not found");1170 GGML_ASSERT(idx_std >= 0 && "image_std not found");1171 const float * mean_data = (const float *) gguf_get_arr_data(ctx_gguf.get(), idx_mean);1172 const float * std_data = (const float *) gguf_get_arr_data(ctx_gguf.get(), idx_std);1173 for (int i = 0; i < 3; ++i) {1174 hparams.image_mean[i] = mean_data[i];1175 hparams.image_std[i] = std_data[i];1176 }1177 }1178 1179 // Load the vision feature layer indices if they are explicitly provided;1180 // if multiple vision feature layers are present, the values will be concatenated1181 // to form the final visual features.1182 // NOTE: gguf conversions should standardize the values of the vision feature layer to1183 // be non-negative, since we use -1 to mark values as unset here.1184 std::vector<int> vision_feature_layer;1185 get_arr_int(KEY_FEATURE_LAYER, vision_feature_layer, false);1186 // convert std::vector to std::unordered_set1187 for (auto & layer : vision_feature_layer) {1188 hparams.vision_feature_layer.insert(layer);1189 }1190 1191 // model-specific params1192 switch (model.proj_type) {1193 case PROJECTOR_TYPE_MLP:1194 case PROJECTOR_TYPE_MLP_NORM:1195 case PROJECTOR_TYPE_LDP:1196 case PROJECTOR_TYPE_LDPV2:1197 case PROJECTOR_TYPE_COGVLM:1198 {1199 hparams.has_llava_projector = model.proj_type != PROJECTOR_TYPE_COGVLM;1200 hparams.image_pad_color = {122, 116, 104};