echodict/llama.cpp
version https://git-lfs.github.com/spec/v1 oid sha256:cfc44b7ba25614df70e6b65e3341cae0310163bd32fd31a6b928a542df433faf size 30786
0773
1#include "mtmd-image.h"2 3#include <algorithm>4#include <cmath>5#include <vector>6 7//8// base implementation9//10 11void mtmd_image_preprocessor::img_u8_to_f32(const clip_image_u8 & src, clip_image_f32 & dst, const float mean[3], const float std[3]) {12 dst.nx = src.nx;13 dst.ny = src.ny;14 dst.buf.resize(src.buf.size());15 16 // TODO @ngxson : seems like this could be done more efficiently on cgraph17 for (size_t i = 0; i < src.buf.size(); ++i) {18 int c = i % 3; // rgb19 dst.buf[i] = (static_cast<float>(src.buf[i]) / 255.0f - mean[c]) / std[c];20 }21}22 23void mtmd_image_preprocessor::img_u8_to_f32(const clip_image_u8 & src, clip_image_f32 & dst) {24 dst.nx = src.nx;25 dst.ny = src.ny;26 dst.buf.resize(src.buf.size());27 28 for (size_t i = 0; i < src.buf.size(); ++i) {29 dst.buf[i] = static_cast<float>(src.buf[i]);30 }31}32 33// set of tools to manipulate images34// in the future, we can have HW acceleration by allowing this struct to access 3rd party lib like imagick or opencv35struct img_tool {36 static void resize(37 const clip_image_u8 & src,38 clip_image_u8 & dst,39 const clip_image_size & target_resolution,40 resize_algo algo,41 bool add_padding = true, // TODO: define the behavior for add_padding = false42 std::array<uint8_t, 3> pad_color = {0, 0, 0}) {43 dst.nx = target_resolution.width;44 dst.ny = target_resolution.height;45 dst.buf.resize(3 * dst.nx * dst.ny);46 47 if (dst.nx == src.nx && dst.ny == src.ny) {48 // no resize needed, simple copy49 dst.buf = src.buf;50 return;51 }52 53 if (!add_padding) {54 // direct resize55 switch (algo) {56 case RESIZE_ALGO_BILINEAR:57 resize_bilinear(src, dst, target_resolution.width, target_resolution.height);58 break;59 case RESIZE_ALGO_BICUBIC:60 resize_bicubic(src, dst, target_resolution.width, target_resolution.height);61 break;62 case RESIZE_ALGO_BICUBIC_PILLOW:63 resize_bicubic_pillow(src, dst, target_resolution.width, target_resolution.height);64 break;65 default:66 throw std::runtime_error("Unsupported resize algorithm");67 }68 } else {69 // resize with padding70 clip_image_u8 resized_image;71 float scale_w = static_cast<float>(target_resolution.width) / src.nx;72 float scale_h = static_cast<float>(target_resolution.height) / src.ny;73 float scale = std::min(scale_w, scale_h);74 int new_width = std::min(static_cast<int>(std::ceil(src.nx * scale)), target_resolution.width);75 int new_height = std::min(static_cast<int>(std::ceil(src.ny * scale)), target_resolution.height);76 77 switch (algo) {78 case RESIZE_ALGO_BILINEAR:79 resize_bilinear(src, resized_image, new_width, new_height);80 break;81 case RESIZE_ALGO_BICUBIC:82 resize_bicubic(src, resized_image, new_width, new_height);83 break;84 case RESIZE_ALGO_BICUBIC_PILLOW:85 resize_bicubic_pillow(src, resized_image, new_width, new_height);86 break;87 default:88 throw std::runtime_error("Unsupported resize algorithm");89 }90 91 // fill dst with pad_color92 fill(dst, pad_color);93 94 int offset_x = (target_resolution.width - new_width) / 2;95 int offset_y = (target_resolution.height - new_height) / 2;96 97 composite(dst, resized_image, offset_x, offset_y);98 }99 }100 101 static void crop(const clip_image_u8 & image, clip_image_u8 & dst, int x, int y, int w, int h) {102 GGML_ASSERT(x >= 0 && y >= 0 && w > 0 && h > 0);103 GGML_ASSERT(x + w <= image.nx && y + h <= image.ny);104 dst.nx = w;105 dst.ny = h;106 dst.buf.resize(3 * w * h);107 108 for (int i = 0; i < h; ++i) {109 for (int j = 0; j < w; ++j) {110 int src_idx = 3 * ((y + i)*image.nx + (x + j));111 int dst_idx = 3 * (i*w + j);112 dst.buf[dst_idx] = image.buf[src_idx];113 dst.buf[dst_idx + 1] = image.buf[src_idx + 1];114 dst.buf[dst_idx + 2] = image.buf[src_idx + 2];115 }116 }117 }118 119 // calculate the size of the **resized** image, while preserving the aspect ratio120 // the calculated size will be aligned to the nearest multiple of align_size121 // if H or W size is larger than longest_edge, it will be resized to longest_edge122 static clip_image_size calc_size_preserved_ratio(const clip_image_size & inp_size, const int align_size, const int longest_edge) {123 GGML_ASSERT(align_size > 0);124 if (inp_size.width <= 0 || inp_size.height <= 0 || longest_edge <= 0) {125 return {0, 0};126 }127 128 float scale = std::min(static_cast<float>(longest_edge) / inp_size.width,129 static_cast<float>(longest_edge) / inp_size.height);130 131 float target_width_f = static_cast<float>(inp_size.width) * scale;132 float target_height_f = static_cast<float>(inp_size.height) * scale;133 134 auto ceil_by_factor = [f = align_size](float x) { return static_cast<int>(std::ceil(x / static_cast<float>(f))) * f; };135 int aligned_width = ceil_by_factor(target_width_f);136 int aligned_height = ceil_by_factor(target_height_f);137 138 return {aligned_width, aligned_height};139 }140 141 // calculate the size of the **resized** image, while preserving the aspect ratio142 // the calculated size will have min_pixels <= W*H <= max_pixels143 // this is referred as "smart_resize" in transformers code144 static clip_image_size calc_size_preserved_ratio(const clip_image_size & inp_size, const int align_size, const int min_pixels, const int max_pixels) {145 GGML_ASSERT(align_size > 0);146 const int width = inp_size.width;147 const int height = inp_size.height;148 149 auto round_by_factor = [f = align_size](float x) { return static_cast<int>(std::round(x / static_cast<float>(f))) * f; };150 auto ceil_by_factor = [f = align_size](float x) { return static_cast<int>(std::ceil(x / static_cast<float>(f))) * f; };151 auto floor_by_factor = [f = align_size](float x) { return static_cast<int>(std::floor(x / static_cast<float>(f))) * f; };152 153 // always align up first154 int h_bar = std::max(align_size, round_by_factor(height));155 int w_bar = std::max(align_size, round_by_factor(width));156 157 if (h_bar * w_bar > max_pixels) {158 const auto beta = std::sqrt(static_cast<float>(height * width) / max_pixels);159 h_bar = std::max(align_size, floor_by_factor(height / beta));160 w_bar = std::max(align_size, floor_by_factor(width / beta));161 } else if (h_bar * w_bar < min_pixels) {162 const auto beta = std::sqrt(static_cast<float>(min_pixels) / (height * width));163 h_bar = ceil_by_factor(height * beta);164 w_bar = ceil_by_factor(width * beta);165 }166 167 return {w_bar, h_bar};168 }169 170 // draw src image into dst image at offset (offset_x, offset_y)171 static void composite(clip_image_u8 & dst, const clip_image_u8 & src, int offset_x, int offset_y) {172 for (int y = 0; y < src.ny; ++y) {173 for (int x = 0; x < src.nx; ++x) {174 int dx = x + offset_x;175 int dy = y + offset_y;176 // skip pixels that would be out of bounds in the destination177 if (dx < 0 || dy < 0 || dx >= dst.nx || dy >= dst.ny) {178 continue;179 }180 size_t dst_idx = 3 * (static_cast<size_t>(dy) * dst.nx + static_cast<size_t>(dx));181 size_t src_idx = 3 * (static_cast<size_t>(y) * src.nx + static_cast<size_t>(x));182 dst.buf[dst_idx + 0] = src.buf[src_idx + 0];183 dst.buf[dst_idx + 1] = src.buf[src_idx + 1];184 dst.buf[dst_idx + 2] = src.buf[src_idx + 2];185 }186 }187 }188 189 // fill the image with a solid color190 static void fill(clip_image_u8 & img, const std::array<uint8_t, 3> & color) {191 for (size_t i = 0; i < img.buf.size(); i += 3) {192 img.buf[i] = color[0];193 img.buf[i + 1] = color[1];194 img.buf[i + 2] = color[2];195 }196 }197 198private:199 // Bilinear resize function200 static void resize_bilinear(const clip_image_u8 & src, clip_image_u8 & dst, int target_width, int target_height) {201 if (src.nx == 0 || src.ny == 0) { dst.nx = dst.ny = 0; dst.buf.clear(); return; }202 if (target_width <= 0) target_width = 1;203 if (target_height <= 0) target_height = 1;204 205 dst.nx = target_width;206 dst.ny = target_height;207 dst.buf.resize(3 * target_width * target_height);208 209 float x_ratio = target_width > 1 ? static_cast<float>(src.nx - 1) / (target_width - 1) : 0.0f;210 float y_ratio = target_height > 1 ? static_cast<float>(src.ny - 1) / (target_height - 1) : 0.0f;211 212 for (int y = 0; y < target_height; ++y) {213 for (int x = 0; x < target_width; ++x) {214 float px = x * x_ratio;215 float py = y * y_ratio;216 217 int x0 = std::min(static_cast<int>(px), src.nx - 1);218 int y0 = std::min(static_cast<int>(py), src.ny - 1);219 int x1 = std::min(x0 + 1, src.nx - 1);220 int y1 = std::min(y0 + 1, src.ny - 1);221 222 float xf = px - x0;223 float yf = py - y0;224 225 for (int c = 0; c < 3; ++c) {226 float top = lerp(static_cast<float>(src.buf[3 * (y0 * src.nx + x0) + c]),227 static_cast<float>(src.buf[3 * (y0 * src.nx + x1) + c]),228 xf);229 float bottom = lerp(static_cast<float>(src.buf[3 * (y1 * src.nx + x0) + c]),230 static_cast<float>(src.buf[3 * (y1 * src.nx + x1) + c]),231 xf);232 dst.buf[3 * (y * target_width + x) + c] = static_cast<uint8_t>(lerp(top, bottom, yf));233 }234 }235 }236 }237 238 // Bicubic resize function239 // part of image will be cropped if the aspect ratio is different240 static bool resize_bicubic(const clip_image_u8 & img, clip_image_u8 & dst, int target_width, int target_height) {241 const int nx = img.nx;242 const int ny = img.ny;243 244 dst.nx = target_width;245 dst.ny = target_height;246 dst.buf.resize(3 * target_width * target_height);247 248 float Cc;249 float C[5] = {};250 float d0, d2, d3, a0, a1, a2, a3;251 int i, j, k, jj;252 int x, y;253 float dx, dy;254 float tx, ty;255 256 tx = (float)nx / (float)target_width;257 ty = (float)ny / (float)target_height;258 259 // Bicubic interpolation; adapted from ViT.cpp, inspired from :260 // -> https://github.com/yglukhov/bicubic-interpolation-image-processing/blob/master/libimage.c#L36261 // -> https://en.wikipedia.org/wiki/Bicubic_interpolation262 263 for (i = 0; i < target_height; i++) {264 for (j = 0; j < target_width; j++) {265 x = (int)(tx * j);266 y = (int)(ty * i);267 268 dx = tx * j - x;269 dy = ty * i - y;270 271 for (k = 0; k < 3; k++) {272 for (jj = 0; jj <= 3; jj++) {273 d0 = img.buf[(clip(y - 1 + jj, 0, ny - 1) * nx + clip(x - 1, 0, nx - 1)) * 3 + k] - img.buf[(clip(y - 1 + jj, 0, ny - 1) * nx + clip(x, 0, nx - 1)) * 3 + k];274 d2 = img.buf[(clip(y - 1 + jj, 0, ny - 1) * nx + clip(x + 1, 0, nx - 1)) * 3 + k] - img.buf[(clip(y - 1 + jj, 0, ny - 1) * nx + clip(x, 0, nx - 1)) * 3 + k];275 d3 = img.buf[(clip(y - 1 + jj, 0, ny - 1) * nx + clip(x + 2, 0, nx - 1)) * 3 + k] - img.buf[(clip(y - 1 + jj, 0, ny - 1) * nx + clip(x, 0, nx - 1)) * 3 + k];276 a0 = img.buf[(clip(y - 1 + jj, 0, ny - 1) * nx + clip(x, 0, nx - 1)) * 3 + k];277 278 a1 = -1.0 / 3 * d0 + d2 - 1.0 / 6 * d3;279 a2 = 1.0 / 2 * d0 + 1.0 / 2 * d2;280 a3 = -1.0 / 6 * d0 - 1.0 / 2 * d2 + 1.0 / 6 * d3;281 282 C[jj] = a0 + a1 * dx + a2 * dx * dx + a3 * dx * dx * dx;283 284 d0 = C[0] - C[1];285 d2 = C[2] - C[1];286 d3 = C[3] - C[1];287 a0 = C[1];288 a1 = -1.0 / 3 * d0 + d2 - 1.0 / 6 * d3;289 a2 = 1.0 / 2 * d0 + 1.0 / 2 * d2;290 a3 = -1.0 / 6 * d0 - 1.0 / 2 * d2 + 1.0 / 6 * d3;291 Cc = a0 + a1 * dy + a2 * dy * dy + a3 * dy * dy * dy;292 293 const uint8_t Cc2 = std::min(std::max(std::round(Cc), 0.0f), 255.0f);294 dst.buf[(i * target_width + j) * 3 + k] = float(Cc2);295 }296 }297 }298 }299 300 return true;301 }302 303 // Bicubic resize function using Pillow's ImagingResample algorithm304 // Adapted from https://github.com/python-pillow/Pillow/blob/main/src/libImaging/Resample.c305 //306 // Key Difference with resize_bicubic:307 // 1. Uses separable filtering: horizontal pass followed by vertical pass308 // 2. Pre-computes normalized filter coefficients for each output pixel309 // 3. Applies convolution using fixed-point integer arithmetic for performance310 static bool resize_bicubic_pillow(const clip_image_u8 & img, clip_image_u8 & dst, int target_width, int target_height) {311 // Fixed-point precision: 22 bits = 32 (int32_t) - 8 (uint8_t pixels) - 2 (headroom for accumulation)312 // This allows encoding fractional weights as integers: weight * 2^22313 const int PRECISION_BITS = 32 - 8 - 2;314 315 // Bicubic filter function with a = -0.5 (Note that GGML/PyTorch takes a = -0.75)316 // Returns filter weight for distance x from pixel center317 // Support: [-2, 2], meaning the filter influences pixels within 2 units of distance318 auto bicubic_filter = [](double x) -> double {319 constexpr double a = -0.5;320 if (x < 0.0) {321 x = -x;322 }323 if (x < 1.0) {324 return ((a + 2.0) * x - (a + 3.0)) * x * x + 1;325 }326 if (x < 2.0) {327 return (((x - 5) * x + 8) * x - 4) * a;328 }329 return 0.0; // Zero outside [-2, 2]330 };331 332 // Filter support radius: bicubic extends 2 pixels in each direction333 constexpr double filter_support = 2.0;334 335 // Clipping function for 8-bit values336 auto clip8 = [](int val) -> uint8_t {337 if (val < 0) return 0;338 if (val > 255) return 255;339 return static_cast<uint8_t>(val);340 };341 342 // Precompute filter coefficients for ONE dimension (horizontal or vertical)343 //344 // Parameters:345 // inSize - Number of pixels in input dimension (e.g., src_width or src_height)346 // outSize - Number of pixels in output dimension (e.g., target_width or target_height)347 // bounds - [OUTPUT] Array of size outSize*2 storing input pixel ranges:348 // bounds[xx*2+0] = first input pixel index for output pixel xx (xmin)349 // bounds[xx*2+1] = number of input pixels for output pixel xx (xcnt)350 // weights - [OUTPUT] Array of size outSize*ksize storing fixed-point filter weights:351 // kk[xx*ksize + x] = weight for input pixel x contributing to output pixel xx352 //353 // Returns: kernel size (ksize) - number of input pixels that contribute to each output pixel354 auto precompute_weights = [&](int inSize, int outSize,355 std::vector<int> & bounds, std::vector<int32_t> & weights) -> int {356 GGML_ASSERT(inSize > 0 && outSize > 0);357 double support, scale, filterscale;358 double center, ww, ss;359 int xx, x, ksize, xmin, xmax, xcnt;360 361 // Calculate scaling factor: ratio of input range to output size362 filterscale = scale = (double)inSize / outSize;363 // For upsampling (scale < 1), keep filterscale = 1 to maintain filter sharpness364 // For downsampling (scale > 1), widen filter to prevent aliasing365 if (filterscale < 1.0) {366 filterscale = 1.0;367 }368 369 // Determine filter support radius and kernel size370 support = filter_support * filterscale; // Widen filter when downsampling371 ksize = static_cast<int>(std::ceil(support)) * 2 + 1; // Total pixels in kernel372 373 std::vector<double> pre_weights(outSize * ksize); // Temporary weights374 bounds.resize(outSize * 2);375 376 // For each output pixel, compute its filter coefficients377 for (xx = 0; xx < outSize; xx++) {378 // Calculate the center position in input space (pixel-center convention: +0.5)379 center = (xx + 0.5) * scale;380 ww = 0.0; // Sum of weights for normalization381 ss = 1.0 / filterscale; // Scale factor for filter function382 383 // Determine the range of input pixels that contribute to this output pixel384 xmin = static_cast<int>(center - support + 0.5);385 if (xmin < 0) {386 xmin = 0;387 }388 389 xmax = static_cast<int>(center + support + 0.5);390 if (xmax > inSize) {391 xmax = inSize;392 }393 394 xcnt = xmax - xmin;395 396 // Compute filter weights for each contributing input pixel397 for (x = 0; x < xcnt; x++) {398 // Distance from input pixel center to output pixel center in input space399 double w = bicubic_filter((x + xmin - center + 0.5) * ss);400 pre_weights[xx * ksize + x] = w;401 ww += w; // Accumulate for normalization402 }403 404 // Normalize weights to sum to 1.0 (preserves brightness)405 for (x = 0; x < xcnt; x++) {406 if (ww != 0.0) {407 pre_weights[xx * ksize + x] /= ww;408 }409 }410 411 // Zero-pad remaining kernel positions412 for (; x < ksize; x++) {413 pre_weights[xx * ksize + x] = 0;414 }415 416 // Store input pixel range for this output pixel417 bounds[xx * 2 + 0] = xmin;418 bounds[xx * 2 + 1] = xcnt;419 }420 421 // Convert floating-point coefficients to fixed-point integers422 // Formula: int32 = round(float * 2^PRECISION_BITS)423 weights.resize(outSize * ksize);424 for (int i = 0; i < outSize * ksize; i++) {425 if (pre_weights[i] < 0) {426 weights[i] = static_cast<int32_t>(-0.5 + pre_weights[i] * (1 << PRECISION_BITS));427 } else {428 weights[i] = static_cast<int32_t>(0.5 + pre_weights[i] * (1 << PRECISION_BITS));429 }430 }431 432 return ksize;433 };434 435 // Horizontal resampling pass436 // Resizes width from imIn.nx to imOut.nx, preserving height437 auto resample_horizontal = [&](const clip_image_u8 & imIn, clip_image_u8 & imOut,438 int ksize, const std::vector<int> & bounds, const std::vector<int32_t> & weights) {439 imOut.ny = imIn.ny;440 imOut.buf.resize(3 * imOut.nx * imOut.ny);441 442 // Process each row independently443 for (int yy = 0; yy < imOut.ny; yy++) {444 // For each output pixel in this row445 for (int xx = 0; xx < imOut.nx; xx++) {446 // Get the range of input pixels and filter coefficients447 int xmin = bounds[xx * 2 + 0]; // First input pixel index448 int xcnt = bounds[xx * 2 + 1]; // Number of input pixels449 450 // Initialize accumulators for RGB channels with rounding bias (0.5 in fixed-point)451 int32_t ss0 = 1 << (PRECISION_BITS - 1);452 int32_t ss1 = 1 << (PRECISION_BITS - 1);453 int32_t ss2 = 1 << (PRECISION_BITS - 1);454 455 // Convolve: sum weighted input pixels456 for (int x = 0; x < xcnt; x++) {457 int src_idx = ((yy * imIn.nx) + (x + xmin)) * 3;458 ss0 += static_cast<uint8_t>(imIn.buf[src_idx + 0]) * weights[xx * ksize + x]; // R channel459 ss1 += static_cast<uint8_t>(imIn.buf[src_idx + 1]) * weights[xx * ksize + x]; // G channel460 ss2 += static_cast<uint8_t>(imIn.buf[src_idx + 2]) * weights[xx * ksize + x]; // B channel461 }462 463 // Convert back from fixed-point (divide by 2^PRECISION_BITS) and clamp to [0,255]464 int dst_idx = (yy * imOut.nx + xx) * 3;465 imOut.buf[dst_idx + 0] = clip8(ss0 >> PRECISION_BITS);466 imOut.buf[dst_idx + 1] = clip8(ss1 >> PRECISION_BITS);467 imOut.buf[dst_idx + 2] = clip8(ss2 >> PRECISION_BITS);468 }469 }470 };471 472 // Vertical resampling pass473 // Resizes height from imIn.ny to imOut.ny, preserving width474 auto resample_vertical = [&](const clip_image_u8 & imIn, clip_image_u8 & imOut,475 int ksize, const std::vector<int> & bounds, const std::vector<int32_t> & weight) {476 imOut.nx = imIn.nx;477 imOut.buf.resize(3 * imOut.nx * imOut.ny);478 479 // For each output row480 for (int yy = 0; yy < imOut.ny; yy++) {481 // Get the range of input rows and filter coefficients482 int ymin = bounds[yy * 2 + 0]; // First input row index483 int ycnt = bounds[yy * 2 + 1]; // Number of input rows484 485 // Process each column in this output row486 for (int xx = 0; xx < imOut.nx; xx++) {487 // Initialize accumulators for RGB channels with rounding bias488 int32_t ss0 = 1 << (PRECISION_BITS - 1);489 int32_t ss1 = 1 << (PRECISION_BITS - 1);490 int32_t ss2 = 1 << (PRECISION_BITS - 1);491 492 // Convolve: sum weighted input pixels vertically493 for (int y = 0; y < ycnt; y++) {494 int src_idx = ((y + ymin) * imIn.nx + xx) * 3;495 ss0 += static_cast<uint8_t>(imIn.buf[src_idx + 0]) * weight[yy * ksize + y]; // R channel496 ss1 += static_cast<uint8_t>(imIn.buf[src_idx + 1]) * weight[yy * ksize + y]; // G channel497 ss2 += static_cast<uint8_t>(imIn.buf[src_idx + 2]) * weight[yy * ksize + y]; // B channel498 }499 500 // Convert back from fixed-point and clamp to [0,255]501 int dst_idx = (yy * imOut.nx + xx) * 3;502 imOut.buf[dst_idx + 0] = clip8(ss0 >> PRECISION_BITS);503 imOut.buf[dst_idx + 1] = clip8(ss1 >> PRECISION_BITS);504 imOut.buf[dst_idx + 2] = clip8(ss2 >> PRECISION_BITS);505 }506 }507 };508 509 // Main resampling logic using separable two-pass approach510 const int src_width = img.nx;511 const int src_height = img.ny;512 513 dst.nx = target_width;514 dst.ny = target_height;515 516 bool need_horizontal = (target_width != src_width);517 bool need_vertical = (target_height != src_height);518 519 // Precompute filter coefficients for both dimensions520 std::vector<int> bounds_horiz, bounds_vert;521 std::vector<int32_t> weights_horiz, weights_vert;522 int ksize_horiz = 0, ksize_vert = 0;523 524 if (need_horizontal) {525 ksize_horiz = precompute_weights(src_width, target_width, bounds_horiz, weights_horiz);526 }527 528 if (need_vertical) {529 ksize_vert = precompute_weights(src_height, target_height, bounds_vert, weights_vert);530 }531 532 // Perform two-pass resampling533 if (need_horizontal && need_vertical) {534 // Both horizontal and vertical535 clip_image_u8 temp;536 temp.nx = target_width;537 resample_horizontal(img, temp, ksize_horiz, bounds_horiz, weights_horiz);538 resample_vertical(temp, dst, ksize_vert, bounds_vert, weights_vert);539 } else if (need_horizontal) {540 // Only horizontal541 resample_horizontal(img, dst, ksize_horiz, bounds_horiz, weights_horiz);542 } else if (need_vertical) {543 // Only vertical544 resample_vertical(img, dst, ksize_vert, bounds_vert, weights_vert);545 } else {546 // No resizing needed - direct copy547 dst.buf = img.buf;548 }549 550 return true;551 }552 553 static inline int clip(int x, int lower, int upper) {554 return std::max(lower, std::min(x, upper));555 }556 557 // Linear interpolation between two points558 static inline float lerp(float s, float e, float t) {559 return s + (e - s) * t;560 }561};562 563 564//565// mtmd_image_preprocessor_llava_uhd566//567 568bool mtmd_image_preprocessor_llava_uhd::preprocess(const clip_image_u8 & img, clip_image_f32_batch & output) {569 const clip_image_size original_size{img.nx, img.ny};570 auto const inst = get_slice_instructions(original_size);571 std::vector<clip_image_u8_ptr> imgs = slice_image(img, inst);572 573 for (size_t i = 0; i < imgs.size(); ++i) {574 // clip_image_save_to_bmp(*imgs[i], "slice_" + std::to_string(i) + ".bmp");575 clip_image_f32_ptr res(clip_image_f32_init());576 img_u8_to_f32(*imgs[i], *res, hparams.image_mean, hparams.image_std);577 output.entries.push_back(std::move(res));578 }579 580 output.grid_x = inst.grid_size.width;581 output.grid_y = inst.grid_size.height;582 return true;583}584 585mtmd_image_preprocessor_llava_uhd::slice_instructions mtmd_image_preprocessor_llava_uhd::get_slice_instructions(const clip_image_size & original_size) {586 mtmd_image_preprocessor_llava_uhd::slice_instructions res;587 const int patch_size = hparams.patch_size;588 const int slice_size = hparams.image_size;589 const int original_width = original_size.width;590 const int original_height = original_size.height;591 592 const bool has_slices = original_size.width > slice_size || original_size.height > slice_size;593 const bool has_pinpoints = !hparams.image_res_candidates.empty();594 595 if (!has_slices) {596 // skip slicing logic597 res.overview_size = clip_image_size{slice_size, slice_size};598 res.refined_size = clip_image_size{0, 0};599 res.grid_size = clip_image_size{0, 0};600 601 return res;602 }603 604 if (has_pinpoints) {605 // has pinpoints, use them to calculate the grid size (e.g. llava-1.6)606 auto refine_size = select_best_resolution(607 original_size,608 hparams.image_res_candidates);609 res.overview_size = clip_image_size{slice_size, slice_size};610 res.refined_size = refine_size;611 res.grid_size = clip_image_size{0, 0};612 613 LOG_DBG("%s: using pinpoints for slicing\n", __func__);614 LOG_DBG("%s: original size: %d x %d, overview size: %d x %d, refined size: %d x %d\n",615 __func__, original_width, original_height,616 res.overview_size.width, res.overview_size.height,617 res.refined_size.width, res.refined_size.height);618 619 for (int y = 0; y < refine_size.height; y += slice_size) {620 for (int x = 0; x < refine_size.width; x += slice_size) {621 slice_coordinates slice;622 slice.x = x;623 slice.y = y;624 slice.size.width = std::min(slice_size, refine_size.width - x);625 slice.size.height = std::min(slice_size, refine_size.height - y);626 res.slices.push_back(slice);627 LOG_DBG("%s: slice %d: x=%d, y=%d, size=%dx%d\n",628 __func__, (int)res.slices.size() - 1,629 slice.x, slice.y, slice.size.width, slice.size.height);630 }631 }632 633 res.grid_size.height = refine_size.height / slice_size;634 res.grid_size.width = refine_size.width / slice_size;635 LOG_DBG("%s: grid size: %d x %d\n", __func__, res.grid_size.width, res.grid_size.height);636 637 return res;638 }639 640 // no pinpoints, dynamically calculate the grid size (e.g. minicpmv)641 642 auto best_size = get_best_resize(original_size, slice_size, patch_size, !has_slices);643 res.overview_size = best_size;644 645 {646 const int max_slice_nums = 9; // TODO: this is only used by minicpmv, maybe remove it647 const float log_ratio = log((float)original_width / original_height);648 const float ratio = (float)original_width * original_height / (slice_size * slice_size);649 const int multiple = fmin(ceil(ratio), max_slice_nums);650 651 auto best_grid = get_best_grid(max_slice_nums, multiple, log_ratio);652 auto refine_size = get_refine_size(original_size, best_grid, slice_size, patch_size, true);653 res.grid_size = best_grid;654 res.refined_size = refine_size;655 656 LOG_DBG("%s: original size: %d x %d, overview size: %d x %d, refined size: %d x %d, grid size: %d x %d\n",657 __func__, original_width, original_height,658 res.overview_size.width, res.overview_size.height,659 res.refined_size.width, res.refined_size.height,660 res.grid_size.width, res.grid_size.height);661 662 int width = refine_size.width;663 int height = refine_size.height;664 int grid_x = int(width / best_grid.width);665 int grid_y = int(height / best_grid.height);666 for (int patches_y = 0, ic = 0;667 patches_y < refine_size.height && ic < best_grid.height;668 patches_y += grid_y, ic += 1) {669 for (int patches_x = 0, jc = 0;670 patches_x < refine_size.width && jc < best_grid.width;671 patches_x += grid_x, jc += 1) {672 slice_coordinates slice;673 slice.x = patches_x;674 slice.y = patches_y;675 slice.size.width = grid_x;676 slice.size.height = grid_y;677 res.slices.push_back(slice);678 LOG_DBG("%s: slice %d: x=%d, y=%d, size=%dx%d\n",679 __func__, (int)res.slices.size() - 1,680 slice.x, slice.y, slice.size.width, slice.size.height);681 }682 }683 }684 685 return res;686}687 688std::vector<clip_image_u8_ptr> mtmd_image_preprocessor_llava_uhd::slice_image(const clip_image_u8 & img, const mtmd_image_preprocessor_llava_uhd::slice_instructions & inst, bool overview_first) {689 std::vector<clip_image_u8_ptr> output;690 691 // resize to overview size692 clip_image_u8_ptr resized_img(clip_image_u8_init());693 img_tool::resize(img, *resized_img, inst.overview_size, hparams.image_resize_algo_ov,694 hparams.image_pad_ov, hparams.image_pad_color_ov);695 if (overview_first) {696 output.push_back(std::move(resized_img));697 }698 699 if (inst.slices.empty()) {700 // no slices, just return the resized image701 if (!overview_first) {702 output.push_back(std::move(resized_img));703 }704 return output;705 }706 707 // resize to refined size708 clip_image_u8_ptr refined_img(clip_image_u8_init());709 img_tool::resize(img, *refined_img, inst.refined_size, hparams.image_resize_algo_rf,710 hparams.image_pad_rf, hparams.image_pad_color_rf);711 712 // create slices713 for (const auto & slice : inst.slices) {714 int x = slice.x;715 int y = slice.y;716 int w = slice.size.width;717 int h = slice.size.height;718 719 clip_image_u8_ptr img_slice(clip_image_u8_init());720 img_tool::crop(*refined_img, *img_slice, x, y, w, h);721 output.push_back(std::move(img_slice));722 }723 724 if (!overview_first) {725 output.push_back(std::move(resized_img));726 }727 728 return output;729}730 731clip_image_size mtmd_image_preprocessor_llava_uhd::get_best_resize(const clip_image_size & original_size, int scale_resolution, int patch_size, bool allow_upscale) {732 int width = original_size.width;733 int height = original_size.height;734 if ((width * height > scale_resolution * scale_resolution) || allow_upscale) {735 float r = static_cast<float>(width) / height;736 height = static_cast<int>(scale_resolution / std::sqrt(r));737 width = static_cast<int>(height * r);738 }739 clip_image_size res;740 res.width = ensure_divide(width, patch_size);741 res.height = ensure_divide(height, patch_size);742 return res;743}744 745clip_image_size mtmd_image_preprocessor_llava_uhd::resize_maintain_aspect_ratio(const clip_image_size & orig, const clip_image_size & target_max) {746 float scale_width = static_cast<float>(target_max.width) / orig.width;747 float scale_height = static_cast<float>(target_max.height) / orig.height;748 float scale = std::min(scale_width, scale_height);749 return clip_image_size{750 static_cast<int>(orig.width * scale),751 static_cast<int>(orig.height * scale),752 };753}754 755clip_image_size mtmd_image_preprocessor_llava_uhd::select_best_resolution(const clip_image_size & original_size, const std::vector<clip_image_size> & possible_resolutions) {756 clip_image_size best_fit;757 int min_wasted_area = std::numeric_limits<int>::max();758 int max_effective_resolution = 0;759 760 for (const clip_image_size & candidate : possible_resolutions) {761 auto target_size = resize_maintain_aspect_ratio(original_size, candidate);762 int effective_resolution = std::min(763 target_size.width * target_size.height,764 original_size.width * original_size.height);765 int wasted_area = (candidate.width * candidate.height) - effective_resolution;766 767 if (effective_resolution > max_effective_resolution || (effective_resolution == max_effective_resolution && wasted_area < min_wasted_area)) {768 max_effective_resolution = effective_resolution;769 min_wasted_area = wasted_area;770 best_fit = candidate;771 }772 773 LOG_DBG("%s: candidate: %d x %d, target: %d x %d, wasted: %d, effective: %d\n", __func__, candidate.width, candidate.height, target_size.width, target_size.height, wasted_area, effective_resolution);774 }775 776 return best_fit;777}778 779int mtmd_image_preprocessor_llava_uhd::ensure_divide(int length, int patch_size) {780 return std::max(static_cast<int>(std::round(static_cast<float>(length) / patch_size) * patch_size), patch_size);781}782 783clip_image_size mtmd_image_preprocessor_llava_uhd::get_refine_size(const clip_image_size & original_size, const clip_image_size & grid, int scale_resolution, int patch_size, bool allow_upscale) {784 int width = original_size.width;785 int height = original_size.height;786 int grid_x = grid.width;787 int grid_y = grid.height;788 789 int refine_width = ensure_divide(width, grid_x);790 int refine_height = ensure_divide(height, grid_y);791 792 clip_image_size grid_size;793 grid_size.width = refine_width / grid_x;794 grid_size.height = refine_height / grid_y;795 796 auto best_grid_size = get_best_resize(grid_size, scale_resolution, patch_size, allow_upscale);797 int best_grid_width = best_grid_size.width;798 int best_grid_height = best_grid_size.height;799 800 clip_image_size refine_size;801 refine_size.width = best_grid_width * grid_x;802 refine_size.height = best_grid_height * grid_y;803 return refine_size;804}805 806clip_image_size mtmd_image_preprocessor_llava_uhd::get_best_grid(const int max_slice_nums, const int multiple, const float log_ratio) {807 std::vector<int> candidate_split_grids_nums;808 for (int i : {multiple - 1, multiple, multiple + 1}) {809 if (i == 1 || i > max_slice_nums) {810 continue;811 }812 candidate_split_grids_nums.push_back(i);813 }814 815 std::vector<clip_image_size> candidate_grids;816 for (int split_grids_nums : candidate_split_grids_nums) {817 int m = 1;818 while (m <= split_grids_nums) {819 if (split_grids_nums % m == 0) {820 candidate_grids.push_back(clip_image_size{m, split_grids_nums / m});821 }822 ++m;823 }824 }825 826 clip_image_size best_grid{1, 1};827 float min_error = std::numeric_limits<float>::infinity();828 for (const auto& grid : candidate_grids) {829 float error = std::abs(log_ratio - std::log(1.0 * grid.width / grid.height));830 if (error < min_error) {831 best_grid = grid;832 min_error = error;833 }834 }835 return best_grid;836}837 838//839// mtmd_image_preprocessor_fixed_size840//841 842bool mtmd_image_preprocessor_fixed_size::preprocess(const clip_image_u8 & img, clip_image_f32_batch & output) {843 clip_image_u8 resized_image;844 int sz = hparams.image_size;845 img_tool::resize(img, resized_image, {sz, sz},846 hparams.image_resize_algo,847 hparams.image_resize_pad,848 hparams.image_pad_color);849 clip_image_f32_ptr img_f32(clip_image_f32_init());850 img_u8_to_f32(resized_image, *img_f32, hparams.image_mean, hparams.image_std);851 output.entries.push_back(std::move(img_f32));852 return true;853}854 855//856// mtmd_image_preprocessor_dyn_size857//858 859bool mtmd_image_preprocessor_dyn_size::preprocess(const clip_image_u8 & img, clip_image_f32_batch & output) {860 GGML_ASSERT(hparams.image_min_pixels > 0 && hparams.image_max_pixels > 0);861 clip_image_u8 resized_image;862 const clip_image_size original_size{img.nx, img.ny};863 // the original pixtral model doesn't have n_merge864 const int cur_merge = hparams.n_merge == 0 ? 1 : hparams.n_merge;865 const clip_image_size target_size = img_tool::calc_size_preserved_ratio(866 original_size,867 hparams.patch_size * cur_merge,868 hparams.image_min_pixels,869 hparams.image_max_pixels);870 img_tool::resize(img, resized_image, target_size,871 hparams.image_resize_algo,872 hparams.image_resize_pad,873 hparams.image_pad_color);874 clip_image_f32_ptr img_f32(clip_image_f32_init());875 img_u8_to_f32(resized_image, *img_f32, hparams.image_mean, hparams.image_std);876 output.entries.push_back(std::move(img_f32));877 return true;878}879 880//881// mtmd_image_preprocessor_longest_edge882//883 884bool mtmd_image_preprocessor_longest_edge::preprocess(const clip_image_u8 & img, clip_image_f32_batch & output) {885 GGML_ASSERT(hparams.image_longest_edge > 0);886 clip_image_u8 resized_image;887 const clip_image_size original_size{img.nx, img.ny};888 // the original pixtral model doesn't have n_merge889 const int cur_merge = hparams.n_merge == 0 ? 1 : hparams.n_merge;890 const clip_image_size target_size = img_tool::calc_size_preserved_ratio(891 original_size,892 hparams.patch_size * cur_merge,893 hparams.image_longest_edge);894 img_tool::resize(img, resized_image, target_size,895 hparams.image_resize_algo,896 hparams.image_resize_pad,897 hparams.image_pad_color);898 clip_image_f32_ptr img_f32(clip_image_f32_init());899 img_u8_to_f32(resized_image, *img_f32, hparams.image_mean, hparams.image_std);900 output.entries.push_back(std::move(img_f32));901 return true;902}903 904//905// mtmd_image_preprocessor_lfm2906//907 908mtmd_image_preprocessor_llava_uhd::slice_instructions mtmd_image_preprocessor_lfm2::get_slice_instructions(const clip_image_size & original_size) {909 mtmd_image_preprocessor_llava_uhd::slice_instructions inst;910 const int align_size = hparams.patch_size * hparams.n_merge;911 inst.overview_size = img_tool::calc_size_preserved_ratio(912 original_size, align_size,913 hparams.image_min_pixels, hparams.image_max_pixels);914 // tile if either dimension exceeds tile_size with tolerance915 const bool needs_tiling = original_size.width > tile_size * max_pixels_tolerance || original_size.height > tile_size * max_pixels_tolerance;916 917 if (!needs_tiling) {918 inst.refined_size = clip_image_size{0, 0};919 inst.grid_size = clip_image_size{0, 0};920 return inst;921 }922 923 const clip_image_size grid = get_grid_layout(original_size.height, original_size.width);924 925 inst.grid_size = grid;926 inst.refined_size = clip_image_size{tile_size * grid.width, tile_size * grid.height};927 928 LOG_DBG("%s: original size: %d x %d, overview size: %d x %d, refined size: %d x %d, grid size: %d x %d\n",929 __func__,930 original_size.width, original_size.height,931 inst.overview_size.width, inst.overview_size.height,932 inst.refined_size.width, inst.refined_size.height,933 grid.width, grid.height);934 935 for (int row = 0; row < grid.height; row++) {936 for (int col = 0; col < grid.width; col++) {937 mtmd_image_preprocessor_llava_uhd::slice_coordinates slice;938 slice.x = col * tile_size;939 slice.y = row * tile_size;940 slice.size = clip_image_size{tile_size, tile_size};941 inst.slices.push_back(slice);942 LOG_DBG("%s: slice %d: x=%d, y=%d, size=%d x %d\n",943 __func__, (int)inst.slices.size() - 1,944 slice.x, slice.y, slice.size.width, slice.size.height);945 }946 }947 948 return inst;949}950 951clip_image_size mtmd_image_preprocessor_lfm2::find_closest_aspect_ratio(952 float aspect_ratio,953 const std::vector<clip_image_size> & target_ratios,954 int width, int height) {955 float best_ratio_diff = std::numeric_limits<float>::max();956 clip_image_size best_ratio = {1, 1};957 const float area = static_cast<float>(width * height);958 959 for (const auto & ratio : target_ratios) {960 const float target_aspect_ratio = static_cast<float>(ratio.width) / ratio.height;961 const float ratio_diff = std::abs(aspect_ratio - target_aspect_ratio);962 if (ratio_diff < best_ratio_diff) {963 best_ratio_diff = ratio_diff;964 best_ratio = ratio;965 } else if (ratio_diff == best_ratio_diff) {966 const float target_area = static_cast<float>(tile_size * tile_size * ratio.width * ratio.height);967 if (area > 0.5f * target_area) {968 best_ratio = ratio;969 }970 }971 }972 return best_ratio;973}974 975std::vector<clip_image_size> mtmd_image_preprocessor_lfm2::get_target_ratios() {976 std::vector<clip_image_size> ratios;977 for (int n = min_tiles; n <= max_tiles; n++) {978 for (int w = 1; w <= n; w++) {979 for (int h = 1; h <= n; h++) {980 if (w * h >= min_tiles && w * h <= max_tiles) {981 bool found = false;982 for (const auto & r : ratios) {983 if (r.width == w && r.height == h) {984 found = true;985 break;986 }987 }988 if (!found) {989 ratios.push_back({w, h});990 }991 }992 }993 }994 }995 std::sort(ratios.begin(), ratios.end(), [](const clip_image_size & a, const clip_image_size & b) {996 return a.width * a.height < b.width * b.height;997 });998 return ratios;999}1000 1001clip_image_size mtmd_image_preprocessor_lfm2::get_grid_layout(int height, int width) {1002 const float aspect_ratio = static_cast<float>(width) / height;1003 const auto ratios = get_target_ratios();1004 return find_closest_aspect_ratio(aspect_ratio, ratios, width, height);1005}1006 1007//1008// mtmd_image_preprocessor_idefics31009//1010 1011bool mtmd_image_preprocessor_idefics3::preprocess(const clip_image_u8 & img, clip_image_f32_batch & output) {1012 // The refined size has two steps:1013 // 1. Resize w/ aspect-ratio preserving such that the longer side is1014 // the preprocessor longest size1015 // 2. Resize w/out preserving aspect ratio such that both sides are1016 // multiples of image_size (always rounding up)1017 //1018 // CITE: https://github.com/huggingface/transformers/blob/main/src/transformers/models/idefics3/image_processing_idefics3.py#L7371019 const clip_image_size original_size{img.nx, img.ny};1020 const clip_image_size refined_size = img_tool::calc_size_preserved_ratio(1021 original_size, hparams.image_size, hparams.image_longest_edge);1022 // LOG_INF("%s: original size: %d x %d, refined size: %d x %d\n",1023 // __func__, original_size.width, original_size.height,1024 // refined_size.width, refined_size.height);1025 1026 mtmd_image_preprocessor_llava_uhd::slice_instructions instructions;1027 instructions.overview_size = clip_image_size{hparams.image_size, hparams.image_size};1028 instructions.refined_size = refined_size;1029 instructions.grid_size = clip_image_size{1030 static_cast<int>(std::ceil(static_cast<float>(refined_size.width) / hparams.image_size)),1031 static_cast<int>(std::ceil(static_cast<float>(refined_size.height) / hparams.image_size)),1032 };1033 for (int y = 0; y < refined_size.height; y += hparams.image_size) {1034 for (int x = 0; x < refined_size.width; x += hparams.image_size) {1035 // LOG_INF("%s: adding slice at x=%d, y=%d\n", __func__, x, y);1036 instructions.slices.push_back(mtmd_image_preprocessor_llava_uhd::slice_coordinates{1037 /* x */x,1038 /* y */y,1039 /* size */clip_image_size{1040 std::min(hparams.image_size, refined_size.width - x),1041 std::min(hparams.image_size, refined_size.height - y)1042 }1043 });1044 }1045 }1046 auto imgs = slice_image(img, instructions);1047 1048 // cast and normalize to f321049 for (size_t i = 0; i < imgs.size(); ++i) {1050 // clip_image_save_to_bmp(*imgs[i], "slice_" + std::to_string(i) + ".bmp");1051 clip_image_f32_ptr res(clip_image_f32_init());1052 img_u8_to_f32(*imgs[i], *res, hparams.image_mean, hparams.image_std);1053 output.entries.push_back(std::move(res));1054 }1055 1056 output.grid_x = instructions.grid_size.width;1057 output.grid_y = instructions.grid_size.height;1058 return true;1059}1060 1061//1062// mtmd_image_preprocessor_internvl1063//1064 1065bool mtmd_image_preprocessor_internvl::preprocess(const clip_image_u8 & img, clip_image_f32_batch & output) {1066 GGML_ASSERT(!hparams.image_res_candidates.empty());1067 const clip_image_size original_size{img.nx, img.ny};1068 auto const inst = get_slice_instructions(original_size);1069 std::vector<clip_image_u8_ptr> imgs = slice_image(img, inst, false);1070 1071 for (size_t i = 0; i < imgs.size(); ++i) {1072 clip_image_f32_ptr res(clip_image_f32_init());1073 img_u8_to_f32(*imgs[i], *res, hparams.image_mean, hparams.image_std);1074 output.entries.push_back(std::move(res));1075 }1076 return true;1077}1078 1079//1080// mtmd_image_preprocessor_deepseekocr1081//1082 1083bool mtmd_image_preprocessor_deepseekocr::preprocess(const clip_image_u8 & img, clip_image_f32_batch & output) {1084 const std::vector native_resolutions = {1085 /*512 tiny , 640 small, */ 1024 /* base */, 1280 /* large */1086 };1087 // original image size1088 const clip_image_size original_size{img.nx, img.ny};1089 const int orig_w = original_size.width;1090 const int orig_h = original_size.height;1091 const int orig_area = orig_h * orig_w;1092 1093 size_t mode_i = 0;1094 int min_diff = orig_area;1095 1096 for (size_t i = 0; i < native_resolutions.size(); i++) {1097 int r = native_resolutions[i];1098 if (std::abs(orig_area - r * r) < min_diff) {1099 mode_i = i;1100 min_diff = std::abs(orig_area - r * r);1101 }1102 }1103 1104 /* Native Resolution (Base/Large) */1105 const int image_size = native_resolutions[mode_i];1106 1107 // scaled and padded image1108 clip_image_u8_ptr scaled_img(clip_image_u8_init());1109 img_tool::resize(img, *scaled_img, clip_image_size{image_size, image_size}, hparams.image_resize_algo);1110 1111 clip_image_f32_ptr res(clip_image_f32_init());1112 img_u8_to_f32(*scaled_img, *res, hparams.image_mean, hparams.image_std);1113 output.entries.push_back(std::move(res));1114 1115 output.grid_x = 1;1116 output.grid_y = 1;1117 return true;1118}1119 1120//1121// mtmd_image_preprocessor_step3vl1122//1123 1124void mtmd_image_preprocessor_step3vl::img_u8_resize_bilinear_to_f32(1125 const clip_image_u8 & src,1126 clip_image_f32 & dst,1127 int target_width,1128 int target_height,1129 const float mean[3],1130 const float std[3]) {1131 if (src.nx == target_width && src.ny == target_height) {1132 img_u8_to_f32(src, dst, mean, std);1133 return;1134 }1135 1136 dst.nx = target_width;1137 dst.ny = target_height;1138 dst.buf.resize(3 * target_width * target_height);1139 1140 const float scale_x = static_cast<float>(src.nx) / target_width;1141 const float scale_y = static_cast<float>(src.ny) / target_height;1142 1143 for (int y = 0; y < target_height; ++y) {1144 const float src_y = (static_cast<float>(y) + 0.5f) * scale_y - 0.5f;1145 const int y0_floor = static_cast<int>(std::floor(src_y));1146 const int y0 = std::max(0, std::min(y0_floor, src.ny - 1));1147 const int y1 = std::max(0, std::min(y0_floor + 1, src.ny - 1));1148 const float ly = src_y - y0_floor;1149 1150 for (int x = 0; x < target_width; ++x) {1151 const float src_x = (static_cast<float>(x) + 0.5f) * scale_x - 0.5f;1152 const int x0_floor = static_cast<int>(std::floor(src_x));1153 const int x0 = std::max(0, std::min(x0_floor, src.nx - 1));1154 const int x1 = std::max(0, std::min(x0_floor + 1, src.nx - 1));1155 const float lx = src_x - x0_floor;1156 1157 const size_t idx00 = 3 * (y0 * src.nx + x0);1158 const size_t idx01 = 3 * (y0 * src.nx + x1);1159 const size_t idx10 = 3 * (y1 * src.nx + x0);1160 const size_t idx11 = 3 * (y1 * src.nx + x1);1161 const size_t idx_dst = 3 * (y * target_width + x);1162 1163 for (int c = 0; c < 3; ++c) {1164 const float v00 = (static_cast<float>(src.buf[idx00 + c]) / 255.0f - mean[c]) / std[c];1165 const float v01 = (static_cast<float>(src.buf[idx01 + c]) / 255.0f - mean[c]) / std[c];1166 const float v10 = (static_cast<float>(src.buf[idx10 + c]) / 255.0f - mean[c]) / std[c];1167 const float v11 = (static_cast<float>(src.buf[idx11 + c]) / 255.0f - mean[c]) / std[c];1168 1169 const float top = v00 + (v01 - v00) * lx;1170 const float bot = v10 + (v11 - v10) * lx;1171 dst.buf[idx_dst + c] = top + (bot - top) * ly;1172 }1173 }1174 }1175}1176 1177int mtmd_image_preprocessor_step3vl::get_image_longest_edge(const clip_hparams & params) {1178 return params.image_longest_edge > 0 ? params.image_longest_edge : default_image_longest_edge;1179}1180 1181int mtmd_image_preprocessor_step3vl::determine_window_size(const clip_hparams & params, int longer, int shorter) {1182 const int image_size = params.image_size;1183 const int crop_size = default_image_crop_size;1184 const float aspect_ratio = static_cast<float>(longer) / shorter;1185 1186 if (longer <= image_size) {1187 return aspect_ratio > small_aspect_ratio_limit ? shorter : 0;1188 }1189 1190 return aspect_ratio > wide_aspect_ratio_limit ? std::min(shorter, crop_size) : crop_size;1191}1192 1193int mtmd_image_preprocessor_step3vl::calc_crop_extent(int length, int window_size) {1194 const float ratio = static_cast<float>(length) / window_size;1195 if (ratio < 1.0f) {1196 return length;1197 }1198 1199 const float decimal = ratio - std::floor(ratio);1200 const int rounded = decimal > crop_rounding_threshold