echodict/llama.cpp

1#pragma once2 3#include "ggml.h"4#include "clip-model.h"5 6#include <vector>7#include <string>8 9#define MTMD_INTERNAL_HEADER10 11// base class, models must inherit from this class12struct mtmd_image_preprocessor {13    const clip_hparams & hparams;14 15    mtmd_image_preprocessor(const clip_ctx * ctx): hparams(*clip_get_hparams(ctx)) {}16 17    virtual ~mtmd_image_preprocessor() = default;18    virtual bool preprocess(const clip_image_u8 & img, clip_image_f32_batch & output) = 0;19 20    void img_u8_to_f32(const clip_image_u8 & src, clip_image_f32 & dst, const float mean[3], const float std[3]);21    void img_u8_to_f32(const clip_image_u8 & src, clip_image_f32 & dst);22};23 24/**25 * implementation of LLaVA-UHD:26 *  - https://arxiv.org/pdf/2403.1170327 *  - https://github.com/thunlp/LLaVA-UHD28 *  - https://github.com/thunlp/LLaVA-UHD/blob/302301bc2175f7e717fb8548516188e89f649753/llava_uhd/train/llava-uhd/slice_logic.py#L11829 *30 * overview:31 *   - an image always have a single overview (downscaled image)32 *   - an image can have 0 or multiple slices, depending on the image size33 *   - each slice can then be considered as a separate image34 *35 * note: the term "slice" and "tile" are used interchangeably36 *37 * for example:38 *39 * [overview] --> [slice 1] --> [slice 2]40 *           |                |41 *           +--> [slice 3] --> [slice 4]42 */43struct mtmd_image_preprocessor_llava_uhd : mtmd_image_preprocessor {44    mtmd_image_preprocessor_llava_uhd(const clip_ctx * ctx) : mtmd_image_preprocessor(ctx) {}45    bool preprocess(const clip_image_u8 & img, clip_image_f32_batch & output) override;46 47    struct slice_coordinates {48        int x;49        int y;50        clip_image_size size;51    };52 53    struct slice_instructions {54        clip_image_size overview_size; // size of downscaled image55        clip_image_size refined_size;  // size of image right before slicing (must be multiple of slice size)56        clip_image_size grid_size;     // grid_size.width * grid_size.height = number of slices57        std::vector<slice_coordinates> slices;58    };59 60    // LFM2 override this function to implement its custom slicing logic61    virtual slice_instructions get_slice_instructions(const clip_image_size & original_size);62 63    std::vector<clip_image_u8_ptr> slice_image(const clip_image_u8 & img, const slice_instructions & inst, bool overview_first = true);64 65private:66    clip_image_size get_best_resize(const clip_image_size & original_size, int scale_resolution, int patch_size, bool allow_upscale = false);67 68    clip_image_size resize_maintain_aspect_ratio(const clip_image_size & orig, const clip_image_size & target_max);69 70    /**71     * Selects the best resolution from a list of possible resolutions based on the original size.72     *73     * For example, when given a list of resolutions:74     *  - 100x10075     *  - 200x10076     *  - 100x20077     *  - 200x20078     *79     * And an input image of size 111x200, then 100x200 is the best fit (least wasted resolution).80     *81     * @param original_size The original size of the image82     * @param possible_resolutions A list of possible resolutions83     * @return The best fit resolution84     */85    clip_image_size select_best_resolution(const clip_image_size & original_size, const std::vector<clip_image_size> & possible_resolutions);86    int ensure_divide(int length, int patch_size);87    clip_image_size get_refine_size(const clip_image_size & original_size, const clip_image_size & grid, int scale_resolution, int patch_size, bool allow_upscale = false);88    clip_image_size get_best_grid(const int max_slice_nums, const int multiple, const float log_ratio);89};90 91// downscale or upscale the input image to fixed size92struct mtmd_image_preprocessor_fixed_size : mtmd_image_preprocessor {93    mtmd_image_preprocessor_fixed_size(const clip_ctx * ctx) : mtmd_image_preprocessor(ctx) {}94    bool preprocess(const clip_image_u8 & img, clip_image_f32_batch & output) override;95};96 97// resize image to multiple of patch_size*n_merge, while preserving aspect ratio98// if image_resize_pad is true, the resized image will be padded, otherwise it will be either stretched or center-cropped depending on image_resize_pad99// this is used by models with native support for dynamic image size, for example: Qwen-VL, Pixtral, Kimi-VL, etc100struct mtmd_image_preprocessor_dyn_size : mtmd_image_preprocessor {101    mtmd_image_preprocessor_dyn_size(const clip_ctx * ctx) : mtmd_image_preprocessor(ctx) {}102    bool preprocess(const clip_image_u8 & img, clip_image_f32_batch & output) override;103};104 105// similar to mtmd_image_preprocessor_dyn_size, but resize the image to have longest edge equal to hparams.image_longest_edge, while preserving aspect ratio106struct mtmd_image_preprocessor_longest_edge : mtmd_image_preprocessor {107    mtmd_image_preprocessor_longest_edge(const clip_ctx * ctx) : mtmd_image_preprocessor(ctx) {}108    bool preprocess(const clip_image_u8 & img, clip_image_f32_batch & output) override;109};110 111// custom llava-uhd slicing logic for LFM2112// ref: https://github.com/huggingface/transformers/blob/v5.1.0/src/transformers/models/lfm2_vl/image_processing_lfm2_vl_fast.py113struct mtmd_image_preprocessor_lfm2 : mtmd_image_preprocessor_llava_uhd {114    // ref: https://huggingface.co/LiquidAI/LFM2.5-VL-1.6B/blob/main/processor_config.json115    static constexpr int   min_tiles            = 2;116    static constexpr int   max_tiles            = 10;117    static constexpr float max_pixels_tolerance = 2.0f;118    static constexpr int   tile_size            = 512;119 120    using mtmd_image_preprocessor_llava_uhd::mtmd_image_preprocessor_llava_uhd;121    slice_instructions get_slice_instructions(const clip_image_size & original_size) override;122 123private:124    clip_image_size find_closest_aspect_ratio(125            float aspect_ratio,126            const std::vector<clip_image_size> & target_ratios,127            int width, int height);128    std::vector<clip_image_size> get_target_ratios();129    clip_image_size get_grid_layout(int height, int width);130};131 132struct mtmd_image_preprocessor_idefics3 : mtmd_image_preprocessor_llava_uhd {133    mtmd_image_preprocessor_idefics3(const clip_ctx * ctx) : mtmd_image_preprocessor_llava_uhd(ctx) {}134    bool preprocess(const clip_image_u8 & img, clip_image_f32_batch & output) override;135};136 137struct mtmd_image_preprocessor_internvl : mtmd_image_preprocessor_llava_uhd {138    mtmd_image_preprocessor_internvl(const clip_ctx * ctx) : mtmd_image_preprocessor_llava_uhd(ctx) {}139    bool preprocess(const clip_image_u8 & img, clip_image_f32_batch & output) override;140};141 142struct mtmd_image_preprocessor_deepseekocr : mtmd_image_preprocessor {143    mtmd_image_preprocessor_deepseekocr(const clip_ctx * ctx) : mtmd_image_preprocessor(ctx) {}144    bool preprocess(const clip_image_u8 & img, clip_image_f32_batch & output) override;145};146 147// custom image preprocessing for Step3VL148// ref: https://huggingface.co/stepfun-ai/Step3-VL-10B/blob/main/processing_step3.py149struct mtmd_image_preprocessor_step3vl : mtmd_image_preprocessor_llava_uhd {150    mtmd_image_preprocessor_step3vl(const clip_ctx * ctx) : mtmd_image_preprocessor_llava_uhd(ctx) {}151    bool preprocess(const clip_image_u8 & img, clip_image_f32_batch & output) override;152    static slice_instructions build_slice_instructions(const clip_hparams & params, const clip_image_size & prepared_size);153 154private:155    static constexpr int   default_image_longest_edge = 3024;156    static constexpr int   default_image_crop_size    = 504;157    static constexpr float small_aspect_ratio_limit   = 1.5f;158    static constexpr float wide_aspect_ratio_limit    = 4.0f;159    static constexpr float crop_rounding_threshold    = 0.2f;160 161    void img_u8_resize_bilinear_to_f32(162            const clip_image_u8 & src,163            clip_image_f32 & dst,164            int target_width,165            int target_height,166            const float mean[3],167            const float std[3]);168    static int get_image_longest_edge(const clip_hparams & params);169    static int determine_window_size(const clip_hparams & params, int longer, int shorter);170    static int calc_crop_extent(int length, int window_size);171    static std::vector<int> calc_grid(int length, int window_size);172    static clip_image_u8 prepare_image(const clip_image_u8 & img, const clip_hparams & params);173    static clip_image_u8 crop_with_black_padding(const clip_image_u8 & image, int x, int y, int w, int h);174};175 176struct mtmd_image_preprocessor_youtuvl : mtmd_image_preprocessor {177    mtmd_image_preprocessor_youtuvl(const clip_ctx * ctx) : mtmd_image_preprocessor(ctx) {}178    bool preprocess(const clip_image_u8 & img, clip_image_f32_batch & output) override;179};180