echodict/llama.cpp

1#ifndef MTMD_H2#define MTMD_H3 4#include "ggml.h"5#include "llama.h"6 7#include <stddef.h>8#include <stdint.h>9#include <stdbool.h>10 11#ifdef __cplusplus12#include <string>13#include <vector>14#include <cinttypes>15#include <memory>16#endif17 18/**19 * libmtmd: A library for multimodal support in llama.cpp.20 *21 * WARNING: This API is experimental and subject to many BREAKING CHANGES.22 *          Issues related to API usage may receive lower priority support.23 *24 * For the usage, see an example in mtmd-cli.cpp25 *26 * For contributors:27 * - Make sure the C API is aligned with the libllama C API (as in llama.h)28 * - Do not include model name (e.g., qwen, gemma) in the API, use generic terms instead29 * - Keep the API minimal, do not expose internal details unless necessary30 *31 * IMPORTANT: The mtmd module does NOT accept pull requests that are fully or predominantly AI-generated.32 * We encourage human contributors to ensure the quality and reliability of the codebase.33 */34 35#ifdef LLAMA_SHARED36#    if defined(_WIN32) && !defined(__MINGW32__)37#        ifdef LLAMA_BUILD38#            define MTMD_API __declspec(dllexport)39#        else40#            define MTMD_API __declspec(dllimport)41#        endif42#    else43#        define MTMD_API __attribute__ ((visibility ("default")))44#    endif45#else46#    define MTMD_API47#endif48 49#ifdef __cplusplus50extern "C" {51#endif52 53enum mtmd_input_chunk_type {54    MTMD_INPUT_CHUNK_TYPE_TEXT,55    MTMD_INPUT_CHUNK_TYPE_IMAGE,56    MTMD_INPUT_CHUNK_TYPE_AUDIO,57};58 59// opaque types60struct mtmd_context;61struct mtmd_bitmap;62struct mtmd_image_tokens;63struct mtmd_input_chunk;64struct mtmd_input_chunks;65 66struct mtmd_input_text {67    const char * text;68    bool add_special;69    bool parse_special;70};71 72//73// C API74//75 76typedef struct mtmd_context      mtmd_context;77typedef struct mtmd_bitmap       mtmd_bitmap;78typedef struct mtmd_image_tokens mtmd_image_tokens;79typedef struct mtmd_input_chunk  mtmd_input_chunk;80typedef struct mtmd_input_chunks mtmd_input_chunks;81typedef struct mtmd_input_text   mtmd_input_text;82 83struct mtmd_context_params {84    bool use_gpu;85    bool print_timings;86    int n_threads;87    const char * image_marker; // deprecated, use media_marker instead88    const char * media_marker;89    enum llama_flash_attn_type flash_attn_type;90    bool warmup; // whether to run a warmup encode pass after initialization91 92    // limit number of image tokens, only for vision models with dynamic resolution93    int image_min_tokens; // minimum number of tokens for image input (default: read from metadata)94    int image_max_tokens; // maximum number of tokens for image input (default: read from metadata)95 96    // callback function passed over to mtmd proper97    ggml_backend_sched_eval_callback cb_eval;98    void * cb_eval_user_data;99};100 101MTMD_API const char * mtmd_default_marker(void);102 103MTMD_API struct mtmd_context_params mtmd_context_params_default(void);104 105// initialize the mtmd context106// return nullptr on failure107MTMD_API mtmd_context * mtmd_init_from_file(const char * mmproj_fname,108                                            const struct llama_model * text_model,109                                            const struct mtmd_context_params ctx_params);110 111MTMD_API void mtmd_free(mtmd_context * ctx);112 113// whether we need to set non-causal mask before llama_decode114// if chunk is nullptr, we assume the default case where chunk is an image chunk115MTMD_API bool mtmd_decode_use_non_causal(const mtmd_context * ctx, const mtmd_input_chunk * chunk);116 117// whether the current model use M-RoPE for llama_decode118MTMD_API bool mtmd_decode_use_mrope(const mtmd_context * ctx);119 120// whether the current model supports vision input121MTMD_API bool mtmd_support_vision(const mtmd_context * ctx);122 123// whether the current model supports audio input124MTMD_API bool mtmd_support_audio(const mtmd_context * ctx);125 126// get audio sample rate in Hz, for example 16000 for Whisper127// return -1 if audio is not supported128MTMD_API int mtmd_get_audio_sample_rate(const mtmd_context * ctx);129 130// mtmd_bitmap131//132// if bitmap is image:133//     length of data must be nx * ny * 3134//     the data is in RGBRGBRGB... format135// if bitmap is audio:136//     length of data must be n_samples * sizeof(float)137//     the data is in float format (PCM F32)138MTMD_API mtmd_bitmap *         mtmd_bitmap_init           (uint32_t nx, uint32_t ny, const unsigned char * data);139MTMD_API mtmd_bitmap *         mtmd_bitmap_init_from_audio(size_t n_samples,         const float         * data);140MTMD_API uint32_t              mtmd_bitmap_get_nx     (const mtmd_bitmap * bitmap);141MTMD_API uint32_t              mtmd_bitmap_get_ny     (const mtmd_bitmap * bitmap);142MTMD_API const unsigned char * mtmd_bitmap_get_data   (const mtmd_bitmap * bitmap);143MTMD_API size_t                mtmd_bitmap_get_n_bytes(const mtmd_bitmap * bitmap);144MTMD_API bool                  mtmd_bitmap_is_audio   (const mtmd_bitmap * bitmap);145MTMD_API void                  mtmd_bitmap_free       (mtmd_bitmap * bitmap);146// bitmap ID is optional, but useful for KV cache tracking147// these getters/setters are dedicated functions, so you can for example calculate the hash of the image based on mtmd_bitmap_get_data()148MTMD_API const char * mtmd_bitmap_get_id(const mtmd_bitmap * bitmap);149MTMD_API void         mtmd_bitmap_set_id(mtmd_bitmap * bitmap, const char * id);150 151 152// mtmd_input_chunks153//154// this is simply a list of mtmd_input_chunk155// the elements can only be populated via mtmd_tokenize()156MTMD_API mtmd_input_chunks *      mtmd_input_chunks_init(void);157MTMD_API size_t                   mtmd_input_chunks_size(const mtmd_input_chunks * chunks);158MTMD_API const mtmd_input_chunk * mtmd_input_chunks_get (const mtmd_input_chunks * chunks, size_t idx);159MTMD_API void                     mtmd_input_chunks_free(mtmd_input_chunks * chunks);160 161// mtmd_input_chunk162//163// the instance will be constructed via mtmd_tokenize()164// it will be freed along with mtmd_input_chunks165MTMD_API enum mtmd_input_chunk_type mtmd_input_chunk_get_type        (const mtmd_input_chunk * chunk);166MTMD_API const llama_token *        mtmd_input_chunk_get_tokens_text (const mtmd_input_chunk * chunk, size_t * n_tokens_output);167MTMD_API const mtmd_image_tokens *  mtmd_input_chunk_get_tokens_image(const mtmd_input_chunk * chunk);168MTMD_API size_t                     mtmd_input_chunk_get_n_tokens    (const mtmd_input_chunk * chunk);169// returns nullptr for ID on text chunk170MTMD_API const char *               mtmd_input_chunk_get_id          (const mtmd_input_chunk * chunk);171// number of temporal positions (equals to max(t,h,w) for M-RoPE; equals to n_tokens otherwise)172MTMD_API llama_pos                  mtmd_input_chunk_get_n_pos       (const mtmd_input_chunk * chunk);173 174// in case you want to use custom logic to handle the chunk (i.e. KV cache management)175// you can move the chunk ownership to your own code by copying it176// remember to free the chunk when you are done with it177MTMD_API mtmd_input_chunk * mtmd_input_chunk_copy(const mtmd_input_chunk * chunk);178MTMD_API void               mtmd_input_chunk_free(mtmd_input_chunk * chunk);179 180 181// mtmd_image_tokens182//183// the instance will be constructed via mtmd_tokenize()184// it will be freed along with mtmd_input_chunk185MTMD_API size_t       mtmd_image_tokens_get_n_tokens(const mtmd_image_tokens * image_tokens); // TODO: deprecate186MTMD_API const char * mtmd_image_tokens_get_id      (const mtmd_image_tokens * image_tokens); // TODO: deprecate187// number of temporal positions (equals to max(t,h,w) for M-RoPE; equals to n_tokens otherwise)188MTMD_API llama_pos    mtmd_image_tokens_get_n_pos   (const mtmd_image_tokens * image_tokens); // TODO: deprecate189 190DEPRECATED(MTMD_API size_t mtmd_image_tokens_get_nx(const mtmd_image_tokens * image_tokens),191           "use mtmd_image_tokens_get_decoder_pos() instead");192DEPRECATED(MTMD_API size_t mtmd_image_tokens_get_ny(const mtmd_image_tokens * image_tokens),193           "use mtmd_image_tokens_get_decoder_pos() instead");194 195struct mtmd_decoder_pos {196    uint32_t t;197    uint32_t x;198    uint32_t y;199    uint32_t z; // unused for now, reserved for future use200};201// get position for decoder attention, to be used by M-RoPE models202// i is the index of the embedding token, ranging from 0 to mtmd_image_tokens_get_n_tokens() - 1203// pos_0 is the absolute position of the first token204// return relative position (for example, embedding 0 will have position (0, 0, 0); remember to adjust it to the current absolute position)205MTMD_API struct mtmd_decoder_pos mtmd_image_tokens_get_decoder_pos(const mtmd_image_tokens * image_tokens, llama_pos pos_0, size_t i);206 207// tokenize an input text prompt and a list of bitmaps (images/audio)208// the prompt must have the input image marker (default: "<__media__>") in it209// the default marker is defined by mtmd_default_marker()210// the marker will be replaced with the image/audio chunk211// for example:212//   "here is an image: <__media__>\ndescribe it in detail."213//   this will gives 3 chunks:214//   1. "here is an image: <start_of_image>"215//   2. (image/audio tokens)216//   3. "<end_of_image>\ndescribe it in detail."217// number of bitmaps must be equal to the number of markers in the prompt218// this function is thread-safe (shared ctx)219// return values:220//   0 on success221//   1 on number of bitmaps not matching the number of markers222//   2 on image preprocessing error223MTMD_API int32_t mtmd_tokenize(mtmd_context * ctx,224                               mtmd_input_chunks * output,225                               const mtmd_input_text * text,226                               const mtmd_bitmap ** bitmaps,227                               size_t n_bitmaps);228 229// returns 0 on success230// TODO: deprecate231MTMD_API int32_t mtmd_encode(mtmd_context * ctx,232                             const mtmd_image_tokens * image_tokens);233 234// returns 0 on success235MTMD_API int32_t mtmd_encode_chunk(mtmd_context * ctx,236                                   const mtmd_input_chunk * chunk);237 238// get output embeddings from the last encode pass239// the reading size (in bytes) is equal to:240// llama_model_n_embd_inp(model) * mtmd_input_chunk_get_n_tokens(chunk) * sizeof(float)241MTMD_API float * mtmd_get_output_embd(mtmd_context * ctx);242 243// Set callback for all future logging events.244// If this is not called, or NULL is supplied, everything is output on stderr.245MTMD_API void mtmd_log_set(ggml_log_callback log_callback, void * user_data);246 247/////////////////////////////////////////248 249// test function, to be used in test-mtmd-c-api.c250MTMD_API mtmd_input_chunks * mtmd_test_create_input_chunks(void);251 252#ifdef __cplusplus253} // extern "C"254#endif255 256//257// C++ wrappers258//259 260#ifdef __cplusplus261 262namespace mtmd {263 264struct mtmd_context_deleter {265    void operator()(mtmd_context * val) { mtmd_free(val); }266};267using context_ptr = std::unique_ptr<mtmd_context, mtmd_context_deleter>;268 269struct mtmd_bitmap_deleter {270    void operator()(mtmd_bitmap * val) { mtmd_bitmap_free(val); }271};272using bitmap_ptr = std::unique_ptr<mtmd_bitmap, mtmd_bitmap_deleter>;273 274struct mtmd_input_chunks_deleter {275    void operator()(mtmd_input_chunks * val) { mtmd_input_chunks_free(val); }276};277using input_chunks_ptr = std::unique_ptr<mtmd_input_chunks, mtmd_input_chunks_deleter>;278 279struct mtmd_input_chunk_deleter {280    void operator()(mtmd_input_chunk * val) { mtmd_input_chunk_free(val); }281};282using input_chunk_ptr = std::unique_ptr<mtmd_input_chunk, mtmd_input_chunk_deleter>;283 284struct bitmap {285    bitmap_ptr ptr;286    bitmap() : ptr(nullptr) {}287    bitmap(mtmd_bitmap * bitmap) : ptr(bitmap) {}288    bitmap(bitmap && other) noexcept : ptr(std::move(other.ptr)) {}289    bitmap(uint32_t nx, uint32_t ny, const unsigned char * data) {290        ptr.reset(mtmd_bitmap_init(nx, ny, data));291    }292    ~bitmap() = default;293    uint32_t nx() const { return mtmd_bitmap_get_nx(ptr.get()); }294    uint32_t ny() const { return mtmd_bitmap_get_ny(ptr.get()); }295    const unsigned char * data() const { return mtmd_bitmap_get_data(ptr.get()); }296    size_t n_bytes() const { return mtmd_bitmap_get_n_bytes(ptr.get()); }297    std::string id() const { return mtmd_bitmap_get_id(ptr.get()); }298    void set_id(const char * id) const { mtmd_bitmap_set_id(ptr.get(), id); }299};300 301struct bitmaps {302    std::vector<bitmap> entries;303    ~bitmaps() = default;304    // return list of pointers to mtmd_bitmap305    // example:306    //   auto bitmaps_c_ptr = bitmaps.c_ptr();307    //   int32_t res = mtmd_tokenize(... bitmaps_c_ptr.data(), bitmaps_c_ptr.size());308    std::vector<const mtmd_bitmap *> c_ptr() {309        std::vector<const mtmd_bitmap *> res(entries.size());310        for (size_t i = 0; i < entries.size(); i++) {311            res[i] = entries[i].ptr.get();312        }313        return res;314    }315};316 317struct input_chunks {318    input_chunks_ptr ptr;319    input_chunks() = default;320    input_chunks(mtmd_input_chunks * chunks) : ptr(chunks) {}321    ~input_chunks() = default;322    size_t size() const { return mtmd_input_chunks_size(ptr.get()); }323    const mtmd_input_chunk * operator[](size_t idx) const {324        return mtmd_input_chunks_get(ptr.get(), idx);325    }326};327 328} // namespace mtmd329 330#endif331 332#endif333