hymenjj/llama-cpp-python-prebuilt

1from __future__ import annotations2 3import os4from ctypes import (5    c_bool,6    c_char_p,7    c_int,8    c_uint8,9    c_float,10    c_void_p,11    POINTER,12    _Pointer,  # type: ignore13    Structure,14)15import pathlib16from typing import (17    Union,18    NewType,19    Optional,20    TYPE_CHECKING,21)22 23import llama_cpp.llama_cpp as llama_cpp24 25from llama_cpp._ctypes_extensions import (26    load_shared_library,27    ctypes_function_for_shared_library,28)29 30if TYPE_CHECKING:31    from llama_cpp._ctypes_extensions import (32        CtypesArray,33    )34 35 36# Specify the base name of the shared library to load37_libllava_base_name = "llava"38_libllava_override_path = os.environ.get("LLAVA_CPP_LIB")39_libllava_base_path = pathlib.Path(os.path.abspath(os.path.dirname(__file__))) / "lib" if _libllava_override_path is None else pathlib.Path()40 41# Load the library42_libllava = load_shared_library(_libllava_base_name, _libllava_base_path)43 44ctypes_function = ctypes_function_for_shared_library(_libllava)45 46 47################################################48# llava.h49################################################50 51# struct clip_ctx;52clip_ctx_p = NewType("clip_ctx_p", int)53clip_ctx_p_ctypes = c_void_p54 55 56# struct llava_image_embed {57#     float * embed;58#     int n_image_pos;59# };60class llava_image_embed(Structure):61    _fields_ = [62        ("embed", POINTER(c_float)),63        ("n_image_pos", c_int),64    ]65 66 67# /** sanity check for clip <-> llava embed size match */68# LLAVA_API bool llava_validate_embed_size(const llama_context * ctx_llama, const clip_ctx * ctx_clip);69@ctypes_function(70    "llava_validate_embed_size",71    [llama_cpp.llama_context_p_ctypes, clip_ctx_p_ctypes],72    c_bool,73)74def llava_validate_embed_size(75    ctx_llama: llama_cpp.llama_context_p, ctx_clip: clip_ctx_p, /76) -> bool:77    ...78 79 80# /** build an image embed from image file bytes */81# LLAVA_API struct llava_image_embed * llava_image_embed_make_with_bytes(struct clip_ctx * ctx_clip, int n_threads, const unsigned char * image_bytes, int image_bytes_length);82@ctypes_function(83    "llava_image_embed_make_with_bytes",84    [clip_ctx_p_ctypes, c_int, POINTER(c_uint8), c_int],85    POINTER(llava_image_embed),86)87def llava_image_embed_make_with_bytes(88    ctx_clip: clip_ctx_p,89    n_threads: Union[c_int, int],90    image_bytes: CtypesArray[c_uint8],91    image_bytes_length: Union[c_int, int],92    /,93) -> "_Pointer[llava_image_embed]":94    ...95 96 97# /** build an image embed from a path to an image filename */98# LLAVA_API struct llava_image_embed * llava_image_embed_make_with_filename(struct clip_ctx * ctx_clip, int n_threads, const char * image_path);99@ctypes_function(100    "llava_image_embed_make_with_filename",101    [clip_ctx_p_ctypes, c_int, c_char_p],102    POINTER(llava_image_embed),103)104def llava_image_embed_make_with_filename(105    ctx_clip: clip_ctx_p, n_threads: Union[c_int, int], image_path: bytes, /106) -> "_Pointer[llava_image_embed]":107    ...108 109 110# LLAVA_API void llava_image_embed_free(struct llava_image_embed * embed);111# /** free an embedding made with llava_image_embed_make_* */112@ctypes_function("llava_image_embed_free", [POINTER(llava_image_embed)], None)113def llava_image_embed_free(embed: "_Pointer[llava_image_embed]", /):114    ...115 116 117# /** write the image represented by embed into the llama context with batch size n_batch, starting at context pos n_past. on completion, n_past points to the next position in the context after the image embed. */118# LLAVA_API bool llava_eval_image_embed(struct llama_context * ctx_llama, const struct llava_image_embed * embed, int n_batch, int * n_past);119@ctypes_function(120    "llava_eval_image_embed",121    [122        llama_cpp.llama_context_p_ctypes,123        POINTER(llava_image_embed),124        c_int,125        POINTER(c_int),126    ],127    c_bool,128)129def llava_eval_image_embed(130    ctx_llama: llama_cpp.llama_context_p,131    embed: "_Pointer[llava_image_embed]",132    n_batch: Union[c_int, int],133    n_past: "_Pointer[c_int]",134    /,135) -> bool:136    ...137 138 139################################################140# clip.h141################################################142 143 144# /** load mmproj model */145# CLIP_API struct clip_ctx * clip_model_load    (const char * fname, int verbosity);146@ctypes_function("clip_model_load", [c_char_p, c_int], clip_ctx_p_ctypes)147def clip_model_load(148    fname: bytes, verbosity: Union[c_int, int], /149) -> Optional[clip_ctx_p]:150    ...151 152 153# /** free mmproj model */154# CLIP_API void clip_free(struct clip_ctx * ctx);155@ctypes_function("clip_free", [clip_ctx_p_ctypes], None)156def clip_free(ctx: clip_ctx_p, /):157    ...158 159