hymenjj/llama-cpp-python-prebuilt
0
1from __future__ import annotations2 3import os4import ctypes5import pathlib6 7from typing import (8 Callable,9 Union,10 NewType,11 Optional,12 TYPE_CHECKING,13)14 15from llama_cpp._ctypes_extensions import (16 load_shared_library,17 byref,18 ctypes_function_for_shared_library,19)20 21if TYPE_CHECKING:22 from llama_cpp._ctypes_extensions import (23 CtypesCData,24 CtypesArray,25 CtypesPointer,26 CtypesVoidPointer,27 CtypesRef,28 CtypesPointerOrRef,29 CtypesFuncPointer,30 )31 32 33# Specify the base name of the shared library to load34_lib_base_name = "llama"35_override_base_path = os.environ.get("LLAMA_CPP_LIB_PATH")36_base_path = pathlib.Path(os.path.abspath(os.path.dirname(__file__))) / "lib" if _override_base_path is None else pathlib.Path(_override_base_path)37# Load the library38_lib = load_shared_library(_lib_base_name, _base_path)39 40ctypes_function = ctypes_function_for_shared_library(_lib)41 42 43# from ggml.h44# // NOTE: always add types at the end of the enum to keep backward compatibility45# enum ggml_type {46# GGML_TYPE_F32 = 0,47# GGML_TYPE_F16 = 1,48# GGML_TYPE_Q4_0 = 2,49# GGML_TYPE_Q4_1 = 3,50# // GGML_TYPE_Q4_2 = 4, support has been removed51# // GGML_TYPE_Q4_3 = 5, support has been removed52# GGML_TYPE_Q5_0 = 6,53# GGML_TYPE_Q5_1 = 7,54# GGML_TYPE_Q8_0 = 8,55# GGML_TYPE_Q8_1 = 9,56# GGML_TYPE_Q2_K = 10,57# GGML_TYPE_Q3_K = 11,58# GGML_TYPE_Q4_K = 12,59# GGML_TYPE_Q5_K = 13,60# GGML_TYPE_Q6_K = 14,61# GGML_TYPE_Q8_K = 15,62# GGML_TYPE_IQ2_XXS = 16,63# GGML_TYPE_IQ2_XS = 17,64# GGML_TYPE_IQ3_XXS = 18,65# GGML_TYPE_IQ1_S = 19,66# GGML_TYPE_IQ4_NL = 20,67# GGML_TYPE_IQ3_S = 21,68# GGML_TYPE_IQ2_S = 22,69# GGML_TYPE_IQ4_XS = 23,70# GGML_TYPE_I8 = 24,71# GGML_TYPE_I16 = 25,72# GGML_TYPE_I32 = 26,73# GGML_TYPE_I64 = 27,74# GGML_TYPE_F64 = 28,75# GGML_TYPE_IQ1_M = 29,76# GGML_TYPE_COUNT,77# };78GGML_TYPE_F32 = 079GGML_TYPE_F16 = 180GGML_TYPE_Q4_0 = 281GGML_TYPE_Q4_1 = 382GGML_TYPE_Q5_0 = 683GGML_TYPE_Q5_1 = 784GGML_TYPE_Q8_0 = 885GGML_TYPE_Q8_1 = 986GGML_TYPE_Q2_K = 1087GGML_TYPE_Q3_K = 1188GGML_TYPE_Q4_K = 1289GGML_TYPE_Q5_K = 1390GGML_TYPE_Q6_K = 1491GGML_TYPE_Q8_K = 1592GGML_TYPE_IQ2_XXS = 1693GGML_TYPE_IQ2_XS = 1794GGML_TYPE_IQ3_XXS = 1895GGML_TYPE_IQ1_S = 1996GGML_TYPE_IQ4_NL = 2097GGML_TYPE_IQ3_S = 2198GGML_TYPE_IQ2_S = 2299GGML_TYPE_IQ4_XS = 23100GGML_TYPE_I8 = 24101GGML_TYPE_I16 = 25102GGML_TYPE_I32 = 26103GGML_TYPE_I64 = 27104GGML_TYPE_F64 = 28105GGML_TYPE_IQ1_M = 29106GGML_TYPE_COUNT = 30107 108# from ggml-backend.h109# typedef bool (*ggml_backend_sched_eval_callback)(struct ggml_tensor * t, bool ask, void * user_data);110ggml_backend_sched_eval_callback = ctypes.CFUNCTYPE(111 ctypes.c_bool, ctypes.c_void_p, ctypes.c_bool, ctypes.c_void_p112)113 114# // Abort callback115# // If not NULL, called before ggml computation116# // If it returns true, the computation is aborted117# typedef bool (*ggml_abort_callback)(void * data);118ggml_abort_callback = ctypes.CFUNCTYPE(ctypes.c_bool, ctypes.c_void_p)119 120# llama.h bindings121 122_lib.llama_max_devices.argtypes = []123_lib.llama_max_devices.restype = ctypes.c_size_t124 125LLAMA_MAX_DEVICES = _lib.llama_max_devices()126 127# define LLAMA_DEFAULT_SEED 0xFFFFFFFF128LLAMA_DEFAULT_SEED = 0xFFFFFFFF129 130# define LLAMA_TOKEN_NULL -1131LLAMA_TOKEN_NULL = -1132 133# define LLAMA_FILE_MAGIC_GGLA 0x67676c61u // 'ggla'134LLAMA_FILE_MAGIC_GGLA = 0x67676C61135 136# define LLAMA_FILE_MAGIC_GGSN 0x6767736eu // 'ggsn'137LLAMA_FILE_MAGIC_GGSN = 0x6767736E138 139# define LLAMA_FILE_MAGIC_GGSQ 0x67677371u // 'ggsq'140LLAMA_FILE_MAGIC_GGSQ = 0x67677371141 142# define LLAMA_SESSION_MAGIC LLAMA_FILE_MAGIC_GGSN143LLAMA_SESSION_MAGIC = LLAMA_FILE_MAGIC_GGSN144# define LLAMA_SESSION_VERSION 9145LLAMA_SESSION_VERSION = 9146 147# define LLAMA_STATE_SEQ_MAGIC LLAMA_FILE_MAGIC_GGSQ148LLAMA_STATE_SEQ_MAGIC = LLAMA_FILE_MAGIC_GGSQ149# define LLAMA_STATE_SEQ_VERSION 2150LLAMA_STATE_SEQ_VERSION = 2151 152# struct llama_vocab;153llama_vocab_p = NewType("llama_vocab_p", int)154llama_vocab_p_ctypes = ctypes.c_void_p155 156# struct llama_model;157llama_model_p = NewType("llama_model_p", int)158llama_model_p_ctypes = ctypes.c_void_p159 160# struct llama_context;161llama_context_p = NewType("llama_context_p", int)162llama_context_p_ctypes = ctypes.c_void_p163 164# typedef struct llama_memory_i * llama_memory_t;165llama_memory_t = NewType("llama_memory_t", int)166llama_memory_t_ctypes = ctypes.c_void_p167 168# struct llama_kv_cache; (DEPRECATED)169llama_kv_cache_p = NewType("llama_kv_cache_p", int)170llama_kv_cache_p_ctypes = ctypes.c_void_p171 172# typedef int32_t llama_pos;173llama_pos = ctypes.c_int32174# typedef int32_t llama_token;175llama_token = ctypes.c_int32176llama_token_p = ctypes.POINTER(llama_token)177# typedef int32_t llama_seq_id;178llama_seq_id = ctypes.c_int32179 180 181# enum llama_vocab_type {182# LLAMA_VOCAB_TYPE_NONE = 0, // For models without vocab183# LLAMA_VOCAB_TYPE_SPM = 1, // LLaMA tokenizer based on byte-level BPE with byte fallback184# LLAMA_VOCAB_TYPE_BPE = 2, // GPT-2 tokenizer based on byte-level BPE185# LLAMA_VOCAB_TYPE_WPM = 3, // BERT tokenizer based on WordPiece186# LLAMA_VOCAB_TYPE_UGM = 4, // T5 tokenizer based on Unigram187# LLAMA_VOCAB_TYPE_RWKV = 5, // RWKV tokenizer based on greedy tokenization188# LLAMA_VOCAB_TYPE_PLAMO2 = 6, // PLaMo-2 tokenizer based on Aho-Corasick with dynamic programming189# };190LLAMA_VOCAB_TYPE_NONE = 0191"""For models without vocab"""192LLAMA_VOCAB_TYPE_SPM = 1193"""LLaMA tokenizer based on byte-level BPE with byte fallback"""194LLAMA_VOCAB_TYPE_BPE = 2195"""GPT-2 tokenizer based on byte-level BPE"""196LLAMA_VOCAB_TYPE_WPM = 3197"""BERT tokenizer based on WordPiece"""198LLAMA_VOCAB_TYPE_UGM = 4199"""T5 tokenizer based on Unigram"""200LLAMA_VOCAB_TYPE_RWKV = 5201"""RWKV tokenizer based on greedy tokenization"""202LLAMA_VOCAB_TYPE_PLAMO2 = 6203"""PLaMo-2 tokenizer based on Aho-Corasick with dynamic programming"""204 205 206# NOTE: Deprecated and will be removed in the future. (already gone in llama.cpp)207# // pre-tokenization types208# enum llama_vocab_pre_type {209# LLAMA_VOCAB_PRE_TYPE_DEFAULT = 0,210# LLAMA_VOCAB_PRE_TYPE_LLAMA3 = 1,211# LLAMA_VOCAB_PRE_TYPE_DEEPSEEK_LLM = 2,212# LLAMA_VOCAB_PRE_TYPE_DEEPSEEK_CODER = 3,213# LLAMA_VOCAB_PRE_TYPE_FALCON = 4,214# LLAMA_VOCAB_PRE_TYPE_MPT = 5,215# LLAMA_VOCAB_PRE_TYPE_STARCODER = 6,216# LLAMA_VOCAB_PRE_TYPE_GPT2 = 7,217# LLAMA_VOCAB_PRE_TYPE_REFACT = 8,218# LLAMA_VOCAB_PRE_TYPE_COMMAND_R = 9,219# LLAMA_VOCAB_PRE_TYPE_STABLELM2 = 10,220# LLAMA_VOCAB_PRE_TYPE_QWEN2 = 11,221# LLAMA_VOCAB_PRE_TYPE_OLMO = 12,222# LLAMA_VOCAB_PRE_TYPE_DBRX = 13,223# LLAMA_VOCAB_PRE_TYPE_SMAUG = 14,224# LLAMA_VOCAB_PRE_TYPE_PORO = 15,225# LLAMA_VOCAB_PRE_TYPE_CHATGLM3 = 16,226# LLAMA_VOCAB_PRE_TYPE_CHATGLM4 = 17,227# LLAMA_VOCAB_PRE_TYPE_VIKING = 18,228# LLAMA_VOCAB_PRE_TYPE_JAIS = 19,229# LLAMA_VOCAB_PRE_TYPE_TEKKEN = 20,230# LLAMA_VOCAB_PRE_TYPE_SMOLLM = 21,231# LLAMA_VOCAB_PRE_TYPE_CODESHELL = 22,232# LLAMA_VOCAB_PRE_TYPE_BLOOM = 23,233# LLAMA_VOCAB_PRE_TYPE_GPT3_FINNISH = 24,234# LLAMA_VOCAB_PRE_TYPE_EXAONE = 25,235# LLAMA_VOCAB_PRE_TYPE_CHAMELEON = 26,236# LLAMA_VOCAB_PRE_TYPE_MINERVA = 27,237# LLAMA_VOCAB_PRE_TYPE_DEEPSEEK3_LLM = 28,238# LLAMA_VOCAB_PRE_TYPE_GPT4O = 29,239# LLAMA_VOCAB_PRE_TYPE_SUPERBPE = 30,240# LLAMA_VOCAB_PRE_TYPE_TRILLION = 31,241# LLAMA_VOCAB_PRE_TYPE_BAILINGMOE = 32,242# LLAMA_VOCAB_PRE_TYPE_LLAMA4 = 33,243# LLAMA_VOCAB_PRE_TYPE_PIXTRAL = 34,244# LLAMA_VOCAB_PRE_TYPE_SEED_CODER = 35,245# };246LLAMA_VOCAB_PRE_TYPE_DEFAULT = 0247LLAMA_VOCAB_PRE_TYPE_LLAMA3 = 1248LLAMA_VOCAB_PRE_TYPE_DEEPSEEK_LLM = 2249LLAMA_VOCAB_PRE_TYPE_DEEPSEEK_CODER = 3250LLAMA_VOCAB_PRE_TYPE_FALCON = 4251LLAMA_VOCAB_PRE_TYPE_MPT = 5252LLAMA_VOCAB_PRE_TYPE_STARCODER = 6253LLAMA_VOCAB_PRE_TYPE_GPT2 = 7254LLAMA_VOCAB_PRE_TYPE_REFACT = 8255LLAMA_VOCAB_PRE_TYPE_COMMAND_R = 9256LLAMA_VOCAB_PRE_TYPE_STABLELM2 = 10257LLAMA_VOCAB_PRE_TYPE_QWEN2 = 11258LLAMA_VOCAB_PRE_TYPE_OLMO = 12259LLAMA_VOCAB_PRE_TYPE_DBRX = 13260LLAMA_VOCAB_PRE_TYPE_SMAUG = 14261LLAMA_VOCAB_PRE_TYPE_PORO = 15262LLAMA_VOCAB_PRE_TYPE_CHATGLM3 = 16263LLAMA_VOCAB_PRE_TYPE_CHATGLM4 = 17264LLAMA_VOCAB_PRE_TYPE_VIKING = 18265LLAMA_VOCAB_PRE_TYPE_JAIS = 19266LLAMA_VOCAB_PRE_TYPE_TEKKEN = 20267LLAMA_VOCAB_PRE_TYPE_SMOLLM = 21268LLAMA_VOCAB_PRE_TYPE_CODESHELL = 22269LLAMA_VOCAB_PRE_TYPE_BLOOM = 23270LLAMA_VOCAB_PRE_TYPE_GPT3_FINNISH = 24271LLAMA_VOCAB_PRE_TYPE_EXAONE = 25272LLAMA_VOCAB_PRE_TYPE_CHAMELEON = 26273LLAMA_VOCAB_PRE_TYPE_MINERVA = 27274LLAMA_VOCAB_PRE_TYPE_DEEPSEEK3_LLM = 28275LLAMA_VOCAB_PRE_TYPE_GPT4O = 29276LLAMA_VOCAB_PRE_TYPE_SUPERBPE = 30277LLAMA_VOCAB_PRE_TYPE_TRILLION = 31278LLAMA_VOCAB_PRE_TYPE_BAILINGMOE = 32279LLAMA_VOCAB_PRE_TYPE_LLAMA4 = 33280LLAMA_VOCAB_PRE_TYPE_PIXTRAL = 34281LLAMA_VOCAB_PRE_TYPE_SEED_CODER = 35282 283 284# // note: these values should be synchronized with ggml_rope285# // TODO: maybe move this enum to ggml.h (ggml_rope_type)286# enum llama_rope_type {287# LLAMA_ROPE_TYPE_NONE = -1,288# LLAMA_ROPE_TYPE_NORM = 0,289# LLAMA_ROPE_TYPE_NEOX = GGML_ROPE_TYPE_NEOX,290# LLAMA_ROPE_TYPE_MROPE = GGML_ROPE_TYPE_MROPE,291# LLAMA_ROPE_TYPE_VISION = GGML_ROPE_TYPE_VISION,292# };293LLAMA_ROPE_TYPE_NONE = -1294LLAMA_ROPE_TYPE_NORM = 0295LLAMA_ROPE_TYPE_NEOX = GGML_ROPE_TYPE_NEOX = 2296LLAMA_ROPE_TYPE_MROPE = GGML_ROPE_TYPE_MROPE = 8297LLAMA_ROPE_TYPE_VISION = GGML_ROPE_TYPE_VISION = 24298 299 300# enum llama_token_type { //TODO: remove, required until per token attributes are available from GGUF file301# LLAMA_TOKEN_TYPE_UNDEFINED = 0,302# LLAMA_TOKEN_TYPE_NORMAL = 1,303# LLAMA_TOKEN_TYPE_UNKNOWN = 2,304# LLAMA_TOKEN_TYPE_CONTROL = 3,305# LLAMA_TOKEN_TYPE_USER_DEFINED = 4,306# LLAMA_TOKEN_TYPE_UNUSED = 5,307# LLAMA_TOKEN_TYPE_BYTE = 6,308# };309LLAMA_TOKEN_TYPE_UNDEFINED = 0310LLAMA_TOKEN_TYPE_NORMAL = 1311LLAMA_TOKEN_TYPE_UNKNOWN = 2312LLAMA_TOKEN_TYPE_CONTROL = 3313LLAMA_TOKEN_TYPE_USER_DEFINED = 4314LLAMA_TOKEN_TYPE_UNUSED = 5315LLAMA_TOKEN_TYPE_BYTE = 6316 317 318# enum llama_token_attr {319# LLAMA_TOKEN_ATTR_UNDEFINED = 0,320# LLAMA_TOKEN_ATTR_UNKNOWN = 1 << 0,321# LLAMA_TOKEN_ATTR_UNUSED = 1 << 1,322# LLAMA_TOKEN_ATTR_NORMAL = 1 << 2,323# LLAMA_TOKEN_ATTR_CONTROL = 1 << 3, // SPECIAL?324# LLAMA_TOKEN_ATTR_USER_DEFINED = 1 << 4,325# LLAMA_TOKEN_ATTR_BYTE = 1 << 5,326# LLAMA_TOKEN_ATTR_NORMALIZED = 1 << 6,327# LLAMA_TOKEN_ATTR_LSTRIP = 1 << 7,328# LLAMA_TOKEN_ATTR_RSTRIP = 1 << 8,329# LLAMA_TOKEN_ATTR_SINGLE_WORD = 1 << 9,330# };331LLAMA_TOKEN_ATTR_UNDEFINED = 0332LLAMA_TOKEN_ATTR_UNKNOWN = 1 << 0333LLAMA_TOKEN_ATTR_UNUSED = 1 << 1334LLAMA_TOKEN_ATTR_NORMAL = 1 << 2335LLAMA_TOKEN_ATTR_CONTROL = 1 << 3336LLAMA_TOKEN_ATTR_USER_DEFINED = 1 << 4337LLAMA_TOKEN_ATTR_BYTE = 1 << 5338LLAMA_TOKEN_ATTR_NORMALIZED = 1 << 6339LLAMA_TOKEN_ATTR_LSTRIP = 1 << 7340LLAMA_TOKEN_ATTR_RSTRIP = 1 << 8341LLAMA_TOKEN_ATTR_SINGLE_WORD = 1 << 9342 343 344# // model file types345# enum llama_ftype {346# LLAMA_FTYPE_ALL_F32 = 0,347# LLAMA_FTYPE_MOSTLY_F16 = 1, // except 1d tensors348# LLAMA_FTYPE_MOSTLY_Q4_0 = 2, // except 1d tensors349# LLAMA_FTYPE_MOSTLY_Q4_1 = 3, // except 1d tensors350# // LLAMA_FTYPE_MOSTLY_Q4_1_SOME_F16 = 4, // tok_embeddings.weight and output.weight are F16351# // LLAMA_FTYPE_MOSTLY_Q4_2 = 5, // support has been removed352# // LLAMA_FTYPE_MOSTLY_Q4_3 = 6, // support has been removed353# LLAMA_FTYPE_MOSTLY_Q8_0 = 7, // except 1d tensors354# LLAMA_FTYPE_MOSTLY_Q5_0 = 8, // except 1d tensors355# LLAMA_FTYPE_MOSTLY_Q5_1 = 9, // except 1d tensors356# LLAMA_FTYPE_MOSTLY_Q2_K = 10, // except 1d tensors357# LLAMA_FTYPE_MOSTLY_Q3_K_S = 11, // except 1d tensors358# LLAMA_FTYPE_MOSTLY_Q3_K_M = 12, // except 1d tensors359# LLAMA_FTYPE_MOSTLY_Q3_K_L = 13, // except 1d tensors360# LLAMA_FTYPE_MOSTLY_Q4_K_S = 14, // except 1d tensors361# LLAMA_FTYPE_MOSTLY_Q4_K_M = 15, // except 1d tensors362# LLAMA_FTYPE_MOSTLY_Q5_K_S = 16, // except 1d tensors363# LLAMA_FTYPE_MOSTLY_Q5_K_M = 17, // except 1d tensors364# LLAMA_FTYPE_MOSTLY_Q6_K = 18, // except 1d tensors365# LLAMA_FTYPE_MOSTLY_IQ2_XXS = 19, // except 1d tensors366# LLAMA_FTYPE_MOSTLY_IQ2_XS = 20, // except 1d tensors367# LLAMA_FTYPE_MOSTLY_Q2_K_S = 21, // except 1d tensors368# LLAMA_FTYPE_MOSTLY_IQ3_XS = 22, // except 1d tensors369# LLAMA_FTYPE_MOSTLY_IQ3_XXS = 23, // except 1d tensors370# LLAMA_FTYPE_MOSTLY_IQ1_S = 24, // except 1d tensors371# LLAMA_FTYPE_MOSTLY_IQ4_NL = 25, // except 1d tensors372# LLAMA_FTYPE_MOSTLY_IQ3_S = 26, // except 1d tensors373# LLAMA_FTYPE_MOSTLY_IQ3_M = 27, // except 1d tensors374# LLAMA_FTYPE_MOSTLY_IQ2_S = 28, // except 1d tensors375# LLAMA_FTYPE_MOSTLY_IQ2_M = 29, // except 1d tensors376# LLAMA_FTYPE_MOSTLY_IQ4_XS = 30, // except 1d tensors377# LLAMA_FTYPE_MOSTLY_IQ1_M = 31, // except 1d tensors378# LLAMA_FTYPE_MOSTLY_BF16 = 32, // except 1d tensors379# //LLAMA_FTYPE_MOSTLY_Q4_0_4_4 = 33, // removed from gguf files, use Q4_0 and runtime repack380# //LLAMA_FTYPE_MOSTLY_Q4_0_4_8 = 34, // removed from gguf files, use Q4_0 and runtime repack381# //LLAMA_FTYPE_MOSTLY_Q4_0_8_8 = 35, // removed from gguf files, use Q4_0 and runtime repack382# LLAMA_FTYPE_MOSTLY_TQ1_0 = 36, // except 1d tensors383# LLAMA_FTYPE_MOSTLY_TQ2_0 = 37, // except 1d tensors384# LLAMA_FTYPE_MOSTLY_MXFP4_MOE = 38, // except 1d tensors385#386# LLAMA_FTYPE_GUESSED = 1024, // not specified in the model file387# };388LLAMA_FTYPE_ALL_F32 = 0389LLAMA_FTYPE_MOSTLY_F16 = 1390LLAMA_FTYPE_MOSTLY_Q4_0 = 2391LLAMA_FTYPE_MOSTLY_Q4_1 = 3392LLAMA_FTYPE_MOSTLY_Q8_0 = 7393LLAMA_FTYPE_MOSTLY_Q5_0 = 8394LLAMA_FTYPE_MOSTLY_Q5_1 = 9395LLAMA_FTYPE_MOSTLY_Q2_K = 10396LLAMA_FTYPE_MOSTLY_Q3_K_S = 11397LLAMA_FTYPE_MOSTLY_Q3_K_M = 12398LLAMA_FTYPE_MOSTLY_Q3_K_L = 13399LLAMA_FTYPE_MOSTLY_Q4_K_S = 14400LLAMA_FTYPE_MOSTLY_Q4_K_M = 15401LLAMA_FTYPE_MOSTLY_Q5_K_S = 16402LLAMA_FTYPE_MOSTLY_Q5_K_M = 17403LLAMA_FTYPE_MOSTLY_Q6_K = 18404LLAMA_FTYPE_MOSTLY_IQ2_XXS = 19405LLAMA_FTYPE_MOSTLY_IQ2_XS = 20406LLAMA_FTYPE_MOSTLY_Q2_K_S = 21407LLAMA_FTYPE_MOSTLY_IQ3_XS = 22408LLAMA_FTYPE_MOSTLY_IQ3_XXS = 23409LLAMA_FTYPE_MOSTLY_IQ1_S = 24410LLAMA_FTYPE_MOSTLY_IQ4_NL = 25411LLAMA_FTYPE_MOSTLY_IQ3_S = 26412LLAMA_FTYPE_MOSTLY_IQ3_M = 27413LLAMA_FTYPE_MOSTLY_IQ2_S = 28414LLAMA_FTYPE_MOSTLY_IQ2_M = 29415LLAMA_FTYPE_MOSTLY_IQ4_XS = 30416LLAMA_FTYPE_MOSTLY_IQ1_M = 31417LLAMA_FTYPE_MOSTLY_BF16 = 32418# LLAMA_FTYPE_MOSTLY_Q4_0_4_4 = 33419# LLAMA_FTYPE_MOSTLY_Q4_0_4_8 = 34420# LLAMA_FTYPE_MOSTLY_Q4_0_8_8 = 35421LLAMA_FTYPE_MOSTLY_TQ1_0 = 36422LLAMA_FTYPE_MOSTLY_TQ2_0 = 37423LLAMA_FTYPE_MOSTLY_MXFP4_MOE = 38424LLAMA_FTYPE_GUESSED = 1024425 426# enum llama_rope_scaling_type {427# LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED = -1,428# LLAMA_ROPE_SCALING_TYPE_NONE = 0,429# LLAMA_ROPE_SCALING_TYPE_LINEAR = 1,430# LLAMA_ROPE_SCALING_TYPE_YARN = 2,431# LLAMA_ROPE_SCALING_TYPE_LONGROPE = 3,432# LLAMA_ROPE_SCALING_TYPE_MAX_VALUE = LLAMA_ROPE_SCALING_TYPE_LONGROPE,433# };434LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED = -1435LLAMA_ROPE_SCALING_TYPE_NONE = 0436LLAMA_ROPE_SCALING_TYPE_LINEAR = 1437LLAMA_ROPE_SCALING_TYPE_YARN = 2438LLAMA_ROPE_SCALING_TYPE_LONGROPE = 3439LLAMA_ROPE_SCALING_TYPE_MAX_VALUE = LLAMA_ROPE_SCALING_TYPE_LONGROPE440 441# enum llama_pooling_type {442# LLAMA_POOLING_TYPE_UNSPECIFIED = -1,443# LLAMA_POOLING_TYPE_NONE = 0,444# LLAMA_POOLING_TYPE_MEAN = 1,445# LLAMA_POOLING_TYPE_CLS = 2,446# LLAMA_POOLING_TYPE_LAST = 3,447# LLAMA_POOLING_TYPE_RANK = 4, // used by reranking models to attach the classification head to the graph448# };449LLAMA_POOLING_TYPE_UNSPECIFIED = -1450LLAMA_POOLING_TYPE_NONE = 0451LLAMA_POOLING_TYPE_MEAN = 1452LLAMA_POOLING_TYPE_CLS = 2453LLAMA_POOLING_TYPE_LAST = 3454LLAMA_POOLING_TYPE_RANK = 4455 456# enum llama_attention_type {457# LLAMA_ATTENTION_TYPE_UNSPECIFIED = -1,458# LLAMA_ATTENTION_TYPE_CAUSAL = 0,459# LLAMA_ATTENTION_TYPE_NON_CAUSAL = 1,460# };461LLAMA_ATTENTION_TYPE_UNSPECIFIED = -1462LLAMA_ATTENTION_TYPE_CAUSAL = 0463LLAMA_ATTENTION_TYPE_NON_CAUSAL = 1464 465 466# enum llama_split_mode {467# LLAMA_SPLIT_MODE_NONE = 0, // single GPU468# LLAMA_SPLIT_MODE_LAYER = 1, // split layers and KV across GPUs469# LLAMA_SPLIT_MODE_ROW = 2, // split layers and KV across GPUs, use tensor parallelism if supported470# };471LLAMA_SPLIT_MODE_NONE = 0472LLAMA_SPLIT_MODE_LAYER = 1473LLAMA_SPLIT_MODE_ROW = 2474 475 476# typedef struct llama_token_data {477# llama_token id; // token id478# float logit; // log-odds of the token479# float p; // probability of the token480# } llama_token_data;481class llama_token_data(ctypes.Structure):482 """Used to store token data483 484 Attributes:485 id (llama_token): token id486 logit (float): log-odds of the token487 p (float): probability of the token"""488 489 if TYPE_CHECKING:490 id: llama_token491 logit: float492 p: float493 494 _fields_ = [495 ("id", llama_token),496 ("logit", ctypes.c_float),497 ("p", ctypes.c_float),498 ]499 500 501llama_token_data_p = ctypes.POINTER(llama_token_data)502 503 504# typedef struct llama_token_data_array {505# // TODO: consider SoA506# // NOTE: this pointer can be modified by the samplers507# llama_token_data * data;508# size_t size;509# int64_t selected; // this is the index in the data array (i.e. not the token id)510# bool sorted;511# } llama_token_data_array;512class llama_token_data_array(ctypes.Structure):513 """Used to sample tokens given logits514 515 Attributes:516 data (ctypes.Array[llama_token_data]): token data517 size (int): size of the array518 selected (int): index in the data array (i.e. not the token id)519 sorted (bool): whether the array is sorted"""520 521 if TYPE_CHECKING:522 data: CtypesArray[llama_token_data]523 size: int524 selected: int525 sorted: bool526 527 _fields_ = [528 ("data", llama_token_data_p),529 ("size", ctypes.c_size_t),530 ("selected", ctypes.c_int64),531 ("sorted", ctypes.c_bool),532 ]533 534 535llama_token_data_array_p = ctypes.POINTER(llama_token_data_array)536 537# typedef bool (*llama_progress_callback)(float progress, void * user_data);538llama_progress_callback = ctypes.CFUNCTYPE(539 ctypes.c_bool, ctypes.c_float, ctypes.c_void_p540)541 542 543# // Input data for llama_encode/llama_decode544# // A llama_batch object can contain input about one or many sequences545# // The provided arrays (i.e. token, embd, pos, etc.) must have size of n_tokens546# //547# // - token : the token ids of the input (used when embd is NULL)548# // - embd : token embeddings (i.e. float vector of size n_embd) (used when token is NULL)549# // - pos : the positions of the respective token in the sequence550# // (if set to NULL, the token position will be tracked automatically by llama_encode/llama_decode)551# // - seq_id : the sequence to which the respective token belongs552# // (if set to NULL, the sequence ID will be assumed to be 0)553# // - logits : if zero, the logits (and/or the embeddings) for the respective token will not be output554# // (if set to NULL:555# // - if embeddings: all tokens are output556# // - if not: only the last token is output557# // )558# //559# typedef struct llama_batch {560# int32_t n_tokens;561 562# llama_token * token;563# float * embd;564# llama_pos * pos;565# int32_t * n_seq_id;566# llama_seq_id ** seq_id;567# int8_t * logits; // TODO: rename this to "output"568# } llama_batch;569class llama_batch(ctypes.Structure):570 """Input data for llama_encode/llama_decode571 572 A llama_batch object can contain input about one or many sequences573 574 The provided arrays (i.e. token, embd, pos, etc.) must have size of n_tokens575 576 Attributes:577 n_tokens (int): number of tokens578 token (ctypes.Array[llama_token]): the token ids of the input (used when embd is NULL)579 embd (ctypes.Array[ctypes.ctypes.c_float]): token embeddings (i.e. float vector of size n_embd) (used when token is NULL)580 pos (ctypes.Array[ctypes.Array[llama_pos]]): the positions of the respective token in the sequence581 seq_id (ctypes.Array[ctypes.Array[llama_seq_id]]): the sequence to which the respective token belongs582 logits (ctypes.Array[ctypes.ctypes.c_int8]): if zero, the logits for the respective token will not be output583 """584 585 if TYPE_CHECKING:586 n_tokens: int587 token: CtypesArray[llama_token]588 embd: CtypesArray[ctypes.c_float]589 pos: CtypesArray[CtypesArray[llama_pos]]590 n_seq_id: CtypesArray[ctypes.c_int]591 seq_id: CtypesArray[CtypesArray[llama_seq_id]]592 logits: CtypesArray[ctypes.c_int8]593 594 _fields_ = [595 ("n_tokens", ctypes.c_int32),596 ("token", ctypes.POINTER(llama_token)),597 ("embd", ctypes.POINTER(ctypes.c_float)),598 ("pos", ctypes.POINTER(llama_pos)),599 ("n_seq_id", ctypes.POINTER(ctypes.c_int32)),600 ("seq_id", ctypes.POINTER(ctypes.POINTER(llama_seq_id))),601 ("logits", ctypes.POINTER(ctypes.c_int8)),602 ]603 604 605# enum llama_model_kv_override_type {606# LLAMA_KV_OVERRIDE_TYPE_INT,607# LLAMA_KV_OVERRIDE_TYPE_FLOAT,608# LLAMA_KV_OVERRIDE_TYPE_BOOL,609# LLAMA_KV_OVERRIDE_TYPE_STR,610# };611LLAMA_KV_OVERRIDE_TYPE_INT = 0612LLAMA_KV_OVERRIDE_TYPE_FLOAT = 1613LLAMA_KV_OVERRIDE_TYPE_BOOL = 2614LLAMA_KV_OVERRIDE_TYPE_STR = 3615 616 617# struct llama_model_kv_override {618# enum llama_model_kv_override_type tag;619 620# char key[128];621 622 623# union {624# int64_t val_i64;625# double val_f64;626# bool val_bool;627# char val_str[128];628# };629# };630class llama_model_kv_override_value(ctypes.Union):631 _fields_ = [632 ("val_i64", ctypes.c_int64),633 ("val_f64", ctypes.c_double),634 ("val_bool", ctypes.c_bool),635 ("val_str", ctypes.c_char * 128),636 ]637 638 if TYPE_CHECKING:639 val_i64: int640 val_f64: float641 val_bool: bool642 val_str: bytes643 644 645class llama_model_kv_override(ctypes.Structure):646 _fields_ = [647 ("tag", ctypes.c_int),648 ("key", ctypes.c_char * 128),649 ("value", llama_model_kv_override_value),650 ]651 652 if TYPE_CHECKING:653 tag: int654 key: bytes655 value: Union[int, float, bool, bytes]656 657 658# struct llama_model_tensor_buft_override {659# const char * pattern;660# ggml_backend_buffer_type_t buft;661# };662 663 664# struct llama_model_params {665# // NULL-terminated list of devices to use for offloading (if NULL, all available devices are used)666# ggml_backend_dev_t * devices;667 668# // NULL-terminated list of buffer types to use for tensors that match a pattern669# const struct llama_model_tensor_buft_override * tensor_buft_overrides;670 671# int32_t n_gpu_layers; // number of layers to store in VRAM672# enum llama_split_mode split_mode; // how to split the model across multiple GPUs673 674# // the GPU that is used for the entire model when split_mode is LLAMA_SPLIT_MODE_NONE675# int32_t main_gpu;676 677# // proportion of the model (layers or rows) to offload to each GPU, size: llama_max_devices()678# const float * tensor_split;679 680# // Called with a progress value between 0.0 and 1.0. Pass NULL to disable.681# // If the provided progress_callback returns true, model loading continues.682# // If it returns false, model loading is immediately aborted.683# llama_progress_callback progress_callback;684 685# // context pointer passed to the progress callback686# void * progress_callback_user_data;687 688# // override key-value pairs of the model meta data689# const struct llama_model_kv_override * kv_overrides;690 691# // Keep the booleans together to avoid misalignment during copy-by-value.692# bool vocab_only; // only load the vocabulary, no weights693# bool use_mmap; // use mmap if possible694# bool use_mlock; // force system to keep model in RAM695# bool check_tensors; // validate model tensor data696# bool use_extra_bufts; // use extra buffer types (used for weight repacking)697# };698class llama_model_params(ctypes.Structure):699 """Parameters for llama_model700 701 Attributes:702 devices (ctypes.Array[ggml_backend_dev_t]): NULL-terminated list of devices to use for offloading (if NULL, all available devices are used)703 tensor_buft_overrides (ctypes.Array[llama_model_tensor_buft_override]): NULL-terminated list of buffer types to use for tensors that match a pattern704 n_gpu_layers (int): number of layers to store in VRAM705 split_mode (int): how to split the model across multiple GPUs706 main_gpu (int): the GPU that is used for the entire model when split_mode is LLAMA_SPLIT_MODE_NONE707 tensor_split (ctypes.Array[ctypes.ctypes.c_float]): proportion of the model (layers or rows) to offload to each GPU, size: llama_max_devices()708 progress_callback (llama_progress_callback): called with a progress value between 0.0 and 1.0. Pass NULL to disable. If the provided progress_callback returns true, model loading continues. If it returns false, model loading is immediately aborted.709 progress_callback_user_data (ctypes.ctypes.c_void_p): context pointer passed to the progress callback710 kv_overrides (ctypes.Array[llama_model_kv_override]): override key-value pairs of the model meta data711 vocab_only (bool): only load the vocabulary, no weights712 use_mmap (bool): use mmap if possible713 use_mlock (bool): force system to keep model in RAM714 check_tensors (bool): validate model tensor data715 use_extra_bufts (bool): use extra buffer types (used for weight repacking)"""716 717 if TYPE_CHECKING:718 devices: CtypesArray[ctypes.c_void_p] # NOTE: unused719 tensor_buft_overrides: CtypesArray[llama_model_tensor_buft_override] # NOTE: unused720 n_gpu_layers: int721 split_mode: int722 main_gpu: int723 tensor_split: CtypesArray[ctypes.c_float]724 progress_callback: Callable[[float, ctypes.c_void_p], bool]725 progress_callback_user_data: ctypes.c_void_p726 kv_overrides: CtypesArray[llama_model_kv_override]727 vocab_only: bool728 use_mmap: bool729 use_mlock: bool730 check_tensors: bool731 use_extra_bufts: bool732 733 _fields_ = [734 ("devices", ctypes.c_void_p), # NOTE: unnused735 ("tensor_buft_overrides", ctypes.c_void_p), # NOTE: unused736 ("n_gpu_layers", ctypes.c_int32),737 ("split_mode", ctypes.c_int),738 ("main_gpu", ctypes.c_int32),739 ("tensor_split", ctypes.POINTER(ctypes.c_float)),740 ("progress_callback", llama_progress_callback),741 ("progress_callback_user_data", ctypes.c_void_p),742 ("kv_overrides", ctypes.POINTER(llama_model_kv_override)),743 ("vocab_only", ctypes.c_bool),744 ("use_mmap", ctypes.c_bool),745 ("use_mlock", ctypes.c_bool),746 ("check_tensors", ctypes.c_bool),747 ("use_extra_bufts", ctypes.c_bool),748 ]749 750 751# // NOTE: changing the default values of parameters marked as [EXPERIMENTAL] may cause crashes or incorrect results in certain configurations752# // https://github.com/ggml-org/llama.cpp/pull/7544753# struct llama_context_params {754# uint32_t n_ctx; // text context, 0 = from model755# uint32_t n_batch; // logical maximum batch size that can be submitted to llama_decode756# uint32_t n_ubatch; // physical maximum batch size757# uint32_t n_seq_max; // max number of sequences (i.e. distinct states for recurrent models)758# int32_t n_threads; // number of threads to use for generation759# int32_t n_threads_batch; // number of threads to use for batch processing760 761# enum llama_rope_scaling_type rope_scaling_type; // RoPE scaling type, from `enum llama_rope_scaling_type`762# enum llama_pooling_type pooling_type; // whether to pool (sum) embedding results by sequence id763# enum llama_attention_type attention_type; // attention type to use for embeddings764 765# // ref: https://github.com/ggml-org/llama.cpp/pull/2054766# float rope_freq_base; // RoPE base frequency, 0 = from model767# float rope_freq_scale; // RoPE frequency scaling factor, 0 = from model768# float yarn_ext_factor; // YaRN extrapolation mix factor, negative = from model769# float yarn_attn_factor; // YaRN magnitude scaling factor770# float yarn_beta_fast; // YaRN low correction dim771# float yarn_beta_slow; // YaRN high correction dim772# uint32_t yarn_orig_ctx; // YaRN original context size773# float defrag_thold; // defragment the KV cache if holes/size > thold, <= 0 disabled (default)774 775# ggml_backend_sched_eval_callback cb_eval;776# void * cb_eval_user_data;777 778# enum ggml_type type_k; // data type for K cache [EXPERIMENTAL]779# enum ggml_type type_v; // data type for V cache [EXPERIMENTAL]780 781# // Abort callback782# // if it returns true, execution of llama_decode() will be aborted783# // currently works only with CPU execution784# ggml_abort_callback abort_callback;785# void * abort_callback_data;786 787# // Keep the booleans together and at the end of the struct to avoid misalignment during copy-by-value.788# bool embeddings; // if true, extract embeddings (together with logits)789# bool offload_kqv; // offload the KQV ops (including the KV cache) to GPU790# bool flash_attn; // use flash attention [EXPERIMENTAL]791# bool no_perf; // measure performance timings792# bool op_offload; // offload host tensor operations to device793# bool swa_full; // use full-size SWA cache (https://github.com/ggml-org/llama.cpp/pull/13194#issuecomment-2868343055)794# // NOTE: setting to false when n_seq_max > 1 can cause bad performance in some cases795# // ref: https://github.com/ggml-org/llama.cpp/pull/13845#issuecomment-2924800573796# bool kv_unified; // use a unified buffer across the input sequences when computing the attention797# // try to disable when n_seq_max > 1 for improved performance when the sequences do not share a large prefix798# // ref: https://github.com/ggml-org/llama.cpp/pull/14363799# };800class llama_context_params(ctypes.Structure):801 """Parameters for llama_context802 803 Attributes:804 n_ctx (int): text context, 0 = from model805 n_batch (int): logical maximum batch size that can be submitted to llama_decode806 n_ubatch (int): physical maximum batch size807 n_seq_max (int): max number of sequences (i.e. distinct states for recurrent models)808 n_threads (int): number of threads to use for generation809 n_threads_batch (int): number of threads to use for batch processing810 rope_scaling_type (int): RoPE scaling type, from `enum llama_rope_scaling_type`811 pooling_type (int): whether to pool (sum) embedding results by sequence id (ignored if no pooling layer)812 attention_type (int): attention type to use for embeddings813 rope_freq_base (float): RoPE base frequency, 0 = from model814 rope_freq_scale (float): RoPE frequency scaling factor, 0 = from model815 yarn_ext_factor (float): YaRN extrapolation mix factor, negative = from model816 yarn_attn_factor (float): YaRN magnitude scaling factor817 yarn_beta_fast (float): YaRN low correction dim818 yarn_beta_slow (float): YaRN high correction dim819 yarn_orig_ctx (int): YaRN original context size820 defrag_thold (float): defragment the KV cache if holes/size > thold, <= 0 disabled (default)821 cb_eval (ggml_backend_sched_eval_callback): callback for scheduling eval822 cb_eval_user_data (ctypes.ctypes.c_void_p): user data for cb_eval823 type_k (int): data type for K cache824 type_v (int): data type for V cache825 abort_callback (ggml_abort_callback): abort callback if it returns true, execution of llama_decode() will be aborted826 abort_callback_data (ctypes.ctypes.c_void_p): data for abort_callback827 embeddings (bool): if true, extract embeddings (together with logits)828 offload_kqv (bool): whether to offload the KQV ops (including the KV cache) to GPU829 flash_attn (bool): whether to use flash attention830 no_perf (bool): whether to measure performance timings831 op_offload (bool): offload host tensor operations to device832 swa_full (bool): use full-size SWA cache833 kv_unified (bool): use a unified buffer across the input sequences when computing the attention834 """835 836 if TYPE_CHECKING:837 n_ctx: int838 n_batch: int839 n_ubatch: int840 n_seq_max: int841 n_threads: int842 n_threads_batch: int843 rope_scaling_type: int844 pooling_type: int845 attention_type: int846 rope_freq_base: float847 rope_freq_scale: float848 yarn_ext_factor: float849 yarn_attn_factor: float850 yarn_beta_fast: float851 yarn_beta_slow: float852 yarn_orig_ctx: int853 defrag_thold: float854 cb_eval: Callable[[ctypes.c_void_p, bool], bool]855 cb_eval_user_data: ctypes.c_void_p856 type_k: int857 type_v: int858 abort_callback: Callable[[ctypes.c_void_p], bool]859 abort_callback_data: ctypes.c_void_p860 embeddings: bool861 offload_kqv: bool862 flash_attn: bool863 no_perf: bool864 op_offload: bool865 swa_full: bool866 kv_unified: bool867 868 _fields_ = [869 ("n_ctx", ctypes.c_uint32),870 ("n_batch", ctypes.c_uint32),871 ("n_ubatch", ctypes.c_uint32),872 ("n_seq_max", ctypes.c_uint32),873 ("n_threads", ctypes.c_int32),874 ("n_threads_batch", ctypes.c_int32),875 ("rope_scaling_type", ctypes.c_int),876 ("pooling_type", ctypes.c_int),877 ("attention_type", ctypes.c_int),878 ("rope_freq_base", ctypes.c_float),879 ("rope_freq_scale", ctypes.c_float),880 ("yarn_ext_factor", ctypes.c_float),881 ("yarn_attn_factor", ctypes.c_float),882 ("yarn_beta_fast", ctypes.c_float),883 ("yarn_beta_slow", ctypes.c_float),884 ("yarn_orig_ctx", ctypes.c_uint32),885 ("defrag_thold", ctypes.c_float),886 ("cb_eval", ggml_backend_sched_eval_callback),887 ("cb_eval_user_data", ctypes.c_void_p),888 ("type_k", ctypes.c_int),889 ("type_v", ctypes.c_int),890 ("abort_callback", ggml_abort_callback),891 ("abort_callback_data", ctypes.c_void_p),892 ("embeddings", ctypes.c_bool),893 ("offload_kqv", ctypes.c_bool),894 ("flash_attn", ctypes.c_bool),895 ("no_perf", ctypes.c_bool),896 ("op_offload", ctypes.c_bool),897 ("swa_full", ctypes.c_bool),898 ("kv_unified", ctypes.c_bool),899 ]900 901 902# // Signature for logging events903# // Note that text includes the new line character at the end for most events.904# // If your logging mechanism cannot handle that, check if the last character is '\n' and strip it905# // if it exists.906# // It might not exist for progress report where '.' is output repeatedly.907# typedef void (*llama_log_callback)(enum llama_log_level level, const char * text, void * user_data);908llama_log_callback = ctypes.CFUNCTYPE(909 None, ctypes.c_int, ctypes.c_char_p, ctypes.c_void_p910)911"""Signature for logging events912Note that text includes the new line character at the end for most events.913If your logging mechanism cannot handle that, check if the last character is '\n' and strip it914if it exists.915It might not exist for progress report where '.' is output repeatedly."""916 917 918# // model quantization parameters919# typedef struct llama_model_quantize_params {920# int32_t nthread; // number of threads to use for quantizing, if <=0 will use std::thread::hardware_concurrency()921# enum llama_ftype ftype; // quantize to this llama_ftype922# enum ggml_type output_tensor_type; // output tensor type923# enum ggml_type token_embedding_type; // token embeddings tensor type924# bool allow_requantize; // allow quantizing non-f32/f16 tensors925# bool quantize_output_tensor; // quantize output.weight926# bool only_copy; // only copy tensors - ftype, allow_requantize and quantize_output_tensor are ignored927# bool pure; // quantize all tensors to the default type928# bool keep_split; // quantize to the same number of shards929# void * imatrix; // pointer to importance matrix data930# void * kv_overrides; // pointer to vector containing overrides931# void * tensor_types; // pointer to vector containing tensor types932# void * prune_layers; // pointer to vector containing layer indices to prune933# } llama_model_quantize_params;934class llama_model_quantize_params(ctypes.Structure):935 """Parameters for llama_model_quantize936 937 Attributes:938 nthread (int): number of threads to use for quantizing, if <=0 will use std::thread::hardware_concurrency()939 ftype (int): quantize to this llama_ftype940 output_tensor_type (int): output tensor type941 token_embedding_type (int): token embeddings tensor type942 allow_requantize (bool): allow quantizing non-f32/f16 tensors943 quantize_output_tensor (bool): quantize output.weight944 only_copy (bool): only copy tensors - ftype, allow_requantize and quantize_output_tensor are ignored945 pure (bool): quantize all tensors to the default type946 keep_split (bool): quantize to the same number of shards947 imatrix (ctypes.c_void_p): pointer to importance matrix data948 kv_overrides (ctypes.c_void_p): pointer to vector containing overrides949 tensor_types (ctypes.c_void_p): pointer to vector containing tensor types950 prune_layers (ctypes.c_void_p): pointer to vector containing layer indices to prune951 """952 953 if TYPE_CHECKING:954 nthread: int955 ftype: int956 output_tensor_type: int957 token_embedding_type: int958 allow_requantize: bool959 quantize_output_tensor: bool960 only_copy: bool961 pure: bool962 keep_split: bool963 imatrix: ctypes.c_void_p964 kv_overrides: ctypes.c_void_p965 tensor_types: ctypes.c_void_p966 prune_layers: ctypes.c_void_p967 968 _fields_ = [969 ("nthread", ctypes.c_int32),970 ("ftype", ctypes.c_int),971 ("output_tensor_type", ctypes.c_int),972 ("token_embedding_type", ctypes.c_int),973 ("allow_requantize", ctypes.c_bool),974 ("quantize_output_tensor", ctypes.c_bool),975 ("only_copy", ctypes.c_bool),976 ("pure", ctypes.c_bool),977 ("keep_split", ctypes.c_bool),978 ("imatrix", ctypes.c_void_p),979 ("kv_overrides", ctypes.c_void_p),980 ("tensor_types", ctypes.c_void_p),981 ("prune_layers", ctypes.c_void_p),982 ]983 984 985# typedef struct llama_logit_bias {986# llama_token token;987# float bias;988# } llama_logit_bias;989class llama_logit_bias(ctypes.Structure):990 """Used to store logit bias991 992 Attributes:993 token (llama_token): token id994 bias (float): bias"""995 996 if TYPE_CHECKING:997 token: llama_token998 bias: float999 1000 _fields_ = [1001 ("token", llama_token),1002 ("bias", ctypes.c_float),1003 ]1004 1005 1006llama_logit_bias_p = ctypes.POINTER(llama_logit_bias)1007 1008 1009# typedef struct llama_sampler_chain_params {1010# bool no_perf; // whether to measure performance timings1011# } llama_sampler_chain_params;1012class llama_sampler_chain_params(ctypes.Structure):1013 """Parameters for llama_sampler_chain1014 1015 Attributes:1016 no_perf (bool): whether to measure performance timings"""1017 1018 if TYPE_CHECKING:1019 no_perf: bool1020 1021 _fields_ = [1022 ("no_perf", ctypes.c_bool),1023 ]1024 1025 1026# // used in chat template1027# typedef struct llama_chat_message {1028# const char * role;1029# const char * content;1030# } llama_chat_message;1031class llama_chat_message(ctypes.Structure):1032 _fields_ = [1033 ("role", ctypes.c_char_p),1034 ("content", ctypes.c_char_p),1035 ]1036 1037 1038# // lora adapter1039# struct llama_adapter_lora;1040llama_adapter_lora_p = ctypes.c_void_p1041llama_adapter_lora_p_ctypes = ctypes.POINTER(ctypes.c_void_p)1042 1043 1044# // Helpers for getting default parameters1045# LLAMA_API struct llama_model_params llama_model_default_params(void);1046@ctypes_function(1047 "llama_model_default_params",1048 [],1049 llama_model_params,1050)1051def llama_model_default_params() -> llama_model_params:1052 """Get default parameters for llama_model"""1053 ...1054 1055 1056# LLAMA_API struct llama_context_params llama_context_default_params(void);1057@ctypes_function(1058 "llama_context_default_params",1059 [],1060 llama_context_params,1061)1062def llama_context_default_params() -> llama_context_params:1063 """Get default parameters for llama_context"""1064 ...1065 1066 1067# LLAMA_API struct llama_sampler_chain_params llama_sampler_chain_default_params(void);1068@ctypes_function(1069 "llama_sampler_chain_default_params",1070 [],1071 llama_sampler_chain_params,1072)1073def llama_sampler_chain_default_params() -> llama_sampler_chain_params:1074 """Get default parameters for llama_sampler_chain"""1075 ...1076 1077 1078# LLAMA_API struct llama_model_quantize_params llama_model_quantize_default_params(void);1079@ctypes_function(1080 "llama_model_quantize_default_params",1081 [],1082 llama_model_quantize_params,1083)1084def llama_model_quantize_default_params() -> llama_model_quantize_params:1085 """Get default parameters for llama_model_quantize"""1086 ...1087 1088 1089# // Initialize the llama + ggml backend1090# // If numa is true, use NUMA optimizations1091# // Call once at the start of the program1092# LLAMA_API void llama_backend_init(void);1093@ctypes_function(1094 "llama_backend_init",1095 [],1096 None,1097)1098def llama_backend_init():1099 """Initialize the llama + ggml backend1100 Call once at the start of the program"""1101 ...1102 1103 1104# // numa strategies1105# enum ggml_numa_strategy {1106# GGML_NUMA_STRATEGY_DISABLED = 0,1107# GGML_NUMA_STRATEGY_DISTRIBUTE = 1,1108# GGML_NUMA_STRATEGY_ISOLATE = 2,1109# GGML_NUMA_STRATEGY_NUMACTL = 3,1110# GGML_NUMA_STRATEGY_MIRROR = 4,1111# GGML_NUMA_STRATEGY_COUNT1112# };1113GGML_NUMA_STRATEGY_DISABLED = 01114GGML_NUMA_STRATEGY_DISTRIBUTE = 11115GGML_NUMA_STRATEGY_ISOLATE = 21116GGML_NUMA_STRATEGY_NUMACTL = 31117GGML_NUMA_STRATEGY_MIRROR = 41118GGML_NUMA_STRATEGY_COUNT = 51119 1120 1121# // Call once at the end of the program - currently only used for MPI1122# LLAMA_API void llama_backend_free(void);1123@ctypes_function(1124 "llama_backend_free",1125 [],1126 None,1127)1128def llama_backend_free():1129 """Call once at the end of the program - currently only used for MPI"""1130 ...1131 1132 1133# //optional:1134# LLAMA_API void llama_numa_init(enum ggml_numa_strategy numa);1135@ctypes_function(1136 "llama_numa_init",1137 [ctypes.c_int],1138 None,1139)1140def llama_numa_init(numa: int, /):1141 ...1142 1143 1144# // Optional: an auto threadpool gets created in ggml if not passed explicitly1145# LLAMA_API void llama_attach_threadpool(1146# struct llama_context * ctx,1147# ggml_threadpool_t threadpool,1148# ggml_threadpool_t threadpool_batch);1149# TODO: Add llama_attach_threadpool1150 1151 1152# LLAMA_API void llama_detach_threadpool(struct llama_context * ctx);1153# TODO: Add llama_detach_threadpool1154 1155 1156# DEPRECATED(LLAMA_API struct llama_model * llama_load_model_from_file(1157# const char * path_model,1158# struct llama_model_params params),1159# "use llama_model_load_from_file instead");1160@ctypes_function(1161 "llama_load_model_from_file",1162 [ctypes.c_char_p, llama_model_params],1163 llama_model_p_ctypes,1164)1165def llama_load_model_from_file(1166 path_model: bytes, params: llama_model_params, /1167) -> Optional[llama_model_p]:1168 ...1169 1170 1171# // Load the model from a file1172# // If the file is split into multiple parts, the file name must follow this pattern: <name>-%05d-of-%05d.gguf1173# // If the split file name does not follow this pattern, use llama_model_load_from_splits1174# LLAMA_API struct llama_model * llama_model_load_from_file(1175# const char * path_model,1176# struct llama_model_params params);1177@ctypes_function(1178 "llama_model_load_from_file",1179 [ctypes.c_char_p, llama_model_params],1180 llama_model_p_ctypes,1181)1182def llama_model_load_from_file(1183 path_model: bytes, params: llama_model_params, /1184) -> Optional[llama_model_p]:1185 """Load the model from a file1186 1187 If the file is split into multiple parts, the file name must follow this pattern: <name>-%05d-of-%05d.gguf1188 1189 If the split file name does not follow this pattern, use llama_model_load_from_splits"""1190 ...1191 1192 1193# // Load the model from multiple splits (support custom naming scheme)1194# // The paths must be in the correct order1195# LLAMA_API struct llama_model * llama_model_load_from_splits(1196# const char ** paths,1197# size_t n_paths,1198# struct llama_model_params params);1199@ctypes_function(1200 "llama_model_load_from_splits",