hymenjj/llama-cpp-python-prebuilt

1from __future__ import annotations2 3from typing import List, Optional, Union, Dict4from typing_extensions import TypedDict, Literal5 6from pydantic import BaseModel, Field7 8import llama_cpp9 10 11model_field = Field(12    description="The model to use for generating completions.", default=None13)14 15max_tokens_field = Field(16    default=16, ge=1, description="The maximum number of tokens to generate."17)18 19min_tokens_field = Field(20    default=0,21    ge=0,22    description="The minimum number of tokens to generate. It may return fewer tokens if another condition is met (e.g. max_tokens, stop).",23)24 25temperature_field = Field(26    default=0.8,27    description="Adjust the randomness of the generated text.\n\n"28    + "Temperature is a hyperparameter that controls the randomness of the generated text. It affects the probability distribution of the model's output tokens. A higher temperature (e.g., 1.5) makes the output more random and creative, while a lower temperature (e.g., 0.5) makes the output more focused, deterministic, and conservative. The default value is 0.8, which provides a balance between randomness and determinism. At the extreme, a temperature of 0 will always pick the most likely next token, leading to identical outputs in each run.",29)30 31top_p_field = Field(32    default=0.95,33    ge=0.0,34    le=1.0,35    description="Limit the next token selection to a subset of tokens with a cumulative probability above a threshold P.\n\n"36    + "Top-p sampling, also known as nucleus sampling, is another text generation method that selects the next token from a subset of tokens that together have a cumulative probability of at least p. This method provides a balance between diversity and quality by considering both the probabilities of tokens and the number of tokens to sample from. A higher value for top_p (e.g., 0.95) will lead to more diverse text, while a lower value (e.g., 0.5) will generate more focused and conservative text.",37)38 39min_p_field = Field(40    default=0.05,41    ge=0.0,42    le=1.0,43    description="Sets a minimum base probability threshold for token selection.\n\n"44    + "The Min-P sampling method was designed as an alternative to Top-P, and aims to ensure a balance of quality and variety. The parameter min_p represents the minimum probability for a token to be considered, relative to the probability of the most likely token. For example, with min_p=0.05 and the most likely token having a probability of 0.9, logits with a value less than 0.045 are filtered out.",45)46 47stop_field = Field(48    default=None,49    description="A list of tokens at which to stop generation. If None, no stop tokens are used.",50)51 52stream_field = Field(53    default=False,54    description="Whether to stream the results as they are generated. Useful for chatbots.",55)56 57top_k_field = Field(58    default=40,59    ge=0,60    description="Limit the next token selection to the K most probable tokens.\n\n"61    + "Top-k sampling is a text generation method that selects the next token only from the top k most likely tokens predicted by the model. It helps reduce the risk of generating low-probability or nonsensical tokens, but it may also limit the diversity of the output. A higher value for top_k (e.g., 100) will consider more tokens and lead to more diverse text, while a lower value (e.g., 10) will focus on the most probable tokens and generate more conservative text.",62)63 64repeat_penalty_field = Field(65    default=1.1,66    ge=0.0,67    description="A penalty applied to each token that is already generated. This helps prevent the model from repeating itself.\n\n"68    + "Repeat penalty is a hyperparameter used to penalize the repetition of token sequences during text generation. It helps prevent the model from generating repetitive or monotonous text. A higher value (e.g., 1.5) will penalize repetitions more strongly, while a lower value (e.g., 0.9) will be more lenient.",69)70 71presence_penalty_field = Field(72    default=0.0,73    ge=-2.0,74    le=2.0,75    description="Positive values penalize new tokens based on whether they appear in the text so far, increasing the model's likelihood to talk about new topics.",76)77 78frequency_penalty_field = Field(79    default=0.0,80    ge=-2.0,81    le=2.0,82    description="Positive values penalize new tokens based on their existing frequency in the text so far, decreasing the model's likelihood to repeat the same line verbatim.",83)84 85mirostat_mode_field = Field(86    default=0,87    ge=0,88    le=2,89    description="Enable Mirostat constant-perplexity algorithm of the specified version (1 or 2; 0 = disabled)",90)91 92mirostat_tau_field = Field(93    default=5.0,94    ge=0.0,95    le=10.0,96    description="Mirostat target entropy, i.e. the target perplexity - lower values produce focused and coherent text, larger values produce more diverse and less coherent text",97)98 99mirostat_eta_field = Field(100    default=0.1, ge=0.001, le=1.0, description="Mirostat learning rate"101)102 103grammar = Field(104    default=None,105    description="A CBNF grammar (as string) to be used for formatting the model's output.",106)107 108 109class CreateCompletionRequest(BaseModel):110    prompt: Union[str, List[str]] = Field(111        default="", description="The prompt to generate completions for."112    )113    suffix: Optional[str] = Field(114        default=None,115        description="A suffix to append to the generated text. If None, no suffix is appended. Useful for chatbots.",116    )117    max_tokens: Optional[int] = Field(118        default=16, ge=0, description="The maximum number of tokens to generate."119    )120    min_tokens: int = min_tokens_field121    temperature: float = temperature_field122    top_p: float = top_p_field123    min_p: float = min_p_field124    echo: bool = Field(125        default=False,126        description="Whether to echo the prompt in the generated text. Useful for chatbots.",127    )128    stop: Optional[Union[str, List[str]]] = stop_field129    stream: bool = stream_field130    logprobs: Optional[int] = Field(131        default=None,132        ge=0,133        description="The number of logprobs to generate. If None, no logprobs are generated.",134    )135    presence_penalty: Optional[float] = presence_penalty_field136    frequency_penalty: Optional[float] = frequency_penalty_field137    logit_bias: Optional[Dict[str, float]] = Field(None)138    seed: Optional[int] = Field(None)139 140    # ignored or currently unsupported141    model: Optional[str] = model_field142    n: Optional[int] = 1143    best_of: Optional[int] = 1144    user: Optional[str] = Field(default=None)145 146    # llama.cpp specific parameters147    top_k: int = top_k_field148    repeat_penalty: float = repeat_penalty_field149    logit_bias_type: Optional[Literal["input_ids", "tokens"]] = Field(None)150    mirostat_mode: int = mirostat_mode_field151    mirostat_tau: float = mirostat_tau_field152    mirostat_eta: float = mirostat_eta_field153    grammar: Optional[str] = None154 155    model_config = {156        "json_schema_extra": {157            "examples": [158                {159                    "prompt": "\n\n### Instructions:\nWhat is the capital of France?\n\n### Response:\n",160                    "stop": ["\n", "###"],161                }162            ]163        }164    }165 166 167class CreateEmbeddingRequest(BaseModel):168    model: Optional[str] = model_field169    input: Union[str, List[str]] = Field(description="The input to embed.")170    user: Optional[str] = Field(default=None)171 172    model_config = {173        "json_schema_extra": {174            "examples": [175                {176                    "input": "The food was delicious and the waiter...",177                }178            ]179        }180    }181 182 183class ChatCompletionRequestMessage(BaseModel):184    role: Literal["system", "user", "assistant", "function"] = Field(185        default="user", description="The role of the message."186    )187    content: Optional[str] = Field(188        default="", description="The content of the message."189    )190 191 192class CreateChatCompletionRequest(BaseModel):193    messages: List[llama_cpp.ChatCompletionRequestMessage] = Field(194        default=[], description="A list of messages to generate completions for."195    )196    functions: Optional[List[llama_cpp.ChatCompletionFunction]] = Field(197        default=None,198        description="A list of functions to apply to the generated completions.",199    )200    function_call: Optional[llama_cpp.ChatCompletionRequestFunctionCall] = Field(201        default=None,202        description="A function to apply to the generated completions.",203    )204    tools: Optional[List[llama_cpp.ChatCompletionTool]] = Field(205        default=None,206        description="A list of tools to apply to the generated completions.",207    )208    tool_choice: Optional[llama_cpp.ChatCompletionToolChoiceOption] = Field(209        default=None,210        description="A tool to apply to the generated completions.",211    )  # TODO: verify212    max_tokens: Optional[int] = Field(213        default=None,214        description="The maximum number of tokens to generate. Defaults to inf",215    )216    min_tokens: int = min_tokens_field217    logprobs: Optional[bool] = Field(218        default=False,219        description="Whether to output the logprobs or not. Default is True",220    )221    top_logprobs: Optional[int] = Field(222        default=None,223        ge=0,224        description="The number of logprobs to generate. If None, no logprobs are generated. logprobs need to set to True.",225    )226    temperature: float = temperature_field227    top_p: float = top_p_field228    min_p: float = min_p_field229    stop: Optional[Union[str, List[str]]] = stop_field230    stream: bool = stream_field231    presence_penalty: Optional[float] = presence_penalty_field232    frequency_penalty: Optional[float] = frequency_penalty_field233    logit_bias: Optional[Dict[str, float]] = Field(None)234    seed: Optional[int] = Field(None)235    response_format: Optional[llama_cpp.ChatCompletionRequestResponseFormat] = Field(236        default=None,237    )238 239    # ignored or currently unsupported240    model: Optional[str] = model_field241    n: Optional[int] = 1242    user: Optional[str] = Field(None)243 244    # llama.cpp specific parameters245    top_k: int = top_k_field246    repeat_penalty: float = repeat_penalty_field247    logit_bias_type: Optional[Literal["input_ids", "tokens"]] = Field(None)248    mirostat_mode: int = mirostat_mode_field249    mirostat_tau: float = mirostat_tau_field250    mirostat_eta: float = mirostat_eta_field251    grammar: Optional[str] = None252 253    model_config = {254        "json_schema_extra": {255            "examples": [256                {257                    "messages": [258                        ChatCompletionRequestMessage(259                            role="system", content="You are a helpful assistant."260                        ).model_dump(),261                        ChatCompletionRequestMessage(262                            role="user", content="What is the capital of France?"263                        ).model_dump(),264                    ]265                }266            ]267        }268    }269 270 271class ModelData(TypedDict):272    id: str273    object: Literal["model"]274    owned_by: str275    permissions: List[str]276 277 278class ModelList(TypedDict):279    object: Literal["list"]280    data: List[ModelData]281 282 283class TokenizeInputRequest(BaseModel):284    model: Optional[str] = model_field285    input: str = Field(description="The input to tokenize.")286 287    model_config = {288        "json_schema_extra": {"examples": [{"input": "How many tokens in this query?"}]}289    }290 291 292class TokenizeInputResponse(BaseModel):293    tokens: List[int] = Field(description="A list of tokens.")294 295    model_config = {"json_schema_extra": {"example": {"tokens": [123, 321, 222]}}}296 297 298class TokenizeInputCountResponse(BaseModel):299    count: int = Field(description="The number of tokens in the input.")300 301    model_config = {"json_schema_extra": {"example": {"count": 5}}}302 303 304class DetokenizeInputRequest(BaseModel):305    model: Optional[str] = model_field306    tokens: List[int] = Field(description="A list of toekns to detokenize.")307 308    model_config = {"json_schema_extra": {"example": [{"tokens": [123, 321, 222]}]}}309 310 311class DetokenizeInputResponse(BaseModel):312    text: str = Field(description="The detokenized text.")313 314    model_config = {315        "json_schema_extra": {"example": {"text": "How many tokens in this query?"}}316    }317