CoolFace
Modelpublic

hymenjj/llama-cpp-python-prebuilt

sourceHugging Faceupdated 7mo agoView on Hugging Face
0likes
llama.cpython-311.pyc712 linesDownload Raw Back to __pycache__
1200�i�w����ddlmZddlZddlZddlZddlZddlZddlZddlZddl	Z	ddl3Z4ddlZddlZddl
Z
ddlmZmZmZmZmZmZmZmZmZmZmZddlmZddlmZddlTddlmZdd	l m!Z!m"Z"m#Z#m$Z$dd5l%m&Z&m'Z'ddl(m)Z)ddl*m+Z+ddl,m-Z-ddl.Z/ddl0mZ1ddl2m3Z4ddl5m6Z6dd
l7m8Z8Gd�d��Z9Gd�d��Z:ee1j;e/j<e1j;e/j=ge1j;e/j=fZ>Gd�dee>��Z?ee1j;e/j<e1j;e/j=ge@fZAGd�deeA��ZBGd�de>��ZCdS)�)�annotationsN)�Any�List�Literal�Optional�Union�	Generator�Sequence�Iterator�Deque�Callable�Dict)�deque)�Path�)�*)�LlamaGrammar)�BaseLlamaCache�6LlamaCache�LlamaDiskCache�
LlamaRAMCache)�BaseLlamaTokenizer�LlamaTokenizer)�LlamaDraftModel)�set_verbose)�suppress_stdout_stderrc,��eZdZdZdZdejddddddejdddddejej	dddd	d7d	ddddddddddd	ddddddddddd�+d�dF�Z8ed�dI���Zed�dK���Z
ed�dM���Zed�dO���Zed�dQ���Zed�dS���Z	d�d�dY�Z		d�d�d]�Zd�d`�Zd�da�Zdb�Zd�dd�Z															d�d�d|�Z																d�d�d~�Z																	d�d�d��Z	d�d�d��Z			d�d�d��Zdd�dhdfdgd	ddgddd	deddd	ddjdidddddfd�d��Zdd�dhdfdgd	ddgddd	deddd	ddjdidddddfd�d��Zdd�dhdfdgd	ddgddd	deddd	ddjdidddddfd�d��Zddddd�dfdedgd	dgdddddd	d	ddjdiddddddfd�d��Z d�d��Z!d��Z"d��Z#d�d��Z$d�d��Z%d�d��Z&d�d��Z'd�d��Z(d�d��Z)d�d��Z*d�d��Z+d�d��Z,d�d„Z-d�dÄZ.d�dĄZ/e0	d�d�dɄ��Z1e0d�d̄��Z2e3				d��dd؄��Z4dS(�Llamaz0High-level Python wrapper for a llama.cpp model.FrNTi�g���?g@@�@)+�n_gpu_layers�9split_mode�main_gpu�tensor_split�10vocab_only�use_mmap�	use_mlock�kv_overrides�seed�n_ctx�n_batch�n_ubatch�	n_threads�n_threads_batch�rope_scaling_type�pooling_type�rope_freq_base�rope_freq_scale�yarn_ext_factor�yarn_attn_factor�yarn_beta_fast�yarn_beta_slow�
yarn_orig_ctx�11logits_all�	embedding�offload_kqv�12flash_attn�13op_offload�swa_full�no_perf�last_n_tokens_size�	lora_base�14lora_scale�	lora_path�numa�chat_format�chat_handler�draft_model�	tokenizer�type_k�type_v�15spm_infill�verbose�16model_path�strr"�intr#r$r%�Optional[List[float]]r&�boolr'r(r)�1Optional[Dict[str, Union[bool, int, float, str]]]r*r+r,r-r.�
Optional[int]r/r0r1r2�floatr3r4r5r6r7r8r9r:r;r<r=�Optional[bool]r>r?r@rA�
Optional[str]rBrCrD�Union[bool, int]rErF�6Optional[llama_chat_format.LlamaChatCompletionHandler]rG�Optional[LlamaDraftModel]rH�Optional[BaseLlamaTokenizer]rIrJrKrLc+17����|,�_tj���_t	|,��t18jsGt|,���5tj	��ddd��n#1swxYwYdt19_t|$t��r |$rtjntj
�_n|$�_�jtj
krAt|,���5tj�j��ddd��n#1swxYwY|�_tj���_|dkrdn|�j_|�j_|�j_|�_d�_�j�rt1�j��tjkrt5dtj�����t6jtjz}.|.|��_�j�j_|�j_|#�|nd�j_|�j_|	�_ |	���t1|	��dz}/tj!|/z���_"tG|	�$����D�]>\}0\}1}2|1�%d	���j"|0_&t|2t��r4tj'�j"|0_(|2�j"|0j)_*�wt|2tV��r4tj,�j"|0_(|2�j"|0j)_-��t|2t\��r5tj/�j"|0_(|2�j"|0j)_0��20t|2tb���r|2�%d	��}3t1|3��d21krt5d|1�d|2�����|3�2d22d
��}3tj3�j"|0_(tij5tVt7j6�j"|0j)��tj7j8j9z��}4t7j5|4t7j:t6j;����}5t7j<|5|3d23����+t5d|1�d|2�����d
�j"d_&�j"�j_ t{||���_>|p#tt�jA��dzd���_B|pt�jA���_C|24ptjD�_EtjF���_G|�jG_H�j>�jG_>t{�j>|
���jG_I�jB�jG_B�jC�jG_C|�|ntjJ�jG_K|�jG_L|dkr|nd�jG_M|dkr|nd�jG_N|dkr|nd�jG_O|dkr|nd�jG_P|dkr|nd�jG_Q|dkr|nd�jG_R|dkr|nd�jG_S|'�|nd�_T|�jG_U|�jG_V|�jG_W|�|�jG_X|�|�jG_Y|)�|)�jG_Z|*�|*�jG_[|�jG_\| �_]d�_^|!�__|"�_`|#�_a|+�_bt�jd�e|��st5d|������j�ftjgt�ji�j�j�j��������_j|(pt�����_l|dkr��jj�m��}t{||���_>�jj�m���jG_H�j>�jG_>t{�j>|
���jG_I�j�ftjgt�jn�jj�jG�j��������_o�j�ftjgt�jp�j>d�jGjH�j��������_qd�_r�jar�tjs�jjjt�ja�%d	�����_r�jr�t�d�ja������fd�}6�j�v|6��tjw�jojx�jr�j`��rt�d�ja������jr?t�tjz���{d	��t�j}���|%�_~|&�_i�_�|'�_�������_���H���_�������_�������_�t�j��j�����_�d�_��tj�|f�tj�����_��tj�|dkr|n|�j�f�tj�����_�t7jd���_�	�jj�����_�nD#�t($r6}7i�_��jrt�d|7��t�j}���Yd}7~7nd}7~7wwxYw�jr#t�d�j���t�j}��������}8�����}9|8dkr�jj��|8��nd }:|9dkr�jj��|9��nd };�t/d!��j��$��D����}<d"�j�vr�j�d"|<d#<�jrE|<rCt�d$d%��|<��������t�j}���|<�$��D]9\}=}>�t5j�|>|:|;|8g�&�������j�|=<�:�j~�‰j��d#|<vr��t5j��j���}%|%�-|%�_~�jrt�d'|%��t�j}���nn�jr`t�d(|<d#��t�j}���t�d)|:��t�j}���t�d*|;��t�j}���d#�_~�j~�8�j�1d+�_~�jr#t�d,�j~��t�j}���d�_�dS)-a�Load a llama.cpp model from `model_path`.25 26        Examples:27            Basic usage28 29            >>> import llama_cpp30            >>> model = llama_cpp.Llama(31            ...     model_path="path/to/model",32            ... )33            >>> print(model("The quick brown fox jumps ", stop=["."])["choices"][0]["text"])34            the lazy dog35 36            Loading a chat model37 38            >>> import llama_cpp39            >>> model = llama_cpp.Llama(40            ...     model_path="path/to/model",41            ...     chat_format="llama-2",42            ... )43            >>> print(model.create_chat_completion(44            ...     messages=[{45            ...         "role": "user",46            ...         "content": "what is the meaning of life?"47            ...     }]48            ... ))49 50        Args:51            model_path: Path to the model.52            n_gpu_layers: Number of layers to offload to GPU (-ngl). If -1, all layers are offloaded.53            split_mode: How to split the model across GPUs. See llama_cpp.LLAMA_SPLIT_* for options.54            main_gpu: main_gpu interpretation depends on split_mode: LLAMA_SPLIT_MODE_NONE: the GPU that is used for the entire model. LLAMA_SPLIT_MODE_ROW: the GPU that is used for small tensors and intermediate results. LLAMA_SPLIT_MODE_LAYER: ignored55            tensor_split: How split tensors should be distributed across GPUs. If None, the model is not split.56            vocab_only: Only load the vocabulary no weights.57            use_mmap: Use mmap if possible.58            use_mlock: Force the system to keep the model in RAM.59            kv_overrides: Key-value overrides for the model.60            seed: RNG seed, -1 for random61            n_ctx: Text context, 0 = from model62            n_batch: Prompt processing maximum batch size63            n_ubatch: Physical batch size64            n_threads: Number of threads to use for generation65            n_threads_batch: Number of threads to use for batch processing66            rope_scaling_type: RoPE scaling type, from `enum llama_rope_scaling_type`. ref: https://github.com/ggerganov/llama.cpp/pull/205467            pooling_type: Pooling type, from `enum llama_pooling_type`.68            rope_freq_base: RoPE base frequency, 0 = from model69            rope_freq_scale: RoPE frequency scaling factor, 0 = from model70            yarn_ext_factor: YaRN extrapolation mix factor, negative = from model71            yarn_attn_factor: YaRN magnitude scaling factor72            yarn_beta_fast: YaRN low correction dim73            yarn_beta_slow: YaRN high correction dim74            yarn_orig_ctx: YaRN original context size75            logits_all: Return logits for all tokens, not just the last token. Must be True for completion to return logprobs.76            embedding: Embedding mode only.77            offload_kqv: Offload K, Q, V to GPU.78            flash_attn: Use flash attention.79            op_offload: offload host tensor operations to device80            swa_full: use full-size SWA cache (https://github.com/ggml-org/llama.cpp/pull/13194#issuecomment-2868343055)81            no_perf: Measure performance timings.82            last_n_tokens_size: Maximum number of tokens to keep in the last_n_tokens deque.83            lora_base: Optional path to base model, useful if using a quantized base model and you want to apply LoRA to an f16 model.84            lora_path: Path to a LoRA file to apply to the model.85            numa: numa policy86            chat_format: String specifying the chat format to use when calling create_chat_completion.87            chat_handler: Optional chat handler to use when calling create_chat_completion.88            draft_model: Optional draft model to use for speculative decoding.89            tokenizer: Optional tokenizer to override the default tokenizer from llama.cpp.90            verbose: Print verbose output to stderr.91            type_k: KV cache data type for K (default: f16)92            type_v: KV cache data type for V (default: f16)93            spm_infill: Use Suffix/Prefix/Middle pattern for infill (instead of Prefix/Suffix/Middle) as some models prefer this.94 95        Raises:96            ValueError: If the model path does not exist.97 98        Returns:99            A Llama instance.100        )�disableNT�����i���zZAttempt to split tensors that exceed maximum supported devices. Current LLAMA_MAX_DEVICES=Fr�utf-8�z101Value for z is too long: �zUnknown value type for z: �rrzModel path does not exist: )�102path_model�paramsrL)�modelrcrL)�n_tokens�embd�	n_seq_maxrLz2Failed to initialize LoRA adapter from lora path: c�Z���j�dStj�j��d�_dS�N)�
_lora_adapter�	llama_cpp�llama_adapter_lora_free��selfs��:/usr/local/lib/python3.11/site-packages/llama_cpp/llama.py�free_lora_adapterz)Llama.__init__.<locals>.free_lora_adapter�s3����%�-��F��1�$�2D�E�E�E�%)��"�"�"�z+Failed to set LoRA adapter from lora path: ��file)�n_vocab��dtypeg$@zFailed to load metadata: zModel metadata: �c3�^K�|](\}}|�d���|dd�|fV��)dS)ztokenizer.chat_template.�103N)�104startswith)�.0�name�templates   ro�	<genexpr>z!Llama.__init__.<locals>.<genexpr>�sZ���� 105� 106���h����9�:�:� 107�
�"�#�#�Y��!� 108� 109� 110� 111� 112� 113rqztokenizer.chat_templatezchat_template.defaultz&Available chat formats from metadata: z, )r}�	eos_token�	bos_token�stop_token_idszGuessed chat format: zUsing gguf chat template: zUsing chat eos_token: zUsing chat bos_token: zllama-2zUsing fallback chat format: )�rL�114contextlib�	ExitStack�_stackrr�_Llama__backend_initializedrrk�llama_backend_init�115isinstancerQ�GGML_NUMA_STRATEGY_DISTRIBUTE�GGML_NUMA_STRATEGY_DISABLEDrD�llama_numa_initrM�llama_model_default_params�model_paramsr"r#r$r%�_c_tensor_split�len�LLAMA_MAX_DEVICES�116ValueError�ctypes�c_floatr&r'r(r)�llama_model_kv_override�_kv_overrides_array�	enumerate�items�encode�key�LLAMA_KV_OVERRIDE_TYPE_BOOL�tag�value�val_boolrO�LLAMA_KV_OVERRIDE_TYPE_INT�val_i64rT�LLAMA_KV_OVERRIDE_TYPE_FLOAT�val_f64rN�ljust�LLAMA_KV_OVERRIDE_TYPE_STR�typing�cast�	addressof�llama_model_kv_override_value�val_str�offset�POINTER�c_char�memmove�minr,�max�multiprocessing�	cpu_countr.r/�LLAMA_DEFAULT_SEED�_seed�llama_context_default_params�context_paramsr+r-�#LLAMA_ROPE_SCALING_TYPE_UNSPECIFIEDr0r1r2r3r4r5r6r7r8�_logits_all�117embeddingsr;r<r=r>rIrJr?r@�cacherArBrCrK�os�path�exists�
enter_context�closing�	internals�118LlamaModel�_modelr�119tokenizer_�n_ctx_train�LlamaContext�_ctx�120LlamaBatch�_batchrj�llama_adapter_lora_initrd�RuntimeError�callback�llama_set_adapter_lora�ctx�print�llama_print_system_info�decode�sys�stderrrErF�_chat_handlersrGrt�_n_vocab�_n_ctx�token_nl�	_token_nl�	token_eos�121_token_eos�LlamaTokenDataArray�_candidatesre�np�ndarray�intc�	input_ids�single�scores�_mirostat_mu�metadata�	Exception�	token_bos�token_get_text�dict�join�keys�llama_chat_format�Jinja2ChatFormatter�to_chat_handler�$guess_chat_format_from_gguf_metadata�_sampler)?rnrMr"r#r$r%r&r'r(r)r*r+r,r-r.r/r0r1r2r3r4r5r6r7r8r9r:r;r<r=r>r?r@rArBrCrDrErFrGrHrIrJrKrL�kwargs�122FloatArray�
kvo_array_len�i�k�v�v_bytes�address�buffer_startrp�e�eos_token_id�bos_token_idrr��template_choicesr|r}s?`                                                              ro�__init__zLlama.__init__<s����V��� �*�,�,����G�����*�	/�'��8�8�8�
/�
/��,�.�.�.�
/�
/�
/�
/�
/�
/�
/�
/�
/�
/�
/����
/�
/�
/�
/�*.�E�'��d�D�!�!�	��;�	�7�7��:�
�I�I��D�I��9�	�=�=�=�'��8�8�8�
5�
5��)�$�)�4�4�4�
5�
5�
5�
5�
5�
5�
5�
5�
5�
5�
5����
5�
5�
5�
5�%���&�@�B�B���&�"�,�,�J�J�,�	
��&�(2���$�%-���"�(���#�����(��4�$�%�%�	�(C�C�C� �O�qz�rM�O�O���� ��)�*E�E�J�#-�:��$�D� �.2�-A�D��*�'1���$�1:�1B�X�X����"�&/���#�)����#���-�-��1�M��1�M�A�(�(�D�$�'�|�'9�'9�';�';�<�<�&
I�&
I�	��6�A�q�23�(�(�7�2C�2C��(��+�/��a��&�&�$I�&�A��,����AB�D�,�Q�/�5�>�>���3�'�'�I�&�@��,����@A�D�,�Q�/�5�=�=���5�)�)�I�&�B��,����@A�D�,�Q�/�5�=�=���3�'�'�I��h�h�w�/�/�G��7�|�|�c�)�)�(�)J�a�)J�)J�q�)J�)J�K�K�K�%�m�m�C��7�7�G�&�@��,����$�k���(��)A�!�)D�)J�K�K�#�A�I�P�Q���G�123$*�;�w���v�}�8U�8U�#V�#V�L��N�$�������%�%G�q�%G�%G�A�%G�%G�H�H�H��
�$��
��-1�-E�D��*��5�'�*�*���"�N�c�/�*C�*E�*E��*J�A�&N�&N���.�M�/�2K�2M�2M����9�Y�9��124�(�D�F�F���$)���!�&*�l���#�'*�4�<��'B�'B���$�(,����%�.2�.B���+�!�,�
���>�	
��-�125,8���(�,��3�3�N�N��	
��*� /�#�5�5�O�O�1�	
��+� /�#�5�5�O�O�1�	
��+�!1�C� 7� 7���Q�	
��,�-��3�3�N�N��	
��*�-��3�3�N�N��	
��*�>K�a�=O�=O�M�M�UV���)�)4�)<�:�:�$���)2���&�*5���'�)3���&��!�-7�D��*���+3�D��(���)/�D��&���)/�D��&�&-���#�"4���/3��126�"���$���"���$����w�~�~�j�)�)�	I��G�:�G�G�H�H�H��k�/�/����$�#���,� �L����
�
�127�128���$�;�~�d�';�';����A�:�:��K�+�+�-�-�E��u�g�.�.�D�L�(,��(?�(?�(A�(A�D��%�*.�,�D��'�+.�t�|�X�+F�+F�D��(��K�-�-����&��+��.� �L����
�
�129�130��	��k�/�/����$�!�\��"�1�7� �L�	���
�
�	131�	132���HL����>�	�!*�!B���!���%�%�g�.�.�"�"�D���!�)�"�Y���Y�Y����
*�
*�
*�
*�
*�
�K� � �!2�3�3�3��/��	�
�t�1�4�?���
�#�R�$�.�R�R�����<�	X��)�3�5�5�<�<�W�E�E�C�J�W�W�W�W�&���(���
�	
��'���������
��j�j�l�l�����������.�.�*�*���$�8���O�O�O�����
�/1�z�5�(�"�'�/R�/R�/R���.0�j� �D�(�(�U�U�g�t�}�E�R�Y�/133�/134�/135���#�N��136�137���	H� �K�0�0�2�2�D�M�M���	H�	H�	H��D�M��|�
H��5�!�5�5�C�J�G�G�G�G����������	H����138�<�	G��4�T�]�4�4�3�:�F�F�F�F��~�~�'�'���~�~�'�'��9E��8J�8J�D�K�&�&�|�4�4�4�PR�	�9E��8J�8J�D�K�&�&�|�4�4�4�PR�	�139 � 140� 141�"&�-�"5�"5�"7�"7� 142� 143� 144�145�146��%��
�5�5�8<�
�)�9��4�5��<�	�,�	��]����CS�CX�CX�CZ�CZ�9[�9[�]�]��Z�
�
�
�
�147/�4�4�6�6�	 �	 �N�D�(�(9�(M�!�#�#� ,�~�	)�)�)�148�o���
���%�%�
��$��!�)�'�+;�;�;�+�P��
���K��&�#.�� ��<�R��?�+�?�?�c�j�Q�Q�Q�Q���<�Q��`�5E�F]�5^�`�`� �Z������>�9�>�>�S�Z�P�P�P�P��>�9�>�>�S�Z�P�P�P�P�#:�� ���#��(9�(A�(�D���|�
��E�4�3C�E�E�C�J�������
�
�
s<�A,�,A0�3A0�$D149�150D�D�;l�m�%,m�m�return�llama_cpp.llama_context_pc��|jjSri)r�r�rms ror�z	Llama.ctx%s���y�}�rq�llama_cpp.llama_model_pc��|jjSri)r�rdrms rordzLlama.model)s
���{� � rq�npt.NDArray[np.intc]c�*�|jd|j�Sri)r�rerms ro�151_input_idszLlama._input_ids-s���~�o��
�o�.�.rq�npt.NDArray[np.single]c�2�|jd|j�dd�fSri)r�rerms ro�_scoresz
Llama._scores1s���{�?�T�]�?�A�A�A�-�.�.rq�152Deque[int]c�v�t|jd|j����|j���S)N��maxlen)rr�re�tolistr�rms ro�eval_tokenszLlama.eval_tokens5s0���T�^�O�d�m�O�4�;�;�=�=�d�k�R�R�R�Rrq�Deque[List[float]]c��t|jd|j�dd�f���|jr|jnd���S)Nrr)rr�rer	r�r�rms ro�eval_logitszLlama.eval_logits9sO����K��$�-�����*�+�2�2�4�4�"&�"2�9�4�;�;��153�154�155�	156rq�text�bytes�add_bos�special�	List[int]c�:�|j�|||��S)aUTokenize a string.157 158        Args:159            text: The utf-8 encoded string to tokenize.160            add_bos: Whether to add a beginning of sequence token.161            special: Whether to tokenize special tokens.162 163        Raises:164            RuntimeError: If the tokenization failed.165 166        Returns:167            A list of tokens.168        )r��tokenize)rnrrrs    rorzLlama.tokenize@s�� ��'�'��g�w�?�?�?rq�tokens�prev_tokens�Optional[List[int]]c�<�|j�|||���S)a?Detokenize a list of tokens.169 170        Args:171            tokens: The list of tokens to detokenize.172            prev_tokens: The list of previous tokens. Offset mapping will be performed if provided.173            special: Whether to detokenize special tokens.174 175        Returns:176            The detokenized string.177        )rr)r��178detokenize)rnrrrs    rorzLlama.detokenizeRs*�� ��)�)���W�*�179�180�	181rqr��Optional[BaseLlamaCache]c��||_dS)zKSet the cache.182 183        Args:184            cache: The cache to set.185        N)r�)rnr�s  ro�	set_cachezLlama.set_cachefs����186187188rqc��||_dS)zOSet the random seed.189 190        Args:191            seed: The random seed.192        N)r�)rnr*s  ro�set_seedzLlama.set_seedns����193194195rqc��d|_dS)zReset the model state.rN)rerms ro�resetzLlama.resetvs
����
�
�
rq�
Sequence[int]c���|j�d|jd��tdt	|��|j��D�]"}||t
t	|��||jz���}|j}t	|��}|j�|||j	���|j�196|j��||j|||z�<|j	rt|}|j}tj�|j���||zf���}||j|||z�dd�f�d��dd�<n	|xj|z
c_��$dS)zfEvaluate a list of tokens.197 198        Args:199            tokens: The list of tokens to evaluate.200        r]r)�batch�n_pastr9)�shapeN)r��kv_cache_seq_rmre�ranger�r,r�r��	set_batchr�r�r�r�r��	ctypeslib�as_array�201get_logitsr��reshape)	rnrr�r#r$re�rows�cols�logitss	         ro�evalz202Llama.evalzs~��	
�	�!�!�"�d�m�R�8�8�8��q�#�f�+�+�t�|�4�4�	&�	&�A��1�s�3�v�;�;��D�L�0@�A�A�A�B�E��]�F��5�z�z�H��K�!�!��F�t�7G�
"�
�
�
�
�I���T�[�)�)�)�9>�D�N�6�F�X�$5�5�6���
����}����.�.��I�(�(�*�*�4�$�;�.�/����NT���F�V�h�%6�6����9�:�B�B�2�F�F�r�r�r�J�J���M�M�X�%�M�M�M�9	&�	&rq�(�ffffff�?皙�����?皙�����?皙�����?�@�top_k�top_p�min_p�	typical_p�temp�repeat_penalty�frequency_penalty�presence_penalty�tfs_z�
mirostat_mode�mirostat_eta�mirostat_tau�penalize_nl�logits_processor�Optional[LogitsProcessorList]�grammar�Optional[LlamaGrammar]c�R���tj��}��d203��fd�}|�|��|��j|||���|�|��j|��|dkr0|���|��j	���n|dkr|�204��n�|205dkr&d}|��j�j	|||��n�|206dkr|�
�j	||��n�d	}td|��}|�|��|�||��|�||��|�||��|�|��|��j	��|S)N�token_data_array�"llama_cpp.llama_token_data_array_pc	���|jj}|jj}tj|j��}tj|ftjdt207jfdt208j	fdt209j	fgd���tj|z�|�����}�D]"}|�j
|j��|jdd�<�#dS)N�id�logit�pT)�align)r%rv�buf)�contents�size�datar�r�r��recarrayrvr�r�rk�llama_token_data�from_addressrrM)rIrR�data_soa�data_soa_addressrT�logit_processorrDrns      ��ro�210apply_funcz'Llama._init_sampler.<locals>.apply_func�s����'�0�5��+�4�9��#)�#3�H�4E�#F�#F� ��;��'��(�����7�B�I�*>��b�i�@P�Q�"����#�3�d�:�H�H�(���
	�	�	��(8�Y�Y�O�(7������(X�(X�H�N�1�1�1�%�%�Y�Yrq)�penalty_last_n�penalty_repeat�penalty_freq�penalty_presentrr�drar)rIrJ)r��LlamaSampler�211add_custom�
add_penaltiesr@�add_grammarr��add_softmax�add_distr��212add_greedy�add_mirostatr��add_mirostat_v2r��	add_top_k�add_typical�	add_top_p�	add_min_p�add_temp)rnr7r8r9r:r;r<r=r>r?r@rArBrCrDrF�samplerrZ�213mirostat_m�n_probs�min_keeps`             `      ro�
_init_samplerzLlama._init_sampler�s����$�(�*�*���'�
Y�
Y�
Y�
Y�
Y�
Y�
Y�$
���z�*�*�*���� �2�)�*�,�	�214	215216	217218	219��������W�5�5�5��#�:�:����!�!�!����T�Z�(�(�(�(�
�S�[�[���� � � � ���!�!� �220��$�$��M��J� � �������!�#�#��'�'��J� � ��������q�'�?�?���!�!�%�(�(�(��#�#�I�x�8�8�8��!�!�%��2�2�2��!�!�%��2�2�2�� � ��&�&�&�� � ���,�,�,��rq�idxc��|jdksJ�d}|j�+d}|�|||||||||	|221|||
||���|_|�222||jz223nd}|j�J�|j�|j|��}|rd|_|S)a0Sample a token from the model.224 225        Args:226            top_k: The top-k sampling parameter.227            top_p: The top-p sampling parameter.228            temp: The temperature parameter.229            repeat_penalty: The repeat penalty parameter.230 231        Returns:232            The sampled token.233        rFNT�r7r8r9r:r;r<r=r>r?r@rBrArCrDrFr])rer�rrr��sampler�)rnr7r8r9r:r;r<r=r>r?r@rArBrCrDrFrs�tmp_sampler�ridx�tokens                    rorvzLlama.sample�s���<�}�q� � � � ����=� ��K� �.�.����#��-�"3�!1��+�)�)�'�!1��/���D�M�$'*�o�s�T�]�"�"�2���x�#�#�#��
�$�$�T�Y��5�5���	!� �D�M��rqr �stopping_criteria�Optional[StoppingCriteriaList]�-Generator[int, Optional[Sequence[int]], None]c#�K�tjd|
z��|_|�|||||||	|234|||
||||���|_|r�|jdkr�d}t
|j|dd���D]\}}||kr|dz
}�|dkrId}||d�}||_|jr/td|�d	t|���d235�tj���|r|�
��|jt|��zdz236}t|��}	|�|��||jkr�|�|||||||	|237|||
|||||�
��}|dz
}|�2||jd|�|j||jz238dd�f��rdS|V�}|���|�|��|�|�|��||jkr:||j|kr)||_|j�d|jd��n||jk��|j��||j|j|jt|��z�<|�|jd|jt|��z���}|�|�t4��d|j|jz239t|��z240������)a�Create a generator of tokens from a prompt.241 242        Examples:243            >>> llama = Llama("models/ggml-7b.bin")244            >>> tokens = llama.tokenize(b"Hello, world!")245            >>> for token in llama.generate(tokens, top_k=40, top_p=0.95, temp=1.0, repeat_penalty=1.0):246            ...     print(llama.detokenize([token]))247 248        Args:249            tokens: The prompt tokens.250            top_k: The top-k sampling parameter.251            top_p: The top-p sampling parameter.252            temp: The temperature parameter.253            repeat_penalty: The repeat penalty parameter.254            reset: Whether to reset the model state.255 256        Yields:257            The generated tokens.258        g@rurNr]rFzLlama.generate: z prefix-match hit, remaining z prompt tokens to evalrrT)r7r8r9r:r;r<r=r>r?r@rBrArDrFrCrs)r�r�r�rrr�re�ziprrLr�r�r�r�r �listr0rvr�clear�append�extendr�r&rGr��astyperOr�)rnrr7r8r9r:r;r<r r=r>r?r@rBrArCrDrzrF�longest_prefix�a�b�259sample_idxry�tokens_or_none�draft_tokenss                          ro�generatezLlama.generate6s{����R#�N�3��+=�>�>����*�*������)�/�-��'�%�%�#�-��+�260�261��
�&�	�T�]�Q�&�&��N��D�O�V�C�R�C�[�9�9�
�
���1���6�6�"�a�'�N�N����!�!�������0�� .��
��<���I�>�I�I�%(��[�[�I�I�I� �Z������	��J�J�L�L�L��]�S��[�[�0�1�4�262��f����/	��I�I�f�����t�}�,�,�������'��#1�&7�%5��"/�!-�!-�%5�#� +�"�!$����&�a��263�$�0�5F�5F��O�L�j�L�1�4�<�264�T�]�@Z�\]�\]�\]�@]�3^�6�6�0��F�',�����������
�
�e�$�$�$�!�-��M�M�.�1�1�1���
�-�-�%�4�?�:�;V�2V�2V�$.�D�M��I�-�-�b�$�-��D�D�D��E�t�}�,�,�H��+�NT���t�}�t�}�s�6�{�{�/J�J�K�#�/�/��N�#@�T�]�S��[�[�%@�#@�A� � ���
�
� �'�'��,�,�C�$�+��
�5��F���C�C�����W/	rq�input�Union[str, List[str]]rd�CreateEmbeddingResponsec���|�|n|j}t|t��r|n|g}|�|d���\}}d�t	|��D��}d||||d�d�S)z�Embed a string.265 266        Args:267            input: The utf-8 encoded string to embed.268 269        Returns:270            An embedding object.271        NT)�return_countc� �g|]\}}d||d���S)r:)�objectr:�index�)r{rs�embs   ro�272<listcomp>z*Llama.create_embedding.<locals>.<listcomp>�s>��!273�!274�!275���S�	&� ��
�
�!276�!277�!278rqr)�
prompt_tokens�total_tokens)r�rSrd�usage)rMr�r�embedr�)rnr�rd�279model_name�embedsr�rSs       ro�create_embeddingzLlama.create_embedding�s���$)�#4�%�%�$�/�280�#�E�4�0�0�=���u�g��281 $�z�z�%�d�z�C�C����!282�!283�&�f�-�-�
!284�!285�!286�����!-� ,���	287�288�	289rq�	normalize�truncater�c�*������������j}������tjk}�jjdurtd����jrtj	�j290j��t|t��r|g}n|}�j���g�d�����fd�}d}	g}291d}d}|D]�}
��|
�d����}|r292|d|�}t%|��}|	|z
}	||krt'd	|�d293|�����||z|kr||294��g}295d}d}�j�|||��|296�|��||z
}|dz
}��||297���jrtj�j298j��t|t��r�dn�}tj�j299j������|r||	fS|S)
z�Embed a string.300 301        Args:302            input: The utf-8 encoded string to embed.303 304        Returns:305            A list of embeddings306        FzCLlama model must be created with embedding=True to call this method�	seq_sizesrc�����tj�307jj���308j��309j���310j����	tjkr|d�t|��D]h\}}tj	�311jj������fd�t|��D��}�rd�|D��}��|���|z
��idStt|����D]V}tj
�312jj|����d��}�rtj|��}��|���WdS)Nrc�B��g|]}��|�zz�|dz�zz���S)rr�)r{�j�n_embd�pos�ptrs  ���ror�z5Llama.embed.<locals>.decode_batch.<locals>.<listcomp>sJ���4�4�4���C�!�f�*�,�s�a�!�e�v�5E�/E�E�F�4�4�4rqc�6�g|]}tj|����Sr�)r��normalize_embedding)r{r�s  ror�z5Llama.embed.<locals>.decode_batch.<locals>.<listcomp>"s0��%�%�%�AB�I�9�!�<�<�%�%�%rq)rk�llama_kv_self_clearr�r�r�r�r �LLAMA_POOLING_TYPE_NONEr��llama_get_embeddingsr'r�r��llama_get_embeddings_seqr�r�)r�r�rRr:r�r�rSr�r�r1rns    @@�����ro�decode_batchz!Llama.embed.<locals>.decode_batchs�������)�$�)�-�8�8�8��I���T�[�)�)�)��K�������y�@�@�@���(��3�3� � �G�A�t�#�8����G�G�C�4�4�4�4�4�4�!&�t���4�4�4�I�!��%�%�FO�%�%�%�	��K�K�	�*�*�*��4�K�C�C� � ��s�9�~�~�.�.�+�+�A�#�<�T�Y�]�A�N�N�C�-0��&��\�I� �M�$-�$A�)�$L�$L�	��K�K�	�*�*�*�*�+�+rqrr^N�Requested tokens (z) exceed batch size of r)r�r)r�r,r1rkr�r�r�r�rL�llama_perf_context_resetr�r�r�rNr�r rr�r�r��add_sequencer��llama_perf_context_printr�)rnr�r�r�r�r,r9�inputsr�r��s_batch�t_batch�p_batchrrre�outputrSr�r1s` `              @@@ror�zLlama.embed�s���������������,���(�(�*�*��!�Y�%F�F�313���)�U�2�2��U���
��<�	>��.�t�y�}�=�=�=��e�S�!�!�	��W�F�F��F�	
�������CE��	+�	+�	+�	+�	+�	+�	+�	+�	+�	+�:���������	�	�D��]�]�4�;�;�w�#7�#7�8�8�F��
*�����)���6�{�{�H��H�$�L��'�!�!� �S��S�S�'�S�S����314��!�G�+�+���W�%�%�%�������
�K�$�$�V�W�j�A�A�A�
�N�N�8�$�$�$��x��G��q�L�G�G�	��W�����<�	>��.�t�y�}�=�=�=�&�u�c�2�2�<��a������%�d�i�m�4�4�4��315316�����	��<�'�'��Mrq��prompt�Union[str, List[int]]�suffix�317max_tokens�temperature�logprobs�echo�stop�Optional[Union[str, List[str]]]�stream�318logit_bias�Optional[Dict[int, float]]�SUnion[Iterator[CreateCompletionResponse], Iterator[CreateCompletionStreamResponse]]c#� #��X�Y�Z�[�\K�|�|jtusJ�dttj������}t	tj����}����}�j���}�j�	��}d}d} d}!�j319�dd��dk}"|dkr|n|g}#|dkr|n����g}$t|t��r|�(�j���r|#dd�dgkrg}#t|t��r|��j���s|dkrg}$d}%|"r|!dkr	|rd|z}d}%t#|��dkrgn|g}&|dkr|�|gngt|t��r;|d	kr3��|�d320��d|dkp|du���ngn|z}'|!dkr=|�;|!g|r3��|�d321��dd���|%d�ngzng}(| dkr|�| gng})|#�jr|(|'z|)zn|'|(z|)zz|$z}*d
}+d},t|322t��r|323nt|324t��r|325gng}326|�|n�j}-|*dd�����gdzkrHt-jd�j��������d�t2��|�Nd�|���D���[d9�[fd�}.t7|.g��}/|�|/}n|�|/��}�jr�j���t#|*���j kr9tCdt#|*���dtEj#�j$�������|�|dkr�j t#|*��z327}|t#|*��z�j kr|n�j t#|*��z328}|329gkr
d�|330D��}0ng}0|��j%durtCd����j&r�	�j&|*}1tN�(|1j)�*��|*��}2tN�(�j+�*��|*��}3|2|3kr7��,|1���jrt[dt\j/���n2#t`$r%�jrt[dt\j/���YnwxYw|���1|��n@��1tej3�j4���5dd����d}4d}5��6|*||||||||||||
|||� ��D�]}6tEj7�jj8|6��r��9|&|*�!��}+d"}4�n�|&�:|6����9|&|*�!���Xtw�Xd#d���D]&\}7}8d$|7z331}7d%D]\}9}:|9|7kr|:|8z|:kr|9|7z332}5��'|5dkr|5dz}5���Xfd&�|0D��};t#|;��dkr*|;d}<�Xd�X�<|<���}+d"}4�n|�r�|&|,d�}=��9|=|*|&d|,�z�!���\t#�\��}>d}?|0D]X}@t{t}t#|@��|>��dd��D])}A�\�?|@d|A���r333|A|?kr|A}?n�*�Yd}B|��Y|=D�]T}6|6|kr�334|Bt#��9|6g|*|&d|,�z�!����z
}B|B|>|?z335kr�n
��9|6g|*|&d|,�z�!���@d336d'�(��}Ct#|��t#��9|&d|,�|*|&d|,�z�!���@d337d'�(����z}Dt#|*��|,z}E�jA|Edz338dd�f}FtN�B|F���*��}Gtt�t�|Gt{t#|G������d)�*����}H�fd+�|Hd|�D��}I|I�E|C|Gt	|6��i����9|6g|*|&d|,�z�!���@d339d'�(��g|Dg|Gt	|6��g|Igd,�}J|,dz
},|d-||-��9|6g|*|&d|,�z�!���@d340d'�(��d|Jdd.�gd/�V���Vn�t#|=��dkr�d}Kt{dt#|=��dz��D]U}A	��9|=d|A�|*|&d|,�z�!��}L|L�@d341��}Md)}Kn#t�$rY�RwxYwnQ|KsnN|Bt#|L��z
}B|B|>|?z342krn2|=|Ad�}=|,|Az
},|d-||-|Mdddd.�gd/�V�t#|=��dk��t#|&��|kr��9|&|*�!��}+d}4n��|�9|�j+�jAddd�f��r��9|&|*�!��}+d"}4�jr�j�G��|�ry|&|,d�}=��9|=|*|&d|,�z�!���\�\fd0�|0D��};t#|;��dkrt}�\fd1�|;D����}Nnt#�\��}Nd}B|=D�]}6|Bt#��9|6g|*|&d|,�z�!����z
}Bd}J|���|6|kr�B��9|6g���@d343d'�(��}Ct#|��t#��9|&d|,�|*|&d|,�z�!����z}Dt#|*��|,zdz344}E�jA|Edd�f}FtN�B|F���*��}Gtt�t�|Gt{t#|G������d)�*����}H�fd2�|Hd|�D��}I|I�E|C|Gt	|6��i����9|6g���@d345d'�(��g|Dg|Gt	|6��g|Igd,�}J|B|Nkrh��9|6g��}O|B|Ndz346krn�|,dz
},|d-||-|Odt#|O��|B|Nz347z348��@d349d'�(��d|Jdd.�gd/�V�n@|,dz
},|d-||-��9|6g���@d350d'�(��d|Jdd.�gd/�V����|d-||-d	dd|4d.�gd/�V��j&rc�jrt[d3t\j/�����H���j&|*|&z<�jrt[d4t\j/���dS�j&rA�jrt[d3t\j/�����H���j&|*|&z<|+�@d351d'�(��}P|	r||Pz}P|!dkr|�|P|z}Pd}J|��1|	rdnt#|��}D|	rdnt#|*dd���}Eg}Qg}Rg}Sg}T|	r.|*|*d����krdndd�|&z�Yn|&�Y�Y�fd5�tw�Y��D��}UtN�B�jA��|Ed�}Vtwt��Y|U|V����D�]@\�Z\}6}C}W|6|kr�|Q�:|Dt#��9�Yd�Z����@d352d'�(����z��|S�:|C��tt�t�|Wt{t#|W������d)�*����}H|R�:|Wt	|6�����Y�Z�fd6�|Hd|�D��}I|I�E|C|Wt	|6��i��|T�:|I����B|	rt#�Y��dkr353d|Rd<d|Td<|S|Q|R|Td,�}J|d-||-|Pd|J|4d.�gt#|*��t#|&��t#|*��t#|&��zd7�d8�V�dS):Nzcmpl-rztokenizer.ggml.add_space_prefix�truer]ru☺rarwr^F)rrrqzDetected duplicate leading "zN" in prompt, this will likely reduce response quality, consider removing it...c�N�i|]"\}}t|��t|����#Sr�)rOrT)r{r�r�s   ro�354<dictcomp>z,Llama._create_completion.<locals>.<dictcomp>�s*��N�N�N�4�1�a�c�!�f�f�e�A�h�h�N�N�Nrqr�r�r�rr�c���tj|��}����D]\}}|||z||<�|Sri)r��copyr�)r�r��355new_scores�input_id�score�logit_bias_maps     �ro�logit_bias_processorz6Llama._create_completion.<locals>.logit_bias_processor�sY��� �W����356�(6�';�';�'=�'=�D�D�O�H�e�+0�6�(�3C�+C�J�x�(�(�!�!rqr�z) exceed context window of c�8�g|]}|�d����S)r^)r�)r{�ss  ror�z,Llama._create_completion.<locals>.<listcomp>	s$��>�>�>�A�a�h�h�w�/�/�>�>�>rqzBlogprobs is not supported for models created with logits_all=Falsez#Llama._create_completion: cache hitrrz$Llama._create_completion: cache missl�length)r7r8r9r:r;r?r@rBrAr=r>r<rzrDrF�rr�������))ra��)r���)���c���g|]}|�v�|��	Sr�r�)r{r��all_texts  �ror�z,Llama._create_completion.<locals>.<listcomp>Rs���C�C�C�a�Q�(�]�]��]�]�]rq�ignore��errorsT)�reversec�n��i|]1\}}��|g���dd���|��2S�r^r�r��rr��r{�logprobr�rns   �ror�z,Llama._create_completion.<locals>.<dictcomp>�sZ���'�'�'�!+���!�O�O�Q�C�0�0�7�7� '��8���&�'�'�'rq)r�text_offset�token_logprobs�top_logprobs�text_completion)rr�r��
finish_reason)rLr��createdrd�choicesc���g|]}|�v�|��	Sr�r�)r{r��remaining_texts  �ror�z,Llama._create_completion.<locals>.<listcomp>�s#���I�I�I�a�Q�.�5H�5H��5H�5H�5Hrqc3�B�K�|]}��|��V��dSri)r�)r{r�r�s  �ror~z+Llama._create_completion.<locals>.<genexpr>�s1�����J�J��.�.�.�t�4�4�J�J�J�J�J�Jrqc�n��i|]1\}}��|g���dd���|��2Sr�r�r�s   �ror�z,Llama._create_completion.<locals>.<dictcomp> sP���#�#�#�&�G�Q������,�,�3�3�G�H�3�M�M�w�#�#�#rqz$Llama._create_completion: cache savez%Llama._create_completion: cache savedc���g|]:\}}��|g�d|�����dd�����;S�Nr�r^r�r�r�)r{r�ry�357all_tokensrns   ��ror�z,Llama._create_completion.<locals>.<listcomp>�sd�������A�u������Z����^��D�D�K�K��H�L�����rqc���i|];\}}��|g�d������dd���|��<Sr�r�)r{r�r�r�rsrns   ���ror�z,Llama._create_completion.<locals>.<dictcomp>�sh���;�;�;�#����O�O�Q�C�Z����5E�O�F�F�M�M���N����;�;�;rq)r��completion_tokensr�)rLr�r�rdr�r��r�r�r�rr�r)I�	__class__rN�uuid�uuid4rO�timer�r��	token_cls�	token_sepr��getr�r�r�
add_bos_token�
add_eos_tokenr�rr�rKrM�warnings�warnr��RuntimeWarningr��LogitsProcessorListr�rLr��
reset_timingsr�r�rk�llama_n_ctxr�r�r�r�longest_token_prefixr�r	r�358load_stater�r�r��KeyErrorr�random�Randomr��randintr��llama_token_is_eog�vocabrr�r�r�r'r��endswithr�r�logits_to_logprobs�sortedr~�update�UnicodeError�
print_timings�359save_state)]rnr�r�r�r�r8r9r:r�r�r�r=r>r<r7r�r*r?r@rBrArdrzrDrFr��
completion_idr�r��cls_token_id�sep_token_id�prefix_token_id�middle_token_id�suffix_token_id�add_space_prefix�360bos_tokens�361eos_tokens�suffix_space_prefixr��
prefix_tokens�
suffix_tokens�
middle_tokensr�r�returned_tokensr�r��_logit_bias_processor�stop_sequences�362cache_item�cache_prefix_len�eval_prefix_lenr��
multibyte_fixryr��char�num�pattern�any_stop�363first_stop�remaining_tokens�remaining_length�first_stop_positionr�r��token_end_position�	token_strr��token_offsetr/�current_logprobs�sorted_logprobs�top_logprob�logprobs_or_none�decode_success�bs�ts�end�	last_text�text_str�text_offsetsr�rr��all_token_strs�all_logprobs�logprobs_tokenr�r�rsr�r�s]`                                                                                       @@@@@ro�_create_completionzLlama._create_completioncs:����������<�~��!1�S�!8�!8�!8�8�8�S�����%6�%6�8�8�
��4�9�;�;�'�'�� �N�N�,�,�� �K�1�1�3�3�� �K�1�1�3�3�� �� �� ���M���?��H�H�F�R�	�2>��1C�1C���� V�364�(�B�.�.�L�L�D�N�N�4D�4D�!365366367���
%�
%�	�*0�.��;�,�,�.�.�+9��"�1�"�~�"��%�%��J��v�t�$�$�	�����)�)�+�+�*8�0<��0B�0B��J�#$���	$��1� 4� 4�� 4��V�^�F�"#��.1��[�[�1�_�_�r�r�<�.��"1�A�!5�!5�&�:L�_���RT��&�#�&�&�368
��R�<�<��
�
��M�M�'�*�*�!�,�q�0�B�F�d�N������R��$369�
�4�!�#�#��(:�!�!�370��D�M�M�&�-�-��"8�"8�%�QV�M�W�W�+�,�,����
���	�"1�A�!5�!5�&�:L�_���RT�	�
��?�E���.��>�>�#�m�3�m�C�	
��

�	��� ���t�T�*�*�W�D�D�*�T�3�:O�:O�0W����UW�	
�$)�#4�%�%�$�/�371���!������!1�!1� 2�Q� 6�6�6��M�l�t�{�/I�/I�$�.�.�JZ�JZ�/[�/[�l�l�l��
�
�
��!�N�N�:�;K�;K�;M�;M�N�N�N�N�	
"�	
"�	
"�	
"�	
"�	
"�%8�9M�8N�$O�$O�!��'�#8� � �#3�#:�#:�;P�#Q�#Q� ��<�	&��I�#�#�%�%�%��}�����,�,��u�S��%7�%7�u�u�T]�Ti�jn�jr�Ts�Ts�u�u���
����q�����s�=�'9�'9�9�J�372�C�
�.�.�.���<�<�
�J��+��M� 2� 2�2�	��2�:�:�>�>��>�>�>�N�N��N���D�$4��$=�$=��T���
��:�	S�
S�!�Z�
�6�373�#(�#=�#=��(�/�/�1�1�=�$�$� �#(�"<�"<��O�*�*�,�,�m�#�#��$�o�5�5��O�O�J�/�/�/��|�V��C�#�*�U�U�U�U����
S�
S�
S��<�S��@�s�z�R�R�R�R���
S�������M�M�$������M�M�&�-��374�3�3�;�;�A�w�G�G�H�H�H� �
��
��]�]��������'�%�%�/�-�)�/�-��!#�375�376�	�	�E�$�+�D�K�,=�u�E�E�
����'8�m��T�T�� &�
����$�$�U�+�+�+����'8�m��T�T�H�%�X�b�c�c�]�3�3�
0�
0���4���E��$B�0�0�L�C���Q�w�w�7�T�>�W�#<�#<�(+�a��
��0��q� � ���"�
��C�C�C�C�>�C�C�C�H��8�}�}�q� � �%�a�[�377�� <�(�.�.��"<�"<� <�=�� &�
����K
�#4�_�5E�5E�#F� �!%���$� -�0A�BR�?�BR�0S� S�"1�"�"��$'�~�#6�#6� �378'(�#�'�"�"�A�"�3�s�1�v�v�/?�#@�#@�!�R�H�H�"�"��)�2�2�1�R�a�R�5�9�9�"� �#6�6�6�67� 3�!�E�"��379&'�"��'�"2�I�I�� �L�0�0�$�*�c� �O�O�!&��,9�"3�4D�_�4D�"E�-F�,���/�/��*�.�,�/B�B���"�E�$(�O�O�"�G�(5�/�0@��0@�A�)B�%4�%�%�!�&���&�:�:�	"�380'*�&�k�k�C� �O�O� 1�2B�?�2B� C�,9�"3�4D�_�4D�"E�-F�,���%�f�W�X�f�>�>�5�5�'��(+�=�'9�'9�O�'K��!%��l�Q�.>����.A�!B��+0�+C�+C�F�+K�+K�+R�+R�+T�+T�(�*.�"� #�$4�e�C�@P�<Q�<Q�6R�6R� S� S�(,����+�+��'�'�'�'�/>�i�x�i�.H�	'�'�'��$�*�*�I�7G��E�381382�7S�+T�U�U�U�!%���%*�G�0=�&7�8H��8H�&I�1J�!0�!"�!"�#)�&���&�"B�"B�'�-8�=�/?��E�383384�/K�.L�-8�M�,�,�(�(�1�,��"/�&7�'.�%/�-1�O�O�).��4A�*;�<L�_�<L�*M�5N�-<�-&�-&�'-�f�W�X�f�&F�&F�-.�0@�59�	!"�	!"�(��������&�.�/�/�!�3�3�).��!&�q�#�.>�*?�*?�!�*C�!D�!D�
"�
"�A�385%�%)�_�_�$4�R�a�R�$8�0=�&7�8H��8H�&I�1J�&5�&"�&"��386&(�Y�Y�w�%7�%7��15�� %���#/�%�%�%� $��%����"�-�"�!�*�c�"�g�g�5�*�-�,�/B�B���"�+;�A�B�B�+?�(�'�1�,��#0�&7�'.�%/�-/�-.�04�59�	!"�!"�(�
�
�
�
�
�7�.�/�/�!�3�3�T�$�%�%��3�3����'8�m��T�T�� (�
���4�387�(�->�->��O�T�\�"�a�a�a�%�0�.388�.389�(��?�?�#4�-�?�P�P�D�"�M��<�	&��I�#�#�%�%�%��u	�0��1A�1A�B��!�_�_� �)�,=�>N��>N�,O�O�-���N�J�I�I�I�>�I�I�I�H��8�}�}�q� � ��J�J�J�J��J�J�J�J�J����.�)�)��!"��)�S
�S
��"�c��O�O���$1�4E�FV��FV�4W�$W�$���'�'��"�BF� ��'���,�,� � $����� 8� 8� ?� ?���!@�!�!�I�#&�f�+�+�����-�.>��.>�?�(5�/�0@��0@�A�)B�(���1�1�#�K�$'�}�#5�#5��#G�!�#K�L�!�\�,����/�:�F�',�'?�'?��'G�'G�'N�'N�'P�'P�$�&*��� 0�%��<L�8M�8M�2N�2N�O�O�$(����'�'�O�#�#�#�#�*9�)�8�)�*D�#�#�#�K� �&�&�	�3C�C��J�J�3O�'P�Q�Q�Q�!�O�O�U�G�4�4�;�;�G�H�;�U�U�#�)4�}�+;�C��J�J�+G�*H�)4�
�
(�(�$�&��,�,� $����� 8� 8�I�)�S�1�W�4�4���#�q�(�O�+�"3�#*�!+�)2�$Q�c�)�n�n�8J�S�8P�&Q�$Q�)"�"(�&���&�"B�"B�)*�,<�15�
��	$������ �E��1�$��'�/�&�'�%)�O�O�U�G�$<�$<�$C�$C� '��%D�%�%�&'�(8�-1�
��	 �������"$�+�"�#�!#�!"�$(�)6�	���
�
�

�

�

��z�
T��<�S��@�s�z�R�R�R�R�@D���@Q�@Q��390�=�+<�<�=��<�T��A��391�S�S�S�S��F��:�	N��|�
O��<�3�:�N�N�N�N�<@�O�O�<M�<M�D�J�}�'8�8�9��;�;�w�x�;�8�8���	)���(�H��Q���6�#5��&�(�H�9=����#�4�!�!��V���K� $�@�1�1�#�m�A�B�B�.?�*@�*@�L�&(�L�46�N� "�F�=?�L��
/�"�}�Q�'7�4�>�>�;K�;K�'K�'K�!�!�QR�"T�"T�U�'�(��392393/�394������!*�*� 5� 5�	���N�!�3�3�D�L�A�A�,�-�-�P�L�;D��J���=�=�<�<�
1�
1�7��7�e�Y���L�(�(���#�#������395�4�C�4�(8�9�9�@�@�#�H�A����������
�
�i�(�(�(�"&���N�E�#�n�2E�2E�,F�,F�G�G�QU����#�#��396�%�%�n�S��Z�Z�&@�A�A�A�;�;�;�;�;�;�'6�i�x�i�&@�	;�;�;���"�"�I�~�c�%�j�j�/I�#J�K�K�K��#�#�K�0�0�0�0��
'��J���!�+�+�$(��q�!�"&��Q�� �+�"0� ,�	 � �� �'���%�� 0�%2�	���"%�]�!3�!3�%(�):�%;�%;� #�M� 2� 2�S�9J�5K�5K� K���397�398�	399�	400�	401�	402�	403s&�B.T404405,T9�8T9�-Ah0�0406h=�<h=�IUnion[CreateCompletionResponse, Iterator[CreateCompletionStreamResponse]]c��|�|||�dn|||||||	|407|||
||||||||||||���}|r|}|St|��}|S)��Generate text from a prompt.408 409        Args:410            prompt: The prompt to generate text from.411            suffix: A suffix to append to the generated text. If None, no suffix is appended.412            max_tokens: The maximum number of tokens to generate. If max_tokens <= 0 or None, the maximum number of tokens to generate is unlimited and depends on n_ctx.413            temperature: The temperature to use for sampling.414            top_p: The top-p value to use for nucleus sampling. Nucleus sampling described in academic paper "The Curious Case of Neural Text Degeneration" https://arxiv.org/abs/1904.09751415            min_p: The min-p value to use for minimum p sampling. Minimum P sampling as described in https://github.com/ggerganov/llama.cpp/pull/3841416            typical_p: The typical-p value to use for sampling. Locally Typical Sampling implementation described in the paper https://arxiv.org/abs/2202.00666.417            logprobs: The number of logprobs to return. If None, no logprobs are returned.418            echo: Whether to echo the prompt.419            stop: A list of strings to stop generation when encountered.420            frequency_penalty: The penalty to apply to tokens based on their frequency in the prompt.421            presence_penalty: The penalty to apply to tokens based on their presence in the prompt.422            repeat_penalty: The penalty to apply to repeated tokens.423            top_k: The top-k value to use for sampling. Top-K sampling described in academic paper "The Curious Case of Neural Text Degeneration" https://arxiv.org/abs/1904.09751424            stream: Whether to stream the results.425            seed: The seed to use for sampling.426            tfs_z: The tail-free sampling parameter. Tail Free Sampling described in https://www.trentonbricken.com/Tail-Free-Sampling/.427            mirostat_mode: The mirostat sampling mode.428            mirostat_tau: The target cross-entropy (or surprise) value you want to achieve for the generated text. A higher value corresponds to more surprising or less predictable text, while a lower value corresponds to less surprising or more predictable text.429            mirostat_eta: The learning rate used to update `mu` based on the error between the target and observed surprisal of the sampled word. A larger learning rate will cause `mu` to be updated more quickly, while a smaller learning rate will result in slower updates.430            model: The name to use for the model in the completion object.431            stopping_criteria: A list of stopping criteria to use.432            logits_processor: A list of logits processors to use.433            grammar: A grammar to use for constrained sampling.434            logit_bias: A logit bias to use.435 436        Raises:437            ValueError: If the requested tokens exceed the context window.438            RuntimeError: If the prompt fails to tokenize or the model fails to evaluate the prompt.439 440        Returns:441            Response object containing the generated text.442        Nr]�r�r�r�r�r8r9r:r�r�r�r=r>r<r7r�r*r?r@rBrArdrzrDrFr�)rA�next)rnr�r�r�r�r8r9r:r�r�r�r=r>r<r7r�r*r?r@rBrArdrzrDrFr��completion_or_chunks�chunks�443completions                             ro�create_completionzLlama.create_completion�s���@ $�6�6���'�/�r�r�Z�#�������/�-�)�����'�%�%��/�-��!�3 7� 444� 445��6�	�?S�F��M�!%�&:�!;�!;�446��rqc�^�|�|||||||||	|447|||
||||||||||||���S)rDrE)rJ)rnr�r�r�r�r8r9r:r�r�r�r=r>r<r7r�r*r?r@rBrArdrzrDrFr�s                          ro�__call__zLlama.__call__0sm��@�%�%���!�#�������/�-�)�����'�%�%��/�-��!�3&�448�449�	450rqg�������?�messages�"List[ChatCompletionRequestMessage]�	functions�&Optional[List[ChatCompletionFunction]]�
function_call�+Optional[ChatCompletionRequestFunctionCall]�tools�"Optional[List[ChatCompletionTool]]�tool_choice�(Optional[ChatCompletionToolChoiceOption]�response_format�-Optional[ChatCompletionRequestResponseFormat]r��QUnion[CreateChatCompletionResponse, Iterator[CreateChatCompletionStreamResponse]]c�:�|jp7|j�|j��pt	j|j��}|did|�d|�d|�d|�d|�d|�d|�d|�d	|�d451|	�d|452�d|�d
|�d|�d|�d|
�d|�d|�d|�d|�d|�d|�d|�d|�d|�d|�d|�d|�d|��S)a�	Generate a chat completion from a list of messages.453 454        Args:455            messages: A list of messages to generate a response for.456            functions: A list of functions to use for the chat completion.457            function_call: A function call to use for the chat completion.458            tools: A list of tools to use for the chat completion.459            tool_choice: A tool choice to use for the chat completion.460            temperature: The temperature to use for sampling.461            top_p: The top-p value to use for nucleus sampling. Nucleus sampling described in academic paper "The Curious Case of Neural Text Degeneration" https://arxiv.org/abs/1904.09751462            top_k: The top-k value to use for sampling. Top-K sampling described in academic paper "The Curious Case of Neural Text Degeneration" https://arxiv.org/abs/1904.09751463            min_p: The min-p value to use for minimum p sampling. Minimum P sampling as described in https://github.com/ggerganov/llama.cpp/pull/3841464            typical_p: The typical-p value to use for sampling. Locally Typical Sampling implementation described in the paper https://arxiv.org/abs/2202.00666.465            stream: Whether to stream the results.466            stop: A list of strings to stop generation when encountered.467            seed: The seed to use for sampling.468            response_format: The response format to use for the chat completion. Use { "type": "json_object" } to contstrain output to only valid json.469            max_tokens: The maximum number of tokens to generate. If max_tokens <= 0 or None, the maximum number of tokens to generate is unlimited and depends on n_ctx.470            presence_penalty: The penalty to apply to tokens based on their presence in the prompt.471            frequency_penalty: The penalty to apply to tokens based on their frequency in the prompt.472            repeat_penalty: The penalty to apply to repeated tokens.473            tfs_z: The tail-free sampling parameter.474            mirostat_mode: The mirostat sampling mode.475            mirostat_tau: The mirostat sampling tau parameter.476            mirostat_eta: The mirostat sampling eta parameter.477            model: The name to use for the model in the completion object.478            logits_processor: A list of logits processors to use.479            grammar: A grammar to use.480            logit_bias: A logit bias to use.481 482        Returns:483            Generated chat completion or a stream of chat completion chunks.484        �llamarMrOrQrSrUr�r8r7r9r:r�r�r�r�r*rWr�r>r=r<r?r@rBrArdrDrFr�r�)rFr�r�rEr��get_chat_completion_handler)rnrMrOrQrSrUr�r8r7r9r:r�r�r*rWr�r>r=r<r?r@rBrArdrDrFr�r�r��handlers                              ro�create_chat_completionzLlama.create_chat_completion�s���F
��
O��"�&�&�t�'7�8�8�
O� �<�T�=M�N�N�	�485�w�486�487�488��$�489��X�490� �i�491�(�-�	492493�%�494�$��
495�$��496��%�497��%�498��%�499� �i�500��X�501�&��502��6�503���504� ��!505�",�O�#506�$"�z�%507�&.�-�'508�(0�/�)509�**�>�+510�,�%�-511�.(�-�/512�0&��1513�2&��3514�4�%�5515�6.�-�7516�8�G�9517�:"�z�;518�	519rq�argsrr�c	���	ddlm}m�|�dd��}t	|t520��sJ�|r�fd�|j|i|��D��S|di|j|i|����S#t$rtd���wxYw)	a�Generate a chat completion with return type based on the the OpenAI v1 API.521 522        OpenAI python package is required to use this method.523 524        You can install it with `pip install openai`.525 526        Args:527            *args: Positional arguments to pass to create_chat_completion.528            **kwargs: Keyword arguments to pass to create_chat_completion.529 530        Returns:531            Generated chat completion or a stream of chat completion chunks.532        r)�ChatCompletion�ChatCompletionChunkr�Fc3�(�K�|]}�di|��V��
dS�Nr�r�)r{�chunkrbs  �ror~z9Llama.create_chat_completion_openai_v1.<locals>.<genexpr>s5�����o�o��+�+�4�4�e�4�4�o�o�o�o�o�orqzzTo use create_chat_completion_openai_v1, you must install the openai package.You can install it with `pip install openai`.Nr�)�openai.types.chatrarbr�r�rQr^�ImportError)rnr_r�rar�rbs     @ro� create_chat_completion_openai_v1z&Llama.create_chat_completion_openai_v1�s����$
	�M�M�M�M�M�M�M�M��Z�Z��%�0�0�F��f�d�+�+�+�+�+��
V�o�o�o�o�B]�$�B]�_c�Bn�gm�Bn�Bn�o�o�o�o�%�~�U�U�(C��(C�T�(T�V�(T�(T�U�U�U���	�	�	��@���
�	���s�AA&�A&�&Bc���td,id|j�d|jj�d|jj�d|jj�d|j�d|jj�d|jj�d|jj	�d	|j533�d534|j�d|jj
�d|j�d
|jj�d|jj�d|jj�d|jj�d|jj�d|jj�d|jj�d|jj�d|jj�d|jj�d|jj�d|jj�d|j�d|jj�d|jj�d|jj�d|jj�d|jj �d|jj!�d |j"�d!|j#�d"|j$�d#|j%�d$|j&�d%|j'�d&|j(�d'|j)�d(|jj*�d)|jj+�d*|j,�d+|j-��S)-NrMr"r#r$r%r&r'r(r)r*r+r,r-r.r/r0r1r2r3r4r5r6r7r8r9r:r;r<r=r>r?r@rArBrCrDrErFrGrIrJrKrLr�).r�rMr�r"r#r$r%r&r'r(r)r�r�r+r,r-r.r/r0r1r2r3r4r5r6r7r8r�r�r;r<r=r>r?r@rArBrCrDrErFrGrIrJrKrLrms ro�__getstate__zLlama.__getstate__s"���5535�5536�5537����5538��*�7�7�5539��(�3�3�	5540541�&�/�/�5542��*�*�
5543��(�3�3�5544��&�/�/�5545��'�1�1�5546��*�*�5547����5548��%�+�+�5549��L�L�5550��(�1�1�5551� �)�3�3�!5552�"!�/�?�?�#5553�$#�1�C�C�%5554�&�,�9�9�'5555�( �.�=�=�)5556�*!�/�?�?�+5557�,!�/�?�?�-5558�."�0�A�A�/5559�0 �.�=�=�15560�2 �.�=�=�35561�4�-�;�;�55562�6�'�'�75563�8�)�4�4�95564�:�+�7�7�;5565�<�*�5�5�=5566�>�*�5�5�?5567�@�(�1�1�A5568�D�'�/�/�E5569�F $�6�6�G5570�J�n�n�K5571�L���M5572�N�n�n�O5573�R���S5574�V�(�(�W5575�X�*�*�Y5576�\�(�(�]5577�`�&�-�-�a5578�b�&�-�-�c5579�f���g5580�h�L�L�i5581�5	582rqc� �|jdi|��dSrd)r�)rn�states  ro�__setstate__zLlama.__setstate__Ls ����
��������rq�583LlamaStatec���|jrtdtj���t	j|jj��}|jrtd|��tj���tj	t|��z��}|jrtdtj���t	j|jj|��}|jrtd|��tj���t|��t|��krtd���tj	t|��z��}tj�
||t|����|jrtd|�d�tj���t|j���|j���|jt'|��||j�	��S)584Nz$Llama.save_state: saving llama staterrz"Llama.save_state: got state size: z!Llama.save_state: allocated statez&Llama.save_state: copied llama state: zFailed to copy llama state datazLlama.save_state: saving z bytes of llama state)r�r�re�llama_state�llama_state_sizer*)rLr�r�r�rk�llama_get_state_sizer�r�r��c_uint8rO�llama_copy_state_datar�r�rnrr�r�rerr�)rn�585state_sizerp�n_bytes�llama_state_compacts     rorzLlama.save_stateOs����<�	K��8�s�z�J�J�J�J��3�D�I�M�B�B�586��<�	V��C�z�C�C�#�*�U�U�U�U��~��J���7�:�:���<�	H��5�C�J�G�G�G�G��1�$�)�-��M�M���<�	W��D�7�D�D�3�:�V�V�V�V��w�<�<�#�j�/�/�)�)��@�A�A�A�%�~��G���<�?�?���� � �!4�k�3�w�<�<�P�P�P��<�	��J�G�J�J�J��Z�
�
�
�
���<�$�$�&�&��n�)�)�+�+��]��1�2�2�$���
587�588�589�	590rqrl�Nonec���|j���|jd|j�dd�f<|j|jd�dd�f}d||dk<|j���|_|j|_|j|_|j}tj|z}|�	|j591��}tj|j
j|��|krtd���dS)NrrzFailed to set llama state data)r�r�rer�r*r�rqr�rs�from_buffer_copyrprk�llama_set_state_datar�r�r�)rnrl�restru�LLamaStateArrayTyperps      rorzLlama.load_statems���+0�<�+<�+<�+>�+>���$�e�n�$�a�a�a�'�(��{�5�>�+�+�Q�Q�Q�.�/����T�A�X����-�-�/�/������
��Z��592��+�593�$�n�z�9��)�:�:�5�;L�M�M���)�$�)�-��E�E��S�S��?�@�@�@�T�Srqc�4�|j���S)zReturn the context window size.)r�r+rms ror+zLlama.n_ctx|s���y��� � � rqc�4�|j���S)zReturn the embedding size.)r�r�rms ror�zLlama.n_embd�s���{�!�!�#�#�#rqc�4�|j���S)zReturn the vocabulary size.)r�rtrms rortz
Llama.n_vocab�s���{�"�"�$�$�$rqrc� �t|��S)z*Return the llama tokenizer for this model.)rrms rorHzLlama.tokenizer�s���d�#�#�#rqc�4�|j���S)z!Return the end-of-sequence token.)r�r�rms ror�zLlama.token_eos�����{�$�$�&�&�&rqc�4�|j���S)z'Return the beginning-of-sequence token.)r�r�rms ror�zLlama.token_bos�r�rqc�4�|j���S)zReturn the newline token.)r�r�rms ror�zLlama.token_nl�s���{�#�#�%�%�%rqc�4�|j���S)zReturn the pooling type.)r�r1rms ror1zLlama.pooling_type�s���y�%�%�'�'�'rqc�8�|j���dS)z&Explicitly free the model from memory.N)r��closerms ror�zLlama.close�s����������rqc�.�|���dSri)r�rms ro�__del__z
Llama.__del__�s���594595�����rqr]r/�#Union[npt.NDArray[np.single], List]�axisc���tj||d���}|jdkrd|tj|��<ntj|��sd}tj||tj���}tj|��}tjd���5tj||d���}tj	|��}ddd��n#1swxYwY||z596S)NT)r��keepdimsrrur�)�divide)597r��amax�ndim�isfinite�subtractr��exp�errstate�sum�log)r/r��logits_maxs�
subtract_maxsr��summed�outs       rorzLlama.logits_to_logprobs�s��598#%�'�&�t�d�"K�"K�"K����a���56�K���[�1�1�1�2�2���[�)�)�	��K���F�K�r�y�I�I�I�
��f�]�#�#��
�[��
)�
)�
)�	!�	!��V�C�d�T�:�:�:�F��&��.�.�C�	!�	!�	!�	!�	!�	!�	!�	!�	!�	!�	!����	!�	!�	!�	!��s�"�"s�,C�C�Cr�r�c�N�d}t||��D]\}}||kr|dz
}�|S)Nrr)r~)r�r�r��_a�_bs     rorzLlama.longest_token_prefix�s@�����!�Q�i�i�	�	�F�B���R�x�x��!�#�����rq�auto�repo_id�filename�additional_files�Optional[List]�	local_dir�&Optional[Union[str, os.PathLike[str]]]�local_dir_use_symlinks�Union[bool, Literal['auto']]�	cache_dir�'Llama'c�����	ddlm}m}	ddlm}599n#t600$rtd���wxYw|601|��|	��}d�|�|d���D��}g}
|D]F}t|���|��}|
�	t|�����G�fd�|
D��}t|��dkr*td	|�d602��dtj|
�������t|��dkr*td
|�d��dtj|�������|\}tt|��j��}t|��j�||�||||���|r�|D]���fd�|
D��}t|��dkr*td	|�d603��dtj|
�������t|��dkr*td
|�d��dtj|�������|\}|||||||�����|�||�||||d���}n t"j�|���}|dd|i|��S)a�Create a Llama model from a pretrained model name or path.604        This method requires the huggingface-hub package.605        You can install it with `pip install huggingface-hub`.606 607        Args:608            repo_id: The model repo id.609            filename: A filename or glob pattern to match the model file in the repo.610            additional_files: A list of filenames or glob patterns to match additional model files in the repo.611            local_dir: The local directory to save the model to.612            local_dir_use_symlinks: Whether to use symlinks when downloading the model.613            **kwargs: Additional keyword arguments to pass to the Llama constructor.614 615        Returns:616            A Llama model.r)�hf_hub_download�HfFileSystem)�validate_repo_idzrLlama.from_pretrained requires the huggingface-hub package. You can install it with `pip install huggingface-hub`.c�L�g|]!}t|t��r|dn|��"S)r|)r�r�)r{rss  ror�z)Llama.from_pretrained.<locals>.<listcomp>�s?��617�618�619��'�t�T�2�2�<�D��L�L��620�621�622rqT)�	recursivec�>��g|]}tj|����|��Sr���fnmatch)r{rsr�s  �ror�z)Llama.from_pretrained.<locals>.<listcomp>�s*���X�X�X�4����h�8W�8W�X�$�X�X�XrqzNo file found in z that match z623 624Available Files:625rzMultiple files found in z626 matching )r�r��	subfolderr�r�r�c�>��g|]}tj|����|��Sr�r�)r{rs�additonal_file_names  �ror�z)Llama.from_pretrained.<locals>.<listcomp>	s,���,v�,v�,v�d�7�?�[_�at�Ku�Ku�,v�T�,v�,v�,vrqN)r�r�r�r�r�r��local_files_onlyrMr�)�huggingface_hubr�r��huggingface_hub.utilsr�rg�lsr�relative_tor�rNr�r��json�dumps�parentr|r�r�r�)�clsr�r�r�r�r�r�r�r�r�r��hffs�files�	file_listrs�rel_path�matching_files�
matching_filer��matching_additional_files�matching_additional_filerMr�s  `                   @ro�from_pretrainedzLlama.from_pretrained�s�����2	�E�E�E�E�E�E�E�E�>�>�>�>�>�>�>���	�	�	��I���
�	����	���!�!�!��|�~�~��627�628�����4��8�8�629�630�631�� "�	��	,�	,�D��D�z�z�-�-�g�6�6�H����S��]�]�+�+�+�+�Y�X�X�X�9�X�X�X���~���!�#�#��=�G�=�=��=�=�%)�Z�	�%:�%:�=�=���
�632�~����"�"��9�7�9�9�h�9�9�%)�Z��%6�%6�9�9���
�633*�����]�+�+�2�3�3�	��
�&�&�+��	������#9��
	634�	635�	636�	637��	�'7�
�
�#�,v�,v�,v�,v�i�,v�,v�,v�)��0�1�1�Q�6�6�$�E�G�E�E�AT�E�E�-1�Z�	�-B�-B�E�E����638�0�1�1�A�5�5�$�A�7�A�A�FY�A�A�-1�Z��->�->�A�A����639/H�+�)� ��#�5�'�'�+A�'�
�������(���!�#�#�'=�#�!%����J�J�����i��:�:�J��s�640�641�!�642��643�644�	645s��-)XrMrNr"rOr#rOr$rOr%rPr&rQr'rQr(rQr)rRr*rOr+rOr,rOr-rOr.rSr/rSr0rSr1rOr2rTr3rTr4rTr5rTr6rTr7rTr8rOr9rQr:rQr;rQr<rQr=rUr>rUr?rQr@rOrArVrBrTrCrVrDrWrErVrFrXrGrYrHrZrIrSrJrSrKrQrLrQ)r�r�)r�r�)r�r�)r�r)r�r)r�r)TF)rrrrQrrQr�r)NF)rrrrrrQr�r)r�r)r*rO)rr!)r1r2r3r r4r rrr rr5r6TNN)r7rOr8rTr9rTr:rTr;rTr<rTr=rTr>rTr?rTr@rOrArTrBrTrCrQrDrErFrG)r1r2r3r r4r rrr rr5r6TNNN) r7rOr8rTr9rTr:rTr;rTr<rTr=rTr>rTr?rTr@rOrArTrBrTrCrQrDrErFrGrsrS)r1r2r3r r4r Trrr rr6r5TNNN)&rr!r7rOr8rTr9rTr:rTr;rTr<rTr rQr=rTr>rTr?rTr@rOrBrTrArTrCrQrDrErzr{rFrGr�r|ri)r�r�rdrVr�r�)FTF)r�r�r�rQr�rQr�rQ)4r�r�r�rVr�rSr�rTr8rTr9rTr:rTr�rSr�rQr�r�r=rTr>rTr<rTr7rOr�rQr*rSr?rTr@rOrBrTrArTrdrVrzr{rDrErFrGr�r�r�r�)4r�r�r�rVr�rSr�rTr8rTr9rTr:rTr�rSr�rQr�r�r=rTr>rTr<rTr7rOr�rQr*rSr?rTr@rOrBrTrArTrdrVrzr{rDrErFrGr�r�r�rB)4r�rNr�rVr�rSr�rTr8rTr9rTr:rTr�rSr�rQr�r�r=rTr>rTr<rTr7rOr�rQr*rSr?rTr@rOrBrTrArTrdrVrzr{rDrErFrGr�r�r�rB):rMrNrOrPrQrRrSrTrUrVr�rTr8rTr7rOr9rTr:rTr�rQr�r�r*rSrWrXr�rSr>rTr=rTr<rTr?rTr@rOrBrTrArTrdrVrDrErFrGr�r�r�rUr�rSr�rY)r_rr�r)r�rn)rlrnr�rx)r�rO)r�r)r�rN)r�rx)r])r/r�r�rOr�r)r�r!r�r!)NNr�N)r�rNr�rVr�r�r�r�r�r�r�r�r�rr�r�)5�__name__�646__module__�__qualname__�__doc__r�rk�LLAMA_SPLIT_MODE_LAYERr�r��LLAMA_POOLING_TYPE_UNSPECIFIEDr��propertyr�rdrrr647r
rrrrr r0rrrvr�r�r�rArJrLr^rhrjrmrrr+r�rtrHr�r�r�r1r�r��staticmethodrr�classmethodr�r�rqrorr7s�������:�:�!���#�:��.2� ���JN��0����#'�)-�
�9�%�D� #�!$�!%�"%� $� #�� �� � �%)�#'��"$�#'��#'�!&�%)�OS�15�26� $� $� ��ug�g�g�g�g�g�R�����X���!�!�!��X�!��/�/�/��X�/��/�/�/��X�/��S�S�S��X�S��648�649�650��X�651�BG�@�@�@�@�@�*,0��	652�653�654�655�656�(�����������#&�#&�#&�#&�N����� #�#&�"%���!�!� �:>�*.�!W�W�W�W�W�v����� #�#&�"%���!�!� �:>�*.�!�#<�<�<�<�<�B����� #��#&�"%���!�!� �:>�<@�*.�'J�J�J�J�J�ZDH�&657�&658�&659�&660�&661�V ��"�w�w�w�w�w�x!%�$&� ����"&��02�#&�"%� #���"���!�!�#�<@�:>�*.�15�5j	662�j	663�j	664�j	665�j	666�^!%�$&� ����"&��02�#&�"%� #���"���!�!�#�<@�:>�*.�15�5_�_�_�_�_�H!%�$&� ����"&��02�#&�"%� #���"���!�!�#�<@�:>�*.�15�5Z667�Z668�Z669�Z670�Z671�~=A�EI�48�@D� ������02�"�IM�$(�"%�#&� #���!�!�#�:>�*.�15�#'�&*�;e672�e673�e674�e675�e676�N����B6677�6678�6679�p���680�681�682�683�<
A�
A�
A�
A�!�!�!�!�$�$�$�$�%�%�%�%�$�$�$�$�'�'�'�'�'�'�'�'�&�&�&�&�(�(�(�(����������AC�#�#�#�#��\�#�"�����\���684,0�<@�?E�<@�|685�|686�|687�|688��[�|689�|690�|691rqrc��eZdZd
d�ZdS)rnr�r�r�rrerOrprrqr*c�Z�||_||_||_||_||_||_dSri)r�r�rerprqr*)rnr�r�rerprqr*s       ror�zLlamaState.__init__@	s4��#������ ��
�&��� 0�����	�	�	rqN)r�r�r�rrerOrprrqrOr*rO)r�r�r�r�r�rqrornrn?	s(�����������rqrnc��eZdZdd�ZdS)	rr�r�r�rr�c�(�|D]}|||��}�|Srir�)rnr�r��	processors    rorLzLogitsProcessorList.__call__W	s+���	2�	2�I��Y�y�&�1�1�F�F��
rqNr��r�r�r�rLr�rqrorrV	s(�����������rqrc��eZdZd	d�ZdS)692�StoppingCriteriaListr�r�r/rr�rQc�>���t��fd�|D����S)Nc�(��g|]}|������Sr�r�)r{rzr�r/s  ��ror�z1StoppingCriteriaList.__call__.<locals>.<listcomp>f	s(���W�W�W�=N�%�%�i��8�8�W�W�Wrq)�any)rnr�r/s ``rorLzStoppingCriteriaList.__call__c	s-�����W�W�W�W�W�RV�W�W�W�X�X�XrqN)r�r�r/rr�rQr�r�rqror�r�b	s.������Y�Y�Y�Y�Y�Yrqr�c��eZdZdd�Zd
d693�ZdS)�MinTokensLogitsProcessor�694min_tokensrOr�c�0�||_||_d|_dSri)r�r�r�)rnr�r�s   ror�z!MinTokensLogitsProcessor.__init__j	s��$���"���!����rqr�r�r�rr�c��|j�t|��|_t|��|jz695|jkrtj||j<|Sri)r�r�r�r��infr�)rnr�r�s   rorLz!MinTokensLogitsProcessor.__call__o	sK����%�!$�Y���D���y�>�>�D�.�.���@�@�&(�f�W�F�4�>�"��
rqN)r�rOr�rOr�)r�r�r�r�rLr�rqror�r�i	s<������"�"�"�"�696�����rqr�)D�697__future__rr�r�r�r�r�r�r�rr�r�r�r�rrrrrr	r698rrr
r�collectionsr�pathlibr�llama_types�
llama_grammarr�llama_cacherrrr�llama_tokenizerrr�llama_cpp.llama_cpprk�llama_cpp.llama_chat_formatr��llama_cpp.llama_speculativer�numpyr��numpy.typing�npt�llama_cpp._internals�699_internalsr��_loggerr�_utilsrrrn�NDArrayr�r��LogitsProcessorrrQ�StoppingCriteriar�r�r�rqro�<module>r�s���"�"�"�"�"�"�	�	�	�	�700701702703�������������
�
�
�
�
�
�
�
�
�
�
�
�����������������������������������������������������������'�'�'�'�'�'�������������@�?�?�?�?�?�?�?�'�'�'�'�'�'�7�7�7�7�7�7�7�7�7�7�7�7�����������(�(�(�(�(�(� � � � � � �*�*�*�*�*�*�E$704�E$705�E$706�E$707�E$708�E$709�E$710�E$711�PH��������$��[����3�;�r�y�1�2�C�K��	�4J�J���712�����$��/�����S�[���1�3�;�r�y�3I�J�D�P�Q��Y�Y�Y�Y�Y�4� 0�1�Y�Y�Y�
�
�
�
�
��
�
�
�
�
rq