CoolFace
Modelpublic

hymenjj/llama-cpp-python-prebuilt

sourceHugging Faceupdated 7mo agoView on Hugging Face
0likes
model.cpython-311.pyc95 linesDownload Raw Back to __pycache__
1200�i�4��x�ddlmZddlZddlmZmZmZmZddlZddl	m3Z4ddlmZddl
mZGd�d��ZdS)�)�annotationsN)�Dict�Optional�Union�List)�
ModelSettingsc�T�eZdZdd�Zddd5�Zdd�Zdd�Zd�Zd�Ze	dd���Z6dS)�7LlamaProxy�models�List[ModelSettings]�return�Nonec�J�t|��dks8Jd���i|_|D]$}|js|j|_||j|j<�%d|_d|_|d|_|jj|_|�|j��|_|j|_dS)NrzNo models provided!)	�len�_model_settings_dict�model_alias�model�_current_model�_current_model_alias�_default_model_settings�_default_model_alias�load_llama_from_model_settings)�selfrrs   �A/usr/local/lib/python3.11/site-packages/llama_cpp/server/model.py�__init__zLlamaProxy.__init__s����6�{�{�Q���� 5����>@��!��	A�	A�E��$�
0�$)�K��!�;@�D�%�e�&7�8�8�9=���37��!�6<�Q�i��$�)-�)E�)Q��!�#�A�A��(�9�10���%)�$=��!�!�!�Nr�
Optional[str]�llama_cpp.Llamac��|�|j}||jvr|j}||jkr|j�|jS|jr|j���d|_|j|}|�|��|_||_|jS�N)rrrr�closer�rr�settingss   r�__call__zLlamaProxy.__call__$s����=��-�E���1�1�1��-�E��D�-�-�-��"�.��*�*���	(���%�%�'�'�'�"����,�U�3��"�A�A�(�K�K���$)��!��"�"r�strc�@�|j|���Sr )r�11model_dump�rrs  r�__getitem__zLlamaProxy.__getitem__8s���(��/�:�:�<�<�<rr#� Union[ModelSettings, str, bytes]c�z�t|ttf��rtj|��}||j|<dSr )�12isinstance�bytesr%r�model_validate_jsonrr"s   r�__setitem__zLlamaProxy.__setitem__;s<���h����-�-�	C�$�8��B�B�H�+3��!�%�(�(�(rc#�&K�|jD]}|V��dSr )rr(s  r�__iter__zLlamaProxy.__iter__@s-�����.�	�	�E��K�K�K�K�	�	rc�N�|jr|j���|`dSdSr )rr!)rs r�freezLlamaProxy.freeDs8����	$���%�%�'�'�'��#�#�#�	$�	$rrc�F�d}|jdkr}|j�13Jd���|j�8tjj�|j|j|j���}�n�tj�|j|j���}�nV|jdkr}|j�14Jd���|j�8tjj�|j|j|j���}�n�tj�|j|j���}�n�|jdkr}|j�15Jd���|j�8tjj	�|j|j|j���}�nstj�	|j|j���}�nF|jdkr}|j�16Jd���|j�8tjj17�|j|j|j���}�n�tj�18|j|j���}�n�|jdkr}|j�19Jd���|j�8tjj�|j|j|j���}�nctj�|j|j���}�n6|jd	kr}|j�20Jd���|j�8tjj�|j|j|j���}�n�tj�|j|j���}�n�|jd21kr}|j�22Jd���|j�8tjj
�|j|j|j���}�nStj�
|j|j���}�n&|jdkr{|j�23Jd���|j�7tjj�|j|j|j���}n�tj�|j|j���}n�|jdkr6|j�24Jd
���tj�|j��}n_|jdkrT|j�25Jd���tj�t'jt+|j������}d}|j�$t,j�|j��}d}|j�t3j|j���}d}|j��t;|jt<��sJ�i}|jD]�}|�d��\}}d|vr�|�d��\}}|dkr|� ��dv||<�V|dkrtC|��||<�o|dkrtE|��||<��|dkr|||<��tGd|�������ddl$}	i}26|j�2|	�%tj&j|j|j'���}ntj&}|j'|27d<|dGi|28�id|j(�d|j)�d|j*�d|j+�d |j,�d!|j-�d"|j.�d#|�d$|j/�d%|j0�d&|j1�d'|j2�d(|j3�d)|j4�d*|j5�d+|j6�d,|j7�d-|j8�d.|j9�d/|j:�d0|j;�d1|j<�d2|j=�d3|j>�d4|j?�d5|j@�d6|jA�d7|jB�d8|jC�d9|jD�d:|jE�d;|jF�d<|j�d=|�d>|�d?|jG�d@|jH�dA|�dB|j���}|jIr�|jJdCkr9|jrt�dD|jL����tjM|jL�E��}
n8|jrt�dF|jL����tjN|jL�E��}
|�O|
��|S)HNz	llava-1-5zclip model not found)�repo_id�filename�verbose)�clip_model_pathr7�obsidianz	llava-1-6�	moondream�	nanollavazllama-3-vision-alphaz
minicpm-v-2.6z29qwen2.5-vlzhf-autotokenizerzAhf_pretrained_model_name_or_path must be set for hf-autotokenizerzhf-tokenizer-configz<hf_tokenizer_config_path must be set for hf-tokenizer-config)�num_pred_tokens�=�:�bool)�true�1�int�floatr%zUnknown value type r)r5r6�30model_path�n_gpu_layers�31split_mode�main_gpu�tensor_split�32vocab_only�use_mmap�	use_mlock�kv_overrides�rpc_servers�seed�n_ctx�n_batch�n_ubatch�	n_threads�n_threads_batch�rope_scaling_type�rope_freq_base�rope_freq_scale�yarn_ext_factor�yarn_attn_factor�yarn_beta_fast�yarn_beta_slow�
yarn_orig_ctx�	mul_mat_q�33logits_all�	embedding�offload_kqv�34flash_attn�last_n_tokens_size�	lora_base�	lora_path�numa�chat_format�chat_handler�draft_model�type_k�type_v�	tokenizerr7�diskzUsing disk cache with size )�capacity_byteszUsing ram cache with size �)Prer8�hf_model_repo_id�	llama_cpp�llama_chat_format�Llava15ChatHandler�from_pretrainedr7�ObsidianChatHandler�Llava16ChatHandler�MoondreamChatHandler�NanoLlavaChatHandler�Llama3VisionAlpha�MiniCPMv26ChatHandler�Qwen25VLChatHandler� hf_pretrained_model_name_or_path�+hf_autotokenizer_to_chat_completion_handler�hf_tokenizer_config_path�.hf_tokenizer_config_to_chat_completion_handler�json�load�open�llama_tokenizer�LlamaHFTokenizerrg�llama_speculative�LlamaPromptLookupDecoding�draft_model_num_pred_tokensrLr,�list�split�lowerrBrC�35ValueError�	functools�partial�LlamarrErFrGrHrIrJrKrMrNrOrPrQrRrSrTrUrVrWrXrYrZr[r\r]r^r_r`rarbrcrdrhri�cache�36cache_type�print�37cache_size�LlamaDiskCache�
LlamaRAMCache�	set_cache)r#rfrjrgrL�kv�key�value�38value_typer��kwargs�	create_fn�_modelr�s              rrz)LlamaProxy.load_llama_from_model_settingsIsk39������;�.�.��+�7�7�9O�7�7�7��(�4��/�B�R�R� (� 9�!)�!9� (� 0�S����� )�:�M�M�$,�$<�h�FV� N� � ����
!�Z�
/�
/��+�7�7�9O�7�7�7��(�4��/�C�S�S� (� 9�!)�!9� (� 0�T����� )�:�N�N�$,�$<�h�FV� O� � ����
!�[�
0�
0��+�7�7�9O�7�7�7��(�4��/�B�R�R� (� 9�!)�!9� (� 0�S����� )�:�M�M�$,�$<�h�FV� N� � ����
!�[�
0�
0��+�7�7�9O�7�7�7��(�4��/�D�T�T� (� 9�!)�!9� (� 0�U����� )�:�O�O�$,�$<�h�FV� P� � ����
!�[�
0�
0��+�7�7�9O�7�7�7��(�4��/�D�T�T� (� 9�!)�!9� (� 0�U����� )�:�O�O�$,�$<�h�FV� P� � ����
!�%;�
;�
;��+�7�7�9O�7�7�7��(�4��/�A�Q�Q� (� 9�!)�!9� (� 0�R����� )�:�L�L�$,�$<�h�FV� M� � ����
!�_�
4�
4��+�7�7�9O�7�7�7��(�4��/�E�U�U� (� 9�!)�!9� (� 0�V����� )�:�P�P�$,�$<�h�FV� Q� � ����
!�\�
1�
1��+�7�7�9O�7�7�7��(�4��/�C�S�S� (� 9�!)�!9� (� 0�T����� )�:�N�N�$,�$<�h�FV� O� � ����
!�%7�
7�
7��9�E�E�R�F�E�E��+�W�W��=���
�L�40�
!�%:�
:�
:��1�=�=�M�>�=�=�$�6�e�e��	�$�x�@�A�A�B�B���L�=A�	��4�@�'�8�H�H��9���I�����+�+�E� (� D����K�KO��� �,��h�3�T�:�:�:�:�:��L��+�

M�

M���X�X�c�]�]�41��U��%�<�<�(-���C�(8�(8�%�J��!�V�+�+�,1�K�K�M�M�]�,J��S�)�)�#�u�,�,�,/��J�J��S�)�)�#�w�.�.�,1�%�L�L��S�)�)�#�u�,�,�,1��S�)�)�(�)K�z�)K�)K�L�L�L� �	�������$�0�!�)�)���/� �1�!��*���I�I�"��I�#+�>�F�<� ���342�343��344�345�346�"�.�.�347� �*�*�	34849�&�&�350�"�.�.�
351� �*�*�352��&�&�353��(�(�354�&��355�!�,�,�356����357��.�.�358��$�$�359� �&�&�!360�"�(�(�#361�$%�4�4�%362�&'�8�8�'363�($�2�2�)364�*%�4�4�+365�,%�4�4�-366�.&�6�6�/367�0$�2�2�1368�2$�2�2�3369�4#�0�0�5370�6�(�(�7371�8 �*�*�9372�:�(�(�;373�<!�,�,�=374�> �*�*�?375�B (�:�:�C376�F�(�(�G377�H�(�(�I378�L���M379�P!�,�,�Q380�R&��S381�V$��W382�Z�?�?�[383�\�?�?�]384�` �i�a385�d�$�$�e386�387��h�>�		$��"�f�,�,��#�O��M��8K�M�M�N�N�N�!�0��@S�T�T�T����#�N��L�x�7J�L�L�M�M�M�!�/�x�?R�S�S�S�����U�#�#�#��
r)rrr
rr )rrr
r)rr%)rr%r#r*)r#rr
r)�__name__�88__module__�__qualname__rr$r)r/r1r3�staticmethodrrmrrr89r90s�������>�>�>�>�*#�#�#�#�#�(=�=�=�=�4�4�4�4�91���$�$�$�92�n�n�n��\�n�n�nrr93)�94__future__rr~�typingrrrrro�llama_cpp.llama_speculativer��llama_cpp.llama_tokenizerr��llama_cpp.server.settingsrr95rmrr�<module>r�s���"�"�"�"�"�"�����.�.�.�.�.�.�.�.�.�.�.�.�����7�7�7�7�7�7�3�3�3�3�3�3�3�3�3�3�3�3�j�j�j�j�j�j�j�j�j�jr