CoolFace
Modelpublic

hymenjj/llama-cpp-python-prebuilt

sourceHugging Faceupdated 7mo agoView on Hugging Face
0likes
app.cpython-311.pyc146 linesDownload Raw Back to __pycache__
1200�itL���UddlmZddlZddlZddlZddlZddlmZddlm	Z	ddlm3Z4mZmZm
Z
ddlZddlZddlmZddlmZmZddlmZmZmZmZmZmZmZdd	lmZdd5lmZddl m!Z!ddl"m#Z#dd
l$m%Z%ddl&m'Z'ddl(m)Z)ddl*m+Z+m,Z,m-Z-m.Z.ddl/m0Z0m1Z1m2Z2m3Z3m4Z4m5Z5m6Z6m7Z7m8Z8ddl9m:Z:ee:���Z;da<de=d<d�d�Z>d�Z?da@de=d<e��ZAe��ZBd�d�ZCd�ZDdaEd e=d!<d�d$�ZF			d�d�d)�ZGd�d2�ZHd�d7�ZId�d;�ZJe!d<�=��ZKee?��eeK��fd�dA�ZLdBZMe;�NdCdDeeL��geejOePfdEdFdGdHdIigdJdK�idGdLdMdNdO�idP�dQ�ieMg�R��e;�NdSd<eeL��geMg�T��d�dW�����ZQe;�NdXdYeeL��geMg�Z��eeD��fd�d\���ZRe;�Nd]d^eeL��geejSePfdEdFdGdHd_igdJdK�idGdLd`dadO�idP�dQ�ieMg�R��edbdcdddedf�dgdhdf�gdi�dj�dkdcdddedf�dgdldf�gdmdnido�dj�dpdcdddedf�dgdqdf�gdrdsdtdudmdLidmdvidw�dxdygdz�d{�d|�gdrdxdsid|�d}�dj�d~dcdddedf�dgdhdf�gdd�d��dj�d������fd�d����ZTe;�Ud�d�eeL��geMg�Z��eeD��fd�d����ZVd�ZWe;�Nd�d�eeL��geWg�Z��eeD��fd�d����ZXe;�Nd�d�eeL��geWg�Z��eeD��fd�d����ZYe;�Nd�d�eeL��geWg�Z��eeD��fd�d����ZZdS)��)�annotationsN)�Lock)�partial)�List�Optional�Union�Dict)�MemoryObjectSendStream)�run_in_threadpool�iterate_in_threadpool)�Depends�FastAPI�	APIRouter�Request�
HTTPException�status�Body)�6Middleware)�CORSMiddleware)�7HTTPBearer)�EventSourceResponse)�RequestIdPlugin)�RawContextMiddleware)�8LlamaProxy)�ConfigFileSettings�Settings�
ModelSettings�ServerSettings)	�CreateCompletionRequest�CreateEmbeddingRequest�CreateChatCompletionRequest�	ModelList�TokenizeInputRequest�TokenizeInputResponse�TokenizeInputCountResponse�DetokenizeInputRequest�DetokenizeInputResponse)�RouteErrorHandler)�route_classzOptional[ServerSettings]�_server_settings�server_settingsrc�9�|adS�N�r*)r+s �?/usr/local/lib/python3.11/site-packages/llama_cpp/server/app.py�set_server_settingsr05s��&����c#�K�tV�dSr-r.�r1r/�get_server_settingsr4:s����10�����r1zOptional[LlamaProxy]�_llama_proxy�model_settings�List[ModelSettings]c�&�t|���adS)N)�models)rr5�r6s r/�set_llama_proxyr;Ds���^�4�4�4�L�L�Lr1c��K�t����d{V��d}	t����d{V��	t���d}tWV�t���n#t���wxYw	|rt���dSdS#|rt���wwxYw)NTF)�llama_outer_lock�acquire�llama_inner_lock�releaser5)�release_outer_locks r/�get_llama_proxyrBIs�����11"�12"�13$�14$�$�$�$�$�$�$�$���15'��&�&�(�(�(�(�(�(�(�(�(�	'��$�$�&�&�&�!&��������$�$�&�&�&�&���$�$�&�&�&�&����&��	'��$�$�&�&�&�&�&�	'�	'���	'��$�$�&�&�&�&�	'���s#�C�%B�*C�B�C�C z+typing.Optional[typing.Callable[[], bytes]]�_ping_message_factory�factory�typing.Callable[[], bytes]c�16�|adSr-)rC)rDs r/�set_ping_message_factoryrG_s��#���r1�settings�Settings | None�ServerSettings | None�List[ModelSettings] | Nonec�`�tj�dd��}|��tj�|��std|�d����t
|d��5}|�d��s|�d��r>ddl}tj17tj|�
|������}n&tj18|�����}tj|��}|j}ddd��n#1swxYwY|�;|�9|�t%��}tj|��}t'j|��g}|�|�19Jd���t)|��t+t,t/��f�	��g}t1|d20t2j|j���}|�t:dgd
dgdg���|�t>��|�J�tA|���|j!rtEd���|S)N�CONFIG_FILEzConfig file z not found!�rbz.yamlz.ymlrz<server_settings and model_settings must be provided together)�pluginsu🦙 llama.cpp Python API)�21middleware�title�version�	root_path�*T)�
allow_origins�allow_credentials�
allow_methods�
allow_headersr:c��t��Sr-)�bytesr3r1r/�<lambda>zcreate_app.<locals>.<lambda>�s22�����r1)#�os�environ�get�path�exists�23ValueError�open�endswith�yamlr�model_validate_json�json�dumps�	safe_load�readr�model_validater9rrr0rrrr�	llama_cpp�__version__rS�add_middlewarer�include_router�routerr;�disable_ping_eventsrG)	rHr+r6�config_file�frd�config_file_settingsrP�apps	         r/�24create_appruds���25�*�.�.���5�5�K����w�~�~�k�*�*�	F��D�K�D�D�D�E�E�E�
�+�t�
$�
$�	9���#�#�G�,�,�
X��0D�0D�V�0L�0L�
X�����'9�'M��J�t�~�~�a�0�0�1�1�(�(�$�$�(:�'M�a�f�f�h�h�'W�'W�$�,�;�<P�Q�Q�O�1�8�N�	9�	9�	9�	9�	9�	9�	9�	9�	9�	9�	9����	9�	9�	9�	9���>�#9����z�z�H�(�7��A�A��'�6�x�@�@�A��	�#��(B�(B�E�)C�(B�B���(�(�(��1�O�<M�<M�;O�P�P�P�Q�J�26��)��%�!�+�	���C������e���e��e��������v�����%�%�%��>�2�2�2�2��*�2� ���1�1�1��Js�&B*D�D �#D �body�5CreateCompletionRequest | CreateChatCompletionRequest�llama_proxyr�27body_model�28str | None�return�llama_cpp.Llamac���|�ttjd����||��}|j�*|jdkrt||j��n|j|d<|j�'tj�	|j��|d<|j29dkrdtjtj|j30|�
����g��}d|vr||d<n|d�|��|S)NzService is not available��status_code�detail�tokens�31logit_bias�grammarr�logits_processor)rr�HTTP_503_SERVICE_UNAVAILABLEr��logit_bias_type�_logit_bias_tokens_to_input_idsr�rk�LlamaGrammar�from_string�32min_tokens�LogitsProcessorList�MinTokensLogitsProcessor�	token_eos�extend)rvrxry�kwargs�llama�_min_tokens_logits_processors      r/�prepare_request_resourcesr��s������;�-�33�34�35�	36�
�K�37�#�#�E���"��#�x�/�/�
,�E�4�?�C�C�C���	�|���|��%�2�>�>�t�|�L�L��y�������'0�'D�
�
/������AR�AR�
S�
S�T�(38�(39�$��V�+�+�)E�F�%�&�&��%�&�-�-�.J�K�K�K��Lr1�requestr�inner_send_chan�"MemoryObjectSendStream[typing.Any]c40��K�tt����}|r|jnd}tjt41����4�d{V��}t
||||��}	|4�d{V��	t||	fi|���d{V��}42t|43��23d{V��}|�	ttj|��������d{V��|�
���d{V��rtj�����|r]t ���rD|�	td������d{V��tj�������6|�	td������d{V��nm#tj��$rT}t%d��tjdd���5t%d|j����|�#1swxYwYYd}~nd}~wwxYwddd���d{V��n#1�d{V��swxYwYddd���d{V��dS#1�d{V��swxYwYdS)	NF)�dataz[DONE]�disconnected�T)�shieldz-Disconnected from client (via refresh/close) )�nextr4�interrupt_requests�44contextlib�asynccontextmanagerrBr�rr�send�dictrfrg�is_disconnected�anyio�get_cancelled_exc_classr=�locked�print�
move_on_after�client)
r�r�rvry�45llama_callr�r+r�rxr��iterator�chunk�es
             r/�get_event_publisherr��s
�����.�0�0�1�1�O�.=�H��*�*�5��?�z�-�o�>�>�@�@��������K�)�$��Z��P�P��"�	�	�	�	�	�	�	�	�
�!2�:�u�!O�!O��!O�!O�O�O�O�O�O�O��#8��#B�#B�@�@�@�@�@�@�@�%�)�.�.�t���E�9J�9J�/K�/K�/K�L�L�L�L�L�L�L�L�L�$�4�4�6�6�6�6�6�6�6�6�@�=�e�;�=�=�?�?�?�)�@�.>�.E�.E�.G�.G�@�-�2�2�4�X�3F�3F�3F�G�G�G�G�G�G�G�G�G�=�e�;�=�=�?�?�?��
$C�&�*�*�4�X�+>�+>�+>�?�?�?�?�?�?�?�?�?�?���0�2�2�
�
�
��n�%�%�%��(��4�8�8�8����X���X�X�����G�	��������������������
����	�	�	�	�	�	�	�	�	�	�	�	�	�	�	�	�	�	�	�	�	�	�	����	�	�	�	���������������������������������s��H5�+H�-#F�E'�C;F�H�G<�(%G7�
G'	�'G+
�+G7�.G+
�/G7�2H�7G<�<H�?H5�46H	�H5�H	�H5�547H?�H?r�r��Dict[str, float]c���i}|���D]G\}}|�d��}|�|dd���D]}||t|��<��H|S)N�utf-8FT)�add_bos�special)�items�encode�tokenize�str)r�r��to_bias�token�score�input_ids      r/r�r��sw��!#�G�"�(�(�*�*�+�+���u����W�%�%�����u�e�T��J�J�	+�	+�H�%*�G�C��M�M�"�"�	+��Nr1F)�48auto_errorr�
authorization�
Optional[str]c��K�|j�dS|r|j|jkr|jSttjd����)NTzInvalid API keyr~)�api_key�credentialsrr�HTTP_401_UNAUTHORIZED)rHr�s  r/�authenticater��s[����49����t��)��2�h�6F�F�F��(�(���0� ����r1z	OpenAI V1z/v1/completions�50Completion�200zSuccessful Response�schemaz$refz-#/components/schemas/CreateCompletionResponsez&Completion response, when stream=False)�anyOfrQ�stringz�Server Side Streaming response, when stream=True. See SSE format: https://developer.mozilla.org/en-US/docs/Web/API/Server-sent_events/Using_server-sent_events#Event_stream_formatzQdata: {... see CreateCompletionResponse ...} \n\n data: ... \n\n ... data: [DONE])�typerQ�example)zapplication/jsonztext/event-stream)�description�content)�summary�dependencies�response_model�	responses�tags�%/v1/engines/copilot-codex/completions)�include_in_schemar�r�r�llama_cpp.Completionc��K�t|jt��rFt|j��dksJ�t|j��dkr
|jdnd|_|jjdkr|jnd}hd�}|�|���}|�dd	��rWtj51d52��\}}t|tt||||tjj|���dt"�
��St%jt(����4�d{V��}t+||||��}|����d{V��r2t/d|j����t3t4jd����t9|fi|���d{V��cddd���d{V��S#1�d{V��swxYwYdS)Nr�r�r�z
copilot-codex>�n�user�best_ofr�r���exclude�streamF�53�r�r�rvryr�r��54��data_sender_callable�sep�ping_message_factory�@Disconnected from client (via refresh/close) before llm invoked �Client closed requestr~)�55isinstance�prompt�list�len�urlr_�model�56model_dumpr^r��create_memory_object_streamrrr�rk�Llama�__call__rCr�r�rBr�r�r�r�rr�HTTP_400_BAD_REQUESTr�	r�rvryr�r��	send_chan�	recv_chanrxr�s	         r/�create_completionr�s�����V�$�+�t�$�$�E��4�;���1�$�$�$�$�(+�D�K�(8�(8�1�(<�(<�d�k�!�n�n�"����;��F�F�F�	
�5758�
�����G��_�_�W�_�
-�
-�F��z�z�(�E�"�"�59�$�@��D�D��	�9�"��!(�#�� )��%�$�?�3��"�"�"��!6�
60�
61�
6263� ?�z�-�o�>�>�@�@�8�8�8�8�8�8�8�K�)�$��Z��P�P���(�(�*�*�*�*�*�*�*�*�	��c�SZ�Sa�c�c�
�
�
� �"�7�.����
�64'�u�7�7��7�7�7�7�7�7�7�7�8�8�8�8�8�8�8�8�8�8�8�8�8�8�8�8�8�8�8�8�8�8�8�8����8�8�8�8�8�8s�*A1F.�.65F8�;F8z/v1/embeddings�	Embedding)r�r�r�r c��|K�t||j��jfi|�dh������d{V��S)Nr�r�)rr��create_embeddingr�)r�rxs  r/r�r�hsj����#���G�M�"�"�3���66�67�68�f�X�69�70.�71.���������r1z/v1/chat/completions�Chatz1#/components/schemas/CreateChatCompletionResponsez�Server Side Streaming response, when stream=TrueSee SSE format: https://developer.mozilla.org/en-US/docs/Web/API/Server-sent_events/Using_server-sent_events#Event_stream_formatzUdata: {... see CreateChatCompletionResponse ...} \n\n data: ... \n\n ... data: [DONE]zChat Completionz
gpt-3.5-turbo�systemzYou are a helpful assistant.)�roler�r�zWhat is the capital of France?)r��messages)r��valuez	JSON Modez Who won the world series in 2020r��json_object)r�r��response_formatzTool CallingzExtract Jason is 30 years old.�function�UserzUser record�object�number)�name�agerr)r��72properties�required)rr��73parameters)r�r�)r�r��tools�tool_choice�LogprobsTr�)r�r��logprobs�top_logprobs)�normal�	json_mode�tool_callingr	)�openapi_examplesr!�llama_cpp.ChatCompletionc��K�|j}hd�}|�|���}|�dd��rWtjd��\}}t|t
t||||tj	j74|���dt���Stj
t����4�d{V��}t||||��}|����d{V��r2t#d	|j����t't(jd75����t-|j76fi|���d{V��cddd���d{V��S#1�d{V��swxYwYdS)N>r�r�r�r�r�r�Fr�r�r�r�r�r�r~)r�r�r^r�r�rrr�rkr��create_chat_completionrCr�r�rBr�r�r�r�rrr�rr�s	         r/rrxsR����\��J����G��_�_�W�_�
-�
-�F��z�z�(�E�"�"�77�$�@��D�D��	�9�"��!(�#�� )��%�$�?�A��"�"�"��!6�
78�
79�
8081� ?�z�-�o�>�>�@�@�O�O�O�O�O�O�O�K�)�$��Z��P�P���(�(�*�*�*�*�*�*�*�*�	��c�SZ�Sa�c�c�
�
�
� �"�7�.����
�82'�u�'C�N�N�v�N�N�N�N�N�N�N�N�O�O�O�O�O�O�O�O�O�O�O�O�O�O�O�O�O�O�O�O�O�O�O�O����O�O�O�O�O�Os�8A6E�83E�Ez84/v1/models�Modelsr"c��$K�dd�|D��d�S)Nr�c��g|]	}|ddgd���85S)r��me)�idr��owned_by�permissionsr3)�.0�model_aliass  r/�86<listcomp>zget_models.<locals>.<listcomp>s=��87�88�89��"�!� �!�	
�
�90�91�92r1)r�r�r3)rxs r/�93get_modelsrs8�����94�95� +�96�97�98���r1�Extrasz/extras/tokenize�Tokenizer#r$c��K�||j���|j�d��d���}t	|���S)Nr�T�r�)r�)r�r��inputr�r$�rvrxr�s   r/r�r�+sN�����[���
$�
$�
-�
-�d�j�.?�.?��.H�.H�RV�
-�
W�
W�F� ��/�/�/�/r1z/extras/tokenize/countzTokenize Countr%c��K�||j���|j�d��d���}t	t|�����S)Nr�Tr )�count)r�r�r!r�r%r�r"s   r/�count_query_tokensr%:sT�����[���
$�
$�
-�
-�d�j�.?�.?��.H�.H�RV�
-�
W�
W�F�%�C��K�K�8�8�8�8r1z/extras/detokenize�99Detokenizer&r'c��K�||j���|j���d��}t	|���S)Nr�)�text)r��100detokenizer��decoder')rvrxr(s   r/r)r)IsH�����;�t�z�"�"�-�-�d�k�:�:�A�A�'�J�J�D�"��-�-�-�-r1)r+r)r6r7)rDrE)NNN)rHrIr+rJr6rK)rvrwrxrryrzr{r|)r�rr�r�rvrwryrz)r�r|r�r�r{r�)rHrr�r�)r�rrvrr{r�)r�r rxr)r�rrvr!r{r)rxrr{r")rvr#rxrr{r$)rvr#rxrr{r%)rvr&rxrr{r')[�101__future__rr\rf�typingr�r�r�	functoolsrrrrr	rk�anyio.streams.memoryr102�starlette.concurrencyrr�fastapir
rrrrrr�fastapi.middlewarer�fastapi.middleware.corsr�fastapi.securityr�sse_starlette.sser�starlette_context.pluginsr�starlette_context.middlewarer�llama_cpp.server.modelr�llama_cpp.server.settingsrrrr�llama_cpp.server.typesrr r!r"r#r$r%r&r'�llama_cpp.server.errorsr(ror*�__annotations__r0r4r5r=r?r;rBrCrGrur�r�r��
bearer_schemer��
openai_v1_tag�post�CreateCompletionResponser�r�r��ChatCompletionrr^r�103extras_tagr�r%r)r3r1r/�<module>rBsG	��"�"�"�"�"�"�"�	�	�	�	�����
�
�
�
�����������������.�.�.�.�.�.�.�.�.�.�.�.���������7�7�7�7�7�7�J�J�J�J�J�J�J�J�U�U�U�U�U�U�U�U�U�U�U�U�U�U�U�U�U�U�)�)�)�)�)�)�2�2�2�2�2�2�'�'�'�'�'�'�1�1�1�1�1�1�5�5�5�5�5�5�=�=�=�=�=�=�������������������104�105�106�107�108�109�110�111�112�113�114�115�116�117�118�119�120�121�122�123�124�125�6�5�5�5�5�5�126��0�	1�	1�	1��-1��1�1�1�1�'�'�'�'�127���&*��)�)�)�)��4�6�6���4�6�6��5�5�5�5�128'�'�'�&FJ��I�I�I�I�$�$�$�$�!%�-1�15�7�7�7�7�7�t����B����D	�	�	�	��129�e�,�,�,�
�!��!4�5�5�#*�7�=�#9�#9������(�
�������'�,�'�'�(���*��	
��130	�0��#�%T�U�"�"J�	��%�� (�"]�#~�	��&���131�132��.��?
� � �B���+���'�,�'�'�(�133��	
���68�68�68�
��C � �N68�r������'�,�'�'�(�134��	
���&�g�o�6�6�����
���������'�,�'�'�(���1�3�6�7�
�0��!'�([��"�135"J�
��	%�� (�"]�$C�	��&���136�137��2��=
���D)-��-�,�!)�6T�U�U�!'�4T�U�U�!���	�	�'�,�!)�6T�U�U�!'�4V�W�W�!�)/�
�'>�
��138�139�*�,�!)�6T�U�U�!'�4T�U�U�!�%/�(.�/<�,4�17��0B�06��/A�3&�3&�28���
/"�/"�)�)����$!+�"�F�%�$�$�/�� � �D&�,�!)�6T�U�U�!'�4T�U�U�!�!%�$&�����oC140�C141�E)�E)�E)�vO�vO�vO�vO�A��@vO�r������'�,�'�'�(�142��	���&�g�o�6�6�����
���"�143�������'�,�'�'�(�144��	
���&�g�o�6�6�0�0�0�0�
��0�������'�,�'�'�(�145��	
���&�g�o�6�6�9�9�9�9�
��9�������'�,�'�'�(�146��	
���&�g�o�6�6�.�.�.�.�
��.�.�.r1