CoolFace
Modelpublic

hymenjj/llama-cpp-python-prebuilt

sourceHugging Faceupdated 7mo agoView on Hugging Face
0likes
llama_grammar.cpython-311.pyc401 linesDownload Raw Back to __pycache__
1200�i�����dZddlmZddlmZddlmZmZmZm	Z	m3Z4mZdZGd�d��Z
	dZd	Zd5Zd6ZdZdZd
Z	ddlZddlZddlmZm	Z	dZejd��Zejd��Zdddd�ZdZ	dHd�ZGd�d��Zeddd��Zedg��edezg��edezg��ed d!d"g��ed#d!g��ed$gd%���ed&d'd(g��ed)d(g��ed*d+�d,�d-D����zd.zg��ed/g��ed0d1g��ed2g��d3�Z ed4g��ed5g��ed6d7d8g��ed9d7g��ed:d8g��ed;d<g��d=�Z!d>Z"d?Z#e$dd@ge �%���e!�%�����Z&e$dA��Z'e$dB��Z(GdC�dD��Z)dIdEe*dFe	ee*fdG�Z+dS)Jz�Python implementation of llama grammar parser directly translated from C++ source file in vendor/llama.cpp/common/grammar-parser.cpp.�)�Path)�groupby)�Any�Set�List�Optional�Tuple�Union�rootc��eZdZdefd�Zed
dededdfd���Zed
deee	fdeddfd	���Z7ed
d8ededdfd���ZdS)�LlamaGrammar�_grammarc�,�||_t|_dS�N)r�LLAMA_GRAMMAR_DEFAULT_ROOT�_root)�selfr�args�kwargss    �B/usr/local/lib/python3.11/site-packages/llama_cpp/llama_grammar.py�__init__zLlamaGrammar.__init__s�� ��
�/��91011�T�grammar�verbose�returnc��||���S)N)r�)�clsrrs   r�from_stringzLlamaGrammar.from_strings���s�G�$�$�$�$r�filec�F�	t|��5}|���}ddd��n#1swxYwYn0#t$r#}t|jj�d|�����d}~wwxYw|r|�||���St
|jj�d����)Nz: error reading grammar file: �rz5: error parsing grammar file: params_grammer is empty)�open�read�	Exception�	from_file�__name__r�12ValueError)rr r�fr�errs      rr&zLlamaGrammar.from_files���	��d���
#�q��&�&�(�(��
#�
#�
#�
#�
#�
#�
#�
#�
#�
#�
#����
#�
#�
#�
#����	�	�	���=�)�N�N��N�N���
�����	����13�	=��?�?�7�G�?�<�<�<���}�%�\�\�\�14�15�	16s-�>�2�>�6�>�6�>�17A+�A&�&A+�json_schemac�J�|�t|��|���S)Nr")r�json_schema_to_gbnf)rr+rs   r�from_json_schemazLlamaGrammar.from_json_schema-s!�����2�;�?�?���Q�Q�QrN)T)r'�18__module__�__qualname__�strr�classmethod�boolrr19rr&r.rrrr
r
s�������0��0�0�0�0��%�%�#�%��%��%�%�%��[�%��20�21�U�3��9�-�22��23��24�25�26��[�27� �R�R�3�R��R��R�R�R��[�R�R�Rrr
z�28root  ::= (expr "=" ws term "\n")+29expr  ::= term ([-+*/] term)*30term  ::= ident | num | "(" ws expr ")" ws31ident ::= [a-z] [a-z0-9_]* ws32num   ::= [0-9]+ ws33ws    ::= [ \t\n]*34ag35root ::= (declaration)*36 37declaration ::= dataType identifier "(" parameter? ")" "{" statement* "}"38 39dataType  ::= "int" ws | "float" ws | "char" ws40identifier ::= [a-zA-Z_] [a-zA-Z_0-9]*41 42parameter ::= dataType identifier43 44statement ::=45    ( dataType identifier ws "=" ws expression ";" ) |46    ( identifier ws "=" ws expression ";" ) |47    ( identifier ws "(" argList? ")" ";" ) |48    ( "return" ws expression ";" ) |49    ( "while" "(" condition ")" "{" statement* "}" ) |50    ( "for" "(" forInit ";" ws condition ";" ws forUpdate ")" "{" statement* "}" ) |51    ( "if" "(" condition ")" "{" statement* "}" ("else" "{" statement* "}")? ) |52    ( singleLineComment ) |53    ( multiLineComment )54 55forInit ::= dataType identifier ws "=" ws expression | identifier ws "=" ws expression56forUpdate ::= identifier ws "=" ws expression57 58condition ::= expression relationOperator expression59relationOperator ::= ("<=" | "<" | "==" | "!=" | ">=" | ">")60 61expression ::= term (("+" | "-") term)*62term ::= factor(("*" | "/") factor)*63 64factor ::= identifier | number | unaryTerm | funcCall | parenExpression65unaryTerm ::= "-" factor66funcCall ::= identifier "(" argList? ")"67parenExpression ::= "(" ws expression ws ")"68 69argList ::= expression ("," ws expression)*70 71number ::= [0-9]+72 73singleLineComment ::= "//" [^\n]* "\n"74multiLineComment ::= "/*" ( [^*] | ("*" [^/]) )* "*/"75 76ws ::= ([ \t\n]+)77aT78root   ::= object79value  ::= object | array | string | number | ("true" | "false" | "null") ws80 81object ::=82  "{" ws (83            string ":" ws value84    ("," ws string ":" ws value)*85  )? "}" ws86 87array  ::=88  "[" ws (89            value90    ("," ws value)*91  )? "]" ws92 93string ::=94  "\"" (95    [^"\\] |96    "\\" (["\\/bfnrt] | "u" [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F]) # escapes97  )* "\"" ws98 99number ::= ("-"? ([0-9] | [1-9] [0-9]*)) ("." [0-9]+)? ([eE] [-+]? [0-9]+)? ws100 101# Optional space: by convention, applied in this grammar after literal chars when allowed102ws ::= ([ \t\n] ws)?103a$104# This is the same as json.gbnf but we restrict whitespaces at the end of the root array105# Useful for generating JSON arrays106 107root   ::= arr108value  ::= object | array | string | number | ("true" | "false" | "null") ws109 110arr  ::=111  "[\n" ws (112            value113    (",\n" ws value)*114  )? "]"115 116object ::=117  "{" ws (118            string ":" ws value119    ("," ws string ":" ws value)*120  )? "}" ws121 122array  ::=123  "[" ws (124            value125    ("," ws value)*126  )? "]" ws127 128string ::=129  "\"" (130    [^"\\\x7F\x00-\x1F] |131    "\\" (["\\/bfnrt] | "u" [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F]) # escapes132  )* "\"" ws133 134number ::= ("-"? ([0-9] | [1-9] [0-9]*)) ("." [0-9]+)? ([eE] [-+]? [0-9]+)? ws135 136# Optional space: by convention, applied in this grammar after literal chars when allowed137ws ::= ([ \t\n] ws)?138aZ139root   ::= object140value  ::= object | array | string | number | ("true" | "false" | "null") ws141 142object ::=143  "{" ws (144            string ":" ws value145    ("," ws string ":" ws value)*146  )? "}" ws147 148array  ::=149  "[" ws (150            value151    ("," ws value)*152  )? "]" ws153 154string ::=155  "\"" (156    [^"\\\x7F\x00-\x1F] |157    "\\" (["\\bfnrt] | "u" [0-9a-fA-F]{4}) # escapes158  )* "\"" ws159 160number ::= ("-"? ([0-9] | [1-9] [0-9]{0,15})) ("." [0-9]+)? ([eE] [-+]? [0-9] [1-9]{0,15})? ws161 162# Optional space: by convention, applied in this grammar after literal chars when allowed163ws ::= | " " | "\n" [ \t]{0,20}164zn165root ::= item+166 167# Excludes various line break characters168item ::= "- " [^\r\n\x0b\x0c\x85\u2028\u2029]+ "\n"169N)rrz" "?z[^a-zA-Z0-9-]+z[\r\n"]z\rz\n�\")�
�170�"Fc������s|dkr|dkr��d�S|dkr|���d�Sd}|dkr9|r��d�dd�|zzdz}n!�rd��d�nd��g|z��}d���fd171�	�|dkr||kr|dz
}|�|�||z172|dk���z
}n)d�r�dznd���d
�}|dkr�r173d��d|�d�}n||�d�z
}|S)Nr��?�+�r7������ Fc���|r	�r��d���n�}|dkrdS|dkrd|�d�S�r|sd|�d�|dz174d����d�Sd|�d�|z���d|zzS)	z�175        - n=4, no sep:             '(a (a (a (a)?)?)?)?'176        - n=4, sep=',', prefix:    '("," a ("," a ("," a ("," a)?)?)?)?'177        - n=4, sep=',', no prefix: '(a ("," a ("," a ("," a)?)?)?)?'178        r>rr<r9�(�)?T��prefix_with_sep)�rstrip)�up_to_nrC�content�	item_rule�opt_repetitions�separator_rules   ���rrHz*_build_repetition.<locals>.opt_repetitionss�����
�#1�
�~�+�+�	�+�+�+��	�179�a�<�<��2�
��\�\�"�w�?�?�?�"�
�	J�O�	J�W�w�W�W����1��d�!S�!S�!S�W�W�W�W�"��N�N�N�W�,�4�4�6�6�$��.�I�IrrBr@�)z*)?�*)F)�join)rG�	min_items�	max_itemsrI�item_rule_is_literal�result�
item_operatorrHs`  `   @r�_build_repetitionrR�s�������#���>�>�i�1�n�n��?�?�?�"�
�!�^�^�	� 1��?�?�?�"�
�F��1�}�}��	�N�$:��I�a��d�O�i�7�8�3�>�F�F�/=�F�+�.�+�+�+�+�3�L�L���i�'���F�J�J�J�J�J�J�J�J�*�1�}�}��i�/�/��#�
�����/�/�)�i�"7��UV��W�W�W�W���X�N�J�N�S�0�0��X�I�X�X�X�
���>�>�n�>�7��7�7�]�7�7�7�F�F���)�)�)�)�F��Mrc� �eZdZddedefd�ZdS)�BuiltinRuleNrF�depsc�&�||_|pg|_dSr)rFrU)rrFrUs   rrzBuiltinRule.__init__7s������J�B��	�	�	rr)r'r/r0r1�listrrrrrTrT6s;����������4������rrTz[0-9]�z("true" | "false") spacez[0-9] z[0-9] | [1-9] zJ("-"? integral-part) ("." decimal-part)? ([eE] [-+]? integral-part)? space�
integral-part�decimal-partz("-"? integral-part) spacez1object | array | string | number | boolean | null)�object�array�string�number�boolean�nullzS"{" space ( string ":" space value ("," space string ":" space value)* )? "}" spacer]�valuez1"[" space ( value ("," space value)* )? "]" space�"\"" z "-" c#� K�|]	}d|zV��180dS)z[0-9a-fA-F]Nr)�.0�ns  r�	<genexpr>rfTs'����C�C�Q�}�q�(�C�C�C�C�C�Cr)��rhrh�� "\"" spacezQ[^"\\] | "\\" (["\\/bfnrt] | "u" [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F])z"\"" char* "\"" space�charz"null" space)r_rZrYr^�integerrar[r\�uuidrkr]r`za[0-9] [0-9] [0-9] [0-9] "-" ( "0" [1-9] | "1" [0-2] ) "-" ( "0" [1-9] | [1-2] [0-9] | "3" [0-1] )z�([01] [0-9] | "2" [0-3]) ":" [0-5] [0-9] ":" [0-5] [0-9] ( "." [0-9] [0-9] [0-9] )? ( "Z" | ( "+" | "-" ) ( [01] [0-9] | "2" [0-3] ) ":" [0-5] [0-9] )z
date "T" time�date�timez"\"" date "\"" spacez"\"" time "\"" spacez"\"" date-time "\"" space�	date-time)rnrorpzdate-stringztime-stringzdate-time-stringz[\U00000000-\U0010FFFF]z[^\x0A\x0D]�dotz|.()[]{}*+?z181[]()|{}*+?c182���eZdZd�Zd�Z	ddededefd�Zd	�Zd183e	defd�Z184d
�Zd�Zd�Z
d�Zd�Zdedefd�Zdeeeefdeededeeeffd�Zd�ZdS)�SchemaConverterc��||_||_||_||_dti|_i|_t��|_dS)N�space)	�_prop_order�_allow_fetch�_dotall�_raw_pattern�185SPACE_RULE�_rules�_refs�set�_refs_being_resolved)r�186prop_order�allow_fetch�dotall�raw_patterns     rrzSchemaConverter.__init__}sJ��%���'������'����Z�187�����188�$'�E�E��!�!�!rc�F�t�d�|��}d|�d�S)Nc�\�t�|�d����S�Nr)�GRAMMAR_LITERAL_ESCAPES�get�group)�ms r�<lambda>z1SchemaConverter._format_literal.<locals>.<lambda>�s��-�1�1�!�'�'�!�*�*�=�=�rr7)�GRAMMAR_LITERAL_ESCAPE_RE�sub)r�literal�escapeds   r�_format_literalzSchemaConverter._format_literal�s/��+�/�/�=�=�w�189�190���7�~�~�~�rTFr�r�rc������t���dks191Jd���dtf����fd��d�dg�d���d�R��S)zp192        not_literal('a') -> '[^a]'193        not_literal('abc') -> '([^a] | "a" ([^b] | "b" ([^c])?)?)?'194        rzEmpty literal not supported�ic3��K��|}�r|dkrd|�d�V�dV�d|�d�V�nd|�d�V�|t���dz195kr9dV���|��V�d	V��|dz��Ed{V��d196V�dSdS)N�_z[^z\\]� | z"\\"? "r7�]r9� (rA)�lenr�)r��cr��maybe_escaped_underscores�recursers  ����rr�z,SchemaConverter.not_literal.<locals>.recurse�s��������197�A�(�
 �Q�#�X�X�#�1�m�m�m�#�#�#�����&�!�&�&�&�&�&�&�&��1�i�i�i�����3�w�<�<�!�#�#�#������*�*�1�-�-�-�-�-��198199200�"�7�1�q�5�>�>�)�)�)�)�)�)�)��201202203204205�$�#rr<r@rJ)r��intrL)rr�r�r�r�s`` `@r�not_literalzSchemaConverter.not_literal�s��������7�|�|�a����!>����
	�s�
	�
	�
	�
	�
	�
	�
	�
	�
	��w�w��.�g�g�a�j�j�.�#�.�.�/�/�/rc�,�t�d|��}||jvs|j||kr|}nQd}|�|��|jvr<|j|�|��|kr'|dz
}|�|��|jvr|j|�|��|k�'|�|��}||j|<|S)N�-rr9)�INVALID_RULE_CHARS_REr�r{)r�name�rule�esc_name�keyr�s      r�	_add_rulezSchemaConverter._add_rule�s���(�,�,�S�$�7�7���4�;�&�&�$�+�h�*?�4�*G�*G��C�C��A�� �Q� � �D�K�/�/��K�8� 0�Q� 0� 0�1�T�9�9��Q���� �Q� � �D�K�/�/��K�8� 0�Q� 0� 0�1�T�9�9��"�q�"�"�C����C���206r�schema�urlc�@�����dtf����fd������S)z�207        Resolves all $ref fields in the given schema, fetching any remote schemas,208        replacing $ref with absolute reference URL and populating self._refs with the209        respective referenced (sub)schema dictionaries.210        rec����t|t��r�fd�|D��St|t���r�|�d��}|��n|�	jv�rd|�d��r��	js211Jd���ddl}|�d��}|d}�	j�|��}|�E�	�	|�|���212��|��}|�	j|<t|��dks|dd	kr|Sn5|�d213��r�}�214�|��}||d<ntd|�����|�d��d�d��dd�D]#}|�||vsJd
|�d|�d|�����||}�$|�	j|<n"|�
��D]
}�|���|S)Nc�&��g|]
}�|����Srr)rd�x�visits  �r�215<listcomp>z?SchemaConverter.resolve_refs.<locals>.visit.<locals>.<listcomp>�s!���,�,�,�Q���a���,�,�,r�$refzhttps://zDFetching remote schemas is not allowed (use --allow-fetch for force)r�#r9r=r<z#/zUnsupported ref �/zError resolving ref z: z not in )�216isinstancerW�dictr�r|�217startswithrw�requests�split�resolve_refs�jsonr�r(�values)re�refr��218frag_split�base_url�target�sel�vr�rr�r�s        ����rr�z+SchemaConverter.resolve_refs.<locals>.visit�s4����!�T�"�"�'
!�,�,�,�,�!�,�,�,�,��A�t�$�$�%
!��e�e�F�m�m���?�s�$�*�'<�'<��~�~�j�1�1�C� �-�b�b�a�b�b�-�'����%(�Y�Y�s�^�^�219�#-�a�=��!%�����!9�!9��!�>�%)�%6�%6� (���S� 1� 1� 6� 6� 8� 8�(�&�&�F�4:�D�J�x�0��z�?�?�a�/�/�:�b�>�R�3G�3G�#)�M�4H�����-�-�C�!'��!$�m�c�m�m��$'��&�	�	�(�)A�C�)A�)A�B�B�B�"�y�y��~�~�b�1�7�7��<�<�Q�R�R�@�-�-��"�.�3�&�=�=�=�N�#�N�N��N�N�f�N�N�4A�=�@�!'�����&,�D�J�s�O�O��X�X�Z�Z�!�!����a������Hr)r�)rr�r�r�s```@rr�zSchemaConverter.resolve_refs�sK������*	�T�*	�*	�*	�*	�*	�*	�*	�*	�*	�X�u�V�}�}�rc�d���d���fd�t|��D����S)Nr�c3�^�K�|]'\}}��|���rdnd�|����V��(dS)r�zalternative-N�r�)rdr��220alt_schemar�rs   ��rrfz7SchemaConverter._generate_union_rule.<locals>.<genexpr>�sc�����
�
�!�A�z��221222�:�$�'T�t�0O����'T�QR�'T�'T�U�U�
�
�
�
�
�
r)rL�	enumerate)rr��alt_schemass`` r�_generate_union_rulez$SchemaConverter._generate_union_rule�sM�����z�z�
�
�
�
�
�%.�{�%;�%;�
�
�
�223�224�	225rc	�������������d��r��d��s226Jd����dd��i�d�t����dttt227fdtfd	��dttt228ff��������fd229������jr������nd������zdz��S)
a�230        Transforms a regular expression pattern into a GBNF rule.231 232        Input: https://json-schema.org/understanding-json-schema/reference/regular_expressions233        Output: https://github.com/ggerganov/llama.cpp/blob/master/grammars/README.md234 235        Unsupported features: negative/positive lookaheads, greedy/non-greedy modifiers.236 237        Mostly a 1:1 translation, except for {x} / {x,} / {x,y} quantifiers for which238        we define sub-rules to keep the output lean.239        �^�$z,Pattern must start with "^" and end with "$"r9r=r�src�$�|\}}|rd|zdzn|S)Nr7r)r��txt�240is_literals   r�to_rulez/SchemaConverter._visit_pattern.<locals>.to_rule241s#�� !��S�*�&0�9�3��9�s�?�?�c�9rc������}g��fd�}��fd�}��k�rX��}|dkr&��|��df���dz
��n|dkr_�dz
���kr&��dksJd���d	��d242��d������d�������d�df���n�|dkr7�dz
�|d
kr�|dz243dksJd|�d��d������|��S|dkr�|}�dz
���krP��dkrD��dkr|���dz�z
}�dz
�n|��z
}�dz
���kr��dk�D��ksJd|�d��d������|dz
}�dz
���|df���n�|dkr��d���dz
��n�|dvr ��d��|zdf�d<�dz
��n�|dk�r�|}�dz
���kr.��dkr"|��z
}�dz
���kr��dk�"��ksJd|�d��d������|dz
}�dz
�d�|dd��d��D��}d
}d}	t|��dkrt|d
��}|}nSt|��dksJ�|d
rt|d
��nd
}|drt|d��nd}n$#t$rt	d |�d!��d����wxYw�d\}	}244|245sI��|	��}|�0����d"t���dz��|	��}|�|	<|}	t|246rd#|	�d#�n|	|||247�$��df�d<n�d%}��krʉ�dkrI��dz248kr@��dz}
|
tvr�dz
�|��z
}�dz
�n�|���dz�z
}�dz
�no��d#kr�j	s|d&z
}�dz
�nQ��tvrA��dz249ks'|d%ks!��dzdks��dztvr|��z
}�dz
�nn��k��|r��|d'f����k��X|��S)(z�250            Parse a unit at index i (advancing it), and return its string representation + whether it's a literal.251            c�\���jrt}nt}��d|��S)Nrq)rx�DOTALL�DOTr�)r�rs �r�get_dotzBSchemaConverter._visit_pattern.<locals>.transform.<locals>.get_dots/����<��!�D�D��D��~�~�f�d�3�3�3rc�J��g}t�d���D]Q\}}|r5|�d�d�|D����df���<|�|���Rt	|��dkr|dSd��fd��D����d	fS)252Nc��|dS)Nr9r)r�s rr�zUSchemaConverter._visit_pattern.<locals>.transform.<locals>.join_seq.<locals>.<lambda>(s253��A�a�D�rr<c3�&K�|]}|dV��
dS)rNr)rdr�s  rrfzVSchemaConverter._visit_pattern.<locals>.transform.<locals>.join_seq.<locals>.<genexpr>*s&����+<�+<�Q�A�a�D�+<�+<�+<�+<�+<�+<rTr9rr>c3�.�K�|]}�|��V��dSrr)rdr�r�s  �rrfzVSchemaConverter._visit_pattern.<locals>.transform.<locals>.join_seq.<locals>.<genexpr>/s+����� 9� 9������� 9� 9� 9� 9� 9� 9rF)r�appendrL�extendr�)�retr��g�seqr�s   ��r�join_seqzCSchemaConverter._visit_pattern.<locals>.transform.<locals>.join_seq%s������%,�S�.�.�%A�%A�&�&�M�J��!�&��254255�B�G�G�+<�+<�!�+<�+<�+<�$<�$<�d�#C�D�D�D�D��256257�1�
�
�
�
��s�8�8�q�=�=��q�6�M���� 9� 9� 9� 9�S� 9� 9� 9�9�9�5�A�Ar�.Fr9r@r:zUnsupported pattern syntax "z" at index z of /r�rJrz Unbalanced parentheses; start = z, i = z, pattern = �[r��\�z$Unbalanced square brackets; start = �|)r�F)rKr;r:r=�{�}z#Unbalanced curly brackets; start = c�6�g|]}|�����Sr)�strip)rdr�s  rr�zESchemaConverter._visit_pattern.<locals>.transform.<locals>.<listcomp>ds ��O�O�O�!�A�G�G�I�I�O�O�Or�,NzInvalid quantifier z in /r�r7)rOr<r4T)r�r�r�r�r(r�r�rR�&ESCAPED_IN_REGEXPS_BUT_NOT_IN_LITERALSry�NON_LITERAL_SET)�startr�r�r��square_brackets�curly_brackets�nums�	min_times�	max_timesr��sub_is_literal�idr��nextr�r��lengthr��patternr�sub_rule_idsr��	transforms              @��������rr�z1SchemaConverter._visit_pattern.<locals>.transformsx�����E�258+-�C�
4�
4�
4�
4�
4�259
B�260
B�261
B�262
B�263
B�264
B��f�*�*��A�J����8�8��J�J���	�	�5�1�2�2�2���F�A�A��#�X�X���F�A��6�z�z�#�A�J�#�-�-�-�c�'�!�*�c�c�QR�c�c�Y`�c�c�c�.�-�-��J�J� ;�G�G�I�I�K�K�$8�$8� ;� ;� ;�U�C�D�D�D�D��#�X�X���F�A���	�	�g�e�a�i�&8�C�&?�&?�&?�_�%�_�_�q�_�_�V]�_�_�'@�&?�?�#�8�:�:�%��#�X�X�&'�O���F�A��f�*�*����s�):�):�"�1�:��-�-�+�w�q�1�q�5�y�/A�A�O���F�A�A�+�w�q�z�9�O���F�A�
�f�*�*����s�):�):��F�265266267�c�e�c�c�1�c�c�Za�c�c�#�268269�#�s�*�O���F�A��J�J���7�8�8�8�8��#�X�X��J�J�|�,�,�,���F�A�A��/�)�)�&�w�s�2�w�/�/�!�3�U�;�C��G���F�A�A��#�X�X�%&�N���F�A��f�*�*����s�):�):�&�'�!�*�4���Q����f�*�*����s�):�):��F�270271272�b�U�b�b�!�b�b�Y`�b�b�#�273274�"�c�)�N���F�A�O�O�~�a��d�/C�/I�/I�#�/N�/N�O�O�O�D� !�I� $�I���t�9�9��>�>�(+�D��G���I�(1�I�I�#&�t�9�9��>�>�>�>�8<�Q��(F��D��G����Q�I�8<�Q��(I��D��G����T�I���%����(�Q�.�Q�Q�w�Q�Q�Q��������275-0��G�)�S�.�)�!�)�-�-�c�2�2���:�!%���4�0Q�0Q�#�l�:K�:K�a�:O�0Q�0Q�SV�!W�!W�B�02�L��-� ��*�*8�A�J��J�J�J�J�c�%�%�1?�	�����C��G�G�!�G��f�*�*�"�1�:��-�-�!�f�q�j�.�.�#*�1�q�5�>�D�#�'M�M�M� !�Q��� '�7�1�:� 5�� !�Q���� '�7�1�q�1�u�9�+=� =�� !�Q����$�Q�Z�3�.�.�t�7H�.�#�u�,�G���F�A�A�$�Q�Z��>�>���!��O�O�&�"�}�}�&�q�1�u�~��4�4�&�q�1�u�~�_�D�D�#�w�q�z�1�G���F�A�A�!�-�f�*�*�.�4��276277�G�T�?�3�3�3�_�f�*�*�b�8�:�:�s
�%A>K$�$!Lrbrj)r��endswithr�r	r1r3r�ry)rr�r�r�r�r�r�r�s```@@@@@r�_visit_patternzSchemaConverter._visit_pattern�s]�����������!�!�#�&�&�	:�7�+;�+;��,278�,279�	:�	:�9�	:�	:�280��!�B�$�-����
���W����	:�u�S�$�Y�'�	:�C�	:�	:�	:�	:�T	�5��d��+�T	�T	�T	�T	�T	�T	�T	�T	�T	�T	�T	�T	�T	�l�~�~���$�F���	�	���$�$�$����	�	��� 4� 4�4�~�E�281�282�	283rc��|�d��d}||jvr`||jvrW|j�|��|j|}|�||��}|j�|��|S)Nr�r=)r�r{r~�addr|r��remove)rr��ref_name�resolveds    r�_resolve_refzSchemaConverter._resolve_ref�s����9�9�S�>�>�"�%���4�;�&�&�3�d�6O�+O�+O��%�)�)�#�.�.�.��z�#��H��z�z�(�H�5�5�H��%�,�,�S�1�1�1��rc�P�|�tj|����Sr)r�r��dumps)rras  r�_generate_constant_rulez'SchemaConverter._generate_constant_rule�s ���#�#�D�J�u�$5�$5�6�6�6rc��284�����|�d��}|�d��}�tvr�dzn�pd}|�d��x}�)��|��|����Sd|vsd|vrE��|���|�d��p|d����St|t��r4��|���d�|D������Sd	|vr/��|��|d	����Sd285|vr=d��fd�|d286D����}��||��S|d
vr�d|vsd|vr�|ddur�t|�dg�����t
|�di���287�������|�����|�d������S|d
vrd|vr{t���g��}���fd�}	|dD],}288d|289vr|290dD]}|	|d�����|	|291d����-��|����|g�����S|dvr�d|vsd|vr�|�d��p|d}t|t��rI��|dd���fd�t|��D����zdz��S��
|���rdnd�d���}
|�d d!��}|�d"��}��|dt|
||d#�$��zdz��S|d%vr d&|vr��|d&|��S|d%vr@t!jd'|pd��r)��|dkrdn|t&d(��S|d%vrF|�d)�t(vr:|�d)�}��|��|t(|����S|d*krd+|vsd,|vrw��d-t&d-��}|�d+d!��}|�d,��}��|d.t|||��zd/z��S|d0kst+|��d!kr5��|��d0t&d0����S|t&vs
Jd1|�������|dkrdn|t&|��S)2N�type�formatr�rr��oneOf�anyOfc��g|]}d|i��S)rr)rd�ts  rr�z)SchemaConverter.visit.<locals>.<listcomp>�s��0R�0R�0R��&�!��0R�0R�0Rr�const�enumr�c3�B�K�|]}��|��V��dSr)r�)rdr�rs  �rrfz(SchemaConverter.visit.<locals>.<genexpr>�s1�����W�W�1�t�;�;�A�>�>�W�W�W�W�W�Wr)Nr[�292properties�additionalPropertiesT�required�allOfc����|�d��x}�
�j|}d|vrN|d���D]5\}}��||f��|r��|���4dSdS)Nr�r293)r�r|�itemsr�r�)�comp_schema�is_requiredr��	prop_name�prop_schemar294rrs     ���r�
add_componentz,SchemaConverter.visit.<locals>.add_component�s����&�?�?�6�2�2�2�C�?�"&�*�S�/�K��;�.�.�2=�l�2K�2Q�2Q�2S�2S�4�4�.�	�;�"�)�)�9�k�*B�C�C�C�&�4�$�L�L��3�3�3��	/�.�4�4rF)r)�additional_properties)Nr\r�prefixItemsz295"[" space � "," space c3�`�K�|](\}}��|���rdnd�d|����V��)dS)r�r<ztuple-Nr�)rdr��itemr�rs   ��rrfz(SchemaConverter.visit.<locals>.<genexpr>se�����)�)�#�A�t��296297�4�D�)P��2E�#�#�2�)P�)P�Q�)P�)P�Q�Q�)�)�)�)�)�)rz298 "]" spacer<r�minItemsr�maxItemsz	"," space)rI)Nr]r�z^uuid[1-5]?$rmz-stringr]�	minLength�	maxLengthrkrbrjr[zUnrecognized schema: )r��RESERVED_NAMESr�r�r�r�rWr�rLr}r�_build_object_ruler�r�rRr��re�match�_add_primitive�PRIMITIVE_RULES�STRING_FORMAT_RULESr�)rr�r��schema_type�
schema_format�	rule_namer�r��hybrid_namerr�ttr�item_rule_namerMrN�	prim_name�	char_rule�min_len�max_lenr299rs` `                 @@rr�zSchemaConverter.visit�s��������j�j��(�(���300301�8�,�,�
�"&�.�"8�"8�D�3�J�J�d�n�f�	��:�:�f�%�%�%�C�2��>�>�)�T�->�->�s�-C�-C�D�D�D�
��
�
�'�V�"3�"3��>�>���)�)�$��302303�7�0C�0C�0V�v�g��W�W���
�304��T�
*�
*�E	��>�>���)�)�$�0R�0R�k�0R�0R�0R�S�S���
�305��
�
��>�>��4�7�7��w��H�H���
��v�
�
��:�:�W�W�W�W��v��W�W�W�X�X�D��>�>�)�T�2�2�2�
�,�
,�
,��F�"�"�&�&�0�0��1�2�$�>�>��6�:�:�j�"�5�5�6�6�H��f�j�j��r�:�:�@�@�B�B�C�C�J��>�>���'�'���$��306307�;Q�0R�0R�����
��,�
,�
,��F�1B�1B��u�u�H��J��K�
4�
4�
4�
4�
4�
4�
4��G�_�
7�
7���a�<�<���j�=�=��%�
�b�e�<�<�<�<�<�=�"�M�!��6�6�6�6�6��>�>���'�'���+�R�(�����
��O�
+�
+��v����&�!8�!8��J�J�w�'�'�@�6�-�+@�E��%��&�&�
��~�~�� �#�(�(�)�)�)�)�)�'0��'7�'7�)�)�)����308#�#����"&���E�d�3U�4�<O�C�C�R�3U�3U�3U�!V�!V��"�J�J�z�1�5�5�	�"�J�J�z�2�2�	��~�~�� �'�&�	�9�[�����#�	#�����,�
,�
,��f�1D�1D��&�&�v�i�'8�)�D�D�D�
�,�
,�
,����]�0�b�2309�2310�
,��&�&�#�v�-�-���=���'���
�
�+�+�+� �)�)�)�-@�@�@�(�1�1�1�I��>�>���#�#�I�/B�9�/M�N�N���
�311�H�
$�
$��6�!�!�[�F�%:�%:��+�+�F�O�F�4K�L�L�I��j�j��a�0�0�G��j�j��-�-�G��>�>���#�I�w��@�@�A� �!���
��X�%�%�3�v�;�;�!�+;�+;��>�>��4�.�.�x���9R�S�S���
�312�/�1�1�1�3S�6�3S�3S�1�1�1��&�&�#�v�-�-���;���,���
rr�r�c��|�||j��}|jD]e}t�|��pt313�|��}|sJd|�d����||jvr|�||���f|S)NzRule z314 not known)r�rFrUr#r�r$r{r")rr�r�re�dep�dep_rules      rr"zSchemaConverter._add_primitiveNs����N�N�4���.�.���9�	3�	3�C�&�*�*�3�/�/�O�3F�3J�3J�3�3O�3O�H��4�4�4�S�4�4�4�4�4�8��$�+�%�%��#�#�C��2�2�2���rr315rrc	������
�����j�d�tt|���fd����D��}i�|D]s\}}��|���rdnd�|����}�����rdnd�|�d���t
j|�����d|�����|<�t�fd�|D��}	�fd	�|D���|d316kst|t��r����rdnd�d�}317��|d318krin||319�d���}��|320�d���321d
td
��d|��z���d<��d��d}|d�
�fd�|	D����z
}�r_|dz
}|	r|dz
}�
���fd��
|d�
�
�fd�tt�����D����z
}|	r|dz
}|dz
}|dz
}|S)Nc�$�g|]
\}}|d��S)rr)rdr��kvs   rr�z6SchemaConverter._build_object_rule.<locals>.<listcomp>as0��322�323�324���2�
�q�E�325�326�327rc�r����|ddt�����|dfS)Nr9r)r�r�)�ikvrs �rr�z4SchemaConverter._build_object_rule.<locals>.<lambda>es-�������A��q�	�3�z�?�?�!K�!K�S�QR�V� T�r�r�r�r<z-kvz space ":" space c���g|]}|�v�|��	Srr�rd�krs  �rr�z6SchemaConverter._build_object_rule.<locals>.<listcomp>rs���C�C�C��Q�(�]�]�!�]�]�]rc���g|]}|�v�|��	Srrr9s  �rr�z6SchemaConverter._build_object_rule.<locals>.<listcomp>ss#���G�G�G��Q�h�5F�5F�!�5F�5F�5FrT�328additionalz-valuer]z ":" space rKz329"{" space rc3�(�K�|]}�|V��
dSrr)rdr:�prop_kv_rule_namess  �rrfz5SchemaConverter._build_object_rule.<locals>.<genexpr>�s)�����"Q�"Q�Q�#5�a�#8�"Q�"Q�"Q�"Q�"Q�"Qrr�z
 "," space ( c	���|^}}�|}|dkr)�	����rdnd�d�|�d�|zdz��}n|rd|�d�}n|}t|��d	kr2|d330�	����rdnd�|�d��|d�
����zz
}|S)NrKr�r<zadditional-kvsz
 ( "," space z )*z( "," space � )?rr>z-restT��first_is_optional)r�r�)331�ksrBr:�rest�kv_rule_name�res�get_recursive_refsr�r>rs332      ����rrGz>SchemaConverter._build_object_rule.<locals>.get_recursive_refs�s�����333��T�1�!�4����8�8��.�.��D��!4���"�D�D�D�'�6�6�6��E��M���C�C�'�'�:��:�:�:�C�C�&�C��t�9�9�q�=�=��3�����>��!4���"�>�a�>�>�>�*�*�4�4�H�H�H�"�"���C��334rr�c3�B�K�|]}��|d�d���V��dS)NFrAr)rdr�rG�optional_propss  ��rrfz5SchemaConverter._build_object_rule.<locals>.<genexpr>�sP��������#�"�>�!�"�"�#5��O�O�O������rz )r@z335 "}" space)rv�sortedr�r�r�r�r�r�r�r�r"r#r�rL�ranger�)rr336rr�r�sorted_propsrr�prop_rule_name�required_props�sub_name�337value_ruler�rGrIr>rs` ``         @@@@rrz"SchemaConverter._build_object_ruleXs����������%�338�339�340���*�%�%�T�T�T�T����341�342�343�� ��&0�	�	�"�I�{�!�Z�Z���F�T�&9�c�c�r�F�9�F�F���N�-1�N�N��<��,���"�<�i�<�<�<��(�(���I�)>�)>�?�?�b�b�R`�b�b�-�-��y�)�)�D�C�C�C�\�C�C�C��G�G�G�G�\�G�G�G�� �D�(�(�J�7L�d�,S�,S�(��?�t� 3����?�?�?�H����+�t�3�3���9N��#�#�#���J�'+�n�n�� � � ��#�#�H�o�h�.G�H�H�,�344�,�,�-�'�'��s�#�345
�!�!�#�&�&�&����
�"�"�"Q�"Q�"Q�"Q�.�"Q�"Q�"Q�Q�Q�Q���	��D�L�D��
(���'��
�
�
�
�
�
�
�
�&
�E�J�J�������s�>�2�2�3�3������
�D��
������E�M�D������rc��d�d�t|j���d����D����S)Nr6c3�*K�|]\}}|�d|��V��dS)z ::= Nr)rdr�r�s   rrfz1SchemaConverter.format_grammar.<locals>.<genexpr>�sH����346�347���d�� � �$� � �348�349�350�351�352�353rc��|dSr�r)r4s rr�z0SchemaConverter.format_grammar.<locals>.<lambda>�s354���A��rr7)rLrJr{r)rs r�format_grammarzSchemaConverter.format_grammar�sS���y�y�355�356�$�T�[�%6�%6�%8�%8�>N�>N�O�O�O�357�358�359�360�361�	362rN)TF)r'r/r0rr�r1r3r�r�r�r�r�r�r�r�r�rTr"rr	rrr363rrTrrrrsrs|s}������	*�	*�	*����LQ�0�0��0�$(�0�	�0�0�0�0�4
�
�
�3�4�3�c�3�3�3�3�j364�365�366�w367�w368�w369�r���7�7�7�S�S�S�j�3��k�����O���s�C�x��)�O��c�(�O��	O�370 %�T�3�Y�/�O�O�O�O�b371�372�373�374�375rrsr�rc�376�|pg}tj|��}d�t|��D��}t|ddd���}|�|d��}|�|d��|���S)Nc��i|]\}}||��	Srr)rd�idxr�s   r�377<dictcomp>z'json_schema_to_gbnf.<locals>.<dictcomp>�s��C�C�C�	��T�$��C�C�CrF)rr�r�r��stdinr<)r��loadsr�rsr�r�rT)r�r�	converters   rr-r-�s����!�r�J�
�Z��
�
�F�C�C�Y�z�-B�-B�C�C�C�J���5��E����I��
#�
#�F�G�
4�
4�F�
�O�O�F�B�����#�#�%�%�%r)NFr),�__doc__�pathlibr�	itertoolsr�typingrrrrr	r378rr
�ARITHMETIC_GBNF�C_GBNF�379CHESS_GBNF�
JAPANESE_GBNF�
JSON_ARR_GBNF�	JSON_GBNF�	LIST_GBNFr�r rz�compiler�r�r�rRrT�_up_to_15_digitsrLr#r$r�r�r}�keysrr�r�rsr1r-rrr�<module>rjsm��L�L�����������������������������$��R�R�R�R�R�R�R�R�>:���+380��Z�381�8�
�8#�
�L
�	�8
�	�j�����	�	�	�	�!�!�!�!�!�!�!�!��382�#��383�#4�5�5��&�B�J�z�2�2��!&�e�%�@�@���384�PU�5�5�5�5�p��������%�$�W�a��4�4���{�5�r�:�:��K��+;� ;�R�@�@� �[�!1�4D�!D�b�I�I��k�T�	�.�)����{�7�/�9J�K�K�
�[�;�B�B�B����k�]�	�7�����[�;�g�Y���
�K��385�,�,�C�C�2B�C�C�C�386C�387C�	D�388�	�	�	
�
�
�K�\�389�
�
��k�2�V�H�=�=��K���+�+�? � ��H
�K�k�390�
�
�
�K�	a�391�
�
���_�v�v�.>�?�?��;�7�&��B�B��;�7�&��B�B�#��$A�K�=�Q�Q�
�
��392%�������U�I�_�)�)�+�+�I�.A�.F�.F�.H�.H�I����393�#�m�$�$��),��\�):�):�&�q394�q395�q396�q397�q398�q399�q400�q401�h	&�	&��	&��$�s�)�1D�	&�	&�	&�	&�	&�	&r