CoolFace
Modelpublic

MutionHydra/HyperAI-Developer-Edition

sourceHugging Faceapache-2.0updated 16d agoView on Hugging Face
0likes588downloads
tokenizer_config.json217 linesDownload Raw Back to root
1{2  "add_prefix_space": false,3  "backend": "tokenizers",4  "bos_token": null,5  "clean_up_tokenization_spaces": false,6  "eos_token": "<|im_end|>",7  "errors": "replace",8  "extra_special_tokens": [9    "<|im_start|>",10    "<|im_end|>",11    "<|object_ref_start|>",12    "<|object_ref_end|>",13    "<|box_start|>",14    "<|box_end|>",15    "<|quad_start|>",16    "<|quad_end|>",17    "<|vision_start|>",18    "<|vision_end|>",19    "<|vision_pad|>",20    "<|image_pad|>",21    "<|video_pad|>"22  ],23  "is_local": false,24  "model_max_length": 32768,25  "pad_token": "<|PAD_TOKEN|>",26  "padding_side": "left",27  "split_special_tokens": false,28  "tokenizer_class": "Qwen2Tokenizer",29  "unk_token": null,30  "added_tokens_decoder": {31    "151643": {32      "content": "<|endoftext|>",33      "single_word": false,34      "lstrip": false,35      "rstrip": false,36      "normalized": false,37      "special": true38    },39    "151644": {40      "content": "<|im_start|>",41      "single_word": false,42      "lstrip": false,43      "rstrip": false,44      "normalized": false,45      "special": true46    },47    "151645": {48      "content": "<|im_end|>",49      "single_word": false,50      "lstrip": false,51      "rstrip": false,52      "normalized": false,53      "special": true54    },55    "151646": {56      "content": "<|object_ref_start|>",57      "single_word": false,58      "lstrip": false,59      "rstrip": false,60      "normalized": false,61      "special": true62    },63    "151647": {64      "content": "<|object_ref_end|>",65      "single_word": false,66      "lstrip": false,67      "rstrip": false,68      "normalized": false,69      "special": true70    },71    "151648": {72      "content": "<|box_start|>",73      "single_word": false,74      "lstrip": false,75      "rstrip": false,76      "normalized": false,77      "special": true78    },79    "151649": {80      "content": "<|box_end|>",81      "single_word": false,82      "lstrip": false,83      "rstrip": false,84      "normalized": false,85      "special": true86    },87    "151650": {88      "content": "<|quad_start|>",89      "single_word": false,90      "lstrip": false,91      "rstrip": false,92      "normalized": false,93      "special": true94    },95    "151651": {96      "content": "<|quad_end|>",97      "single_word": false,98      "lstrip": false,99      "rstrip": false,100      "normalized": false,101      "special": true102    },103    "151652": {104      "content": "<|vision_start|>",105      "single_word": false,106      "lstrip": false,107      "rstrip": false,108      "normalized": false,109      "special": true110    },111    "151653": {112      "content": "<|vision_end|>",113      "single_word": false,114      "lstrip": false,115      "rstrip": false,116      "normalized": false,117      "special": true118    },119    "151654": {120      "content": "<|vision_pad|>",121      "single_word": false,122      "lstrip": false,123      "rstrip": false,124      "normalized": false,125      "special": true126    },127    "151655": {128      "content": "<|image_pad|>",129      "single_word": false,130      "lstrip": false,131      "rstrip": false,132      "normalized": false,133      "special": true134    },135    "151656": {136      "content": "<|video_pad|>",137      "single_word": false,138      "lstrip": false,139      "rstrip": false,140      "normalized": false,141      "special": true142    },143    "151657": {144      "content": "<tool_call>",145      "single_word": false,146      "lstrip": false,147      "rstrip": false,148      "normalized": false,149      "special": false150    },151    "151658": {152      "content": "</tool_call>",153      "single_word": false,154      "lstrip": false,155      "rstrip": false,156      "normalized": false,157      "special": false158    },159    "151659": {160      "content": "<|fim_prefix|>",161      "single_word": false,162      "lstrip": false,163      "rstrip": false,164      "normalized": false,165      "special": false166    },167    "151660": {168      "content": "<|fim_middle|>",169      "single_word": false,170      "lstrip": false,171      "rstrip": false,172      "normalized": false,173      "special": false174    },175    "151661": {176      "content": "<|fim_suffix|>",177      "single_word": false,178      "lstrip": false,179      "rstrip": false,180      "normalized": false,181      "special": false182    },183    "151662": {184      "content": "<|fim_pad|>",185      "single_word": false,186      "lstrip": false,187      "rstrip": false,188      "normalized": false,189      "special": false190    },191    "151663": {192      "content": "<|repo_name|>",193      "single_word": false,194      "lstrip": false,195      "rstrip": false,196      "normalized": false,197      "special": false198    },199    "151664": {200      "content": "<|file_sep|>",201      "single_word": false,202      "lstrip": false,203      "rstrip": false,204      "normalized": false,205      "special": false206    },207    "151665": {208      "content": "<|PAD_TOKEN|>",209      "single_word": false,210      "lstrip": false,211      "rstrip": false,212      "normalized": false,213      "special": true214    }215  },216  "chat_template": "{%- if tools %}\n    {{- '<|im_start|>system\\n' }}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- messages[0]['content'] }}\n    {%- else %}\n        {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}\n    {%- endif %}\n    {{- \"\\n\\n# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>\" }}\n    {%- for tool in tools %}\n        {{- \"\\n\" }}\n        {{- tool | tojson }}\n    {%- endfor %}\n    {{- \"\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}\\n</tool_call><|im_end|>\\n\" }}\n{%- else %}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- '<|im_start|>system\\n' + messages[0]['content'] + '<|im_end|>\\n' }}\n    {%- else %}\n        {{- '<|im_start|>system\\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\\n' }}\n    {%- endif %}\n{%- endif %}\n{%- for message in messages %}\n    {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) or (message.role == \"assistant\" and not message.tool_calls) %}\n        {{- '<|im_start|>' + message.role + '\\n' + message.content + '<|im_end|>' + '\\n' }}\n    {%- elif message.role == \"assistant\" %}\n        {{- '<|im_start|>' + message.role }}\n        {%- if message.content %}\n            {{- '\\n' + message.content }}\n        {%- endif %}\n        {%- for tool_call in message.tool_calls %}\n            {%- if tool_call.function is defined %}\n                {%- set tool_call = tool_call.function %}\n            {%- endif %}\n            {{- '\\n<tool_call>\\n{\"name\": \"' }}\n            {{- tool_call.name }}\n            {{- '\", \"arguments\": ' }}\n            {{- tool_call.arguments | tojson }}\n            {{- '}\\n</tool_call>' }}\n        {%- endfor %}\n        {{- '<|im_end|>\\n' }}\n    {%- elif message.role == \"tool\" %}\n        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != \"tool\") %}\n            {{- '<|im_start|>user' }}\n        {%- endif %}\n        {{- '\\n<tool_response>\\n' }}\n        {{- message.content }}\n        {{- '\\n</tool_response>' }}\n        {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n            {{- '<|im_end|>\\n' }}\n        {%- endif %}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '<|im_start|>assistant\\n' }}\n{%- endif %}\n"217}