CoolFace
Modelpublic

Kompaktion/action

sourceHugging Faceapache-2.0updated 2mo agoView on Hugging Face
0likes7downloads
tokenizer_config.json301 linesDownload Raw Back to root
1{2  "add_prefix_space": false,3  "audio_bos_token": "<|audio_start|>",4  "audio_eos_token": "<|audio_end|>",5  "audio_token": "<|audio_pad|>",6  "backend": "tokenizers",7  "bos_token": null,8  "clean_up_tokenization_spaces": false,9  "eos_token": "<|im_end|>",10  "errors": "replace",11  "image_token": "<|image_pad|>",12  "is_local": true,13  "local_files_only": false,14  "model_max_length": 262144,15  "model_specific_special_tokens": {16    "audio_bos_token": "<|audio_start|>",17    "audio_eos_token": "<|audio_end|>",18    "audio_token": "<|audio_pad|>",19    "image_token": "<|image_pad|>",20    "video_token": "<|video_pad|>",21    "vision_bos_token": "<|vision_start|>",22    "vision_eos_token": "<|vision_end|>"23  },24  "pad_token": "<|endoftext|>",25  "padding_side": "left",26  "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+",27  "processor_class": "Qwen3VLProcessor",28  "split_special_tokens": false,29  "tokenizer_class": "Qwen2Tokenizer",30  "unk_token": null,31  "video_token": "<|video_pad|>",32  "vision_bos_token": "<|vision_start|>",33  "vision_eos_token": "<|vision_end|>",34  "added_tokens_decoder": {35    "248044": {36      "content": "<|endoftext|>",37      "single_word": false,38      "lstrip": false,39      "rstrip": false,40      "normalized": false,41      "special": true42    },43    "248045": {44      "content": "<|im_start|>",45      "single_word": false,46      "lstrip": false,47      "rstrip": false,48      "normalized": false,49      "special": true50    },51    "248046": {52      "content": "<|im_end|>",53      "single_word": false,54      "lstrip": false,55      "rstrip": false,56      "normalized": false,57      "special": true58    },59    "248047": {60      "content": "<|object_ref_start|>",61      "single_word": false,62      "lstrip": false,63      "rstrip": false,64      "normalized": false,65      "special": true66    },67    "248048": {68      "content": "<|object_ref_end|>",69      "single_word": false,70      "lstrip": false,71      "rstrip": false,72      "normalized": false,73      "special": true74    },75    "248049": {76      "content": "<|box_start|>",77      "single_word": false,78      "lstrip": false,79      "rstrip": false,80      "normalized": false,81      "special": true82    },83    "248050": {84      "content": "<|box_end|>",85      "single_word": false,86      "lstrip": false,87      "rstrip": false,88      "normalized": false,89      "special": true90    },91    "248051": {92      "content": "<|quad_start|>",93      "single_word": false,94      "lstrip": false,95      "rstrip": false,96      "normalized": false,97      "special": true98    },99    "248052": {100      "content": "<|quad_end|>",101      "single_word": false,102      "lstrip": false,103      "rstrip": false,104      "normalized": false,105      "special": true106    },107    "248053": {108      "content": "<|vision_start|>",109      "single_word": false,110      "lstrip": false,111      "rstrip": false,112      "normalized": false,113      "special": true114    },115    "248054": {116      "content": "<|vision_end|>",117      "single_word": false,118      "lstrip": false,119      "rstrip": false,120      "normalized": false,121      "special": true122    },123    "248055": {124      "content": "<|vision_pad|>",125      "single_word": false,126      "lstrip": false,127      "rstrip": false,128      "normalized": false,129      "special": true130    },131    "248056": {132      "content": "<|image_pad|>",133      "single_word": false,134      "lstrip": false,135      "rstrip": false,136      "normalized": false,137      "special": true138    },139    "248057": {140      "content": "<|video_pad|>",141      "single_word": false,142      "lstrip": false,143      "rstrip": false,144      "normalized": false,145      "special": true146    },147    "248058": {148      "content": "<tool_call>",149      "single_word": false,150      "lstrip": false,151      "rstrip": false,152      "normalized": false,153      "special": false154    },155    "248059": {156      "content": "</tool_call>",157      "single_word": false,158      "lstrip": false,159      "rstrip": false,160      "normalized": false,161      "special": false162    },163    "248060": {164      "content": "<|fim_prefix|>",165      "single_word": false,166      "lstrip": false,167      "rstrip": false,168      "normalized": false,169      "special": false170    },171    "248061": {172      "content": "<|fim_middle|>",173      "single_word": false,174      "lstrip": false,175      "rstrip": false,176      "normalized": false,177      "special": false178    },179    "248062": {180      "content": "<|fim_suffix|>",181      "single_word": false,182      "lstrip": false,183      "rstrip": false,184      "normalized": false,185      "special": false186    },187    "248063": {188      "content": "<|fim_pad|>",189      "single_word": false,190      "lstrip": false,191      "rstrip": false,192      "normalized": false,193      "special": false194    },195    "248064": {196      "content": "<|repo_name|>",197      "single_word": false,198      "lstrip": false,199      "rstrip": false,200      "normalized": false,201      "special": false202    },203    "248065": {204      "content": "<|file_sep|>",205      "single_word": false,206      "lstrip": false,207      "rstrip": false,208      "normalized": false,209      "special": false210    },211    "248066": {212      "content": "<tool_response>",213      "single_word": false,214      "lstrip": false,215      "rstrip": false,216      "normalized": false,217      "special": false218    },219    "248067": {220      "content": "</tool_response>",221      "single_word": false,222      "lstrip": false,223      "rstrip": false,224      "normalized": false,225      "special": false226    },227    "248068": {228      "content": "<think>",229      "single_word": false,230      "lstrip": false,231      "rstrip": false,232      "normalized": false,233      "special": false234    },235    "248069": {236      "content": "</think>",237      "single_word": false,238      "lstrip": false,239      "rstrip": false,240      "normalized": false,241      "special": false242    },243    "248070": {244      "content": "<|audio_start|>",245      "single_word": false,246      "lstrip": false,247      "rstrip": false,248      "normalized": false,249      "special": true250    },251    "248071": {252      "content": "<|audio_end|>",253      "single_word": false,254      "lstrip": false,255      "rstrip": false,256      "normalized": false,257      "special": true258    },259    "248072": {260      "content": "<tts_pad>",261      "single_word": false,262      "lstrip": false,263      "rstrip": false,264      "normalized": false,265      "special": true266    },267    "248073": {268      "content": "<tts_text_bos>",269      "single_word": false,270      "lstrip": false,271      "rstrip": false,272      "normalized": false,273      "special": true274    },275    "248074": {276      "content": "<tts_text_eod>",277      "single_word": false,278      "lstrip": false,279      "rstrip": false,280      "normalized": false,281      "special": true282    },283    "248075": {284      "content": "<tts_text_bos_single>",285      "single_word": false,286      "lstrip": false,287      "rstrip": false,288      "normalized": false,289      "special": true290    },291    "248076": {292      "content": "<|audio_pad|>",293      "single_word": false,294      "lstrip": false,295      "rstrip": false,296      "normalized": false,297      "special": true298    }299  },300  "chat_template": "{%- set image_count = namespace(value=0) %}\n{%- set video_count = namespace(value=0) %}\n{%- macro render_content(content, do_vision_count, is_system_content=false) %}\n    {%- if content is string %}\n        {{- content }}\n    {%- elif content is iterable and content is not mapping %}\n        {%- for item in content %}\n            {%- if 'image' in item or 'image_url' in item or item.type == 'image' %}\n                {%- if is_system_content %}\n                    {{- raise_exception('System message cannot contain images.') }}\n                {%- endif %}\n                {%- if do_vision_count %}\n                    {%- set image_count.value = image_count.value + 1 %}\n                {%- endif %}\n                {%- if add_vision_id %}\n                    {{- 'Picture ' ~ image_count.value ~ ': ' }}\n                {%- endif %}\n                {{- '<|vision_start|><|image_pad|><|vision_end|>' }}\n            {%- elif 'video' in item or item.type == 'video' %}\n                {%- if is_system_content %}\n                    {{- raise_exception('System message cannot contain videos.') }}\n                {%- endif %}\n                {%- if do_vision_count %}\n                    {%- set video_count.value = video_count.value + 1 %}\n                {%- endif %}\n                {%- if add_vision_id %}\n                    {{- 'Video ' ~ video_count.value ~ ': ' }}\n                {%- endif %}\n                {{- '<|vision_start|><|video_pad|><|vision_end|>' }}\n            {%- elif 'text' in item %}\n                {{- item.text }}\n            {%- else %}\n                {{- raise_exception('Unexpected item type in content.') }}\n            {%- endif %}\n        {%- endfor %}\n    {%- elif content is none or content is undefined %}\n        {{- '' }}\n    {%- else %}\n        {{- raise_exception('Unexpected content type.') }}\n    {%- endif %}\n{%- endmacro %}\n{%- if not messages %}\n    {{- raise_exception('No messages provided.') }}\n{%- endif %}\n{%- if tools and tools is iterable and tools is not mapping %}\n    {{- '<|im_start|>system\\n' }}\n    {{- \"# Tools\\n\\nYou have access to the following functions:\\n\\n<tools>\" }}\n    {%- for tool in tools %}\n        {{- \"\\n\" }}\n        {{- tool | tojson }}\n    {%- endfor %}\n    {{- \"\\n</tools>\" }}\n    {{- '\\n\\nIf you choose to call a function ONLY reply in the following format with NO suffix:\\n\\n<tool_call>\\n<function=example_function_name>\\n<parameter=example_parameter_1>\\nvalue_1\\n</parameter>\\n<parameter=example_parameter_2>\\nThis is the value for the second parameter\\nthat can span\\nmultiple lines\\n</parameter>\\n</function>\\n</tool_call>\\n\\n<IMPORTANT>\\nReminder:\\n- Function calls MUST follow the specified format: an inner <function=...></function> block must be nested within <tool_call></tool_call> XML tags\\n- Required parameters MUST be specified\\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\\n</IMPORTANT>' }}\n    {%- if messages[0].role == 'system' %}\n        {%- set content = render_content(messages[0].content, false, true)|trim %}\n        {%- if content %}\n            {{- '\\n\\n' + content }}\n        {%- endif %}\n    {%- endif %}\n    {{- '<|im_end|>\\n' }}\n{%- else %}\n    {%- if messages[0].role == 'system' %}\n        {%- set content = render_content(messages[0].content, false, true)|trim %}\n        {{- '<|im_start|>system\\n' + content + '<|im_end|>\\n' }}\n    {%- endif %}\n{%- endif %}\n{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}\n{%- for message in messages[::-1] %}\n    {%- set index = (messages|length - 1) - loop.index0 %}\n    {%- if ns.multi_step_tool and message.role == \"user\" %}\n        {%- set content = render_content(message.content, false)|trim %}\n        {%- if not(content.startswith('<tool_response>') and content.endswith('</tool_response>')) %}\n            {%- set ns.multi_step_tool = false %}\n            {%- set ns.last_query_index = index %}\n        {%- endif %}\n    {%- endif %}\n{%- endfor %}\n{%- if ns.multi_step_tool %}\n    {{- raise_exception('No user query found in messages.') }}\n{%- endif %}\n{%- for message in messages %}\n    {%- set content = render_content(message.content, true)|trim %}\n    {%- if message.role == \"system\" %}\n        {%- if not loop.first %}\n            {{- raise_exception('System message must be at the beginning.') }}\n        {%- endif %}\n    {%- elif message.role == \"user\" %}\n        {{- '<|im_start|>' + message.role + '\\n' + content + '<|im_end|>' + '\\n' }}\n    {%- elif message.role == \"assistant\" %}\n        {%- set reasoning_content = '' %}\n        {%- if message.reasoning_content is string %}\n            {%- set reasoning_content = message.reasoning_content %}\n        {%- else %}\n            {%- if '</think>' in content %}\n                {%- set reasoning_content = content.split('</think>')[0].rstrip('\\n').split('<think>')[-1].lstrip('\\n') %}\n                {%- set content = content.split('</think>')[-1].lstrip('\\n') %}\n            {%- endif %}\n        {%- endif %}\n        {%- set reasoning_content = reasoning_content|trim %}\n        {{- '<|im_start|>' + message.role + '\\n<think>\\n' + reasoning_content + '\\n</think>\\n\\n' + content }}\n        {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %}\n            {%- for tool_call in message.tool_calls %}\n                {%- if tool_call.function is defined %}\n                    {%- set tool_call = tool_call.function %}\n                {%- endif %}\n                {%- if loop.first %}\n                    {%- if content|trim %}\n                        {{- '\\n\\n<tool_call>\\n<function=' + tool_call.name + '>\\n' }}\n                    {%- else %}\n                        {{- '<tool_call>\\n<function=' + tool_call.name + '>\\n' }}\n                    {%- endif %}\n                {%- else %}\n                    {{- '\\n<tool_call>\\n<function=' + tool_call.name + '>\\n' }}\n                {%- endif %}\n                {%- if tool_call.arguments is defined %}\n                    {%- for args_name, args_value in tool_call.arguments|items %}\n                        {{- '<parameter=' + args_name + '>\\n' }}\n                        {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %}\n                        {{- args_value }}\n                        {{- '\\n</parameter>\\n' }}\n                    {%- endfor %}\n                {%- endif %}\n                {{- '</function>\\n</tool_call>' }}\n            {%- endfor %}\n        {%- endif %}\n        {{- '<|im_end|>\\n' }}\n    {%- elif message.role == \"tool\" %}\n        {%- if loop.previtem and loop.previtem.role != \"tool\" %}\n            {{- '<|im_start|>user' }}\n        {%- endif %}\n        {{- '\\n<tool_response>\\n' }}\n        {{- content }}\n        {{- '\\n</tool_response>' }}\n        {%- if not loop.last and loop.nextitem.role != \"tool\" %}\n            {{- '<|im_end|>\\n' }}\n        {%- elif loop.last %}\n            {{- '<|im_end|>\\n' }}\n        {%- endif %}\n    {%- else %}\n        {{- raise_exception('Unexpected message role.') }}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '<|im_start|>assistant\\n' }}\n    {%- if enable_thinking is defined and enable_thinking is false %}\n        {{- '<think>\\n\\n</think>\\n\\n' }}\n    {%- else %}\n        {{- '<think>\\n' }}\n    {%- endif %}\n{%- endif %}\n"301}