CoolFace
Modelpublic

nikhil061307/gemma4-e4b-icd-coding

sourceHugging Faceapache-2.0updated 5mo agoView on Hugging Face
0likes24downloads
tokenizer_config.json96 linesDownload Raw Back to root
1{2  "audio_token": "<|audio|>",3  "backend": "tokenizers",4  "boa_token": "<|audio>",5  "boi_token": "<|image>",6  "bos_token": "<bos>",7  "eoa_token": "<audio|>",8  "eoc_token": "<channel|>",9  "eoi_token": "<image|>",10  "eos_token": "<turn|>",11  "eot_token": "<turn|>",12  "escape_token": "<|\"|>",13  "etc_token": "<tool_call|>",14  "etd_token": "<tool|>",15  "etr_token": "<tool_response|>",16  "extra_special_tokens": [17    "<|video|>"18  ],19  "image_token": "<|image|>",20  "is_local": false,21  "mask_token": "<mask>",22  "model_max_length": 131072,23  "model_specific_special_tokens": {24    "audio_token": "<|audio|>",25    "boa_token": "<|audio>",26    "boi_token": "<|image>",27    "eoa_token": "<audio|>",28    "eoc_token": "<channel|>",29    "eoi_token": "<image|>",30    "eot_token": "<turn|>",31    "escape_token": "<|\"|>",32    "etc_token": "<tool_call|>",33    "etd_token": "<tool|>",34    "etr_token": "<tool_response|>",35    "image_token": "<|image|>",36    "soc_token": "<|channel>",37    "sot_token": "<|turn>",38    "stc_token": "<|tool_call>",39    "std_token": "<|tool>",40    "str_token": "<|tool_response>",41    "think_token": "<|think|>"42  },43  "pad_token": "<pad>",44  "padding_side": "right",45  "processor_class": "Gemma4Processor",46  "response_schema": {47    "properties": {48      "content": {49        "type": "string"50      },51      "role": {52        "const": "assistant"53      },54      "thinking": {55        "type": "string"56      },57      "tool_calls": {58        "items": {59          "properties": {60            "function": {61              "properties": {62                "arguments": {63                  "additionalProperties": {},64                  "type": "object",65                  "x-parser": "gemma4-tool-call"66                },67                "name": {68                  "type": "string"69                }70              },71              "type": "object",72              "x-regex": "call\\:(?P<name>\\w+)(?P<arguments>\\{.*\\})"73            },74            "type": {75              "const": "function"76            }77          },78          "type": "object"79        },80        "type": "array",81        "x-regex-iterator": "<\\|tool_call>(.*?)<tool_call\\|>"82      }83    },84    "type": "object",85    "x-regex": "(\\<\\|channel\\>thought\\n(?P<thinking>.*?)\\<channel\\|\\>)?(?P<content>(?:(?!\\<\\|tool_call\\>)(?!\\<turn\\|\\>).)+)?(?P<tool_calls>\\<\\|tool_call\\>.*\\<tool_call\\|\\>)?(?:\\<turn\\|\\>)?"86  },87  "soc_token": "<|channel>",88  "sot_token": "<|turn>",89  "stc_token": "<|tool_call>",90  "std_token": "<|tool>",91  "str_token": "<|tool_response>",92  "think_token": "<|think|>",93  "tokenizer_class": "GemmaTokenizer",94  "unk_token": "<unk>",95  "chat_template": "{{ bos_token }}{%- if messages[0]['role'] == 'system' -%}\n    {%- set first_user_prefix = messages[0]['content'] + '\n\n' -%}\n    {%- set loop_messages = messages[1:] -%}\n{%- else -%}\n    {%- set first_user_prefix = \"\" -%}\n    {%- set loop_messages = messages -%}\n{%- endif -%}\n{%- for message in loop_messages -%}\n    {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%}\n        {{ raise_exception(\"Conversation roles must alternate user/assistant/user/assistant/...\") }}\n    {%- endif -%}\n    {%- if (message['role'] == 'assistant') -%}\n        {%- set role = \"model\" -%}\n    {%- else -%}\n        {%- set role = message['role'] -%}\n    {%- endif -%}\n    {{ '<|turn>' + role + '\n' + (first_user_prefix if loop.first else \"\") }}\n    {%- if role == \"model\" -%}\n        {{ '<|channel>thought\n<channel|>' }}\n    {%- endif -%}\n    {%- if message['content'] is string -%}\n        {{ message['content'] | trim }}\n    {%- elif message['content'] is iterable -%}\n        {%- for item in message['content'] -%}\n            {%- if item['type'] == 'audio' -%}\n                {{ '<|audio|>' }}\n            {%- elif item['type'] == 'image' -%}\n                {{ '<|image|>' }}\n            {%- elif item['type'] == 'video' -%}\n                {{ '<|video|>' }}\n            {%- elif item['type'] == 'text' -%}\n                {{ item['text'] | trim }}\n            {%- endif -%}\n        {%- endfor -%}\n    {%- else -%}\n        {{ raise_exception(\"Invalid content type\") }}\n    {%- endif -%}\n    {{ '<turn|>\n' }}\n{%- endfor -%}\n{%- if add_generation_prompt -%}\n    {{'<|turn>model\n'}}\n{%- endif -%}\n"96}