CoolFace
Modelpublic

Cyclone-Labs/Starlight-Synthesis-31B

sourceHugging Faceapache-2.0updated 16d agoView on Hugging Face
3likes136downloads
tokenizer_config.json142 linesDownload Raw Back to root
1{2  "audio_token": "<|audio|>",3  "backend": "tokenizers",4  "boa_token": "<|audio>",5  "boi_token": "<|image>",6  "bos_token": "<bos>",7  "eoa_token": "<audio|>",8  "eoc_token": "<channel|>",9  "eoi_token": "<image|>",10  "eos_token": "<eos>",11  "eot_token": "<turn|>",12  "escape_token": "<|\"|>",13  "etc_token": "<tool_call|>",14  "etd_token": "<tool|>",15  "etr_token": "<tool_response|>",16  "extra_special_tokens": [17    "<|video|>"18  ],19  "image_token": "<|image|>",20  "is_local": true,21  "mask_token": "<mask>",22  "model_max_length": 1000000000000000019884624838656,23  "model_specific_special_tokens": {24    "audio_token": "<|audio|>",25    "boa_token": "<|audio>",26    "boi_token": "<|image>",27    "eoa_token": "<audio|>",28    "eoc_token": "<channel|>",29    "eoi_token": "<image|>",30    "eot_token": "<turn|>",31    "escape_token": "<|\"|>",32    "etc_token": "<tool_call|>",33    "etd_token": "<tool|>",34    "etr_token": "<tool_response|>",35    "image_token": "<|image|>",36    "soc_token": "<|channel>",37    "sot_token": "<|turn>",38    "stc_token": "<|tool_call>",39    "std_token": "<|tool>",40    "str_token": "<|tool_response>",41    "think_token": "<|think|>"42  },43  "pad_token": "<pad>",44  "padding_side": "left",45  "processor_class": "Gemma4Processor",46  "response_schema": {47    "properties": {48      "content": {49        "type": "string"50      },51      "role": {52        "const": "assistant"53      },54      "thinking": {55        "type": "string"56      },57      "tool_calls": {58        "items": {59          "properties": {60            "function": {61              "properties": {62                "arguments": {63                  "additionalProperties": {},64                  "type": "object",65                  "x-parser": "gemma4-tool-call"66                },67                "name": {68                  "type": "string"69                }70              },71              "type": "object",72              "x-regex": "call\\:(?P<name>\\w+)(?P<arguments>\\{.*\\})"73            },74            "type": {75              "const": "function"76            }77          },78          "type": "object"79        },80        "type": "array",81        "x-regex-iterator": "<\\|tool_call>(.*?)<tool_call\\|>"82      }83    },84    "type": "object",85    "x-regex": "(\\<\\|channel\\>thought\\n(?P<thinking>.*?)\\<channel\\|\\>)?(?P<tool_calls>\\<\\|tool_call\\>.*\\<tool_call\\|\\>)?(?P<content>(?:(?!\\<turn\\|\\>)(?!\\<\\|tool_response\\>).)+)?(?:\\<turn\\|\\>|\\<\\|tool_response\\>)?"86  },87  "response_template": {88    "defaults": {89      "role": "assistant"90    },91    "fields": {92      "content": {93        "close": [94          "<turn|>",95          "<|tool_response>",96          "<eos>"97        ],98        "content": "text"99      },100      "thinking": {101        "close": "<channel|>",102        "content": "text",103        "open": "<|channel>thought\n"104      },105      "tool_calls": {106        "close": "<tool_call|>",107        "content": "json",108        "content_args": {109          "string_delims": [110            [111              "<|\"|>",112              "<|\"|>"113            ]114          ],115          "unquoted_keys": true116        },117        "open_pattern": "<\\|tool_call>call:(?P<name>\\w+)",118        "repeats": true,119        "transform": {120          "function": {121            "arguments": "{content}",122            "name": "{name}"123          },124          "type": "function"125        }126      }127    },128    "start_anchor": [129      "<|turn>model\n",130      "<tool_response|>"131    ]132  },133  "soc_token": "<|channel>",134  "sot_token": "<|turn>",135  "stc_token": "<|tool_call>",136  "std_token": "<|tool>",137  "str_token": "<|tool_response>",138  "think_token": "<|think|>",139  "tokenizer_class": "GemmaTokenizer",140  "unk_token": "<unk>"141}142