CoolFace
Modelpublic

EventHorizon-AI/Light-R1-14B-DS-abliterated

sourceHugging Faceapache-2.0updated 1y agoView on Hugging Face
1likes21downloads
tokenizer_config.json197 linesDownload Raw Back to root
1{
2  "add_bos_token": true,
3  "add_eos_token": false,
4  "add_prefix_space": null,
5  "added_tokens_decoder": {
6    "151643": {
7      "content": "<|end▁of▁sentence|>",
8      "lstrip": false,
9      "normalized": false,
10      "rstrip": false,
11      "single_word": false,
12      "special": true
13    },
14    "151644": {
15      "content": "<|User|>",
16      "lstrip": false,
17      "normalized": false,
18      "rstrip": false,
19      "single_word": false,
20      "special": false
21    },
22    "151645": {
23      "content": "<|Assistant|>",
24      "lstrip": false,
25      "normalized": false,
26      "rstrip": false,
27      "single_word": false,
28      "special": false
29    },
30    "151646": {
31      "content": "<|begin▁of▁sentence|>",
32      "lstrip": false,
33      "normalized": false,
34      "rstrip": false,
35      "single_word": false,
36      "special": true
37    },
38    "151647": {
39      "content": "<|EOT|>",
40      "lstrip": false,
41      "normalized": false,
42      "rstrip": false,
43      "single_word": false,
44      "special": false
45    },
46    "151648": {
47      "content": "<think>",
48      "lstrip": false,
49      "normalized": false,
50      "rstrip": false,
51      "single_word": false,
52      "special": false
53    },
54    "151649": {
55      "content": "</think>",
56      "lstrip": false,
57      "normalized": false,
58      "rstrip": false,
59      "single_word": false,
60      "special": false
61    },
62    "151650": {
63      "content": "<|quad_start|>",
64      "lstrip": false,
65      "normalized": false,
66      "rstrip": false,
67      "single_word": false,
68      "special": true
69    },
70    "151651": {
71      "content": "<|quad_end|>",
72      "lstrip": false,
73      "normalized": false,
74      "rstrip": false,
75      "single_word": false,
76      "special": true
77    },
78    "151652": {
79      "content": "<|vision_start|>",
80      "lstrip": false,
81      "normalized": false,
82      "rstrip": false,
83      "single_word": false,
84      "special": true
85    },
86    "151653": {
87      "content": "<|vision_end|>",
88      "lstrip": false,
89      "normalized": false,
90      "rstrip": false,
91      "single_word": false,
92      "special": true
93    },
94    "151654": {
95      "content": "<|vision_pad|>",
96      "lstrip": false,
97      "normalized": false,
98      "rstrip": false,
99      "single_word": false,
100      "special": true
101    },
102    "151655": {
103      "content": "<|image_pad|>",
104      "lstrip": false,
105      "normalized": false,
106      "rstrip": false,
107      "single_word": false,
108      "special": true
109    },
110    "151656": {
111      "content": "<|video_pad|>",
112      "lstrip": false,
113      "normalized": false,
114      "rstrip": false,
115      "single_word": false,
116      "special": true
117    },
118    "151657": {
119      "content": "<tool_call>",
120      "lstrip": false,
121      "normalized": false,
122      "rstrip": false,
123      "single_word": false,
124      "special": false
125    },
126    "151658": {
127      "content": "</tool_call>",
128      "lstrip": false,
129      "normalized": false,
130      "rstrip": false,
131      "single_word": false,
132      "special": false
133    },
134    "151659": {
135      "content": "<|fim_prefix|>",
136      "lstrip": false,
137      "normalized": false,
138      "rstrip": false,
139      "single_word": false,
140      "special": false
141    },
142    "151660": {
143      "content": "<|fim_middle|>",
144      "lstrip": false,
145      "normalized": false,
146      "rstrip": false,
147      "single_word": false,
148      "special": false
149    },
150    "151661": {
151      "content": "<|fim_suffix|>",
152      "lstrip": false,
153      "normalized": false,
154      "rstrip": false,
155      "single_word": false,
156      "special": false
157    },
158    "151662": {
159      "content": "<|fim_pad|>",
160      "lstrip": false,
161      "normalized": false,
162      "rstrip": false,
163      "single_word": false,
164      "special": false
165    },
166    "151663": {
167      "content": "<|repo_name|>",
168      "lstrip": false,
169      "normalized": false,
170      "rstrip": false,
171      "single_word": false,
172      "special": false
173    },
174    "151664": {
175      "content": "<|file_sep|>",
176      "lstrip": false,
177      "normalized": false,
178      "rstrip": false,
179      "single_word": false,
180      "special": false
181    }
182  },
183  "bos_token": "<|begin▁of▁sentence|>",
184  "chat_template": "{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% set ns = namespace(is_first=false, is_tool=false, is_output_first=true, system_prompt='') %}{%- for message in messages %}{%- if message['role'] == 'system' %}{% set ns.system_prompt = message['content'] %}{%- endif %}{%- endfor %}{{bos_token}}{{ns.system_prompt}}{%- for message in messages %}{%- if message['role'] == 'user' %}{%- set ns.is_tool = false -%}{{'<|User|>' + message['content']}}{%- endif %}{%- if message['role'] == 'assistant' and message['content'] is none %}{%- set ns.is_tool = false -%}{%- for tool in message['tool_calls']%}{%- if not ns.is_first %}{{'<|Assistant|><|tool▁calls▁begin|><|tool▁call▁begin|>' + tool['type'] + '<|tool▁sep|>' + tool['function']['name'] + '\\n' + '```json' + '\\n' + tool['function']['arguments'] + '\\n' + '```' + '<|tool▁call▁end|>'}}{%- set ns.is_first = true -%}{%- else %}{{'\\n' + '<|tool▁call▁begin|>' + tool['type'] + '<|tool▁sep|>' + tool['function']['name'] + '\\n' + '```json' + '\\n' + tool['function']['arguments'] + '\\n' + '```' + '<|tool▁call▁end|>'}}{{'<|tool▁calls▁end|><|end▁of▁sentence|>'}}{%- endif %}{%- endfor %}{%- endif %}{%- if message['role'] == 'assistant' and message['content'] is not none %}{%- if ns.is_tool %}{{'<|tool▁outputs▁end|>' + message['content'] + '<|end▁of▁sentence|>'}}{%- set ns.is_tool = false -%}{%- else %}{% set content = message['content'] %}{% if '</think>' in content %}{% set content = content.split('</think>')[-1] %}{% endif %}{{'<|Assistant|>' + content + '<|end▁of▁sentence|>'}}{%- endif %}{%- endif %}{%- if message['role'] == 'tool' %}{%- set ns.is_tool = true -%}{%- if ns.is_output_first %}{{'<|tool▁outputs▁begin|><|tool▁output▁begin|>' + message['content'] + '<|tool▁output▁end|>'}}{%- set ns.is_output_first = false %}{%- else %}{{'\\n<|tool▁output▁begin|>' + message['content'] + '<|tool▁output▁end|>'}}{%- endif %}{%- endif %}{%- endfor -%}{% if ns.is_tool %}{{'<|tool▁outputs▁end|>'}}{% endif %}{% if add_generation_prompt and not ns.is_tool %}{{'<|Assistant|>'}}{% endif %}",
185  "clean_up_tokenization_spaces": false,
186  "eos_token": "<|end▁of▁sentence|>",
187  "extra_special_tokens": {},
188  "legacy": true,
189  "model_max_length": 20000,
190  "pad_token": "<|end▁of▁sentence|>",
191  "padding_side": "right",
192  "sp_model_kwargs": {},
193  "tokenizer_class": "LlamaTokenizerFast",
194  "unk_token": null,
195  "use_default_system_prompt": false
196}
197