hf-internal-testing/tokenizers-test-data
tokenizers-test-data Test and benchmark fixtures for huggingface/tokenizers, pulled on demand by the repo Makefiles (make test / make bench / make fixtures via hf download). Layout fixtures/ — multilingual + modality corpora for cross-language encode benchmarks. Organized, documented, and reproducible: see fixtures/FIXTURES.md for provenance and fixtures/fixtures_manifest.json for exact sources, pinned revisions, and sizes. Rebuild any file with… See the full description on the dataset page: https://huggingface.co/datasets/hf-internal-testing/tokenizers-test-data.
019k
1{2 "version": "1.0",3 "truncation": null,4 "padding": null,5 "added_tokens": [6 {7 "id": 0,8 "content": "<s>",9 "single_word": false,10 "lstrip": false,11 "rstrip": false,12 "normalized": false,13 "special": true14 },15 {16 "id": 1,17 "content": "<pad>",18 "single_word": false,19 "lstrip": false,20 "rstrip": false,21 "normalized": false,22 "special": true23 },24 {25 "id": 2,26 "content": "</s>",27 "single_word": false,28 "lstrip": false,29 "rstrip": false,30 "normalized": false,31 "special": true32 },33 {34 "id": 3,35 "content": "<unk>",36 "single_word": false,37 "lstrip": false,38 "rstrip": false,39 "normalized": false,40 "special": true41 },42 {43 "id": 4,44 "content": "<mask>",45 "single_word": false,46 "lstrip": false,47 "rstrip": false,48 "normalized": false,49 "special": true50 }51 ],52 "normalizer": {53 "type": "Sequence",54 "normalizers": [55 {56 "type": "Strip",57 "strip_left": true,58 "strip_right": true59 },60 {61 "type": "NFC"62 }63 ]64 },65 "pre_tokenizer": {66 "type": "ByteLevel",67 "add_prefix_space": true,68 "use_regex": true69 },70 "post_processor": {71 "type": "ByteLevel",72 "add_prefix_space": true,73 "trim_offsets": true74 },75 "decoder": {76 "type": "ByteLevel"77 },78 "model": {79 "type": "BPE",80 "dropout": null,81 "unk_token": null,82 "continuing_subword_prefix": null,83 "end_of_word_suffix": null,84 "fuse_unk": false,85 "byte_fallback": false,86 "ignore_merges": false,87 "vocab": {88 "<s>": 0,89 "<pad>": 1,90 "</s>": 2,91 "<unk>": 3,92 "<mask>": 4,93 "!": 5,94 "\"": 6,95 "#": 7,96 "'": 8,97 "(": 9,98 ")": 10,99 "*": 11,100 ",": 12,101 "-": 13,102 ".": 14,103 "0": 15,104 "1": 16,105 "2": 17,106 "5": 18,107 "6": 19,108 "7": 20,109 "8": 21,110 "9": 22,111 ":": 23,112 ";": 24,113 "?": 25,114 "A": 26,115 "B": 27,116 "C": 28,117 "D": 29,118 "E": 30,119 "F": 31,120 "G": 32,121 "H": 33,122 "I": 34,123 "J": 35,124 "K": 36,125 "L": 37,126 "M": 38,127 "N": 39,128 "O": 40,129 "P": 41,130 "R": 42,131 "S": 43,132 "T": 44,133 "U": 45,134 "V": 46,135 "W": 47,136 "X": 48,137 "Y": 49,138 "[": 50,139 "]": 51,140 "a": 52,141 "b": 53,142 "c": 54,143 "d": 55,144 "e": 56,145 "f": 57,146 "g": 58,147 "h": 59,148 "i": 60,149 "j": 61,150 "k": 62,151 "l": 63,152 "m": 64,153 "n": 65,154 "o": 66,155 "p": 67,156 "q": 68,157 "r": 69,158 "s": 70,159 "t": 71,160 "u": 72,161 "v": 73,162 "w": 74,163 "x": 75,164 "y": 76,165 "z": 77,166 "ĉ": 78,167 "Ġ": 79,168 "Ġt": 80,169 "Ġa": 81,170 "he": 82,171 "in": 83,172 "Ġo": 84,173 "Ġh": 85,174 "Ġs": 86,175 "er": 87,176 "Ġw": 88,177 "en": 89,178 "Ġthe": 90,179 "re": 91,180 "nd": 92,181 "is": 93,182 "ed": 94,183 "Ġm": 95,184 "Ġof": 96,185 "ou": 97,186 "on": 98,187 "it": 99188 },189 "merges": [190 [191 "Ġ",192 "t"193 ],194 [195 "Ġ",196 "a"197 ],198 [199 "h",200 "e"201 ],202 [203 "i",204 "n"205 ],206 [207 "Ġ",208 "o"209 ],210 [211 "Ġ",212 "h"213 ],214 [215 "Ġ",216 "s"217 ],218 [219 "e",220 "r"221 ],222 [223 "Ġ",224 "w"225 ],226 [227 "e",228 "n"229 ],230 [231 "Ġt",232 "he"233 ],234 [235 "r",236 "e"237 ],238 [239 "n",240 "d"241 ],242 [243 "i",244 "s"245 ],246 [247 "e",248 "d"249 ],250 [251 "Ġ",252 "m"253 ],254 [255 "Ġo",256 "f"257 ],258 [259 "o",260 "u"261 ],262 [263 "o",264 "n"265 ],266 [267 "i",268 "t"269 ]270 ]271 }272}