Banaxi-Tech/pico-50
051
1{2 "version": "1.0",3 "truncation": null,4 "padding": null,5 "added_tokens": [6 {7 "id": 0,8 "content": "<|pad|>",9 "single_word": false,10 "lstrip": false,11 "rstrip": false,12 "normalized": false,13 "special": true14 },15 {16 "id": 1,17 "content": "<|bos|>",18 "single_word": false,19 "lstrip": false,20 "rstrip": false,21 "normalized": false,22 "special": true23 },24 {25 "id": 2,26 "content": "<|eos|>",27 "single_word": false,28 "lstrip": false,29 "rstrip": false,30 "normalized": false,31 "special": true32 },33 {34 "id": 3,35 "content": "<|unk|>",36 "single_word": false,37 "lstrip": false,38 "rstrip": false,39 "normalized": false,40 "special": true41 }42 ],43 "normalizer": {44 "type": "NFKC"45 },46 "pre_tokenizer": {47 "type": "Sequence",48 "pretokenizers": [49 {50 "type": "Split",51 "pattern": {52 "Regex": "\\d"53 },54 "behavior": "Isolated",55 "invert": false56 },57 {58 "type": "ByteLevel",59 "add_prefix_space": false,60 "trim_offsets": true,61 "use_regex": true62 }63 ]64 },65 "post_processor": {66 "type": "TemplateProcessing",67 "single": [68 {69 "Sequence": {70 "id": "A",71 "type_id": 072 }73 }74 ],75 "pair": [76 {77 "Sequence": {78 "id": "A",79 "type_id": 080 }81 },82 {83 "Sequence": {84 "id": "B",85 "type_id": 186 }87 }88 ],89 "special_tokens": {}90 },91 "decoder": {92 "type": "ByteLevel",93 "add_prefix_space": true,94 "trim_offsets": true,95 "use_regex": true96 },97 "model": {98 "type": "BPE",99 "dropout": null,100 "unk_token": "<|unk|>",101 "continuing_subword_prefix": null,102 "end_of_word_suffix": null,103 "fuse_unk": false,104 "byte_fallback": false,105 "ignore_merges": false,106 "vocab": {107 "<|pad|>": 0,108 "<|bos|>": 1,109 "<|eos|>": 2,110 "<|unk|>": 3,111 "!": 4,112 "\"": 5,113 "#": 6,114 "$": 7,115 "%": 8,116 "&": 9,117 "'": 10,118 "(": 11,119 ")": 12,120 "*": 13,121 "+": 14,122 ",": 15,123 "-": 16,124 ".": 17,125 "/": 18,126 "0": 19,127 "1": 20,128 "2": 21,129 "3": 22,130 "4": 23,131 "5": 24,132 "6": 25,133 "7": 26,134 "8": 27,135 "9": 28,136 ":": 29,137 ";": 30,138 "<": 31,139 "=": 32,140 ">": 33,141 "?": 34,142 "@": 35,143 "A": 36,144 "B": 37,145 "C": 38,146 "D": 39,147 "E": 40,148 "F": 41,149 "G": 42,150 "H": 43,151 "I": 44,152 "J": 45,153 "K": 46,154 "L": 47,155 "M": 48,156 "N": 49,157 "O": 50,158 "P": 51,159 "Q": 52,160 "R": 53,161 "S": 54,162 "T": 55,163 "U": 56,164 "V": 57,165 "W": 58,166 "X": 59,167 "Y": 60,168 "Z": 61,169 "[": 62,170 "\\": 63,171 "]": 64,172 "^": 65,173 "_": 66,174 "`": 67,175 "a": 68,176 "b": 69,177 "c": 70,178 "d": 71,179 "e": 72,180 "f": 73,181 "g": 74,182 "h": 75,183 "i": 76,184 "j": 77,185 "k": 78,186 "l": 79,187 "m": 80,188 "n": 81,189 "o": 82,190 "p": 83,191 "q": 84,192 "r": 85,193 "s": 86,194 "t": 87,195 "u": 88,196 "v": 89,197 "w": 90,198 "x": 91,199 "y": 92,200 "z": 93,201 "{": 94,202 "|": 95,203 "}": 96,204 "~": 97,205 "¡": 98,206 "¢": 99,207 "£": 100,208 "¤": 101,209 "¥": 102,210 "¦": 103,211 "§": 104,212 "¨": 105,213 "©": 106,214 "ª": 107,215 "«": 108,216 "¬": 109,217 "®": 110,218 "¯": 111,219 "°": 112,220 "±": 113,221 "²": 114,222 "³": 115,223 "´": 116,224 "µ": 117,225 "¶": 118,226 "·": 119,227 "¸": 120,228 "¹": 121,229 "º": 122,230 "»": 123,231 "¼": 124,232 "½": 125,233 "¾": 126,234 "¿": 127,235 "À": 128,236 "Á": 129,237 "Â": 130,238 "Ã": 131,239 "Ä": 132,240 "Å": 133,241 "Æ": 134,242 "Ç": 135,243 "È": 136,244 "É": 137,245 "Ê": 138,246 "Ë": 139,247 "Ì": 140,248 "Í": 141,249 "Î": 142,250 "Ï": 143,251 "Ð": 144,252 "Ñ": 145,253 "Ò": 146,254 "Ó": 147,255 "Ô": 148,256 "Õ": 149,257 "Ö": 150,258 "×": 151,259 "Ø": 152,260 "Ù": 153,261 "Ú": 154,262 "Û": 155,263 "Ü": 156,264 "Ý": 157,265 "Þ": 158,266 "ß": 159,267 "à": 160,268 "á": 161,269 "â": 162,270 "ã": 163,271 "ä": 164,272 "å": 165,273 "æ": 166,274 "ç": 167,275 "è": 168,276 "é": 169,277 "ê": 170,278 "ë": 171,279 "ì": 172,280 "í": 173,281 "î": 174,282 "ï": 175,283 "ð": 176,284 "ñ": 177,285 "ò": 178,286 "ó": 179,287 "ô": 180,288 "õ": 181,289 "ö": 182,290 "÷": 183,291 "ø": 184,292 "ù": 185,293 "ú": 186,294 "û": 187,295 "ü": 188,296 "ý": 189,297 "þ": 190,298 "ÿ": 191,299 "Ā": 192,300 "ā": 193,301 "Ă": 194,302 "ă": 195,303 "Ą": 196,304 "ą": 197,305 "Ć": 198,306 "ć": 199,307 "Ĉ": 200,308 "ĉ": 201,309 "Ċ": 202,310 "ċ": 203,311 "Č": 204,312 "č": 205,313 "Ď": 206,314 "ď": 207,315 "Đ": 208,316 "đ": 209,317 "Ē": 210,318 "ē": 211,319 "Ĕ": 212,320 "ĕ": 213,321 "Ė": 214,322 "ė": 215,323 "Ę": 216,324 "ę": 217,325 "Ě": 218,326 "ě": 219,327 "Ĝ": 220,328 "ĝ": 221,329 "Ğ": 222,330 "ğ": 223,331 "Ġ": 224,332 "ġ": 225,333 "Ģ": 226,334 "ģ": 227,335 "Ĥ": 228,336 "ĥ": 229,337 "Ħ": 230,338 "ħ": 231,339 "Ĩ": 232,340 "ĩ": 233,341 "Ī": 234,342 "ī": 235,343 "Ĭ": 236,344 "ĭ": 237,345 "Į": 238,346 "į": 239,347 "İ": 240,348 "ı": 241,349 "IJ": 242,350 "ij": 243,351 "Ĵ": 244,352 "ĵ": 245,353 "Ķ": 246,354 "ķ": 247,355 "ĸ": 248,356 "Ĺ": 249,357 "ĺ": 250,358 "Ļ": 251,359 "ļ": 252,360 "Ľ": 253,361 "ľ": 254,362 "Ŀ": 255,363 "ŀ": 256,364 "Ł": 257,365 "ł": 258,366 "Ń": 259,367 "Ġt": 260,368 "Ġa": 261,369 "in": 262,370 "he": 263,371 "re": 264,372 "on": 265,373 "Ġthe": 266,374 "er": 267,375 "Ġs": 268,376 "at": 269,377 "Ġo": 270,378 "en": 271,379 "Ġc": 272,380 "Ġw": 273,381 "es": 274,382 "nd": 275,383 "it": 276,384 "ing": 277,385 "is": 278,386 "or": 279,387 "ou": 280,388 "al": 281,389 "Ġp": 282,390 "ed": 283,391 "Ġf": 284,392 "an": 285,393 "Ġb": 286,394 "Ġin": 287,395 "ar": 288,396 "Ġof": 289,397 "Ġm": 290,398 "Ġand": 291,399 "Ġto": 292,400 "Ġd": 293,401 "ion": 294,402 "ic": 295,403 "le": 296,404 "om": 297,405 "Ġh": 298,406 "as": 299,407 "Ġth": 300,408 "ent": 301,409 "Ġe": 302,410 "Ġre": 303,411 "ct": 304,412 "il": 305,413 "ro": 306,414 "Ġl": 307,415 "st": 308,416 "ve": 309,417 "Ġn": 310,418 "ly": 311,419 "et": 312,420 "Ġy": 313,421 "Ġg": 314,422 "id": 315,423 "im": 316,424 "ĠT": 317,425 "se": 318,426 "ation": 319,427 "Ġbe": 320,428 "ol": 321,429 "ce": 322,430 "ur": 323,431 "ut": 324,432 "ow": 325,433 "ra": 326,434 "ot": 327,435 "ig": 328,436 "Ġyou": 329,437 "Ġis": 330,438 "Ġu": 331,439 "ĠS": 332,440 "ĠA": 333,441 "ĠI": 334,442 "Ġfor": 335,443 "ch": 336,444 "ver": 337,445 "Ġon": 338,446 "Ġst": 339,447 "am": 340,448 "ay": 341,449 "ĠC": 342,450 "Ġthat": 343,451 "ir": 344,452 "ith": 345,453 "el": 346,454 "Ġhe": 347,455 "ul": 348,456 "ad": 349,457 "âĢ": 350,458 "Ġcon": 351,459 "Ġwith": 352,460 "Ġas": 353,461 "ter": 354,462 "if": 355,463 "Ġwh": 356,464 "Ġan": 357,465 "Ġit": 358,466 "ĠM": 359,467 "Ġor": 360,468 "Ġpro": 361,469 "Ġal": 362,470 "her": 363,471 "ers": 364,472 "em": 365,473 "od": 366,474 "ess": 367,475 "Ġex": 368,476 "ate": 369,477 "ĠB": 370,478 "ag": 371,479 "ist": 372,480 "Ġcom": 373,481 "ri": 374,482 "ĠP": 375,483 "us": 376,484 "ill": 377,485 "res": 378,486 "pp": 379,487 "ab": 380,488 "ke": 381,489 "th": 382,490 "ĠH": 383491 },492 "merges": [493 [494 "Ġ",495 "t"496 ],497 [498 "Ġ",499 "a"500 ],501 [502 "i",503 "n"504 ],505 [506 "h",507 "e"508 ],509 [510 "r",511 "e"512 ],513 [514 "o",515 "n"516 ],517 [518 "Ġt",519 "he"520 ],521 [522 "e",523 "r"524 ],525 [526 "Ġ",527 "s"528 ],529 [530 "a",531 "t"532 ],533 [534 "Ġ",535 "o"536 ],537 [538 "e",539 "n"540 ],541 [542 "Ġ",543 "c"544 ],545 [546 "Ġ",547 "w"548 ],549 [550 "e",551 "s"552 ],553 [554 "n",555 "d"556 ],557 [558 "i",559 "t"560 ],561 [562 "in",563 "g"564 ],565 [566 "i",567 "s"568 ],569 [570 "o",571 "r"572 ],573 [574 "o",575 "u"576 ],577 [578 "a",579 "l"580 ],581 [582 "Ġ",583 "p"584 ],585 [586 "e",587 "d"588 ],589 [590 "Ġ",591 "f"592 ],593 [594 "a",595 "n"596 ],597 [598 "Ġ",599 "b"600 ],601 [602 "Ġ",603 "in"604 ],605 [606 "a",607 "r"608 ],609 [610 "Ġo",611 "f"612 ],613 [614 "Ġ",615 "m"616 ],617 [618 "Ġa",619 "nd"620 ],621 [622 "Ġt",623 "o"624 ],625 [626 "Ġ",627 "d"628 ],629 [630 "i",631 "on"632 ],633 [634 "i",635 "c"636 ],637 [638 "l",639 "e"640 ],641 [642 "o",643 "m"644 ],645 [646 "Ġ",647 "h"648 ],649 [650 "a",651 "s"652 ],653 [654 "Ġt",655 "h"656 ],657 [658 "en",659 "t"660 ],661 [662 "Ġ",663 "e"664 ],665 [666 "Ġ",667 "re"668 ],669 [670 "c",671 "t"672 ],673 [674 "i",675 "l"676 ],677 [678 "r",679 "o"680 ],681 [682 "Ġ",683 "l"684 ],685 [686 "s",687 "t"688 ],689 [690 "v",691 "e"692 ],693 [694 "Ġ",695 "n"696 ],697 [698 "l",699 "y"700 ],701 [702 "e",703 "t"704 ],705 [706 "Ġ",707 "y"708 ],709 [710 "Ġ",711 "g"712 ],713 [714 "i",715 "d"716 ],717 [718 "i",719 "m"720 ],721 [722 "Ġ",723 "T"724 ],725 [726 "s",727 "e"728 ],729 [730 "at",731 "ion"732 ],733 [734 "Ġb",735 "e"736 ],737 [738 "o",739 "l"740 ],741 [742 "c",743 "e"744 ],745 [746 "u",747 "r"748 ],749 [750 "u",751 "t"752 ],753 [754 "o",755 "w"756 ],757 [758 "r",759 "a"760 ],761 [762 "o",763 "t"764 ],765 [766 "i",767 "g"768 ],769 [770 "Ġy",771 "ou"772 ],773 [774 "Ġ",775 "is"776 ],777 [778 "Ġ",779 "u"780 ],781 [782 "Ġ",783 "S"784 ],785 [786 "Ġ",787 "A"788 ],789 [790 "Ġ",791 "I"792 ],793 [794 "Ġf",795 "or"796 ],797 [798 "c",799 "h"800 ],801 [802 "v",803 "er"804 ],805 [806 "Ġ",807 "on"808 ],809 [810 "Ġs",811 "t"812 ],813 [814 "a",815 "m"816 ],817 [818 "a",819 "y"820 ],821 [822 "Ġ",823 "C"824 ],825 [826 "Ġth",827 "at"828 ],829 [830 "i",831 "r"832 ],833 [834 "it",835 "h"836 ],837 [838 "e",839 "l"840 ],841 [842 "Ġ",843 "he"844 ],845 [846 "u",847 "l"848 ],849 [850 "a",851 "d"852 ],853 [854 "â",855 "Ģ"856 ],857 [858 "Ġc",859 "on"860 ],861 [862 "Ġw",863 "ith"864 ],865 [866 "Ġa",867 "s"868 ],869 [870 "t",871 "er"872 ],873 [874 "i",875 "f"876 ],877 [878 "Ġw",879 "h"880 ],881 [882 "Ġa",883 "n"884 ],885 [886 "Ġ",887 "it"888 ],889 [890 "Ġ",891 "M"892 ],893 [894 "Ġo",895 "r"896 ],897 [898 "Ġp",899 "ro"900 ],901 [902 "Ġa",903 "l"904 ],905 [906 "he",907 "r"908 ],909 [910 "er",911 "s"912 ],913 [914 "e",915 "m"916 ],917 [918 "o",919 "d"920 ],921 [922 "es",923 "s"924 ],925 [926 "Ġe",927 "x"928 ],929 [930 "at",931 "e"932 ],933 [934 "Ġ",935 "B"936 ],937 [938 "a",939 "g"940 ],941 [942 "is",943 "t"944 ],945 [946 "Ġc",947 "om"948 ],949 [950 "r",951 "i"952 ],953 [954 "Ġ",955 "P"956 ],957 [958 "u",959 "s"960 ],961 [962 "il",963 "l"964 ],965 [966 "re",967 "s"968 ],969 [970 "p",971 "p"972 ],973 [974 "a",975 "b"976 ],977 [978 "k",979 "e"980 ],981 [982 "t",983 "h"984 ],985 [986 "Ġ",987 "H"988 ]989 ]990 }991}