NithishAI/deepseek-xcodec-tokenizer
0
1{2 "version": "1.0",3 "truncation": null,4 "padding": null,5 "added_tokens": [6 {7 "id": 100000,8 "content": "<｜begin▁of▁sentence｜>",9 "single_word": false,10 "lstrip": false,11 "rstrip": false,12 "normalized": true,13 "special": true14 },15 {16 "id": 100001,17 "content": "<｜end▁of▁sentence｜>",18 "single_word": false,19 "lstrip": false,20 "rstrip": false,21 "normalized": true,22 "special": true23 },24 {25 "id": 100002,26 "content": "<|TEXT_START|>",27 "single_word": false,28 "lstrip": false,29 "rstrip": false,30 "normalized": true,31 "special": false32 },33 {34 "id": 100003,35 "content": "<|TEXT_END|>",36 "single_word": false,37 "lstrip": false,38 "rstrip": false,39 "normalized": true,40 "special": false41 },42 {43 "id": 100004,44 "content": "<|SPEECH_START|>",45 "single_word": false,46 "lstrip": false,47 "rstrip": false,48 "normalized": true,49 "special": false50 },51 {52 "id": 100005,53 "content": "<|SPEECH_END|>",54 "single_word": false,55 "lstrip": false,56 "rstrip": false,57 "normalized": true,58 "special": false59 },60 {61 "id": 100006,62 "content": "<|s_0|>",63 "single_word": false,64 "lstrip": false,65 "rstrip": false,66 "normalized": true,67 "special": false68 },69 {70 "id": 100007,71 "content": "<|s_1|>",72 "single_word": false,73 "lstrip": false,74 "rstrip": false,75 "normalized": true,76 "special": false77 },78 {79 "id": 100008,80 "content": "<|s_2|>",81 "single_word": false,82 "lstrip": false,83 "rstrip": false,84 "normalized": true,85 "special": false86 },87 {88 "id": 100009,89 "content": "<|s_3|>",90 "single_word": false,91 "lstrip": false,92 "rstrip": false,93 "normalized": true,94 "special": false95 },96 {97 "id": 100010,98 "content": "<|s_4|>",99 "single_word": false,100 "lstrip": false,101 "rstrip": false,102 "normalized": true,103 "special": false104 },105 {106 "id": 100011,107 "content": "<|s_5|>",108 "single_word": false,109 "lstrip": false,110 "rstrip": false,111 "normalized": true,112 "special": false113 },114 {115 "id": 100012,116 "content": "<|s_6|>",117 "single_word": false,118 "lstrip": false,119 "rstrip": false,120 "normalized": true,121 "special": false122 },123 {124 "id": 100013,125 "content": "<|s_7|>",126 "single_word": false,127 "lstrip": false,128 "rstrip": false,129 "normalized": true,130 "special": false131 },132 {133 "id": 100014,134 "content": "<|s_8|>",135 "single_word": false,136 "lstrip": false,137 "rstrip": false,138 "normalized": true,139 "special": false140 },141 {142 "id": 100015,143 "content": "<|s_9|>",144 "single_word": false,145 "lstrip": false,146 "rstrip": false,147 "normalized": true,148 "special": false149 },150 {151 "id": 100016,152 "content": "<|s_10|>",153 "single_word": false,154 "lstrip": false,155 "rstrip": false,156 "normalized": true,157 "special": false158 },159 {160 "id": 100017,161 "content": "<|s_11|>",162 "single_word": false,163 "lstrip": false,164 "rstrip": false,165 "normalized": true,166 "special": false167 },168 {169 "id": 100018,170 "content": "<|s_12|>",171 "single_word": false,172 "lstrip": false,173 "rstrip": false,174 "normalized": true,175 "special": false176 },177 {178 "id": 100019,179 "content": "<|s_13|>",180 "single_word": false,181 "lstrip": false,182 "rstrip": false,183 "normalized": true,184 "special": false185 },186 {187 "id": 100020,188 "content": "<|s_14|>",189 "single_word": false,190 "lstrip": false,191 "rstrip": false,192 "normalized": true,193 "special": false194 },195 {196 "id": 100021,197 "content": "<|s_15|>",198 "single_word": false,199 "lstrip": false,200 "rstrip": false,201 "normalized": true,202 "special": false203 },204 {205 "id": 100022,206 "content": "<|s_16|>",207 "single_word": false,208 "lstrip": false,209 "rstrip": false,210 "normalized": true,211 "special": false212 },213 {214 "id": 100023,215 "content": "<|s_17|>",216 "single_word": false,217 "lstrip": false,218 "rstrip": false,219 "normalized": true,220 "special": false221 },222 {223 "id": 100024,224 "content": "<|s_18|>",225 "single_word": false,226 "lstrip": false,227 "rstrip": false,228 "normalized": true,229 "special": false230 },231 {232 "id": 100025,233 "content": "<|s_19|>",234 "single_word": false,235 "lstrip": false,236 "rstrip": false,237 "normalized": true,238 "special": false239 },240 {241 "id": 100026,242 "content": "<|s_20|>",243 "single_word": false,244 "lstrip": false,245 "rstrip": false,246 "normalized": true,247 "special": false248 },249 {250 "id": 100027,251 "content": "<|s_21|>",252 "single_word": false,253 "lstrip": false,254 "rstrip": false,255 "normalized": true,256 "special": false257 },258 {259 "id": 100028,260 "content": "<|s_22|>",261 "single_word": false,262 "lstrip": false,263 "rstrip": false,264 "normalized": true,265 "special": false266 },267 {268 "id": 100029,269 "content": "<|s_23|>",270 "single_word": false,271 "lstrip": false,272 "rstrip": false,273 "normalized": true,274 "special": false275 },276 {277 "id": 100030,278 "content": "<|s_24|>",279 "single_word": false,280 "lstrip": false,281 "rstrip": false,282 "normalized": true,283 "special": false284 },285 {286 "id": 100031,287 "content": "<|s_25|>",288 "single_word": false,289 "lstrip": false,290 "rstrip": false,291 "normalized": true,292 "special": false293 },294 {295 "id": 100032,296 "content": "<|s_26|>",297 "single_word": false,298 "lstrip": false,299 "rstrip": false,300 "normalized": true,301 "special": false302 },303 {304 "id": 100033,305 "content": "<|s_27|>",306 "single_word": false,307 "lstrip": false,308 "rstrip": false,309 "normalized": true,310 "special": false311 },312 {313 "id": 100034,314 "content": "<|s_28|>",315 "single_word": false,316 "lstrip": false,317 "rstrip": false,318 "normalized": true,319 "special": false320 },321 {322 "id": 100035,323 "content": "<|s_29|>",324 "single_word": false,325 "lstrip": false,326 "rstrip": false,327 "normalized": true,328 "special": false329 },330 {331 "id": 100036,332 "content": "<|s_30|>",333 "single_word": false,334 "lstrip": false,335 "rstrip": false,336 "normalized": true,337 "special": false338 },339 {340 "id": 100037,341 "content": "<|s_31|>",342 "single_word": false,343 "lstrip": false,344 "rstrip": false,345 "normalized": true,346 "special": false347 },348 {349 "id": 100038,350 "content": "<|s_32|>",351 "single_word": false,352 "lstrip": false,353 "rstrip": false,354 "normalized": true,355 "special": false356 },357 {358 "id": 100039,359 "content": "<|s_33|>",360 "single_word": false,361 "lstrip": false,362 "rstrip": false,363 "normalized": true,364 "special": false365 },366 {367 "id": 100040,368 "content": "<|s_34|>",369 "single_word": false,370 "lstrip": false,371 "rstrip": false,372 "normalized": true,373 "special": false374 },375 {376 "id": 100041,377 "content": "<|s_35|>",378 "single_word": false,379 "lstrip": false,380 "rstrip": false,381 "normalized": true,382 "special": false383 },384 {385 "id": 100042,386 "content": "<|s_36|>",387 "single_word": false,388 "lstrip": false,389 "rstrip": false,390 "normalized": true,391 "special": false392 },393 {394 "id": 100043,395 "content": "<|s_37|>",396 "single_word": false,397 "lstrip": false,398 "rstrip": false,399 "normalized": true,400 "special": false401 },402 {403 "id": 100044,404 "content": "<|s_38|>",405 "single_word": false,406 "lstrip": false,407 "rstrip": false,408 "normalized": true,409 "special": false410 },411 {412 "id": 100045,413 "content": "<|s_39|>",414 "single_word": false,415 "lstrip": false,416 "rstrip": false,417 "normalized": true,418 "special": false419 },420 {421 "id": 100046,422 "content": "<|s_40|>",423 "single_word": false,424 "lstrip": false,425 "rstrip": false,426 "normalized": true,427 "special": false428 },429 {430 "id": 100047,431 "content": "<|s_41|>",432 "single_word": false,433 "lstrip": false,434 "rstrip": false,435 "normalized": true,436 "special": false437 },438 {439 "id": 100048,440 "content": "<|s_42|>",441 "single_word": false,442 "lstrip": false,443 "rstrip": false,444 "normalized": true,445 "special": false446 },447 {448 "id": 100049,449 "content": "<|s_43|>",450 "single_word": false,451 "lstrip": false,452 "rstrip": false,453 "normalized": true,454 "special": false455 },456 {457 "id": 100050,458 "content": "<|s_44|>",459 "single_word": false,460 "lstrip": false,461 "rstrip": false,462 "normalized": true,463 "special": false464 },465 {466 "id": 100051,467 "content": "<|s_45|>",468 "single_word": false,469 "lstrip": false,470 "rstrip": false,471 "normalized": true,472 "special": false473 },474 {475 "id": 100052,476 "content": "<|s_46|>",477 "single_word": false,478 "lstrip": false,479 "rstrip": false,480 "normalized": true,481 "special": false482 },483 {484 "id": 100053,485 "content": "<|s_47|>",486 "single_word": false,487 "lstrip": false,488 "rstrip": false,489 "normalized": true,490 "special": false491 },492 {493 "id": 100054,494 "content": "<|s_48|>",495 "single_word": false,496 "lstrip": false,497 "rstrip": false,498 "normalized": true,499 "special": false500 },501 {502 "id": 100055,503 "content": "<|s_49|>",504 "single_word": false,505 "lstrip": false,506 "rstrip": false,507 "normalized": true,508 "special": false509 },510 {511 "id": 100056,512 "content": "<|s_50|>",513 "single_word": false,514 "lstrip": false,515 "rstrip": false,516 "normalized": true,517 "special": false518 },519 {520 "id": 100057,521 "content": "<|s_51|>",522 "single_word": false,523 "lstrip": false,524 "rstrip": false,525 "normalized": true,526 "special": false527 },528 {529 "id": 100058,530 "content": "<|s_52|>",531 "single_word": false,532 "lstrip": false,533 "rstrip": false,534 "normalized": true,535 "special": false536 },537 {538 "id": 100059,539 "content": "<|s_53|>",540 "single_word": false,541 "lstrip": false,542 "rstrip": false,543 "normalized": true,544 "special": false545 },546 {547 "id": 100060,548 "content": "<|s_54|>",549 "single_word": false,550 "lstrip": false,551 "rstrip": false,552 "normalized": true,553 "special": false554 },555 {556 "id": 100061,557 "content": "<|s_55|>",558 "single_word": false,559 "lstrip": false,560 "rstrip": false,561 "normalized": true,562 "special": false563 },564 {565 "id": 100062,566 "content": "<|s_56|>",567 "single_word": false,568 "lstrip": false,569 "rstrip": false,570 "normalized": true,571 "special": false572 },573 {574 "id": 100063,575 "content": "<|s_57|>",576 "single_word": false,577 "lstrip": false,578 "rstrip": false,579 "normalized": true,580 "special": false581 },582 {583 "id": 100064,584 "content": "<|s_58|>",585 "single_word": false,586 "lstrip": false,587 "rstrip": false,588 "normalized": true,589 "special": false590 },591 {592 "id": 100065,593 "content": "<|s_59|>",594 "single_word": false,595 "lstrip": false,596 "rstrip": false,597 "normalized": true,598 "special": false599 },600 {601 "id": 100066,602 "content": "<|s_60|>",603 "single_word": false,604 "lstrip": false,605 "rstrip": false,606 "normalized": true,607 "special": false608 },609 {610 "id": 100067,611 "content": "<|s_61|>",612 "single_word": false,613 "lstrip": false,614 "rstrip": false,615 "normalized": true,616 "special": false617 },618 {619 "id": 100068,620 "content": "<|s_62|>",621 "single_word": false,622 "lstrip": false,623 "rstrip": false,624 "normalized": true,625 "special": false626 },627 {628 "id": 100069,629 "content": "<|s_63|>",630 "single_word": false,631 "lstrip": false,632 "rstrip": false,633 "normalized": true,634 "special": false635 },636 {637 "id": 100070,638 "content": "<|s_64|>",639 "single_word": false,640 "lstrip": false,641 "rstrip": false,642 "normalized": true,643 "special": false644 },645 {646 "id": 100071,647 "content": "<|s_65|>",648 "single_word": false,649 "lstrip": false,650 "rstrip": false,651 "normalized": true,652 "special": false653 },654 {655 "id": 100072,656 "content": "<|s_66|>",657 "single_word": false,658 "lstrip": false,659 "rstrip": false,660 "normalized": true,661 "special": false662 },663 {664 "id": 100073,665 "content": "<|s_67|>",666 "single_word": false,667 "lstrip": false,668 "rstrip": false,669 "normalized": true,670 "special": false671 },672 {673 "id": 100074,674 "content": "<|s_68|>",675 "single_word": false,676 "lstrip": false,677 "rstrip": false,678 "normalized": true,679 "special": false680 },681 {682 "id": 100075,683 "content": "<|s_69|>",684 "single_word": false,685 "lstrip": false,686 "rstrip": false,687 "normalized": true,688 "special": false689 },690 {691 "id": 100076,692 "content": "<|s_70|>",693 "single_word": false,694 "lstrip": false,695 "rstrip": false,696 "normalized": true,697 "special": false698 },699 {700 "id": 100077,701 "content": "<|s_71|>",702 "single_word": false,703 "lstrip": false,704 "rstrip": false,705 "normalized": true,706 "special": false707 },708 {709 "id": 100078,710 "content": "<|s_72|>",711 "single_word": false,712 "lstrip": false,713 "rstrip": false,714 "normalized": true,715 "special": false716 },717 {718 "id": 100079,719 "content": "<|s_73|>",720 "single_word": false,721 "lstrip": false,722 "rstrip": false,723 "normalized": true,724 "special": false725 },726 {727 "id": 100080,728 "content": "<|s_74|>",729 "single_word": false,730 "lstrip": false,731 "rstrip": false,732 "normalized": true,733 "special": false734 },735 {736 "id": 100081,737 "content": "<|s_75|>",738 "single_word": false,739 "lstrip": false,740 "rstrip": false,741 "normalized": true,742 "special": false743 },744 {745 "id": 100082,746 "content": "<|s_76|>",747 "single_word": false,748 "lstrip": false,749 "rstrip": false,750 "normalized": true,751 "special": false752 },753 {754 "id": 100083,755 "content": "<|s_77|>",756 "single_word": false,757 "lstrip": false,758 "rstrip": false,759 "normalized": true,760 "special": false761 },762 {763 "id": 100084,764 "content": "<|s_78|>",765 "single_word": false,766 "lstrip": false,767 "rstrip": false,768 "normalized": true,769 "special": false770 },771 {772 "id": 100085,773 "content": "<|s_79|>",774 "single_word": false,775 "lstrip": false,776 "rstrip": false,777 "normalized": true,778 "special": false779 },780 {781 "id": 100086,782 "content": "<|s_80|>",783 "single_word": false,784 "lstrip": false,785 "rstrip": false,786 "normalized": true,787 "special": false788 },789 {790 "id": 100087,791 "content": "<|s_81|>",792 "single_word": false,793 "lstrip": false,794 "rstrip": false,795 "normalized": true,796 "special": false797 },798 {799 "id": 100088,800 "content": "<|s_82|>",801 "single_word": false,802 "lstrip": false,803 "rstrip": false,804 "normalized": true,805 "special": false806 },807 {808 "id": 100089,809 "content": "<|s_83|>",810 "single_word": false,811 "lstrip": false,812 "rstrip": false,813 "normalized": true,814 "special": false815 },816 {817 "id": 100090,818 "content": "<|s_84|>",819 "single_word": false,820 "lstrip": false,821 "rstrip": false,822 "normalized": true,823 "special": false824 },825 {826 "id": 100091,827 "content": "<|s_85|>",828 "single_word": false,829 "lstrip": false,830 "rstrip": false,831 "normalized": true,832 "special": false833 },834 {835 "id": 100092,836 "content": "<|s_86|>",837 "single_word": false,838 "lstrip": false,839 "rstrip": false,840 "normalized": true,841 "special": false842 },843 {844 "id": 100093,845 "content": "<|s_87|>",846 "single_word": false,847 "lstrip": false,848 "rstrip": false,849 "normalized": true,850 "special": false851 },852 {853 "id": 100094,854 "content": "<|s_88|>",855 "single_word": false,856 "lstrip": false,857 "rstrip": false,858 "normalized": true,859 "special": false860 },861 {862 "id": 100095,863 "content": "<|s_89|>",864 "single_word": false,865 "lstrip": false,866 "rstrip": false,867 "normalized": true,868 "special": false869 },870 {871 "id": 100096,872 "content": "<|s_90|>",873 "single_word": false,874 "lstrip": false,875 "rstrip": false,876 "normalized": true,877 "special": false878 },879 {880 "id": 100097,881 "content": "<|s_91|>",882 "single_word": false,883 "lstrip": false,884 "rstrip": false,885 "normalized": true,886 "special": false887 },888 {889 "id": 100098,890 "content": "<|s_92|>",891 "single_word": false,892 "lstrip": false,893 "rstrip": false,894 "normalized": true,895 "special": false896 },897 {898 "id": 100099,899 "content": "<|s_93|>",900 "single_word": false,901 "lstrip": false,902 "rstrip": false,903 "normalized": true,904 "special": false905 },906 {907 "id": 100100,908 "content": "<|s_94|>",909 "single_word": false,910 "lstrip": false,911 "rstrip": false,912 "normalized": true,913 "special": false914 },915 {916 "id": 100101,917 "content": "<|s_95|>",918 "single_word": false,919 "lstrip": false,920 "rstrip": false,921 "normalized": true,922 "special": false923 },924 {925 "id": 100102,926 "content": "<|s_96|>",927 "single_word": false,928 "lstrip": false,929 "rstrip": false,930 "normalized": true,931 "special": false932 },933 {934 "id": 100103,935 "content": "<|s_97|>",936 "single_word": false,937 "lstrip": false,938 "rstrip": false,939 "normalized": true,940 "special": false941 },942 {943 "id": 100104,944 "content": "<|s_98|>",945 "single_word": false,946 "lstrip": false,947 "rstrip": false,948 "normalized": true,949 "special": false950 },951 {952 "id": 100105,953 "content": "<|s_99|>",954 "single_word": false,955 "lstrip": false,956 "rstrip": false,957 "normalized": true,958 "special": false959 },960 {961 "id": 100106,962 "content": "<|s_100|>",963 "single_word": false,964 "lstrip": false,965 "rstrip": false,966 "normalized": true,967 "special": false968 },969 {970 "id": 100107,971 "content": "<|s_101|>",972 "single_word": false,973 "lstrip": false,974 "rstrip": false,975 "normalized": true,976 "special": false977 },978 {979 "id": 100108,980 "content": "<|s_102|>",981 "single_word": false,982 "lstrip": false,983 "rstrip": false,984 "normalized": true,985 "special": false986 },987 {988 "id": 100109,989 "content": "<|s_103|>",990 "single_word": false,991 "lstrip": false,992 "rstrip": false,993 "normalized": true,994 "special": false995 },996 {997 "id": 100110,998 "content": "<|s_104|>",999 "single_word": false,1000 "lstrip": false,1001 "rstrip": false,1002 "normalized": true,1003 "special": false1004 },1005 {1006 "id": 100111,1007 "content": "<|s_105|>",1008 "single_word": false,1009 "lstrip": false,1010 "rstrip": false,1011 "normalized": true,1012 "special": false1013 },1014 {1015 "id": 100112,1016 "content": "<|s_106|>",1017 "single_word": false,1018 "lstrip": false,1019 "rstrip": false,1020 "normalized": true,1021 "special": false1022 },1023 {1024 "id": 100113,1025 "content": "<|s_107|>",1026 "single_word": false,1027 "lstrip": false,1028 "rstrip": false,1029 "normalized": true,1030 "special": false1031 },1032 {1033 "id": 100114,1034 "content": "<|s_108|>",1035 "single_word": false,1036 "lstrip": false,1037 "rstrip": false,1038 "normalized": true,1039 "special": false1040 },1041 {1042 "id": 100115,1043 "content": "<|s_109|>",1044 "single_word": false,1045 "lstrip": false,1046 "rstrip": false,1047 "normalized": true,1048 "special": false1049 },1050 {1051 "id": 100116,1052 "content": "<|s_110|>",1053 "single_word": false,1054 "lstrip": false,1055 "rstrip": false,1056 "normalized": true,1057 "special": false1058 },1059 {1060 "id": 100117,1061 "content": "<|s_111|>",1062 "single_word": false,1063 "lstrip": false,1064 "rstrip": false,1065 "normalized": true,1066 "special": false1067 },1068 {1069 "id": 100118,1070 "content": "<|s_112|>",1071 "single_word": false,1072 "lstrip": false,1073 "rstrip": false,1074 "normalized": true,1075 "special": false1076 },1077 {1078 "id": 100119,1079 "content": "<|s_113|>",1080 "single_word": false,1081 "lstrip": false,1082 "rstrip": false,1083 "normalized": true,1084 "special": false1085 },1086 {1087 "id": 100120,1088 "content": "<|s_114|>",1089 "single_word": false,1090 "lstrip": false,1091 "rstrip": false,1092 "normalized": true,1093 "special": false1094 },1095 {1096 "id": 100121,1097 "content": "<|s_115|>",1098 "single_word": false,1099 "lstrip": false,1100 "rstrip": false,1101 "normalized": true,1102 "special": false1103 },1104 {1105 "id": 100122,1106 "content": "<|s_116|>",1107 "single_word": false,1108 "lstrip": false,1109 "rstrip": false,1110 "normalized": true,1111 "special": false1112 },1113 {1114 "id": 100123,1115 "content": "<|s_117|>",1116 "single_word": false,1117 "lstrip": false,1118 "rstrip": false,1119 "normalized": true,1120 "special": false1121 },1122 {1123 "id": 100124,1124 "content": "<|s_118|>",1125 "single_word": false,1126 "lstrip": false,1127 "rstrip": false,1128 "normalized": true,1129 "special": false1130 },1131 {1132 "id": 100125,1133 "content": "<|s_119|>",1134 "single_word": false,1135 "lstrip": false,1136 "rstrip": false,1137 "normalized": true,1138 "special": false1139 },1140 {1141 "id": 100126,1142 "content": "<|s_120|>",1143 "single_word": false,1144 "lstrip": false,1145 "rstrip": false,1146 "normalized": true,1147 "special": false1148 },1149 {1150 "id": 100127,1151 "content": "<|s_121|>",1152 "single_word": false,1153 "lstrip": false,1154 "rstrip": false,1155 "normalized": true,1156 "special": false1157 },1158 {1159 "id": 100128,1160 "content": "<|s_122|>",1161 "single_word": false,1162 "lstrip": false,1163 "rstrip": false,1164 "normalized": true,1165 "special": false1166 },1167 {1168 "id": 100129,1169 "content": "<|s_123|>",1170 "single_word": false,1171 "lstrip": false,1172 "rstrip": false,1173 "normalized": true,1174 "special": false1175 },1176 {1177 "id": 100130,1178 "content": "<|s_124|>",1179 "single_word": false,1180 "lstrip": false,1181 "rstrip": false,1182 "normalized": true,1183 "special": false1184 },1185 {1186 "id": 100131,1187 "content": "<|s_125|>",1188 "single_word": false,1189 "lstrip": false,1190 "rstrip": false,1191 "normalized": true,1192 "special": false1193 },1194 {1195 "id": 100132,1196 "content": "<|s_126|>",1197 "single_word": false,1198 "lstrip": false,1199 "rstrip": false,1200 "normalized": true,