NithishAI/deepseek-xcodec-tokenizer
0
1{2 "add_bos_token": true,3 "add_eos_token": false,4 "add_prefix_space": null,5 "added_tokens_decoder": {6 "100000": {7 "content": "<｜begin▁of▁sentence｜>",8 "lstrip": false,9 "normalized": true,10 "rstrip": false,11 "single_word": false,12 "special": true13 },14 "100001": {15 "content": "<｜end▁of▁sentence｜>",16 "lstrip": false,17 "normalized": true,18 "rstrip": false,19 "single_word": false,20 "special": true21 },22 "100002": {23 "content": "<|TEXT_START|>",24 "lstrip": false,25 "normalized": true,26 "rstrip": false,27 "single_word": false,28 "special": false29 },30 "100003": {31 "content": "<|TEXT_END|>",32 "lstrip": false,33 "normalized": true,34 "rstrip": false,35 "single_word": false,36 "special": false37 },38 "100004": {39 "content": "<|SPEECH_START|>",40 "lstrip": false,41 "normalized": true,42 "rstrip": false,43 "single_word": false,44 "special": false45 },46 "100005": {47 "content": "<|SPEECH_END|>",48 "lstrip": false,49 "normalized": true,50 "rstrip": false,51 "single_word": false,52 "special": false53 },54 "100006": {55 "content": "<|s_0|>",56 "lstrip": false,57 "normalized": true,58 "rstrip": false,59 "single_word": false,60 "special": false61 },62 "100007": {63 "content": "<|s_1|>",64 "lstrip": false,65 "normalized": true,66 "rstrip": false,67 "single_word": false,68 "special": false69 },70 "100008": {71 "content": "<|s_2|>",72 "lstrip": false,73 "normalized": true,74 "rstrip": false,75 "single_word": false,76 "special": false77 },78 "100009": {79 "content": "<|s_3|>",80 "lstrip": false,81 "normalized": true,82 "rstrip": false,83 "single_word": false,84 "special": false85 },86 "100010": {87 "content": "<|s_4|>",88 "lstrip": false,89 "normalized": true,90 "rstrip": false,91 "single_word": false,92 "special": false93 },94 "100011": {95 "content": "<|s_5|>",96 "lstrip": false,97 "normalized": true,98 "rstrip": false,99 "single_word": false,100 "special": false101 },102 "100012": {103 "content": "<|s_6|>",104 "lstrip": false,105 "normalized": true,106 "rstrip": false,107 "single_word": false,108 "special": false109 },110 "100013": {111 "content": "<|s_7|>",112 "lstrip": false,113 "normalized": true,114 "rstrip": false,115 "single_word": false,116 "special": false117 },118 "100014": {119 "content": "<|s_8|>",120 "lstrip": false,121 "normalized": true,122 "rstrip": false,123 "single_word": false,124 "special": false125 },126 "100015": {127 "content": "<|s_9|>",128 "lstrip": false,129 "normalized": true,130 "rstrip": false,131 "single_word": false,132 "special": false133 },134 "100016": {135 "content": "<|s_10|>",136 "lstrip": false,137 "normalized": true,138 "rstrip": false,139 "single_word": false,140 "special": false141 },142 "100017": {143 "content": "<|s_11|>",144 "lstrip": false,145 "normalized": true,146 "rstrip": false,147 "single_word": false,148 "special": false149 },150 "100018": {151 "content": "<|s_12|>",152 "lstrip": false,153 "normalized": true,154 "rstrip": false,155 "single_word": false,156 "special": false157 },158 "100019": {159 "content": "<|s_13|>",160 "lstrip": false,161 "normalized": true,162 "rstrip": false,163 "single_word": false,164 "special": false165 },166 "100020": {167 "content": "<|s_14|>",168 "lstrip": false,169 "normalized": true,170 "rstrip": false,171 "single_word": false,172 "special": false173 },174 "100021": {175 "content": "<|s_15|>",176 "lstrip": false,177 "normalized": true,178 "rstrip": false,179 "single_word": false,180 "special": false181 },182 "100022": {183 "content": "<|s_16|>",184 "lstrip": false,185 "normalized": true,186 "rstrip": false,187 "single_word": false,188 "special": false189 },190 "100023": {191 "content": "<|s_17|>",192 "lstrip": false,193 "normalized": true,194 "rstrip": false,195 "single_word": false,196 "special": false197 },198 "100024": {199 "content": "<|s_18|>",200 "lstrip": false,201 "normalized": true,202 "rstrip": false,203 "single_word": false,204 "special": false205 },206 "100025": {207 "content": "<|s_19|>",208 "lstrip": false,209 "normalized": true,210 "rstrip": false,211 "single_word": false,212 "special": false213 },214 "100026": {215 "content": "<|s_20|>",216 "lstrip": false,217 "normalized": true,218 "rstrip": false,219 "single_word": false,220 "special": false221 },222 "100027": {223 "content": "<|s_21|>",224 "lstrip": false,225 "normalized": true,226 "rstrip": false,227 "single_word": false,228 "special": false229 },230 "100028": {231 "content": "<|s_22|>",232 "lstrip": false,233 "normalized": true,234 "rstrip": false,235 "single_word": false,236 "special": false237 },238 "100029": {239 "content": "<|s_23|>",240 "lstrip": false,241 "normalized": true,242 "rstrip": false,243 "single_word": false,244 "special": false245 },246 "100030": {247 "content": "<|s_24|>",248 "lstrip": false,249 "normalized": true,250 "rstrip": false,251 "single_word": false,252 "special": false253 },254 "100031": {255 "content": "<|s_25|>",256 "lstrip": false,257 "normalized": true,258 "rstrip": false,259 "single_word": false,260 "special": false261 },262 "100032": {263 "content": "<|s_26|>",264 "lstrip": false,265 "normalized": true,266 "rstrip": false,267 "single_word": false,268 "special": false269 },270 "100033": {271 "content": "<|s_27|>",272 "lstrip": false,273 "normalized": true,274 "rstrip": false,275 "single_word": false,276 "special": false277 },278 "100034": {279 "content": "<|s_28|>",280 "lstrip": false,281 "normalized": true,282 "rstrip": false,283 "single_word": false,284 "special": false285 },286 "100035": {287 "content": "<|s_29|>",288 "lstrip": false,289 "normalized": true,290 "rstrip": false,291 "single_word": false,292 "special": false293 },294 "100036": {295 "content": "<|s_30|>",296 "lstrip": false,297 "normalized": true,298 "rstrip": false,299 "single_word": false,300 "special": false301 },302 "100037": {303 "content": "<|s_31|>",304 "lstrip": false,305 "normalized": true,306 "rstrip": false,307 "single_word": false,308 "special": false309 },310 "100038": {311 "content": "<|s_32|>",312 "lstrip": false,313 "normalized": true,314 "rstrip": false,315 "single_word": false,316 "special": false317 },318 "100039": {319 "content": "<|s_33|>",320 "lstrip": false,321 "normalized": true,322 "rstrip": false,323 "single_word": false,324 "special": false325 },326 "100040": {327 "content": "<|s_34|>",328 "lstrip": false,329 "normalized": true,330 "rstrip": false,331 "single_word": false,332 "special": false333 },334 "100041": {335 "content": "<|s_35|>",336 "lstrip": false,337 "normalized": true,338 "rstrip": false,339 "single_word": false,340 "special": false341 },342 "100042": {343 "content": "<|s_36|>",344 "lstrip": false,345 "normalized": true,346 "rstrip": false,347 "single_word": false,348 "special": false349 },350 "100043": {351 "content": "<|s_37|>",352 "lstrip": false,353 "normalized": true,354 "rstrip": false,355 "single_word": false,356 "special": false357 },358 "100044": {359 "content": "<|s_38|>",360 "lstrip": false,361 "normalized": true,362 "rstrip": false,363 "single_word": false,364 "special": false365 },366 "100045": {367 "content": "<|s_39|>",368 "lstrip": false,369 "normalized": true,370 "rstrip": false,371 "single_word": false,372 "special": false373 },374 "100046": {375 "content": "<|s_40|>",376 "lstrip": false,377 "normalized": true,378 "rstrip": false,379 "single_word": false,380 "special": false381 },382 "100047": {383 "content": "<|s_41|>",384 "lstrip": false,385 "normalized": true,386 "rstrip": false,387 "single_word": false,388 "special": false389 },390 "100048": {391 "content": "<|s_42|>",392 "lstrip": false,393 "normalized": true,394 "rstrip": false,395 "single_word": false,396 "special": false397 },398 "100049": {399 "content": "<|s_43|>",400 "lstrip": false,401 "normalized": true,402 "rstrip": false,403 "single_word": false,404 "special": false405 },406 "100050": {407 "content": "<|s_44|>",408 "lstrip": false,409 "normalized": true,410 "rstrip": false,411 "single_word": false,412 "special": false413 },414 "100051": {415 "content": "<|s_45|>",416 "lstrip": false,417 "normalized": true,418 "rstrip": false,419 "single_word": false,420 "special": false421 },422 "100052": {423 "content": "<|s_46|>",424 "lstrip": false,425 "normalized": true,426 "rstrip": false,427 "single_word": false,428 "special": false429 },430 "100053": {431 "content": "<|s_47|>",432 "lstrip": false,433 "normalized": true,434 "rstrip": false,435 "single_word": false,436 "special": false437 },438 "100054": {439 "content": "<|s_48|>",440 "lstrip": false,441 "normalized": true,442 "rstrip": false,443 "single_word": false,444 "special": false445 },446 "100055": {447 "content": "<|s_49|>",448 "lstrip": false,449 "normalized": true,450 "rstrip": false,451 "single_word": false,452 "special": false453 },454 "100056": {455 "content": "<|s_50|>",456 "lstrip": false,457 "normalized": true,458 "rstrip": false,459 "single_word": false,460 "special": false461 },462 "100057": {463 "content": "<|s_51|>",464 "lstrip": false,465 "normalized": true,466 "rstrip": false,467 "single_word": false,468 "special": false469 },470 "100058": {471 "content": "<|s_52|>",472 "lstrip": false,473 "normalized": true,474 "rstrip": false,475 "single_word": false,476 "special": false477 },478 "100059": {479 "content": "<|s_53|>",480 "lstrip": false,481 "normalized": true,482 "rstrip": false,483 "single_word": false,484 "special": false485 },486 "100060": {487 "content": "<|s_54|>",488 "lstrip": false,489 "normalized": true,490 "rstrip": false,491 "single_word": false,492 "special": false493 },494 "100061": {495 "content": "<|s_55|>",496 "lstrip": false,497 "normalized": true,498 "rstrip": false,499 "single_word": false,500 "special": false501 },502 "100062": {503 "content": "<|s_56|>",504 "lstrip": false,505 "normalized": true,506 "rstrip": false,507 "single_word": false,508 "special": false509 },510 "100063": {511 "content": "<|s_57|>",512 "lstrip": false,513 "normalized": true,514 "rstrip": false,515 "single_word": false,516 "special": false517 },518 "100064": {519 "content": "<|s_58|>",520 "lstrip": false,521 "normalized": true,522 "rstrip": false,523 "single_word": false,524 "special": false525 },526 "100065": {527 "content": "<|s_59|>",528 "lstrip": false,529 "normalized": true,530 "rstrip": false,531 "single_word": false,532 "special": false533 },534 "100066": {535 "content": "<|s_60|>",536 "lstrip": false,537 "normalized": true,538 "rstrip": false,539 "single_word": false,540 "special": false541 },542 "100067": {543 "content": "<|s_61|>",544 "lstrip": false,545 "normalized": true,546 "rstrip": false,547 "single_word": false,548 "special": false549 },550 "100068": {551 "content": "<|s_62|>",552 "lstrip": false,553 "normalized": true,554 "rstrip": false,555 "single_word": false,556 "special": false557 },558 "100069": {559 "content": "<|s_63|>",560 "lstrip": false,561 "normalized": true,562 "rstrip": false,563 "single_word": false,564 "special": false565 },566 "100070": {567 "content": "<|s_64|>",568 "lstrip": false,569 "normalized": true,570 "rstrip": false,571 "single_word": false,572 "special": false573 },574 "100071": {575 "content": "<|s_65|>",576 "lstrip": false,577 "normalized": true,578 "rstrip": false,579 "single_word": false,580 "special": false581 },582 "100072": {583 "content": "<|s_66|>",584 "lstrip": false,585 "normalized": true,586 "rstrip": false,587 "single_word": false,588 "special": false589 },590 "100073": {591 "content": "<|s_67|>",592 "lstrip": false,593 "normalized": true,594 "rstrip": false,595 "single_word": false,596 "special": false597 },598 "100074": {599 "content": "<|s_68|>",600 "lstrip": false,601 "normalized": true,602 "rstrip": false,603 "single_word": false,604 "special": false605 },606 "100075": {607 "content": "<|s_69|>",608 "lstrip": false,609 "normalized": true,610 "rstrip": false,611 "single_word": false,612 "special": false613 },614 "100076": {615 "content": "<|s_70|>",616 "lstrip": false,617 "normalized": true,618 "rstrip": false,619 "single_word": false,620 "special": false621 },622 "100077": {623 "content": "<|s_71|>",624 "lstrip": false,625 "normalized": true,626 "rstrip": false,627 "single_word": false,628 "special": false629 },630 "100078": {631 "content": "<|s_72|>",632 "lstrip": false,633 "normalized": true,634 "rstrip": false,635 "single_word": false,636 "special": false637 },638 "100079": {639 "content": "<|s_73|>",640 "lstrip": false,641 "normalized": true,642 "rstrip": false,643 "single_word": false,644 "special": false645 },646 "100080": {647 "content": "<|s_74|>",648 "lstrip": false,649 "normalized": true,650 "rstrip": false,651 "single_word": false,652 "special": false653 },654 "100081": {655 "content": "<|s_75|>",656 "lstrip": false,657 "normalized": true,658 "rstrip": false,659 "single_word": false,660 "special": false661 },662 "100082": {663 "content": "<|s_76|>",664 "lstrip": false,665 "normalized": true,666 "rstrip": false,667 "single_word": false,668 "special": false669 },670 "100083": {671 "content": "<|s_77|>",672 "lstrip": false,673 "normalized": true,674 "rstrip": false,675 "single_word": false,676 "special": false677 },678 "100084": {679 "content": "<|s_78|>",680 "lstrip": false,681 "normalized": true,682 "rstrip": false,683 "single_word": false,684 "special": false685 },686 "100085": {687 "content": "<|s_79|>",688 "lstrip": false,689 "normalized": true,690 "rstrip": false,691 "single_word": false,692 "special": false693 },694 "100086": {695 "content": "<|s_80|>",696 "lstrip": false,697 "normalized": true,698 "rstrip": false,699 "single_word": false,700 "special": false701 },702 "100087": {703 "content": "<|s_81|>",704 "lstrip": false,705 "normalized": true,706 "rstrip": false,707 "single_word": false,708 "special": false709 },710 "100088": {711 "content": "<|s_82|>",712 "lstrip": false,713 "normalized": true,714 "rstrip": false,715 "single_word": false,716 "special": false717 },718 "100089": {719 "content": "<|s_83|>",720 "lstrip": false,721 "normalized": true,722 "rstrip": false,723 "single_word": false,724 "special": false725 },726 "100090": {727 "content": "<|s_84|>",728 "lstrip": false,729 "normalized": true,730 "rstrip": false,731 "single_word": false,732 "special": false733 },734 "100091": {735 "content": "<|s_85|>",736 "lstrip": false,737 "normalized": true,738 "rstrip": false,739 "single_word": false,740 "special": false741 },742 "100092": {743 "content": "<|s_86|>",744 "lstrip": false,745 "normalized": true,746 "rstrip": false,747 "single_word": false,748 "special": false749 },750 "100093": {751 "content": "<|s_87|>",752 "lstrip": false,753 "normalized": true,754 "rstrip": false,755 "single_word": false,756 "special": false757 },758 "100094": {759 "content": "<|s_88|>",760 "lstrip": false,761 "normalized": true,762 "rstrip": false,763 "single_word": false,764 "special": false765 },766 "100095": {767 "content": "<|s_89|>",768 "lstrip": false,769 "normalized": true,770 "rstrip": false,771 "single_word": false,772 "special": false773 },774 "100096": {775 "content": "<|s_90|>",776 "lstrip": false,777 "normalized": true,778 "rstrip": false,779 "single_word": false,780 "special": false781 },782 "100097": {783 "content": "<|s_91|>",784 "lstrip": false,785 "normalized": true,786 "rstrip": false,787 "single_word": false,788 "special": false789 },790 "100098": {791 "content": "<|s_92|>",792 "lstrip": false,793 "normalized": true,794 "rstrip": false,795 "single_word": false,796 "special": false797 },798 "100099": {799 "content": "<|s_93|>",800 "lstrip": false,801 "normalized": true,802 "rstrip": false,803 "single_word": false,804 "special": false805 },806 "100100": {807 "content": "<|s_94|>",808 "lstrip": false,809 "normalized": true,810 "rstrip": false,811 "single_word": false,812 "special": false813 },814 "100101": {815 "content": "<|s_95|>",816 "lstrip": false,817 "normalized": true,818 "rstrip": false,819 "single_word": false,820 "special": false821 },822 "100102": {823 "content": "<|s_96|>",824 "lstrip": false,825 "normalized": true,826 "rstrip": false,827 "single_word": false,828 "special": false829 },830 "100103": {831 "content": "<|s_97|>",832 "lstrip": false,833 "normalized": true,834 "rstrip": false,835 "single_word": false,836 "special": false837 },838 "100104": {839 "content": "<|s_98|>",840 "lstrip": false,841 "normalized": true,842 "rstrip": false,843 "single_word": false,844 "special": false845 },846 "100105": {847 "content": "<|s_99|>",848 "lstrip": false,849 "normalized": true,850 "rstrip": false,851 "single_word": false,852 "special": false853 },854 "100106": {855 "content": "<|s_100|>",856 "lstrip": false,857 "normalized": true,858 "rstrip": false,859 "single_word": false,860 "special": false861 },862 "100107": {863 "content": "<|s_101|>",864 "lstrip": false,865 "normalized": true,866 "rstrip": false,867 "single_word": false,868 "special": false869 },870 "100108": {871 "content": "<|s_102|>",872 "lstrip": false,873 "normalized": true,874 "rstrip": false,875 "single_word": false,876 "special": false877 },878 "100109": {879 "content": "<|s_103|>",880 "lstrip": false,881 "normalized": true,882 "rstrip": false,883 "single_word": false,884 "special": false885 },886 "100110": {887 "content": "<|s_104|>",888 "lstrip": false,889 "normalized": true,890 "rstrip": false,891 "single_word": false,892 "special": false893 },894 "100111": {895 "content": "<|s_105|>",896 "lstrip": false,897 "normalized": true,898 "rstrip": false,899 "single_word": false,900 "special": false901 },902 "100112": {903 "content": "<|s_106|>",904 "lstrip": false,905 "normalized": true,906 "rstrip": false,907 "single_word": false,908 "special": false909 },910 "100113": {911 "content": "<|s_107|>",912 "lstrip": false,913 "normalized": true,914 "rstrip": false,915 "single_word": false,916 "special": false917 },918 "100114": {919 "content": "<|s_108|>",920 "lstrip": false,921 "normalized": true,922 "rstrip": false,923 "single_word": false,924 "special": false925 },926 "100115": {927 "content": "<|s_109|>",928 "lstrip": false,929 "normalized": true,930 "rstrip": false,931 "single_word": false,932 "special": false933 },934 "100116": {935 "content": "<|s_110|>",936 "lstrip": false,937 "normalized": true,938 "rstrip": false,939 "single_word": false,940 "special": false941 },942 "100117": {943 "content": "<|s_111|>",944 "lstrip": false,945 "normalized": true,946 "rstrip": false,947 "single_word": false,948 "special": false949 },950 "100118": {951 "content": "<|s_112|>",952 "lstrip": false,953 "normalized": true,954 "rstrip": false,955 "single_word": false,956 "special": false957 },958 "100119": {959 "content": "<|s_113|>",960 "lstrip": false,961 "normalized": true,962 "rstrip": false,963 "single_word": false,964 "special": false965 },966 "100120": {967 "content": "<|s_114|>",968 "lstrip": false,969 "normalized": true,970 "rstrip": false,971 "single_word": false,972 "special": false973 },974 "100121": {975 "content": "<|s_115|>",976 "lstrip": false,977 "normalized": true,978 "rstrip": false,979 "single_word": false,980 "special": false981 },982 "100122": {983 "content": "<|s_116|>",984 "lstrip": false,985 "normalized": true,986 "rstrip": false,987 "single_word": false,988 "special": false989 },990 "100123": {991 "content": "<|s_117|>",992 "lstrip": false,993 "normalized": true,994 "rstrip": false,995 "single_word": false,996 "special": false997 },998 "100124": {999 "content": "<|s_118|>",1000 "lstrip": false,1001 "normalized": true,1002 "rstrip": false,1003 "single_word": false,1004 "special": false1005 },1006 "100125": {1007 "content": "<|s_119|>",1008 "lstrip": false,1009 "normalized": true,1010 "rstrip": false,1011 "single_word": false,1012 "special": false1013 },1014 "100126": {1015 "content": "<|s_120|>",1016 "lstrip": false,1017 "normalized": true,1018 "rstrip": false,1019 "single_word": false,1020 "special": false1021 },1022 "100127": {1023 "content": "<|s_121|>",1024 "lstrip": false,1025 "normalized": true,1026 "rstrip": false,1027 "single_word": false,1028 "special": false1029 },1030 "100128": {1031 "content": "<|s_122|>",1032 "lstrip": false,1033 "normalized": true,1034 "rstrip": false,1035 "single_word": false,1036 "special": false1037 },1038 "100129": {1039 "content": "<|s_123|>",1040 "lstrip": false,1041 "normalized": true,1042 "rstrip": false,1043 "single_word": false,1044 "special": false1045 },1046 "100130": {1047 "content": "<|s_124|>",1048 "lstrip": false,1049 "normalized": true,1050 "rstrip": false,1051 "single_word": false,1052 "special": false1053 },1054 "100131": {1055 "content": "<|s_125|>",1056 "lstrip": false,1057 "normalized": true,1058 "rstrip": false,1059 "single_word": false,1060 "special": false1061 },1062 "100132": {1063 "content": "<|s_126|>",1064 "lstrip": false,1065 "normalized": true,1066 "rstrip": false,1067 "single_word": false,1068 "special": false1069 },1070 "100133": {1071 "content": "<|s_127|>",1072 "lstrip": false,1073 "normalized": true,1074 "rstrip": false,1075 "single_word": false,1076 "special": false1077 },1078 "100134": {1079 "content": "<|s_128|>",1080 "lstrip": false,1081 "normalized": true,1082 "rstrip": false,1083 "single_word": false,1084 "special": false1085 },1086 "100135": {1087 "content": "<|s_129|>",1088 "lstrip": false,1089 "normalized": true,1090 "rstrip": false,1091 "single_word": false,1092 "special": false1093 },1094 "100136": {1095 "content": "<|s_130|>",1096 "lstrip": false,1097 "normalized": true,1098 "rstrip": false,1099 "single_word": false,1100 "special": false1101 },1102 "100137": {1103 "content": "<|s_131|>",1104 "lstrip": false,1105 "normalized": true,1106 "rstrip": false,1107 "single_word": false,1108 "special": false1109 },1110 "100138": {1111 "content": "<|s_132|>",1112 "lstrip": false,1113 "normalized": true,1114 "rstrip": false,1115 "single_word": false,1116 "special": false1117 },1118 "100139": {1119 "content": "<|s_133|>",1120 "lstrip": false,1121 "normalized": true,1122 "rstrip": false,1123 "single_word": false,1124 "special": false1125 },1126 "100140": {1127 "content": "<|s_134|>",1128 "lstrip": false,1129 "normalized": true,1130 "rstrip": false,1131 "single_word": false,1132 "special": false1133 },1134 "100141": {1135 "content": "<|s_135|>",1136 "lstrip": false,1137 "normalized": true,1138 "rstrip": false,1139 "single_word": false,1140 "special": false1141 },1142 "100142": {1143 "content": "<|s_136|>",1144 "lstrip": false,1145 "normalized": true,1146 "rstrip": false,1147 "single_word": false,1148 "special": false1149 },1150 "100143": {1151 "content": "<|s_137|>",1152 "lstrip": false,1153 "normalized": true,1154 "rstrip": false,1155 "single_word": false,1156 "special": false1157 },1158 "100144": {1159 "content": "<|s_138|>",1160 "lstrip": false,1161 "normalized": true,1162 "rstrip": false,1163 "single_word": false,1164 "special": false1165 },1166 "100145": {1167 "content": "<|s_139|>",1168 "lstrip": false,1169 "normalized": true,1170 "rstrip": false,1171 "single_word": false,1172 "special": false1173 },1174 "100146": {1175 "content": "<|s_140|>",1176 "lstrip": false,1177 "normalized": true,1178 "rstrip": false,1179 "single_word": false,1180 "special": false1181 },1182 "100147": {1183 "content": "<|s_141|>",1184 "lstrip": false,1185 "normalized": true,1186 "rstrip": false,1187 "single_word": false,1188 "special": false1189 },1190 "100148": {1191 "content": "<|s_142|>",1192 "lstrip": false,1193 "normalized": true,1194 "rstrip": false,1195 "single_word": false,1196 "special": false1197 },1198 "100149": {1199 "content": "<|s_143|>",1200 "lstrip": false,