RowRed/ComplexityRouter
021
1{2 "version": "1.0",3 "truncation": {4 "direction": "Right",5 "max_length": 256,6 "strategy": "LongestFirst",7 "stride": 08 },9 "padding": null,10 "added_tokens": [11 {12 "id": 0,13 "content": "[PAD]",14 "single_word": false,15 "lstrip": false,16 "rstrip": false,17 "normalized": false,18 "special": true19 },20 {21 "id": 1,22 "content": "[CLS]",23 "single_word": false,24 "lstrip": false,25 "rstrip": false,26 "normalized": false,27 "special": true28 },29 {30 "id": 2,31 "content": "[SEP]",32 "single_word": false,33 "lstrip": false,34 "rstrip": false,35 "normalized": false,36 "special": true37 },38 {39 "id": 3,40 "content": "[UNK]",41 "single_word": false,42 "lstrip": false,43 "rstrip": false,44 "normalized": false,45 "special": true46 },47 {48 "id": 128000,49 "content": "[MASK]",50 "single_word": false,51 "lstrip": false,52 "rstrip": false,53 "normalized": false,54 "special": true55 }56 ],57 "normalizer": {58 "type": "Sequence",59 "normalizers": [60 {61 "type": "Replace",62 "pattern": {63 "Regex": "\\s{2,}|[\\n\\r\\t]"64 },65 "content": " "66 },67 {68 "type": "NFC"69 },70 {71 "type": "Strip",72 "strip_left": false,73 "strip_right": true74 }75 ]76 },77 "pre_tokenizer": {78 "type": "Sequence",79 "pretokenizers": [80 {81 "type": "Metaspace",82 "replacement": "▁",83 "prepend_scheme": "always",84 "split": true85 }86 ]87 },88 "post_processor": {89 "type": "TemplateProcessing",90 "single": [91 {92 "SpecialToken": {93 "id": "[CLS]",94 "type_id": 095 }96 },97 {98 "Sequence": {99 "id": "A",100 "type_id": 0101 }102 },103 {104 "SpecialToken": {105 "id": "[SEP]",106 "type_id": 0107 }108 }109 ],110 "pair": [111 {112 "SpecialToken": {113 "id": "[CLS]",114 "type_id": 0115 }116 },117 {118 "Sequence": {119 "id": "A",120 "type_id": 0121 }122 },123 {124 "SpecialToken": {125 "id": "[SEP]",126 "type_id": 0127 }128 },129 {130 "Sequence": {131 "id": "B",132 "type_id": 1133 }134 },135 {136 "SpecialToken": {137 "id": "[SEP]",138 "type_id": 1139 }140 }141 ],142 "special_tokens": {143 "[CLS]": {144 "id": "[CLS]",145 "ids": [146 1147 ],148 "tokens": [149 "[CLS]"150 ]151 },152 "[SEP]": {153 "id": "[SEP]",154 "ids": [155 2156 ],157 "tokens": [158 "[SEP]"159 ]160 }161 }162 },163 "decoder": {164 "type": "Metaspace",165 "replacement": "▁",166 "prepend_scheme": "always",167 "split": true168 },169 "model": {170 "type": "Unigram",171 "unk_id": 3,172 "vocab": [173 [174 "[PAD]",175 0.0176 ],177 [178 "[CLS]",179 0.0180 ],181 [182 "[SEP]",183 0.0184 ],185 [186 "[UNK]",187 0.0188 ],189 [190 "<0x00>",191 0.0192 ],193 [194 "<0x01>",195 0.0196 ],197 [198 "<0x02>",199 0.0200 ],201 [202 "<0x03>",203 0.0204 ],205 [206 "<0x04>",207 0.0208 ],209 [210 "<0x05>",211 0.0212 ],213 [214 "<0x06>",215 0.0216 ],217 [218 "<0x07>",219 0.0220 ],221 [222 "<0x08>",223 0.0224 ],225 [226 "<0x09>",227 0.0228 ],229 [230 "<0x0A>",231 0.0232 ],233 [234 "<0x0B>",235 0.0236 ],237 [238 "<0x0C>",239 0.0240 ],241 [242 "<0x0D>",243 0.0244 ],245 [246 "<0x0E>",247 0.0248 ],249 [250 "<0x0F>",251 0.0252 ],253 [254 "<0x10>",255 0.0256 ],257 [258 "<0x11>",259 0.0260 ],261 [262 "<0x12>",263 0.0264 ],265 [266 "<0x13>",267 0.0268 ],269 [270 "<0x14>",271 0.0272 ],273 [274 "<0x15>",275 0.0276 ],277 [278 "<0x16>",279 0.0280 ],281 [282 "<0x17>",283 0.0284 ],285 [286 "<0x18>",287 0.0288 ],289 [290 "<0x19>",291 0.0292 ],293 [294 "<0x1A>",295 0.0296 ],297 [298 "<0x1B>",299 0.0300 ],301 [302 "<0x1C>",303 0.0304 ],305 [306 "<0x1D>",307 0.0308 ],309 [310 "<0x1E>",311 0.0312 ],313 [314 "<0x1F>",315 0.0316 ],317 [318 "<0x20>",319 0.0320 ],321 [322 "<0x21>",323 0.0324 ],325 [326 "<0x22>",327 0.0328 ],329 [330 "<0x23>",331 0.0332 ],333 [334 "<0x24>",335 0.0336 ],337 [338 "<0x25>",339 0.0340 ],341 [342 "<0x26>",343 0.0344 ],345 [346 "<0x27>",347 0.0348 ],349 [350 "<0x28>",351 0.0352 ],353 [354 "<0x29>",355 0.0356 ],357 [358 "<0x2A>",359 0.0360 ],361 [362 "<0x2B>",363 0.0364 ],365 [366 "<0x2C>",367 0.0368 ],369 [370 "<0x2D>",371 0.0372 ],373 [374 "<0x2E>",375 0.0376 ],377 [378 "<0x2F>",379 0.0380 ],381 [382 "<0x30>",383 0.0384 ],385 [386 "<0x31>",387 0.0388 ],389 [390 "<0x32>",391 0.0392 ],393 [394 "<0x33>",395 0.0396 ],397 [398 "<0x34>",399 0.0400 ],401 [402 "<0x35>",403 0.0404 ],405 [406 "<0x36>",407 0.0408 ],409 [410 "<0x37>",411 0.0412 ],413 [414 "<0x38>",415 0.0416 ],417 [418 "<0x39>",419 0.0420 ],421 [422 "<0x3A>",423 0.0424 ],425 [426 "<0x3B>",427 0.0428 ],429 [430 "<0x3C>",431 0.0432 ],433 [434 "<0x3D>",435 0.0436 ],437 [438 "<0x3E>",439 0.0440 ],441 [442 "<0x3F>",443 0.0444 ],445 [446 "<0x40>",447 0.0448 ],449 [450 "<0x41>",451 0.0452 ],453 [454 "<0x42>",455 0.0456 ],457 [458 "<0x43>",459 0.0460 ],461 [462 "<0x44>",463 0.0464 ],465 [466 "<0x45>",467 0.0468 ],469 [470 "<0x46>",471 0.0472 ],473 [474 "<0x47>",475 0.0476 ],477 [478 "<0x48>",479 0.0480 ],481 [482 "<0x49>",483 0.0484 ],485 [486 "<0x4A>",487 0.0488 ],489 [490 "<0x4B>",491 0.0492 ],493 [494 "<0x4C>",495 0.0496 ],497 [498 "<0x4D>",499 0.0500 ],501 [502 "<0x4E>",503 0.0504 ],505 [506 "<0x4F>",507 0.0508 ],509 [510 "<0x50>",511 0.0512 ],513 [514 "<0x51>",515 0.0516 ],517 [518 "<0x52>",519 0.0520 ],521 [522 "<0x53>",523 0.0524 ],525 [526 "<0x54>",527 0.0528 ],529 [530 "<0x55>",531 0.0532 ],533 [534 "<0x56>",535 0.0536 ],537 [538 "<0x57>",539 0.0540 ],541 [542 "<0x58>",543 0.0544 ],545 [546 "<0x59>",547 0.0548 ],549 [550 "<0x5A>",551 0.0552 ],553 [554 "<0x5B>",555 0.0556 ],557 [558 "<0x5C>",559 0.0560 ],561 [562 "<0x5D>",563 0.0564 ],565 [566 "<0x5E>",567 0.0568 ],569 [570 "<0x5F>",571 0.0572 ],573 [574 "<0x60>",575 0.0576 ],577 [578 "<0x61>",579 0.0580 ],581 [582 "<0x62>",583 0.0584 ],585 [586 "<0x63>",587 0.0588 ],589 [590 "<0x64>",591 0.0592 ],593 [594 "<0x65>",595 0.0596 ],597 [598 "<0x66>",599 0.0600 ],601 [602 "<0x67>",603 0.0604 ],605 [606 "<0x68>",607 0.0608 ],609 [610 "<0x69>",611 0.0612 ],613 [614 "<0x6A>",615 0.0616 ],617 [618 "<0x6B>",619 0.0620 ],621 [622 "<0x6C>",623 0.0624 ],625 [626 "<0x6D>",627 0.0628 ],629 [630 "<0x6E>",631 0.0632 ],633 [634 "<0x6F>",635 0.0636 ],637 [638 "<0x70>",639 0.0640 ],641 [642 "<0x71>",643 0.0644 ],645 [646 "<0x72>",647 0.0648 ],649 [650 "<0x73>",651 0.0652 ],653 [654 "<0x74>",655 0.0656 ],657 [658 "<0x75>",659 0.0660 ],661 [662 "<0x76>",663 0.0664 ],665 [666 "<0x77>",667 0.0668 ],669 [670 "<0x78>",671 0.0672 ],673 [674 "<0x79>",675 0.0676 ],677 [678 "<0x7A>",679 0.0680 ],681 [682 "<0x7B>",683 0.0684 ],685 [686 "<0x7C>",687 0.0688 ],689 [690 "<0x7D>",691 0.0692 ],693 [694 "<0x7E>",695 0.0696 ],697 [698 "<0x7F>",699 0.0700 ],701 [702 "<0x80>",703 0.0704 ],705 [706 "<0x81>",707 0.0708 ],709 [710 "<0x82>",711 0.0712 ],713 [714 "<0x83>",715 0.0716 ],717 [718 "<0x84>",719 0.0720 ],721 [722 "<0x85>",723 0.0724 ],725 [726 "<0x86>",727 0.0728 ],729 [730 "<0x87>",731 0.0732 ],733 [734 "<0x88>",735 0.0736 ],737 [738 "<0x89>",739 0.0740 ],741 [742 "<0x8A>",743 0.0744 ],745 [746 "<0x8B>",747 0.0748 ],749 [750 "<0x8C>",751 0.0752 ],753 [754 "<0x8D>",755 0.0756 ],757 [758 "<0x8E>",759 0.0760 ],761 [762 "<0x8F>",763 0.0764 ],765 [766 "<0x90>",767 0.0768 ],769 [770 "<0x91>",771 0.0772 ],773 [774 "<0x92>",775 0.0776 ],777 [778 "<0x93>",779 0.0780 ],781 [782 "<0x94>",783 0.0784 ],785 [786 "<0x95>",787 0.0788 ],789 [790 "<0x96>",791 0.0792 ],793 [794 "<0x97>",795 0.0796 ],797 [798 "<0x98>",799 0.0800 ],801 [802 "<0x99>",803 0.0804 ],805 [806 "<0x9A>",807 0.0808 ],809 [810 "<0x9B>",811 0.0812 ],813 [814 "<0x9C>",815 0.0816 ],817 [818 "<0x9D>",819 0.0820 ],821 [822 "<0x9E>",823 0.0824 ],825 [826 "<0x9F>",827 0.0828 ],829 [830 "<0xA0>",831 0.0832 ],833 [834 "<0xA1>",835 0.0836 ],837 [838 "<0xA2>",839 0.0840 ],841 [842 "<0xA3>",843 0.0844 ],845 [846 "<0xA4>",847 0.0848 ],849 [850 "<0xA5>",851 0.0852 ],853 [854 "<0xA6>",855 0.0856 ],857 [858 "<0xA7>",859 0.0860 ],861 [862 "<0xA8>",863 0.0864 ],865 [866 "<0xA9>",867 0.0868 ],869 [870 "<0xAA>",871 0.0872 ],873 [874 "<0xAB>",875 0.0876 ],877 [878 "<0xAC>",879 0.0880 ],881 [882 "<0xAD>",883 0.0884 ],885 [886 "<0xAE>",887 0.0888 ],889 [890 "<0xAF>",891 0.0892 ],893 [894 "<0xB0>",895 0.0896 ],897 [898 "<0xB1>",899 0.0900 ],901 [902 "<0xB2>",903 0.0904 ],905 [906 "<0xB3>",907 0.0908 ],909 [910 "<0xB4>",911 0.0912 ],913 [914 "<0xB5>",915 0.0916 ],917 [918 "<0xB6>",919 0.0920 ],921 [922 "<0xB7>",923 0.0924 ],925 [926 "<0xB8>",927 0.0928 ],929 [930 "<0xB9>",931 0.0932 ],933 [934 "<0xBA>",935 0.0936 ],937 [938 "<0xBB>",939 0.0940 ],941 [942 "<0xBC>",943 0.0944 ],945 [946 "<0xBD>",947 0.0948 ],949 [950 "<0xBE>",951 0.0952 ],953 [954 "<0xBF>",955 0.0956 ],957 [958 "<0xC0>",959 0.0960 ],961 [962 "<0xC1>",963 0.0964 ],965 [966 "<0xC2>",967 0.0968 ],969 [970 "<0xC3>",971 0.0972 ],973 [974 "<0xC4>",975 0.0976 ],977 [978 "<0xC5>",979 0.0980 ],981 [982 "<0xC6>",983 0.0984 ],985 [986 "<0xC7>",987 0.0988 ],989 [990 "<0xC8>",991 0.0992 ],993 [994 "<0xC9>",995 0.0996 ],997 [998 "<0xCA>",999 0.01000 ],1001 [1002 "<0xCB>",1003 0.01004 ],1005 [1006 "<0xCC>",1007 0.01008 ],1009 [1010 "<0xCD>",1011 0.01012 ],1013 [1014 "<0xCE>",1015 0.01016 ],1017 [1018 "<0xCF>",1019 0.01020 ],1021 [1022 "<0xD0>",1023 0.01024 ],1025 [1026 "<0xD1>",1027 0.01028 ],1029 [1030 "<0xD2>",1031 0.01032 ],1033 [1034 "<0xD3>",1035 0.01036 ],1037 [1038 "<0xD4>",1039 0.01040 ],1041 [1042 "<0xD5>",1043 0.01044 ],1045 [1046 "<0xD6>",1047 0.01048 ],1049 [1050 "<0xD7>",1051 0.01052 ],1053 [1054 "<0xD8>",1055 0.01056 ],1057 [1058 "<0xD9>",1059 0.01060 ],1061 [1062 "<0xDA>",1063 0.01064 ],1065 [1066 "<0xDB>",1067 0.01068 ],1069 [1070 "<0xDC>",1071 0.01072 ],1073 [1074 "<0xDD>",1075 0.01076 ],1077 [1078 "<0xDE>",1079 0.01080 ],1081 [1082 "<0xDF>",1083 0.01084 ],1085 [1086 "<0xE0>",1087 0.01088 ],1089 [1090 "<0xE1>",1091 0.01092 ],1093 [1094 "<0xE2>",1095 0.01096 ],1097 [1098 "<0xE3>",1099 0.01100 ],1101 [1102 "<0xE4>",1103 0.01104 ],1105 [1106 "<0xE5>",1107 0.01108 ],1109 [1110 "<0xE6>",1111 0.01112 ],1113 [1114 "<0xE7>",1115 0.01116 ],1117 [1118 "<0xE8>",1119 0.01120 ],1121 [1122 "<0xE9>",1123 0.01124 ],1125 [1126 "<0xEA>",1127 0.01128 ],1129 [1130 "<0xEB>",1131 0.01132 ],1133 [1134 "<0xEC>",1135 0.01136 ],1137 [1138 "<0xED>",1139 0.01140 ],1141 [1142 "<0xEE>",1143 0.01144 ],1145 [1146 "<0xEF>",1147 0.01148 ],1149 [1150 "<0xF0>",1151 0.01152 ],1153 [1154 "<0xF1>",1155 0.01156 ],1157 [1158 "<0xF2>",1159 0.01160 ],1161 [1162 "<0xF3>",1163 0.01164 ],1165 [1166 "<0xF4>",1167 0.01168 ],1169 [1170 "<0xF5>",1171 0.01172 ],1173 [1174 "<0xF6>",1175 0.01176 ],1177 [1178 "<0xF7>",1179 0.01180 ],1181 [1182 "<0xF8>",1183 0.01184 ],1185 [1186 "<0xF9>",1187 0.01188 ],1189 [1190 "<0xFA>",1191 0.01192 ],1193 [1194 "<0xFB>",1195 0.01196 ],1197 [1198 "<0xFC>",1199 0.01200 ],