underscore2/modernbert_large_slop_writing
05
1{2 "version": "1.0",3 "truncation": {4 "direction": "Right",5 "max_length": 8192,6 "strategy": "LongestFirst",7 "stride": 08 },9 "padding": null,10 "added_tokens": [11 {12 "id": 0,13 "content": "|||IP_ADDRESS|||",14 "single_word": false,15 "lstrip": false,16 "rstrip": false,17 "normalized": true,18 "special": false19 },20 {21 "id": 1,22 "content": "<|padding|>",23 "single_word": false,24 "lstrip": false,25 "rstrip": false,26 "normalized": false,27 "special": true28 },29 {30 "id": 50254,31 "content": " ",32 "single_word": false,33 "lstrip": false,34 "rstrip": false,35 "normalized": true,36 "special": false37 },38 {39 "id": 50255,40 "content": " ",41 "single_word": false,42 "lstrip": false,43 "rstrip": false,44 "normalized": true,45 "special": false46 },47 {48 "id": 50256,49 "content": " ",50 "single_word": false,51 "lstrip": false,52 "rstrip": false,53 "normalized": true,54 "special": false55 },56 {57 "id": 50257,58 "content": " ",59 "single_word": false,60 "lstrip": false,61 "rstrip": false,62 "normalized": true,63 "special": false64 },65 {66 "id": 50258,67 "content": " ",68 "single_word": false,69 "lstrip": false,70 "rstrip": false,71 "normalized": true,72 "special": false73 },74 {75 "id": 50259,76 "content": " ",77 "single_word": false,78 "lstrip": false,79 "rstrip": false,80 "normalized": true,81 "special": false82 },83 {84 "id": 50260,85 "content": " ",86 "single_word": false,87 "lstrip": false,88 "rstrip": false,89 "normalized": true,90 "special": false91 },92 {93 "id": 50261,94 "content": " ",95 "single_word": false,96 "lstrip": false,97 "rstrip": false,98 "normalized": true,99 "special": false100 },101 {102 "id": 50262,103 "content": " ",104 "single_word": false,105 "lstrip": false,106 "rstrip": false,107 "normalized": true,108 "special": false109 },110 {111 "id": 50263,112 "content": " ",113 "single_word": false,114 "lstrip": false,115 "rstrip": false,116 "normalized": true,117 "special": false118 },119 {120 "id": 50264,121 "content": " ",122 "single_word": false,123 "lstrip": false,124 "rstrip": false,125 "normalized": true,126 "special": false127 },128 {129 "id": 50265,130 "content": " ",131 "single_word": false,132 "lstrip": false,133 "rstrip": false,134 "normalized": true,135 "special": false136 },137 {138 "id": 50266,139 "content": " ",140 "single_word": false,141 "lstrip": false,142 "rstrip": false,143 "normalized": true,144 "special": false145 },146 {147 "id": 50267,148 "content": " ",149 "single_word": false,150 "lstrip": false,151 "rstrip": false,152 "normalized": true,153 "special": false154 },155 {156 "id": 50268,157 "content": " ",158 "single_word": false,159 "lstrip": false,160 "rstrip": false,161 "normalized": true,162 "special": false163 },164 {165 "id": 50269,166 "content": " ",167 "single_word": false,168 "lstrip": false,169 "rstrip": false,170 "normalized": true,171 "special": false172 },173 {174 "id": 50270,175 "content": " ",176 "single_word": false,177 "lstrip": false,178 "rstrip": false,179 "normalized": true,180 "special": false181 },182 {183 "id": 50271,184 "content": " ",185 "single_word": false,186 "lstrip": false,187 "rstrip": false,188 "normalized": true,189 "special": false190 },191 {192 "id": 50272,193 "content": " ",194 "single_word": false,195 "lstrip": false,196 "rstrip": false,197 "normalized": true,198 "special": false199 },200 {201 "id": 50273,202 "content": " ",203 "single_word": false,204 "lstrip": false,205 "rstrip": false,206 "normalized": true,207 "special": false208 },209 {210 "id": 50274,211 "content": " ",212 "single_word": false,213 "lstrip": false,214 "rstrip": false,215 "normalized": true,216 "special": false217 },218 {219 "id": 50275,220 "content": " ",221 "single_word": false,222 "lstrip": false,223 "rstrip": false,224 "normalized": true,225 "special": false226 },227 {228 "id": 50276,229 "content": " ",230 "single_word": false,231 "lstrip": false,232 "rstrip": false,233 "normalized": true,234 "special": false235 },236 {237 "id": 50277,238 "content": "|||EMAIL_ADDRESS|||",239 "single_word": false,240 "lstrip": false,241 "rstrip": false,242 "normalized": true,243 "special": false244 },245 {246 "id": 50278,247 "content": "|||PHONE_NUMBER|||",248 "single_word": false,249 "lstrip": false,250 "rstrip": false,251 "normalized": true,252 "special": false253 },254 {255 "id": 50279,256 "content": "<|endoftext|>",257 "single_word": false,258 "lstrip": false,259 "rstrip": false,260 "normalized": false,261 "special": true262 },263 {264 "id": 50280,265 "content": "[UNK]",266 "single_word": false,267 "lstrip": false,268 "rstrip": false,269 "normalized": false,270 "special": true271 },272 {273 "id": 50281,274 "content": "[CLS]",275 "single_word": false,276 "lstrip": false,277 "rstrip": false,278 "normalized": false,279 "special": true280 },281 {282 "id": 50282,283 "content": "[SEP]",284 "single_word": false,285 "lstrip": false,286 "rstrip": false,287 "normalized": false,288 "special": true289 },290 {291 "id": 50283,292 "content": "[PAD]",293 "single_word": false,294 "lstrip": false,295 "rstrip": false,296 "normalized": false,297 "special": true298 },299 {300 "id": 50284,301 "content": "[MASK]",302 "single_word": false,303 "lstrip": true,304 "rstrip": false,305 "normalized": false,306 "special": true307 },308 {309 "id": 50285,310 "content": "[unused0]",311 "single_word": false,312 "lstrip": false,313 "rstrip": false,314 "normalized": true,315 "special": false316 },317 {318 "id": 50286,319 "content": "[unused1]",320 "single_word": false,321 "lstrip": false,322 "rstrip": false,323 "normalized": true,324 "special": false325 },326 {327 "id": 50287,328 "content": "[unused2]",329 "single_word": false,330 "lstrip": false,331 "rstrip": false,332 "normalized": true,333 "special": false334 },335 {336 "id": 50288,337 "content": "[unused3]",338 "single_word": false,339 "lstrip": false,340 "rstrip": false,341 "normalized": true,342 "special": false343 },344 {345 "id": 50289,346 "content": "[unused4]",347 "single_word": false,348 "lstrip": false,349 "rstrip": false,350 "normalized": true,351 "special": false352 },353 {354 "id": 50290,355 "content": "[unused5]",356 "single_word": false,357 "lstrip": false,358 "rstrip": false,359 "normalized": true,360 "special": false361 },362 {363 "id": 50291,364 "content": "[unused6]",365 "single_word": false,366 "lstrip": false,367 "rstrip": false,368 "normalized": true,369 "special": false370 },371 {372 "id": 50292,373 "content": "[unused7]",374 "single_word": false,375 "lstrip": false,376 "rstrip": false,377 "normalized": true,378 "special": false379 },380 {381 "id": 50293,382 "content": "[unused8]",383 "single_word": false,384 "lstrip": false,385 "rstrip": false,386 "normalized": true,387 "special": false388 },389 {390 "id": 50294,391 "content": "[unused9]",392 "single_word": false,393 "lstrip": false,394 "rstrip": false,395 "normalized": true,396 "special": false397 },398 {399 "id": 50295,400 "content": "[unused10]",401 "single_word": false,402 "lstrip": false,403 "rstrip": false,404 "normalized": true,405 "special": false406 },407 {408 "id": 50296,409 "content": "[unused11]",410 "single_word": false,411 "lstrip": false,412 "rstrip": false,413 "normalized": true,414 "special": false415 },416 {417 "id": 50297,418 "content": "[unused12]",419 "single_word": false,420 "lstrip": false,421 "rstrip": false,422 "normalized": true,423 "special": false424 },425 {426 "id": 50298,427 "content": "[unused13]",428 "single_word": false,429 "lstrip": false,430 "rstrip": false,431 "normalized": true,432 "special": false433 },434 {435 "id": 50299,436 "content": "[unused14]",437 "single_word": false,438 "lstrip": false,439 "rstrip": false,440 "normalized": true,441 "special": false442 },443 {444 "id": 50300,445 "content": "[unused15]",446 "single_word": false,447 "lstrip": false,448 "rstrip": false,449 "normalized": true,450 "special": false451 },452 {453 "id": 50301,454 "content": "[unused16]",455 "single_word": false,456 "lstrip": false,457 "rstrip": false,458 "normalized": true,459 "special": false460 },461 {462 "id": 50302,463 "content": "[unused17]",464 "single_word": false,465 "lstrip": false,466 "rstrip": false,467 "normalized": true,468 "special": false469 },470 {471 "id": 50303,472 "content": "[unused18]",473 "single_word": false,474 "lstrip": false,475 "rstrip": false,476 "normalized": true,477 "special": false478 },479 {480 "id": 50304,481 "content": "[unused19]",482 "single_word": false,483 "lstrip": false,484 "rstrip": false,485 "normalized": true,486 "special": false487 },488 {489 "id": 50305,490 "content": "[unused20]",491 "single_word": false,492 "lstrip": false,493 "rstrip": false,494 "normalized": true,495 "special": false496 },497 {498 "id": 50306,499 "content": "[unused21]",500 "single_word": false,501 "lstrip": false,502 "rstrip": false,503 "normalized": true,504 "special": false505 },506 {507 "id": 50307,508 "content": "[unused22]",509 "single_word": false,510 "lstrip": false,511 "rstrip": false,512 "normalized": true,513 "special": false514 },515 {516 "id": 50308,517 "content": "[unused23]",518 "single_word": false,519 "lstrip": false,520 "rstrip": false,521 "normalized": true,522 "special": false523 },524 {525 "id": 50309,526 "content": "[unused24]",527 "single_word": false,528 "lstrip": false,529 "rstrip": false,530 "normalized": true,531 "special": false532 },533 {534 "id": 50310,535 "content": "[unused25]",536 "single_word": false,537 "lstrip": false,538 "rstrip": false,539 "normalized": true,540 "special": false541 },542 {543 "id": 50311,544 "content": "[unused26]",545 "single_word": false,546 "lstrip": false,547 "rstrip": false,548 "normalized": true,549 "special": false550 },551 {552 "id": 50312,553 "content": "[unused27]",554 "single_word": false,555 "lstrip": false,556 "rstrip": false,557 "normalized": true,558 "special": false559 },560 {561 "id": 50313,562 "content": "[unused28]",563 "single_word": false,564 "lstrip": false,565 "rstrip": false,566 "normalized": true,567 "special": false568 },569 {570 "id": 50314,571 "content": "[unused29]",572 "single_word": false,573 "lstrip": false,574 "rstrip": false,575 "normalized": true,576 "special": false577 },578 {579 "id": 50315,580 "content": "[unused30]",581 "single_word": false,582 "lstrip": false,583 "rstrip": false,584 "normalized": true,585 "special": false586 },587 {588 "id": 50316,589 "content": "[unused31]",590 "single_word": false,591 "lstrip": false,592 "rstrip": false,593 "normalized": true,594 "special": false595 },596 {597 "id": 50317,598 "content": "[unused32]",599 "single_word": false,600 "lstrip": false,601 "rstrip": false,602 "normalized": true,603 "special": false604 },605 {606 "id": 50318,607 "content": "[unused33]",608 "single_word": false,609 "lstrip": false,610 "rstrip": false,611 "normalized": true,612 "special": false613 },614 {615 "id": 50319,616 "content": "[unused34]",617 "single_word": false,618 "lstrip": false,619 "rstrip": false,620 "normalized": true,621 "special": false622 },623 {624 "id": 50320,625 "content": "[unused35]",626 "single_word": false,627 "lstrip": false,628 "rstrip": false,629 "normalized": true,630 "special": false631 },632 {633 "id": 50321,634 "content": "[unused36]",635 "single_word": false,636 "lstrip": false,637 "rstrip": false,638 "normalized": true,639 "special": false640 },641 {642 "id": 50322,643 "content": "[unused37]",644 "single_word": false,645 "lstrip": false,646 "rstrip": false,647 "normalized": true,648 "special": false649 },650 {651 "id": 50323,652 "content": "[unused38]",653 "single_word": false,654 "lstrip": false,655 "rstrip": false,656 "normalized": true,657 "special": false658 },659 {660 "id": 50324,661 "content": "[unused39]",662 "single_word": false,663 "lstrip": false,664 "rstrip": false,665 "normalized": true,666 "special": false667 },668 {669 "id": 50325,670 "content": "[unused40]",671 "single_word": false,672 "lstrip": false,673 "rstrip": false,674 "normalized": true,675 "special": false676 },677 {678 "id": 50326,679 "content": "[unused41]",680 "single_word": false,681 "lstrip": false,682 "rstrip": false,683 "normalized": true,684 "special": false685 },686 {687 "id": 50327,688 "content": "[unused42]",689 "single_word": false,690 "lstrip": false,691 "rstrip": false,692 "normalized": true,693 "special": false694 },695 {696 "id": 50328,697 "content": "[unused43]",698 "single_word": false,699 "lstrip": false,700 "rstrip": false,701 "normalized": true,702 "special": false703 },704 {705 "id": 50329,706 "content": "[unused44]",707 "single_word": false,708 "lstrip": false,709 "rstrip": false,710 "normalized": true,711 "special": false712 },713 {714 "id": 50330,715 "content": "[unused45]",716 "single_word": false,717 "lstrip": false,718 "rstrip": false,719 "normalized": true,720 "special": false721 },722 {723 "id": 50331,724 "content": "[unused46]",725 "single_word": false,726 "lstrip": false,727 "rstrip": false,728 "normalized": true,729 "special": false730 },731 {732 "id": 50332,733 "content": "[unused47]",734 "single_word": false,735 "lstrip": false,736 "rstrip": false,737 "normalized": true,738 "special": false739 },740 {741 "id": 50333,742 "content": "[unused48]",743 "single_word": false,744 "lstrip": false,745 "rstrip": false,746 "normalized": true,747 "special": false748 },749 {750 "id": 50334,751 "content": "[unused49]",752 "single_word": false,753 "lstrip": false,754 "rstrip": false,755 "normalized": true,756 "special": false757 },758 {759 "id": 50335,760 "content": "[unused50]",761 "single_word": false,762 "lstrip": false,763 "rstrip": false,764 "normalized": true,765 "special": false766 },767 {768 "id": 50336,769 "content": "[unused51]",770 "single_word": false,771 "lstrip": false,772 "rstrip": false,773 "normalized": true,774 "special": false775 },776 {777 "id": 50337,778 "content": "[unused52]",779 "single_word": false,780 "lstrip": false,781 "rstrip": false,782 "normalized": true,783 "special": false784 },785 {786 "id": 50338,787 "content": "[unused53]",788 "single_word": false,789 "lstrip": false,790 "rstrip": false,791 "normalized": true,792 "special": false793 },794 {795 "id": 50339,796 "content": "[unused54]",797 "single_word": false,798 "lstrip": false,799 "rstrip": false,800 "normalized": true,801 "special": false802 },803 {804 "id": 50340,805 "content": "[unused55]",806 "single_word": false,807 "lstrip": false,808 "rstrip": false,809 "normalized": true,810 "special": false811 },812 {813 "id": 50341,814 "content": "[unused56]",815 "single_word": false,816 "lstrip": false,817 "rstrip": false,818 "normalized": true,819 "special": false820 },821 {822 "id": 50342,823 "content": "[unused57]",824 "single_word": false,825 "lstrip": false,826 "rstrip": false,827 "normalized": true,828 "special": false829 },830 {831 "id": 50343,832 "content": "[unused58]",833 "single_word": false,834 "lstrip": false,835 "rstrip": false,836 "normalized": true,837 "special": false838 },839 {840 "id": 50344,841 "content": "[unused59]",842 "single_word": false,843 "lstrip": false,844 "rstrip": false,845 "normalized": true,846 "special": false847 },848 {849 "id": 50345,850 "content": "[unused60]",851 "single_word": false,852 "lstrip": false,853 "rstrip": false,854 "normalized": true,855 "special": false856 },857 {858 "id": 50346,859 "content": "[unused61]",860 "single_word": false,861 "lstrip": false,862 "rstrip": false,863 "normalized": true,864 "special": false865 },866 {867 "id": 50347,868 "content": "[unused62]",869 "single_word": false,870 "lstrip": false,871 "rstrip": false,872 "normalized": true,873 "special": false874 },875 {876 "id": 50348,877 "content": "[unused63]",878 "single_word": false,879 "lstrip": false,880 "rstrip": false,881 "normalized": true,882 "special": false883 },884 {885 "id": 50349,886 "content": "[unused64]",887 "single_word": false,888 "lstrip": false,889 "rstrip": false,890 "normalized": true,891 "special": false892 },893 {894 "id": 50350,895 "content": "[unused65]",896 "single_word": false,897 "lstrip": false,898 "rstrip": false,899 "normalized": true,900 "special": false901 },902 {903 "id": 50351,904 "content": "[unused66]",905 "single_word": false,906 "lstrip": false,907 "rstrip": false,908 "normalized": true,909 "special": false910 },911 {912 "id": 50352,913 "content": "[unused67]",914 "single_word": false,915 "lstrip": false,916 "rstrip": false,917 "normalized": true,918 "special": false919 },920 {921 "id": 50353,922 "content": "[unused68]",923 "single_word": false,924 "lstrip": false,925 "rstrip": false,926 "normalized": true,927 "special": false928 },929 {930 "id": 50354,931 "content": "[unused69]",932 "single_word": false,933 "lstrip": false,934 "rstrip": false,935 "normalized": true,936 "special": false937 },938 {939 "id": 50355,940 "content": "[unused70]",941 "single_word": false,942 "lstrip": false,943 "rstrip": false,944 "normalized": true,945 "special": false946 },947 {948 "id": 50356,949 "content": "[unused71]",950 "single_word": false,951 "lstrip": false,952 "rstrip": false,953 "normalized": true,954 "special": false955 },956 {957 "id": 50357,958 "content": "[unused72]",959 "single_word": false,960 "lstrip": false,961 "rstrip": false,962 "normalized": true,963 "special": false964 },965 {966 "id": 50358,967 "content": "[unused73]",968 "single_word": false,969 "lstrip": false,970 "rstrip": false,971 "normalized": true,972 "special": false973 },974 {975 "id": 50359,976 "content": "[unused74]",977 "single_word": false,978 "lstrip": false,979 "rstrip": false,980 "normalized": true,981 "special": false982 },983 {984 "id": 50360,985 "content": "[unused75]",986 "single_word": false,987 "lstrip": false,988 "rstrip": false,989 "normalized": true,990 "special": false991 },992 {993 "id": 50361,994 "content": "[unused76]",995 "single_word": false,996 "lstrip": false,997 "rstrip": false,998 "normalized": true,999 "special": false1000 },1001 {1002 "id": 50362,1003 "content": "[unused77]",1004 "single_word": false,1005 "lstrip": false,1006 "rstrip": false,1007 "normalized": true,1008 "special": false1009 },1010 {1011 "id": 50363,1012 "content": "[unused78]",1013 "single_word": false,1014 "lstrip": false,1015 "rstrip": false,1016 "normalized": true,1017 "special": false1018 },1019 {1020 "id": 50364,1021 "content": "[unused79]",1022 "single_word": false,1023 "lstrip": false,1024 "rstrip": false,1025 "normalized": true,1026 "special": false1027 },1028 {1029 "id": 50365,1030 "content": "[unused80]",1031 "single_word": false,1032 "lstrip": false,1033 "rstrip": false,1034 "normalized": true,1035 "special": false1036 },1037 {1038 "id": 50366,1039 "content": "[unused81]",1040 "single_word": false,1041 "lstrip": false,1042 "rstrip": false,1043 "normalized": true,1044 "special": false1045 },1046 {1047 "id": 50367,1048 "content": "[unused82]",1049 "single_word": false,1050 "lstrip": false,1051 "rstrip": false,1052 "normalized": true,1053 "special": false1054 }1055 ],1056 "normalizer": {1057 "type": "NFC"1058 },1059 "pre_tokenizer": {1060 "type": "ByteLevel",1061 "add_prefix_space": false,1062 "trim_offsets": true,1063 "use_regex": true1064 },1065 "post_processor": {1066 "type": "TemplateProcessing",1067 "single": [1068 {1069 "SpecialToken": {1070 "id": "[CLS]",1071 "type_id": 01072 }1073 },1074 {1075 "Sequence": {1076 "id": "A",1077 "type_id": 01078 }1079 },1080 {1081 "SpecialToken": {1082 "id": "[SEP]",1083 "type_id": 01084 }1085 }1086 ],1087 "pair": [1088 {1089 "SpecialToken": {1090 "id": "[CLS]",1091 "type_id": 01092 }1093 },1094 {1095 "Sequence": {1096 "id": "A",1097 "type_id": 01098 }1099 },1100 {1101 "SpecialToken": {1102 "id": "[SEP]",1103 "type_id": 01104 }1105 },1106 {1107 "Sequence": {1108 "id": "B",1109 "type_id": 01110 }1111 },1112 {1113 "SpecialToken": {1114 "id": "[SEP]",1115 "type_id": 01116 }1117 }1118 ],1119 "special_tokens": {1120 "[CLS]": {1121 "id": "[CLS]",1122 "ids": [1123 502811124 ],1125 "tokens": [1126 "[CLS]"1127 ]1128 },1129 "[MASK]": {1130 "id": "[MASK]",1131 "ids": [1132 502841133 ],1134 "tokens": [1135 "[MASK]"1136 ]1137 },1138 "[PAD]": {1139 "id": "[PAD]",1140 "ids": [1141 502831142 ],1143 "tokens": [1144 "[PAD]"1145 ]1146 },1147 "[SEP]": {1148 "id": "[SEP]",1149 "ids": [1150 502821151 ],1152 "tokens": [1153 "[SEP]"1154 ]1155 },1156 "[UNK]": {1157 "id": "[UNK]",1158 "ids": [1159 502801160 ],1161 "tokens": [1162 "[UNK]"1163 ]1164 }1165 }1166 },1167 "decoder": {1168 "type": "ByteLevel",1169 "add_prefix_space": false,1170 "trim_offsets": true,1171 "use_regex": true1172 },1173 "model": {1174 "type": "BPE",1175 "dropout": null,1176 "unk_token": null,1177 "continuing_subword_prefix": null,1178 "end_of_word_suffix": null,1179 "fuse_unk": false,1180 "byte_fallback": false,1181 "ignore_merges": false,1182 "vocab": {1183 "|||IP_ADDRESS|||": 0,1184 "<|padding|>": 1,1185 "!": 2,1186 "\"": 3,1187 "#": 4,1188 "$": 5,1189 "%": 6,1190 "&": 7,1191 "'": 8,1192 "(": 9,1193 ")": 10,1194 "*": 11,1195 "+": 12,1196 ",": 13,1197 "-": 14,1198 ".": 15,1199 "/": 16,1200 "0": 17,