FrameByFrame/programming-language-identification-100plus
122
1{2 "version": "1.0",3 "truncation": {4 "direction": "Right",5 "max_length": 512,6 "strategy": "LongestFirst",7 "stride": 08 },9 "padding": {10 "strategy": "BatchLongest",11 "direction": "Right",12 "pad_to_multiple_of": null,13 "pad_id": 50283,14 "pad_type_id": 0,15 "pad_token": "[PAD]"16 },17 "added_tokens": [18 {19 "id": 0,20 "content": "|||IP_ADDRESS|||",21 "single_word": false,22 "lstrip": false,23 "rstrip": false,24 "normalized": true,25 "special": false26 },27 {28 "id": 1,29 "content": "<|padding|>",30 "single_word": false,31 "lstrip": false,32 "rstrip": false,33 "normalized": false,34 "special": true35 },36 {37 "id": 50254,38 "content": " ",39 "single_word": false,40 "lstrip": false,41 "rstrip": false,42 "normalized": true,43 "special": false44 },45 {46 "id": 50255,47 "content": " ",48 "single_word": false,49 "lstrip": false,50 "rstrip": false,51 "normalized": true,52 "special": false53 },54 {55 "id": 50256,56 "content": " ",57 "single_word": false,58 "lstrip": false,59 "rstrip": false,60 "normalized": true,61 "special": false62 },63 {64 "id": 50257,65 "content": " ",66 "single_word": false,67 "lstrip": false,68 "rstrip": false,69 "normalized": true,70 "special": false71 },72 {73 "id": 50258,74 "content": " ",75 "single_word": false,76 "lstrip": false,77 "rstrip": false,78 "normalized": true,79 "special": false80 },81 {82 "id": 50259,83 "content": " ",84 "single_word": false,85 "lstrip": false,86 "rstrip": false,87 "normalized": true,88 "special": false89 },90 {91 "id": 50260,92 "content": " ",93 "single_word": false,94 "lstrip": false,95 "rstrip": false,96 "normalized": true,97 "special": false98 },99 {100 "id": 50261,101 "content": " ",102 "single_word": false,103 "lstrip": false,104 "rstrip": false,105 "normalized": true,106 "special": false107 },108 {109 "id": 50262,110 "content": " ",111 "single_word": false,112 "lstrip": false,113 "rstrip": false,114 "normalized": true,115 "special": false116 },117 {118 "id": 50263,119 "content": " ",120 "single_word": false,121 "lstrip": false,122 "rstrip": false,123 "normalized": true,124 "special": false125 },126 {127 "id": 50264,128 "content": " ",129 "single_word": false,130 "lstrip": false,131 "rstrip": false,132 "normalized": true,133 "special": false134 },135 {136 "id": 50265,137 "content": " ",138 "single_word": false,139 "lstrip": false,140 "rstrip": false,141 "normalized": true,142 "special": false143 },144 {145 "id": 50266,146 "content": " ",147 "single_word": false,148 "lstrip": false,149 "rstrip": false,150 "normalized": true,151 "special": false152 },153 {154 "id": 50267,155 "content": " ",156 "single_word": false,157 "lstrip": false,158 "rstrip": false,159 "normalized": true,160 "special": false161 },162 {163 "id": 50268,164 "content": " ",165 "single_word": false,166 "lstrip": false,167 "rstrip": false,168 "normalized": true,169 "special": false170 },171 {172 "id": 50269,173 "content": " ",174 "single_word": false,175 "lstrip": false,176 "rstrip": false,177 "normalized": true,178 "special": false179 },180 {181 "id": 50270,182 "content": " ",183 "single_word": false,184 "lstrip": false,185 "rstrip": false,186 "normalized": true,187 "special": false188 },189 {190 "id": 50271,191 "content": " ",192 "single_word": false,193 "lstrip": false,194 "rstrip": false,195 "normalized": true,196 "special": false197 },198 {199 "id": 50272,200 "content": " ",201 "single_word": false,202 "lstrip": false,203 "rstrip": false,204 "normalized": true,205 "special": false206 },207 {208 "id": 50273,209 "content": " ",210 "single_word": false,211 "lstrip": false,212 "rstrip": false,213 "normalized": true,214 "special": false215 },216 {217 "id": 50274,218 "content": " ",219 "single_word": false,220 "lstrip": false,221 "rstrip": false,222 "normalized": true,223 "special": false224 },225 {226 "id": 50275,227 "content": " ",228 "single_word": false,229 "lstrip": false,230 "rstrip": false,231 "normalized": true,232 "special": false233 },234 {235 "id": 50276,236 "content": " ",237 "single_word": false,238 "lstrip": false,239 "rstrip": false,240 "normalized": true,241 "special": false242 },243 {244 "id": 50277,245 "content": "|||EMAIL_ADDRESS|||",246 "single_word": false,247 "lstrip": false,248 "rstrip": false,249 "normalized": true,250 "special": false251 },252 {253 "id": 50278,254 "content": "|||PHONE_NUMBER|||",255 "single_word": false,256 "lstrip": false,257 "rstrip": false,258 "normalized": true,259 "special": false260 },261 {262 "id": 50279,263 "content": "<|endoftext|>",264 "single_word": false,265 "lstrip": false,266 "rstrip": false,267 "normalized": false,268 "special": true269 },270 {271 "id": 50280,272 "content": "[UNK]",273 "single_word": false,274 "lstrip": false,275 "rstrip": false,276 "normalized": false,277 "special": true278 },279 {280 "id": 50281,281 "content": "[CLS]",282 "single_word": false,283 "lstrip": false,284 "rstrip": false,285 "normalized": false,286 "special": true287 },288 {289 "id": 50282,290 "content": "[SEP]",291 "single_word": false,292 "lstrip": false,293 "rstrip": false,294 "normalized": false,295 "special": true296 },297 {298 "id": 50283,299 "content": "[PAD]",300 "single_word": false,301 "lstrip": false,302 "rstrip": false,303 "normalized": false,304 "special": true305 },306 {307 "id": 50284,308 "content": "[MASK]",309 "single_word": false,310 "lstrip": true,311 "rstrip": false,312 "normalized": false,313 "special": true314 },315 {316 "id": 50285,317 "content": "[unused0]",318 "single_word": false,319 "lstrip": false,320 "rstrip": false,321 "normalized": true,322 "special": false323 },324 {325 "id": 50286,326 "content": "[unused1]",327 "single_word": false,328 "lstrip": false,329 "rstrip": false,330 "normalized": true,331 "special": false332 },333 {334 "id": 50287,335 "content": "[unused2]",336 "single_word": false,337 "lstrip": false,338 "rstrip": false,339 "normalized": true,340 "special": false341 },342 {343 "id": 50288,344 "content": "[unused3]",345 "single_word": false,346 "lstrip": false,347 "rstrip": false,348 "normalized": true,349 "special": false350 },351 {352 "id": 50289,353 "content": "[unused4]",354 "single_word": false,355 "lstrip": false,356 "rstrip": false,357 "normalized": true,358 "special": false359 },360 {361 "id": 50290,362 "content": "[unused5]",363 "single_word": false,364 "lstrip": false,365 "rstrip": false,366 "normalized": true,367 "special": false368 },369 {370 "id": 50291,371 "content": "[unused6]",372 "single_word": false,373 "lstrip": false,374 "rstrip": false,375 "normalized": true,376 "special": false377 },378 {379 "id": 50292,380 "content": "[unused7]",381 "single_word": false,382 "lstrip": false,383 "rstrip": false,384 "normalized": true,385 "special": false386 },387 {388 "id": 50293,389 "content": "[unused8]",390 "single_word": false,391 "lstrip": false,392 "rstrip": false,393 "normalized": true,394 "special": false395 },396 {397 "id": 50294,398 "content": "[unused9]",399 "single_word": false,400 "lstrip": false,401 "rstrip": false,402 "normalized": true,403 "special": false404 },405 {406 "id": 50295,407 "content": "[unused10]",408 "single_word": false,409 "lstrip": false,410 "rstrip": false,411 "normalized": true,412 "special": false413 },414 {415 "id": 50296,416 "content": "[unused11]",417 "single_word": false,418 "lstrip": false,419 "rstrip": false,420 "normalized": true,421 "special": false422 },423 {424 "id": 50297,425 "content": "[unused12]",426 "single_word": false,427 "lstrip": false,428 "rstrip": false,429 "normalized": true,430 "special": false431 },432 {433 "id": 50298,434 "content": "[unused13]",435 "single_word": false,436 "lstrip": false,437 "rstrip": false,438 "normalized": true,439 "special": false440 },441 {442 "id": 50299,443 "content": "[unused14]",444 "single_word": false,445 "lstrip": false,446 "rstrip": false,447 "normalized": true,448 "special": false449 },450 {451 "id": 50300,452 "content": "[unused15]",453 "single_word": false,454 "lstrip": false,455 "rstrip": false,456 "normalized": true,457 "special": false458 },459 {460 "id": 50301,461 "content": "[unused16]",462 "single_word": false,463 "lstrip": false,464 "rstrip": false,465 "normalized": true,466 "special": false467 },468 {469 "id": 50302,470 "content": "[unused17]",471 "single_word": false,472 "lstrip": false,473 "rstrip": false,474 "normalized": true,475 "special": false476 },477 {478 "id": 50303,479 "content": "[unused18]",480 "single_word": false,481 "lstrip": false,482 "rstrip": false,483 "normalized": true,484 "special": false485 },486 {487 "id": 50304,488 "content": "[unused19]",489 "single_word": false,490 "lstrip": false,491 "rstrip": false,492 "normalized": true,493 "special": false494 },495 {496 "id": 50305,497 "content": "[unused20]",498 "single_word": false,499 "lstrip": false,500 "rstrip": false,501 "normalized": true,502 "special": false503 },504 {505 "id": 50306,506 "content": "[unused21]",507 "single_word": false,508 "lstrip": false,509 "rstrip": false,510 "normalized": true,511 "special": false512 },513 {514 "id": 50307,515 "content": "[unused22]",516 "single_word": false,517 "lstrip": false,518 "rstrip": false,519 "normalized": true,520 "special": false521 },522 {523 "id": 50308,524 "content": "[unused23]",525 "single_word": false,526 "lstrip": false,527 "rstrip": false,528 "normalized": true,529 "special": false530 },531 {532 "id": 50309,533 "content": "[unused24]",534 "single_word": false,535 "lstrip": false,536 "rstrip": false,537 "normalized": true,538 "special": false539 },540 {541 "id": 50310,542 "content": "[unused25]",543 "single_word": false,544 "lstrip": false,545 "rstrip": false,546 "normalized": true,547 "special": false548 },549 {550 "id": 50311,551 "content": "[unused26]",552 "single_word": false,553 "lstrip": false,554 "rstrip": false,555 "normalized": true,556 "special": false557 },558 {559 "id": 50312,560 "content": "[unused27]",561 "single_word": false,562 "lstrip": false,563 "rstrip": false,564 "normalized": true,565 "special": false566 },567 {568 "id": 50313,569 "content": "[unused28]",570 "single_word": false,571 "lstrip": false,572 "rstrip": false,573 "normalized": true,574 "special": false575 },576 {577 "id": 50314,578 "content": "[unused29]",579 "single_word": false,580 "lstrip": false,581 "rstrip": false,582 "normalized": true,583 "special": false584 },585 {586 "id": 50315,587 "content": "[unused30]",588 "single_word": false,589 "lstrip": false,590 "rstrip": false,591 "normalized": true,592 "special": false593 },594 {595 "id": 50316,596 "content": "[unused31]",597 "single_word": false,598 "lstrip": false,599 "rstrip": false,600 "normalized": true,601 "special": false602 },603 {604 "id": 50317,605 "content": "[unused32]",606 "single_word": false,607 "lstrip": false,608 "rstrip": false,609 "normalized": true,610 "special": false611 },612 {613 "id": 50318,614 "content": "[unused33]",615 "single_word": false,616 "lstrip": false,617 "rstrip": false,618 "normalized": true,619 "special": false620 },621 {622 "id": 50319,623 "content": "[unused34]",624 "single_word": false,625 "lstrip": false,626 "rstrip": false,627 "normalized": true,628 "special": false629 },630 {631 "id": 50320,632 "content": "[unused35]",633 "single_word": false,634 "lstrip": false,635 "rstrip": false,636 "normalized": true,637 "special": false638 },639 {640 "id": 50321,641 "content": "[unused36]",642 "single_word": false,643 "lstrip": false,644 "rstrip": false,645 "normalized": true,646 "special": false647 },648 {649 "id": 50322,650 "content": "[unused37]",651 "single_word": false,652 "lstrip": false,653 "rstrip": false,654 "normalized": true,655 "special": false656 },657 {658 "id": 50323,659 "content": "[unused38]",660 "single_word": false,661 "lstrip": false,662 "rstrip": false,663 "normalized": true,664 "special": false665 },666 {667 "id": 50324,668 "content": "[unused39]",669 "single_word": false,670 "lstrip": false,671 "rstrip": false,672 "normalized": true,673 "special": false674 },675 {676 "id": 50325,677 "content": "[unused40]",678 "single_word": false,679 "lstrip": false,680 "rstrip": false,681 "normalized": true,682 "special": false683 },684 {685 "id": 50326,686 "content": "[unused41]",687 "single_word": false,688 "lstrip": false,689 "rstrip": false,690 "normalized": true,691 "special": false692 },693 {694 "id": 50327,695 "content": "[unused42]",696 "single_word": false,697 "lstrip": false,698 "rstrip": false,699 "normalized": true,700 "special": false701 },702 {703 "id": 50328,704 "content": "[unused43]",705 "single_word": false,706 "lstrip": false,707 "rstrip": false,708 "normalized": true,709 "special": false710 },711 {712 "id": 50329,713 "content": "[unused44]",714 "single_word": false,715 "lstrip": false,716 "rstrip": false,717 "normalized": true,718 "special": false719 },720 {721 "id": 50330,722 "content": "[unused45]",723 "single_word": false,724 "lstrip": false,725 "rstrip": false,726 "normalized": true,727 "special": false728 },729 {730 "id": 50331,731 "content": "[unused46]",732 "single_word": false,733 "lstrip": false,734 "rstrip": false,735 "normalized": true,736 "special": false737 },738 {739 "id": 50332,740 "content": "[unused47]",741 "single_word": false,742 "lstrip": false,743 "rstrip": false,744 "normalized": true,745 "special": false746 },747 {748 "id": 50333,749 "content": "[unused48]",750 "single_word": false,751 "lstrip": false,752 "rstrip": false,753 "normalized": true,754 "special": false755 },756 {757 "id": 50334,758 "content": "[unused49]",759 "single_word": false,760 "lstrip": false,761 "rstrip": false,762 "normalized": true,763 "special": false764 },765 {766 "id": 50335,767 "content": "[unused50]",768 "single_word": false,769 "lstrip": false,770 "rstrip": false,771 "normalized": true,772 "special": false773 },774 {775 "id": 50336,776 "content": "[unused51]",777 "single_word": false,778 "lstrip": false,779 "rstrip": false,780 "normalized": true,781 "special": false782 },783 {784 "id": 50337,785 "content": "[unused52]",786 "single_word": false,787 "lstrip": false,788 "rstrip": false,789 "normalized": true,790 "special": false791 },792 {793 "id": 50338,794 "content": "[unused53]",795 "single_word": false,796 "lstrip": false,797 "rstrip": false,798 "normalized": true,799 "special": false800 },801 {802 "id": 50339,803 "content": "[unused54]",804 "single_word": false,805 "lstrip": false,806 "rstrip": false,807 "normalized": true,808 "special": false809 },810 {811 "id": 50340,812 "content": "[unused55]",813 "single_word": false,814 "lstrip": false,815 "rstrip": false,816 "normalized": true,817 "special": false818 },819 {820 "id": 50341,821 "content": "[unused56]",822 "single_word": false,823 "lstrip": false,824 "rstrip": false,825 "normalized": true,826 "special": false827 },828 {829 "id": 50342,830 "content": "[unused57]",831 "single_word": false,832 "lstrip": false,833 "rstrip": false,834 "normalized": true,835 "special": false836 },837 {838 "id": 50343,839 "content": "[unused58]",840 "single_word": false,841 "lstrip": false,842 "rstrip": false,843 "normalized": true,844 "special": false845 },846 {847 "id": 50344,848 "content": "[unused59]",849 "single_word": false,850 "lstrip": false,851 "rstrip": false,852 "normalized": true,853 "special": false854 },855 {856 "id": 50345,857 "content": "[unused60]",858 "single_word": false,859 "lstrip": false,860 "rstrip": false,861 "normalized": true,862 "special": false863 },864 {865 "id": 50346,866 "content": "[unused61]",867 "single_word": false,868 "lstrip": false,869 "rstrip": false,870 "normalized": true,871 "special": false872 },873 {874 "id": 50347,875 "content": "[unused62]",876 "single_word": false,877 "lstrip": false,878 "rstrip": false,879 "normalized": true,880 "special": false881 },882 {883 "id": 50348,884 "content": "[unused63]",885 "single_word": false,886 "lstrip": false,887 "rstrip": false,888 "normalized": true,889 "special": false890 },891 {892 "id": 50349,893 "content": "[unused64]",894 "single_word": false,895 "lstrip": false,896 "rstrip": false,897 "normalized": true,898 "special": false899 },900 {901 "id": 50350,902 "content": "[unused65]",903 "single_word": false,904 "lstrip": false,905 "rstrip": false,906 "normalized": true,907 "special": false908 },909 {910 "id": 50351,911 "content": "[unused66]",912 "single_word": false,913 "lstrip": false,914 "rstrip": false,915 "normalized": true,916 "special": false917 },918 {919 "id": 50352,920 "content": "[unused67]",921 "single_word": false,922 "lstrip": false,923 "rstrip": false,924 "normalized": true,925 "special": false926 },927 {928 "id": 50353,929 "content": "[unused68]",930 "single_word": false,931 "lstrip": false,932 "rstrip": false,933 "normalized": true,934 "special": false935 },936 {937 "id": 50354,938 "content": "[unused69]",939 "single_word": false,940 "lstrip": false,941 "rstrip": false,942 "normalized": true,943 "special": false944 },945 {946 "id": 50355,947 "content": "[unused70]",948 "single_word": false,949 "lstrip": false,950 "rstrip": false,951 "normalized": true,952 "special": false953 },954 {955 "id": 50356,956 "content": "[unused71]",957 "single_word": false,958 "lstrip": false,959 "rstrip": false,960 "normalized": true,961 "special": false962 },963 {964 "id": 50357,965 "content": "[unused72]",966 "single_word": false,967 "lstrip": false,968 "rstrip": false,969 "normalized": true,970 "special": false971 },972 {973 "id": 50358,974 "content": "[unused73]",975 "single_word": false,976 "lstrip": false,977 "rstrip": false,978 "normalized": true,979 "special": false980 },981 {982 "id": 50359,983 "content": "[unused74]",984 "single_word": false,985 "lstrip": false,986 "rstrip": false,987 "normalized": true,988 "special": false989 },990 {991 "id": 50360,992 "content": "[unused75]",993 "single_word": false,994 "lstrip": false,995 "rstrip": false,996 "normalized": true,997 "special": false998 },999 {1000 "id": 50361,1001 "content": "[unused76]",1002 "single_word": false,1003 "lstrip": false,1004 "rstrip": false,1005 "normalized": true,1006 "special": false1007 },1008 {1009 "id": 50362,1010 "content": "[unused77]",1011 "single_word": false,1012 "lstrip": false,1013 "rstrip": false,1014 "normalized": true,1015 "special": false1016 },1017 {1018 "id": 50363,1019 "content": "[unused78]",1020 "single_word": false,1021 "lstrip": false,1022 "rstrip": false,1023 "normalized": true,1024 "special": false1025 },1026 {1027 "id": 50364,1028 "content": "[unused79]",1029 "single_word": false,1030 "lstrip": false,1031 "rstrip": false,1032 "normalized": true,1033 "special": false1034 },1035 {1036 "id": 50365,1037 "content": "[unused80]",1038 "single_word": false,1039 "lstrip": false,1040 "rstrip": false,1041 "normalized": true,1042 "special": false1043 },1044 {1045 "id": 50366,1046 "content": "[unused81]",1047 "single_word": false,1048 "lstrip": false,1049 "rstrip": false,1050 "normalized": true,1051 "special": false1052 },1053 {1054 "id": 50367,1055 "content": "[unused82]",1056 "single_word": false,1057 "lstrip": false,1058 "rstrip": false,1059 "normalized": true,1060 "special": false1061 }1062 ],1063 "normalizer": {1064 "type": "NFC"1065 },1066 "pre_tokenizer": {1067 "type": "ByteLevel",1068 "add_prefix_space": false,1069 "trim_offsets": true,1070 "use_regex": true1071 },1072 "post_processor": {1073 "type": "TemplateProcessing",1074 "single": [1075 {1076 "SpecialToken": {1077 "id": "[CLS]",1078 "type_id": 01079 }1080 },1081 {1082 "Sequence": {1083 "id": "A",1084 "type_id": 01085 }1086 },1087 {1088 "SpecialToken": {1089 "id": "[SEP]",1090 "type_id": 01091 }1092 }1093 ],1094 "pair": [1095 {1096 "SpecialToken": {1097 "id": "[CLS]",1098 "type_id": 01099 }1100 },1101 {1102 "Sequence": {1103 "id": "A",1104 "type_id": 01105 }1106 },1107 {1108 "SpecialToken": {1109 "id": "[SEP]",1110 "type_id": 01111 }1112 },1113 {1114 "Sequence": {1115 "id": "B",1116 "type_id": 01117 }1118 },1119 {1120 "SpecialToken": {1121 "id": "[SEP]",1122 "type_id": 01123 }1124 }1125 ],1126 "special_tokens": {1127 "[CLS]": {1128 "id": "[CLS]",1129 "ids": [1130 502811131 ],1132 "tokens": [1133 "[CLS]"1134 ]1135 },1136 "[MASK]": {1137 "id": "[MASK]",1138 "ids": [1139 502841140 ],1141 "tokens": [1142 "[MASK]"1143 ]1144 },1145 "[PAD]": {1146 "id": "[PAD]",1147 "ids": [1148 502831149 ],1150 "tokens": [1151 "[PAD]"1152 ]1153 },1154 "[SEP]": {1155 "id": "[SEP]",1156 "ids": [1157 502821158 ],1159 "tokens": [1160 "[SEP]"1161 ]1162 },1163 "[UNK]": {1164 "id": "[UNK]",1165 "ids": [1166 502801167 ],1168 "tokens": [1169 "[UNK]"1170 ]1171 }1172 }1173 },1174 "decoder": {1175 "type": "ByteLevel",1176 "add_prefix_space": false,1177 "trim_offsets": true,1178 "use_regex": true1179 },1180 "model": {1181 "type": "BPE",1182 "dropout": null,1183 "unk_token": null,1184 "continuing_subword_prefix": null,1185 "end_of_word_suffix": null,1186 "fuse_unk": false,1187 "byte_fallback": false,1188 "ignore_merges": false,1189 "vocab": {1190 "|||IP_ADDRESS|||": 0,1191 "<|padding|>": 1,1192 "!": 2,1193 "\"": 3,1194 "#": 4,1195 "$": 5,1196 "%": 6,1197 "&": 7,1198 "'": 8,1199 "(": 9,1200 ")": 10,