onalla/idT5AQG-semantic
02
1{2 "version": "1.0",3 "truncation": {4 "direction": "Right",5 "max_length": 512,6 "strategy": "LongestFirst",7 "stride": 08 },9 "padding": null,10 "added_tokens": [11 {12 "id": 0,13 "content": "<pad>",14 "single_word": false,15 "lstrip": false,16 "rstrip": false,17 "normalized": false,18 "special": true19 },20 {21 "id": 1,22 "content": "</s>",23 "single_word": false,24 "lstrip": false,25 "rstrip": false,26 "normalized": false,27 "special": true28 },29 {30 "id": 2,31 "content": "<unk>",32 "single_word": false,33 "lstrip": false,34 "rstrip": false,35 "normalized": false,36 "special": true37 },38 {39 "id": 32000,40 "content": "<extra_id_99>",41 "single_word": false,42 "lstrip": false,43 "rstrip": false,44 "normalized": false,45 "special": true46 },47 {48 "id": 32001,49 "content": "<extra_id_98>",50 "single_word": false,51 "lstrip": false,52 "rstrip": false,53 "normalized": false,54 "special": true55 },56 {57 "id": 32002,58 "content": "<extra_id_97>",59 "single_word": false,60 "lstrip": false,61 "rstrip": false,62 "normalized": false,63 "special": true64 },65 {66 "id": 32003,67 "content": "<extra_id_96>",68 "single_word": false,69 "lstrip": false,70 "rstrip": false,71 "normalized": false,72 "special": true73 },74 {75 "id": 32004,76 "content": "<extra_id_95>",77 "single_word": false,78 "lstrip": false,79 "rstrip": false,80 "normalized": false,81 "special": true82 },83 {84 "id": 32005,85 "content": "<extra_id_94>",86 "single_word": false,87 "lstrip": false,88 "rstrip": false,89 "normalized": false,90 "special": true91 },92 {93 "id": 32006,94 "content": "<extra_id_93>",95 "single_word": false,96 "lstrip": false,97 "rstrip": false,98 "normalized": false,99 "special": true100 },101 {102 "id": 32007,103 "content": "<extra_id_92>",104 "single_word": false,105 "lstrip": false,106 "rstrip": false,107 "normalized": false,108 "special": true109 },110 {111 "id": 32008,112 "content": "<extra_id_91>",113 "single_word": false,114 "lstrip": false,115 "rstrip": false,116 "normalized": false,117 "special": true118 },119 {120 "id": 32009,121 "content": "<extra_id_90>",122 "single_word": false,123 "lstrip": false,124 "rstrip": false,125 "normalized": false,126 "special": true127 },128 {129 "id": 32010,130 "content": "<extra_id_89>",131 "single_word": false,132 "lstrip": false,133 "rstrip": false,134 "normalized": false,135 "special": true136 },137 {138 "id": 32011,139 "content": "<extra_id_88>",140 "single_word": false,141 "lstrip": false,142 "rstrip": false,143 "normalized": false,144 "special": true145 },146 {147 "id": 32012,148 "content": "<extra_id_87>",149 "single_word": false,150 "lstrip": false,151 "rstrip": false,152 "normalized": false,153 "special": true154 },155 {156 "id": 32013,157 "content": "<extra_id_86>",158 "single_word": false,159 "lstrip": false,160 "rstrip": false,161 "normalized": false,162 "special": true163 },164 {165 "id": 32014,166 "content": "<extra_id_85>",167 "single_word": false,168 "lstrip": false,169 "rstrip": false,170 "normalized": false,171 "special": true172 },173 {174 "id": 32015,175 "content": "<extra_id_84>",176 "single_word": false,177 "lstrip": false,178 "rstrip": false,179 "normalized": false,180 "special": true181 },182 {183 "id": 32016,184 "content": "<extra_id_83>",185 "single_word": false,186 "lstrip": false,187 "rstrip": false,188 "normalized": false,189 "special": true190 },191 {192 "id": 32017,193 "content": "<extra_id_82>",194 "single_word": false,195 "lstrip": false,196 "rstrip": false,197 "normalized": false,198 "special": true199 },200 {201 "id": 32018,202 "content": "<extra_id_81>",203 "single_word": false,204 "lstrip": false,205 "rstrip": false,206 "normalized": false,207 "special": true208 },209 {210 "id": 32019,211 "content": "<extra_id_80>",212 "single_word": false,213 "lstrip": false,214 "rstrip": false,215 "normalized": false,216 "special": true217 },218 {219 "id": 32020,220 "content": "<extra_id_79>",221 "single_word": false,222 "lstrip": false,223 "rstrip": false,224 "normalized": false,225 "special": true226 },227 {228 "id": 32021,229 "content": "<extra_id_78>",230 "single_word": false,231 "lstrip": false,232 "rstrip": false,233 "normalized": false,234 "special": true235 },236 {237 "id": 32022,238 "content": "<extra_id_77>",239 "single_word": false,240 "lstrip": false,241 "rstrip": false,242 "normalized": false,243 "special": true244 },245 {246 "id": 32023,247 "content": "<extra_id_76>",248 "single_word": false,249 "lstrip": false,250 "rstrip": false,251 "normalized": false,252 "special": true253 },254 {255 "id": 32024,256 "content": "<extra_id_75>",257 "single_word": false,258 "lstrip": false,259 "rstrip": false,260 "normalized": false,261 "special": true262 },263 {264 "id": 32025,265 "content": "<extra_id_74>",266 "single_word": false,267 "lstrip": false,268 "rstrip": false,269 "normalized": false,270 "special": true271 },272 {273 "id": 32026,274 "content": "<extra_id_73>",275 "single_word": false,276 "lstrip": false,277 "rstrip": false,278 "normalized": false,279 "special": true280 },281 {282 "id": 32027,283 "content": "<extra_id_72>",284 "single_word": false,285 "lstrip": false,286 "rstrip": false,287 "normalized": false,288 "special": true289 },290 {291 "id": 32028,292 "content": "<extra_id_71>",293 "single_word": false,294 "lstrip": false,295 "rstrip": false,296 "normalized": false,297 "special": true298 },299 {300 "id": 32029,301 "content": "<extra_id_70>",302 "single_word": false,303 "lstrip": false,304 "rstrip": false,305 "normalized": false,306 "special": true307 },308 {309 "id": 32030,310 "content": "<extra_id_69>",311 "single_word": false,312 "lstrip": false,313 "rstrip": false,314 "normalized": false,315 "special": true316 },317 {318 "id": 32031,319 "content": "<extra_id_68>",320 "single_word": false,321 "lstrip": false,322 "rstrip": false,323 "normalized": false,324 "special": true325 },326 {327 "id": 32032,328 "content": "<extra_id_67>",329 "single_word": false,330 "lstrip": false,331 "rstrip": false,332 "normalized": false,333 "special": true334 },335 {336 "id": 32033,337 "content": "<extra_id_66>",338 "single_word": false,339 "lstrip": false,340 "rstrip": false,341 "normalized": false,342 "special": true343 },344 {345 "id": 32034,346 "content": "<extra_id_65>",347 "single_word": false,348 "lstrip": false,349 "rstrip": false,350 "normalized": false,351 "special": true352 },353 {354 "id": 32035,355 "content": "<extra_id_64>",356 "single_word": false,357 "lstrip": false,358 "rstrip": false,359 "normalized": false,360 "special": true361 },362 {363 "id": 32036,364 "content": "<extra_id_63>",365 "single_word": false,366 "lstrip": false,367 "rstrip": false,368 "normalized": false,369 "special": true370 },371 {372 "id": 32037,373 "content": "<extra_id_62>",374 "single_word": false,375 "lstrip": false,376 "rstrip": false,377 "normalized": false,378 "special": true379 },380 {381 "id": 32038,382 "content": "<extra_id_61>",383 "single_word": false,384 "lstrip": false,385 "rstrip": false,386 "normalized": false,387 "special": true388 },389 {390 "id": 32039,391 "content": "<extra_id_60>",392 "single_word": false,393 "lstrip": false,394 "rstrip": false,395 "normalized": false,396 "special": true397 },398 {399 "id": 32040,400 "content": "<extra_id_59>",401 "single_word": false,402 "lstrip": false,403 "rstrip": false,404 "normalized": false,405 "special": true406 },407 {408 "id": 32041,409 "content": "<extra_id_58>",410 "single_word": false,411 "lstrip": false,412 "rstrip": false,413 "normalized": false,414 "special": true415 },416 {417 "id": 32042,418 "content": "<extra_id_57>",419 "single_word": false,420 "lstrip": false,421 "rstrip": false,422 "normalized": false,423 "special": true424 },425 {426 "id": 32043,427 "content": "<extra_id_56>",428 "single_word": false,429 "lstrip": false,430 "rstrip": false,431 "normalized": false,432 "special": true433 },434 {435 "id": 32044,436 "content": "<extra_id_55>",437 "single_word": false,438 "lstrip": false,439 "rstrip": false,440 "normalized": false,441 "special": true442 },443 {444 "id": 32045,445 "content": "<extra_id_54>",446 "single_word": false,447 "lstrip": false,448 "rstrip": false,449 "normalized": false,450 "special": true451 },452 {453 "id": 32046,454 "content": "<extra_id_53>",455 "single_word": false,456 "lstrip": false,457 "rstrip": false,458 "normalized": false,459 "special": true460 },461 {462 "id": 32047,463 "content": "<extra_id_52>",464 "single_word": false,465 "lstrip": false,466 "rstrip": false,467 "normalized": false,468 "special": true469 },470 {471 "id": 32048,472 "content": "<extra_id_51>",473 "single_word": false,474 "lstrip": false,475 "rstrip": false,476 "normalized": false,477 "special": true478 },479 {480 "id": 32049,481 "content": "<extra_id_50>",482 "single_word": false,483 "lstrip": false,484 "rstrip": false,485 "normalized": false,486 "special": true487 },488 {489 "id": 32050,490 "content": "<extra_id_49>",491 "single_word": false,492 "lstrip": false,493 "rstrip": false,494 "normalized": false,495 "special": true496 },497 {498 "id": 32051,499 "content": "<extra_id_48>",500 "single_word": false,501 "lstrip": false,502 "rstrip": false,503 "normalized": false,504 "special": true505 },506 {507 "id": 32052,508 "content": "<extra_id_47>",509 "single_word": false,510 "lstrip": false,511 "rstrip": false,512 "normalized": false,513 "special": true514 },515 {516 "id": 32053,517 "content": "<extra_id_46>",518 "single_word": false,519 "lstrip": false,520 "rstrip": false,521 "normalized": false,522 "special": true523 },524 {525 "id": 32054,526 "content": "<extra_id_45>",527 "single_word": false,528 "lstrip": false,529 "rstrip": false,530 "normalized": false,531 "special": true532 },533 {534 "id": 32055,535 "content": "<extra_id_44>",536 "single_word": false,537 "lstrip": false,538 "rstrip": false,539 "normalized": false,540 "special": true541 },542 {543 "id": 32056,544 "content": "<extra_id_43>",545 "single_word": false,546 "lstrip": false,547 "rstrip": false,548 "normalized": false,549 "special": true550 },551 {552 "id": 32057,553 "content": "<extra_id_42>",554 "single_word": false,555 "lstrip": false,556 "rstrip": false,557 "normalized": false,558 "special": true559 },560 {561 "id": 32058,562 "content": "<extra_id_41>",563 "single_word": false,564 "lstrip": false,565 "rstrip": false,566 "normalized": false,567 "special": true568 },569 {570 "id": 32059,571 "content": "<extra_id_40>",572 "single_word": false,573 "lstrip": false,574 "rstrip": false,575 "normalized": false,576 "special": true577 },578 {579 "id": 32060,580 "content": "<extra_id_39>",581 "single_word": false,582 "lstrip": false,583 "rstrip": false,584 "normalized": false,585 "special": true586 },587 {588 "id": 32061,589 "content": "<extra_id_38>",590 "single_word": false,591 "lstrip": false,592 "rstrip": false,593 "normalized": false,594 "special": true595 },596 {597 "id": 32062,598 "content": "<extra_id_37>",599 "single_word": false,600 "lstrip": false,601 "rstrip": false,602 "normalized": false,603 "special": true604 },605 {606 "id": 32063,607 "content": "<extra_id_36>",608 "single_word": false,609 "lstrip": false,610 "rstrip": false,611 "normalized": false,612 "special": true613 },614 {615 "id": 32064,616 "content": "<extra_id_35>",617 "single_word": false,618 "lstrip": false,619 "rstrip": false,620 "normalized": false,621 "special": true622 },623 {624 "id": 32065,625 "content": "<extra_id_34>",626 "single_word": false,627 "lstrip": false,628 "rstrip": false,629 "normalized": false,630 "special": true631 },632 {633 "id": 32066,634 "content": "<extra_id_33>",635 "single_word": false,636 "lstrip": false,637 "rstrip": false,638 "normalized": false,639 "special": true640 },641 {642 "id": 32067,643 "content": "<extra_id_32>",644 "single_word": false,645 "lstrip": false,646 "rstrip": false,647 "normalized": false,648 "special": true649 },650 {651 "id": 32068,652 "content": "<extra_id_31>",653 "single_word": false,654 "lstrip": false,655 "rstrip": false,656 "normalized": false,657 "special": true658 },659 {660 "id": 32069,661 "content": "<extra_id_30>",662 "single_word": false,663 "lstrip": false,664 "rstrip": false,665 "normalized": false,666 "special": true667 },668 {669 "id": 32070,670 "content": "<extra_id_29>",671 "single_word": false,672 "lstrip": false,673 "rstrip": false,674 "normalized": false,675 "special": true676 },677 {678 "id": 32071,679 "content": "<extra_id_28>",680 "single_word": false,681 "lstrip": false,682 "rstrip": false,683 "normalized": false,684 "special": true685 },686 {687 "id": 32072,688 "content": "<extra_id_27>",689 "single_word": false,690 "lstrip": false,691 "rstrip": false,692 "normalized": false,693 "special": true694 },695 {696 "id": 32073,697 "content": "<extra_id_26>",698 "single_word": false,699 "lstrip": false,700 "rstrip": false,701 "normalized": false,702 "special": true703 },704 {705 "id": 32074,706 "content": "<extra_id_25>",707 "single_word": false,708 "lstrip": false,709 "rstrip": false,710 "normalized": false,711 "special": true712 },713 {714 "id": 32075,715 "content": "<extra_id_24>",716 "single_word": false,717 "lstrip": false,718 "rstrip": false,719 "normalized": false,720 "special": true721 },722 {723 "id": 32076,724 "content": "<extra_id_23>",725 "single_word": false,726 "lstrip": false,727 "rstrip": false,728 "normalized": false,729 "special": true730 },731 {732 "id": 32077,733 "content": "<extra_id_22>",734 "single_word": false,735 "lstrip": false,736 "rstrip": false,737 "normalized": false,738 "special": true739 },740 {741 "id": 32078,742 "content": "<extra_id_21>",743 "single_word": false,744 "lstrip": false,745 "rstrip": false,746 "normalized": false,747 "special": true748 },749 {750 "id": 32079,751 "content": "<extra_id_20>",752 "single_word": false,753 "lstrip": false,754 "rstrip": false,755 "normalized": false,756 "special": true757 },758 {759 "id": 32080,760 "content": "<extra_id_19>",761 "single_word": false,762 "lstrip": false,763 "rstrip": false,764 "normalized": false,765 "special": true766 },767 {768 "id": 32081,769 "content": "<extra_id_18>",770 "single_word": false,771 "lstrip": false,772 "rstrip": false,773 "normalized": false,774 "special": true775 },776 {777 "id": 32082,778 "content": "<extra_id_17>",779 "single_word": false,780 "lstrip": false,781 "rstrip": false,782 "normalized": false,783 "special": true784 },785 {786 "id": 32083,787 "content": "<extra_id_16>",788 "single_word": false,789 "lstrip": false,790 "rstrip": false,791 "normalized": false,792 "special": true793 },794 {795 "id": 32084,796 "content": "<extra_id_15>",797 "single_word": false,798 "lstrip": false,799 "rstrip": false,800 "normalized": false,801 "special": true802 },803 {804 "id": 32085,805 "content": "<extra_id_14>",806 "single_word": false,807 "lstrip": false,808 "rstrip": false,809 "normalized": false,810 "special": true811 },812 {813 "id": 32086,814 "content": "<extra_id_13>",815 "single_word": false,816 "lstrip": false,817 "rstrip": false,818 "normalized": false,819 "special": true820 },821 {822 "id": 32087,823 "content": "<extra_id_12>",824 "single_word": false,825 "lstrip": false,826 "rstrip": false,827 "normalized": false,828 "special": true829 },830 {831 "id": 32088,832 "content": "<extra_id_11>",833 "single_word": false,834 "lstrip": false,835 "rstrip": false,836 "normalized": false,837 "special": true838 },839 {840 "id": 32089,841 "content": "<extra_id_10>",842 "single_word": false,843 "lstrip": false,844 "rstrip": false,845 "normalized": false,846 "special": true847 },848 {849 "id": 32090,850 "content": "<extra_id_9>",851 "single_word": false,852 "lstrip": false,853 "rstrip": false,854 "normalized": false,855 "special": true856 },857 {858 "id": 32091,859 "content": "<extra_id_8>",860 "single_word": false,861 "lstrip": false,862 "rstrip": false,863 "normalized": false,864 "special": true865 },866 {867 "id": 32092,868 "content": "<extra_id_7>",869 "single_word": false,870 "lstrip": false,871 "rstrip": false,872 "normalized": false,873 "special": true874 },875 {876 "id": 32093,877 "content": "<extra_id_6>",878 "single_word": false,879 "lstrip": false,880 "rstrip": false,881 "normalized": false,882 "special": true883 },884 {885 "id": 32094,886 "content": "<extra_id_5>",887 "single_word": false,888 "lstrip": false,889 "rstrip": false,890 "normalized": false,891 "special": true892 },893 {894 "id": 32095,895 "content": "<extra_id_4>",896 "single_word": false,897 "lstrip": false,898 "rstrip": false,899 "normalized": false,900 "special": true901 },902 {903 "id": 32096,904 "content": "<extra_id_3>",905 "single_word": false,906 "lstrip": false,907 "rstrip": false,908 "normalized": false,909 "special": true910 },911 {912 "id": 32097,913 "content": "<extra_id_2>",914 "single_word": false,915 "lstrip": false,916 "rstrip": false,917 "normalized": false,918 "special": true919 },920 {921 "id": 32098,922 "content": "<extra_id_1>",923 "single_word": false,924 "lstrip": false,925 "rstrip": false,926 "normalized": false,927 "special": true928 },929 {930 "id": 32099,931 "content": "<extra_id_0>",932 "single_word": false,933 "lstrip": false,934 "rstrip": false,935 "normalized": false,936 "special": true937 }938 ],939 "normalizer": null,940 "pre_tokenizer": {941 "type": "Sequence",942 "pretokenizers": [943 {944 "type": "WhitespaceSplit"945 },946 {947 "type": "Metaspace",948 "replacement": "▁",949 "prepend_scheme": "always",950 "split": true951 }952 ]953 },954 "post_processor": {955 "type": "TemplateProcessing",956 "single": [957 {958 "Sequence": {959 "id": "A",960 "type_id": 0961 }962 },963 {964 "SpecialToken": {965 "id": "</s>",966 "type_id": 0967 }968 }969 ],970 "pair": [971 {972 "Sequence": {973 "id": "A",974 "type_id": 0975 }976 },977 {978 "SpecialToken": {979 "id": "</s>",980 "type_id": 0981 }982 },983 {984 "Sequence": {985 "id": "B",986 "type_id": 0987 }988 },989 {990 "SpecialToken": {991 "id": "</s>",992 "type_id": 0993 }994 }995 ],996 "special_tokens": {997 "</s>": {998 "id": "</s>",999 "ids": [1000 11001 ],1002 "tokens": [1003 "</s>"1004 ]1005 }1006 }1007 },1008 "decoder": {1009 "type": "Metaspace",1010 "replacement": "▁",1011 "prepend_scheme": "always",1012 "split": true1013 },1014 "model": {1015 "type": "Unigram",1016 "unk_id": 2,1017 "vocab": [1018 [1019 "<pad>",1020 0.01021 ],1022 [1023 "</s>",1024 0.01025 ],1026 [1027 "<unk>",1028 0.01029 ],1030 [1031 ".",1032 0.01033 ],1034 [1035 "(",1036 0.01037 ],1038 [1039 ")",1040 0.01041 ],1042 [1043 "\"",1044 0.01045 ],1046 [1047 "-",1048 0.01049 ],1050 [1051 "–",1052 0.01053 ],1054 [1055 "£",1056 0.01057 ],1058 [1059 "€",1060 0.01061 ],1062 [1063 "#",1064 0.01065 ],1066 [1067 "'",1068 0.01069 ],1070 [1071 "a",1072 -2.321918010711671073 ],1074 [1075 ",",1076 -3.31079268455505371077 ],1078 [1079 "▁",1080 -3.3870470523834231081 ],1082 [1083 "▁dan",1084 -4.0218544006347661085 ],1086 [1087 "▁yang",1088 -4.0516090393066411089 ],1090 [1091 "▁di",1092 -4.17873096466064451093 ],1094 [1095 "▁dalam",1096 -5.0780787467956541097 ],1098 [1099 "▁ini",1100 -5.0833439826965331101 ],1102 [1103 "▁untuk",1104 -5.1651163101196291105 ],1106 [1107 "▁dengan",1108 -5.1708183288574221109 ],1110 [1111 "▁pada",1112 -5.2891373634338381113 ],1114 [1115 "▁itu",1116 -5.4216794967651371117 ],1118 [1119 "s",1120 -5.57548046112060551121 ],1122 [1123 "nya",1124 -5.6227574348449711125 ],1126 [1127 "▁,",1128 -5.65400218963623051129 ],1130 [1131 "▁kepada",1132 -5.6727280616760251133 ],1134 [1135 "▁tidak",1136 -5.6835947036743161137 ],1138 [1139 "▁ke",1140 -5.7762260437011721141 ],1142 [1143 "▁tahun",1144 -5.7977895736694341145 ],1146 [1147 "▁oleh",1148 -5.8319120407104491149 ],1150 [1151 "▁telah",1152 -5.8575315475463871153 ],1154 [1155 "▁merupakan",1156 -5.8582162857055661157 ],1158 [1159 "▁adalah",1160 -5.8647289276123051161 ],1162 [1163 "an",1164 -5.8726353645324711165 ],1166 [1167 "▁dari",1168 -5.8764820098876951169 ],1170 [1171 "▁akan",1172 -5.883253097534181173 ],1174 [1175 "▁sebuah",1176 -5.9140896797180181177 ],1178 [1179 "▁Malaysia",1180 -5.9507384300231931181 ],1182 [1183 "▁sebagai",1184 -5.9816722869873051185 ],1186 [1187 "▁juga",1188 -5.9994087219238281189 ],1190 [1191 "▁bagi",1192 -6.0628776550292971193 ],1194 [1195 "▁the",1196 -6.1072249412536621197 ],1198 [1199 "▁mereka",1200 -6.173074245452881