hf-tiny-model-private/tiny-random-BigBirdForCausalLM
043
1{2 "version": "1.0",3 "truncation": null,4 "padding": null,5 "added_tokens": [6 {7 "id": 0,8 "content": "<pad>",9 "single_word": false,10 "lstrip": false,11 "rstrip": false,12 "normalized": true,13 "special": true14 },15 {16 "id": 1,17 "content": "</s>",18 "single_word": false,19 "lstrip": false,20 "rstrip": false,21 "normalized": true,22 "special": true23 },24 {25 "id": 2,26 "content": "<s>",27 "single_word": false,28 "lstrip": false,29 "rstrip": false,30 "normalized": true,31 "special": true32 },33 {34 "id": 65,35 "content": "[CLS]",36 "single_word": false,37 "lstrip": false,38 "rstrip": false,39 "normalized": true,40 "special": true41 },42 {43 "id": 66,44 "content": "[SEP]",45 "single_word": false,46 "lstrip": false,47 "rstrip": false,48 "normalized": true,49 "special": true50 },51 {52 "id": 67,53 "content": "[MASK]",54 "single_word": false,55 "lstrip": true,56 "rstrip": false,57 "normalized": true,58 "special": true59 },60 {61 "id": 100,62 "content": "<unk>",63 "single_word": false,64 "lstrip": false,65 "rstrip": false,66 "normalized": true,67 "special": true68 }69 ],70 "normalizer": {71 "type": "Sequence",72 "normalizers": [73 {74 "type": "Replace",75 "pattern": {76 "Regex": " {2,}"77 },78 "content": " "79 }80 ]81 },82 "pre_tokenizer": {83 "type": "Metaspace",84 "replacement": "▁",85 "add_prefix_space": true86 },87 "post_processor": {88 "type": "TemplateProcessing",89 "single": [90 {91 "SpecialToken": {92 "id": "[CLS]",93 "type_id": 094 }95 },96 {97 "Sequence": {98 "id": "A",99 "type_id": 0100 }101 },102 {103 "SpecialToken": {104 "id": "[SEP]",105 "type_id": 0106 }107 }108 ],109 "pair": [110 {111 "SpecialToken": {112 "id": "[CLS]",113 "type_id": 0114 }115 },116 {117 "Sequence": {118 "id": "A",119 "type_id": 0120 }121 },122 {123 "SpecialToken": {124 "id": "[SEP]",125 "type_id": 0126 }127 },128 {129 "Sequence": {130 "id": "B",131 "type_id": 1132 }133 },134 {135 "SpecialToken": {136 "id": "[SEP]",137 "type_id": 1138 }139 }140 ],141 "special_tokens": {142 "[CLS]": {143 "id": "[CLS]",144 "ids": [145 65146 ],147 "tokens": [148 "[CLS]"149 ]150 },151 "[SEP]": {152 "id": "[SEP]",153 "ids": [154 66155 ],156 "tokens": [157 "[SEP]"158 ]159 }160 }161 },162 "decoder": {163 "type": "Metaspace",164 "replacement": "▁",165 "add_prefix_space": true166 },167 "model": {168 "type": "Unigram",169 "unk_id": 100,170 "vocab": [171 [172 "<pad>",173 0.0174 ],175 [176 "</s>",177 0.0178 ],179 [180 "<s>",181 0.0182 ],183 [184 "<sep_0>",185 0.0186 ],187 [188 "<sep_1>",189 0.0190 ],191 [192 "<sep_2>",193 0.0194 ],195 [196 "<sep_3>",197 0.0198 ],199 [200 "<sep_4>",201 0.0202 ],203 [204 "<no_saic_raw_sp>",205 0.0206 ],207 [208 "<length_0>",209 0.0210 ],211 [212 "<length_1>",213 0.0214 ],215 [216 "<length_2>",217 0.0218 ],219 [220 "<length_3>",221 0.0222 ],223 [224 "<length_4>",225 0.0226 ],227 [228 "<length_5>",229 0.0230 ],231 [232 "<length_6>",233 0.0234 ],235 [236 "<length_7>",237 0.0238 ],239 [240 "<length_8>",241 0.0242 ],243 [244 "<length_9>",245 0.0246 ],247 [248 "<length_10>",249 0.0250 ],251 [252 "<length_11>",253 0.0254 ],255 [256 "<length_12>",257 0.0258 ],259 [260 "<length_13>",261 0.0262 ],263 [264 "<length_14>",265 0.0266 ],267 [268 "<length_15>",269 0.0270 ],271 [272 "<length_16>",273 0.0274 ],275 [276 "<length_17>",277 0.0278 ],279 [280 "<length_18>",281 0.0282 ],283 [284 "<length_19>",285 0.0286 ],287 [288 "<length_20>",289 0.0290 ],291 [292 "<length_s>",293 0.0294 ],295 [296 "<length_m>",297 0.0298 ],299 [300 "<length_l>",301 0.0302 ],303 [304 "<::::>",305 0.0306 ],307 [308 "<space>",309 0.0310 ],311 [312 "<unused_34>",313 0.0314 ],315 [316 "<unused_35>",317 0.0318 ],319 [320 "<unused_36>",321 0.0322 ],323 [324 "<no_query_sp>",325 0.0326 ],327 [328 "<no_passage_raw_sp>",329 0.0330 ],331 [332 "<unused_39>",333 0.0334 ],335 [336 "<unused_40>",337 0.0338 ],339 [340 "<unused_41>",341 0.0342 ],343 [344 "<unused_42>",345 0.0346 ],347 [348 "<unused_43>",349 0.0350 ],351 [352 "<unused_44>",353 0.0354 ],355 [356 "<unused_45>",357 0.0358 ],359 [360 "<unused_46>",361 0.0362 ],363 [364 "<unused_47>",365 0.0366 ],367 [368 "<unused_48>",369 0.0370 ],371 [372 "<unused_49>",373 0.0374 ],375 [376 "<paraphrasing_0>",377 0.0378 ],379 [380 "<paraphrasing_1>",381 0.0382 ],383 [384 "<paraphrasing_2>",385 0.0386 ],387 [388 "<paraphrasing_3>",389 0.0390 ],391 [392 "<paraphrasing_4>",393 0.0394 ],395 [396 "<paraphrasing_5>",397 0.0398 ],399 [400 "<paraphrasing_6>",401 0.0402 ],403 [404 "<paraphrasing_7>",405 0.0406 ],407 [408 "<paraphrasing_8>",409 0.0410 ],411 [412 "<empty>",413 0.0414 ],415 [416 "<row>",417 0.0418 ],419 [420 "<cn>",421 0.0422 ],423 [424 "<t>",425 0.0426 ],427 [428 "<ans>",429 0.0430 ],431 [432 "[CLS]",433 0.0434 ],435 [436 "[SEP]",437 0.0438 ],439 [440 "[MASK]",441 0.0442 ],443 [444 "<unused_67>",445 0.0446 ],447 [448 "<unused_68>",449 0.0450 ],451 [452 "<unused_69>",453 0.0454 ],455 [456 "<unused_70>",457 0.0458 ],459 [460 "<unused_71>",461 0.0462 ],463 [464 "<unused_72>",465 0.0466 ],467 [468 "<unused_73>",469 0.0470 ],471 [472 "<unused_74>",473 0.0474 ],475 [476 "<unused_75>",477 0.0478 ],479 [480 "<unused_76>",481 0.0482 ],483 [484 "<unused_77>",485 0.0486 ],487 [488 "<unused_78>",489 0.0490 ],491 [492 "<unused_79>",493 0.0494 ],495 [496 "<unused_80>",497 0.0498 ],499 [500 "<unused_81>",501 0.0502 ],503 [504 "<unused_82>",505 0.0506 ],507 [508 "<unused_83>",509 0.0510 ],511 [512 "<unused_84>",513 0.0514 ],515 [516 "<unused_85>",517 0.0518 ],519 [520 "<unused_86>",521 0.0522 ],523 [524 "<unused_87>",525 0.0526 ],527 [528 "<unused_88>",529 0.0530 ],531 [532 "<unused_89>",533 0.0534 ],535 [536 "<unused_90>",537 0.0538 ],539 [540 "<unused_91>",541 0.0542 ],543 [544 "<unused_92>",545 0.0546 ],547 [548 "<unused_93>",549 0.0550 ],551 [552 "<unused_94>",553 0.0554 ],555 [556 "<unused_95>",557 0.0558 ],559 [560 "<unused_96>",561 0.0562 ],563 [564 "<unused_97>",565 0.0566 ],567 [568 "<unused_98>",569 0.0570 ],571 [572 "<unk>",573 0.0574 ],575 [576 "!",577 0.0578 ],579 [580 "\"",581 0.0582 ],583 [584 "#",585 0.0586 ],587 [588 "$",589 0.0590 ],591 [592 "%",593 0.0594 ],595 [596 "&",597 0.0598 ],599 [600 "'",601 0.0602 ],603 [604 "(",605 0.0606 ],607 [608 ")",609 0.0610 ],611 [612 "*",613 0.0614 ],615 [616 "+",617 0.0618 ],619 [620 ",",621 0.0622 ],623 [624 "-",625 0.0626 ],627 [628 ".",629 0.0630 ],631 [632 "/",633 0.0634 ],635 [636 "0",637 0.0638 ],639 [640 "1",641 0.0642 ],643 [644 "2",645 0.0646 ],647 [648 "3",649 0.0650 ],651 [652 "4",653 0.0654 ],655 [656 "5",657 0.0658 ],659 [660 "6",661 0.0662 ],663 [664 "7",665 0.0666 ],667 [668 "8",669 0.0670 ],671 [672 "9",673 0.0674 ],675 [676 ":",677 0.0678 ],679 [680 ";",681 0.0682 ],683 [684 "<",685 0.0686 ],687 [688 "=",689 0.0690 ],691 [692 ">",693 0.0694 ],695 [696 "?",697 0.0698 ],699 [700 "@",701 0.0702 ],703 [704 "A",705 0.0706 ],707 [708 "B",709 0.0710 ],711 [712 "C",713 0.0714 ],715 [716 "D",717 0.0718 ],719 [720 "E",721 0.0722 ],723 [724 "F",725 0.0726 ],727 [728 "G",729 0.0730 ],731 [732 "H",733 0.0734 ],735 [736 "I",737 0.0738 ],739 [740 "J",741 0.0742 ],743 [744 "K",745 0.0746 ],747 [748 "L",749 0.0750 ],751 [752 "M",753 0.0754 ],755 [756 "N",757 0.0758 ],759 [760 "O",761 0.0762 ],763 [764 "P",765 0.0766 ],767 [768 "Q",769 0.0770 ],771 [772 "R",773 0.0774 ],775 [776 "S",777 0.0778 ],779 [780 "T",781 0.0782 ],783 [784 "U",785 0.0786 ],787 [788 "V",789 0.0790 ],791 [792 "W",793 0.0794 ],795 [796 "X",797 0.0798 ],799 [800 "Y",801 0.0802 ],803 [804 "Z",805 0.0806 ],807 [808 "[",809 0.0810 ],811 [812 "\\",813 0.0814 ],815 [816 "]",817 0.0818 ],819 [820 "^",821 0.0822 ],823 [824 "_",825 0.0826 ],827 [828 "`",829 0.0830 ],831 [832 "a",833 0.0834 ],835 [836 "b",837 0.0838 ],839 [840 "c",841 0.0842 ],843 [844 "d",845 0.0846 ],847 [848 "e",849 0.0850 ],851 [852 "f",853 0.0854 ],855 [856 "g",857 0.0858 ],859 [860 "h",861 0.0862 ],863 [864 "i",865 0.0866 ],867 [868 "j",869 0.0870 ],871 [872 "k",873 0.0874 ],875 [876 "l",877 0.0878 ],879 [880 "m",881 0.0882 ],883 [884 "n",885 0.0886 ],887 [888 "o",889 0.0890 ],891 [892 "p",893 0.0894 ],895 [896 "q",897 0.0898 ],899 [900 "r",901 0.0902 ],903 [904 "s",905 0.0906 ],907 [908 "t",909 0.0910 ],911 [912 "u",913 0.0914 ],915 [916 "v",917 0.0918 ],919 [920 "w",921 0.0922 ],923 [924 "x",925 0.0926 ],927 [928 "y",929 0.0930 ],931 [932 "z",933 0.0934 ],935 [936 "{",937 0.0938 ],939 [940 "|",941 0.0942 ],943 [944 "}",945 0.0946 ],947 [948 "~",949 0.0950 ],951 [952 "¡",953 0.0954 ],955 [956 "¢",957 0.0958 ],959 [960 "£",961 0.0962 ],963 [964 "¤",965 0.0966 ],967 [968 "¥",969 0.0970 ],971 [972 "¦",973 0.0974 ],975 [976 "§",977 0.0978 ],979 [980 "¨",981 0.0982 ],983 [984 "©",985 0.0986 ],987 [988 "ª",989 0.0990 ],991 [992 "«",993 0.0994 ],995 [996 "¬",997 0.0998 ],999 [1000 "®",1001 0.01002 ],1003 [1004 "¯",1005 0.01006 ],1007 [1008 "°",1009 0.01010 ],1011 [1012 "±",1013 0.01014 ],1015 [1016 "²",1017 0.01018 ],1019 [1020 "³",1021 0.01022 ],1023 [1024 "´",1025 0.01026 ],1027 [1028 "µ",1029 0.01030 ],1031 [1032 "¶",1033 0.01034 ],1035 [1036 "·",1037 0.01038 ],1039 [1040 "¸",1041 0.01042 ],1043 [1044 "¹",1045 0.01046 ],1047 [1048 "º",1049 0.01050 ],1051 [1052 "»",1053 0.01054 ],1055 [1056 "¼",1057 0.01058 ],1059 [1060 "½",1061 0.01062 ],1063 [1064 "¾",1065 0.01066 ],1067 [1068 "¿",1069 0.01070 ],1071 [1072 "À",1073 0.01074 ],1075 [1076 "Á",1077 0.01078 ],1079 [1080 "Â",1081 0.01082 ],1083 [1084 "Ã",1085 0.01086 ],1087 [1088 "Ä",1089 0.01090 ],1091 [1092 "Å",1093 0.01094 ],1095 [1096 "Æ",1097 0.01098 ],1099 [1100 "Ç",1101 0.01102 ],1103 [1104 "È",1105 0.01106 ],1107 [1108 "É",1109 0.01110 ],1111 [1112 "Ê",1113 0.01114 ],1115 [1116 "Ë",1117 0.01118 ],1119 [1120 "Ì",1121 0.01122 ],1123 [1124 "Í",1125 0.01126 ],1127 [1128 "Î",1129 0.01130 ],1131 [1132 "Ï",1133 0.01134 ],1135 [1136 "Ð",1137 0.01138 ],1139 [1140 "Ñ",1141 0.01142 ],1143 [1144 "Ò",1145 0.01146 ],1147 [1148 "Ó",1149 0.01150 ],1151 [1152 "Ô",1153 0.01154 ],1155 [1156 "Õ",1157 0.01158 ],1159 [1160 "Ö",1161 0.01162 ],1163 [1164 "×",1165 0.01166 ],1167 [1168 "Ø",1169 0.01170 ],1171 [1172 "Ù",1173 0.01174 ],1175 [1176 "Ú",1177 0.01178 ],1179 [1180 "Û",1181 0.01182 ],1183 [1184 "Ü",1185 0.01186 ],1187 [1188 "Ý",1189 0.01190 ],1191 [1192 "Þ",1193 0.01194 ],1195 [1196 "ß",1197 0.01198 ],1199 [1200 "à",