AnjaliSarawgi/model-fullset-batch4
021
1{2 "version": "1.0",3 "truncation": null,4 "padding": null,5 "added_tokens": [6 {7 "id": 500,8 "content": "[SEP]",9 "single_word": false,10 "lstrip": false,11 "rstrip": false,12 "normalized": false,13 "special": true14 },15 {16 "id": 501,17 "content": "[UNK]",18 "single_word": false,19 "lstrip": false,20 "rstrip": false,21 "normalized": false,22 "special": true23 },24 {25 "id": 502,26 "content": "[PAD]",27 "single_word": false,28 "lstrip": false,29 "rstrip": false,30 "normalized": false,31 "special": true32 },33 {34 "id": 503,35 "content": "[CLS]",36 "single_word": false,37 "lstrip": false,38 "rstrip": false,39 "normalized": false,40 "special": true41 }42 ],43 "normalizer": null,44 "pre_tokenizer": {45 "type": "ByteLevel",46 "add_prefix_space": false,47 "trim_offsets": true,48 "use_regex": true49 },50 "post_processor": {51 "type": "ByteLevel",52 "add_prefix_space": true,53 "trim_offsets": false,54 "use_regex": true55 },56 "decoder": {57 "type": "ByteLevel",58 "add_prefix_space": true,59 "trim_offsets": true,60 "use_regex": true61 },62 "model": {63 "type": "BPE",64 "dropout": null,65 "unk_token": null,66 "continuing_subword_prefix": null,67 "end_of_word_suffix": null,68 "fuse_unk": false,69 "byte_fallback": false,70 "ignore_merges": false,71 "vocab": {72 "!": 0,73 "\"": 1,74 "#": 2,75 "$": 3,76 "%": 4,77 "&": 5,78 "'": 6,79 "(": 7,80 ")": 8,81 "*": 9,82 "+": 10,83 ",": 11,84 "-": 12,85 ".": 13,86 "/": 14,87 "0": 15,88 "1": 16,89 "2": 17,90 "3": 18,91 "4": 19,92 "5": 20,93 "6": 21,94 "7": 22,95 "8": 23,96 "9": 24,97 ":": 25,98 ";": 26,99 "<": 27,100 "=": 28,101 ">": 29,102 "?": 30,103 "@": 31,104 "A": 32,105 "B": 33,106 "C": 34,107 "D": 35,108 "E": 36,109 "F": 37,110 "G": 38,111 "H": 39,112 "I": 40,113 "J": 41,114 "K": 42,115 "L": 43,116 "M": 44,117 "N": 45,118 "O": 46,119 "P": 47,120 "Q": 48,121 "R": 49,122 "S": 50,123 "T": 51,124 "U": 52,125 "V": 53,126 "W": 54,127 "X": 55,128 "Y": 56,129 "Z": 57,130 "[": 58,131 "\\": 59,132 "]": 60,133 "^": 61,134 "_": 62,135 "`": 63,136 "a": 64,137 "b": 65,138 "c": 66,139 "d": 67,140 "e": 68,141 "f": 69,142 "g": 70,143 "h": 71,144 "i": 72,145 "j": 73,146 "k": 74,147 "l": 75,148 "m": 76,149 "n": 77,150 "o": 78,151 "p": 79,152 "q": 80,153 "r": 81,154 "s": 82,155 "t": 83,156 "u": 84,157 "v": 85,158 "w": 86,159 "x": 87,160 "y": 88,161 "z": 89,162 "{": 90,163 "|": 91,164 "}": 92,165 "~": 93,166 "¡": 94,167 "¢": 95,168 "£": 96,169 "¤": 97,170 "¥": 98,171 "¦": 99,172 "§": 100,173 "¨": 101,174 "©": 102,175 "ª": 103,176 "«": 104,177 "¬": 105,178 "®": 106,179 "¯": 107,180 "°": 108,181 "±": 109,182 "²": 110,183 "³": 111,184 "´": 112,185 "µ": 113,186 "¶": 114,187 "·": 115,188 "¸": 116,189 "¹": 117,190 "º": 118,191 "»": 119,192 "¼": 120,193 "½": 121,194 "¾": 122,195 "¿": 123,196 "À": 124,197 "Á": 125,198 "Â": 126,199 "Ã": 127,200 "Ä": 128,201 "Å": 129,202 "Æ": 130,203 "Ç": 131,204 "È": 132,205 "É": 133,206 "Ê": 134,207 "Ë": 135,208 "Ì": 136,209 "Í": 137,210 "Î": 138,211 "Ï": 139,212 "Ð": 140,213 "Ñ": 141,214 "Ò": 142,215 "Ó": 143,216 "Ô": 144,217 "Õ": 145,218 "Ö": 146,219 "×": 147,220 "Ø": 148,221 "Ù": 149,222 "Ú": 150,223 "Û": 151,224 "Ü": 152,225 "Ý": 153,226 "Þ": 154,227 "ß": 155,228 "à": 156,229 "á": 157,230 "â": 158,231 "ã": 159,232 "ä": 160,233 "å": 161,234 "æ": 162,235 "ç": 163,236 "è": 164,237 "é": 165,238 "ê": 166,239 "ë": 167,240 "ì": 168,241 "í": 169,242 "î": 170,243 "ï": 171,244 "ð": 172,245 "ñ": 173,246 "ò": 174,247 "ó": 175,248 "ô": 176,249 "õ": 177,250 "ö": 178,251 "÷": 179,252 "ø": 180,253 "ù": 181,254 "ú": 182,255 "û": 183,256 "ü": 184,257 "ý": 185,258 "þ": 186,259 "ÿ": 187,260 "Ā": 188,261 "ā": 189,262 "Ă": 190,263 "ă": 191,264 "Ą": 192,265 "ą": 193,266 "Ć": 194,267 "ć": 195,268 "Ĉ": 196,269 "ĉ": 197,270 "Ċ": 198,271 "ċ": 199,272 "Č": 200,273 "č": 201,274 "Ď": 202,275 "ď": 203,276 "Đ": 204,277 "đ": 205,278 "Ē": 206,279 "ē": 207,280 "Ĕ": 208,281 "ĕ": 209,282 "Ė": 210,283 "ė": 211,284 "Ę": 212,285 "ę": 213,286 "Ě": 214,287 "ě": 215,288 "Ĝ": 216,289 "ĝ": 217,290 "Ğ": 218,291 "ğ": 219,292 "Ġ": 220,293 "ġ": 221,294 "Ģ": 222,295 "ģ": 223,296 "Ĥ": 224,297 "ĥ": 225,298 "Ħ": 226,299 "ħ": 227,300 "Ĩ": 228,301 "ĩ": 229,302 "Ī": 230,303 "ī": 231,304 "Ĭ": 232,305 "ĭ": 233,306 "Į": 234,307 "į": 235,308 "İ": 236,309 "ı": 237,310 "IJ": 238,311 "ij": 239,312 "Ĵ": 240,313 "ĵ": 241,314 "Ķ": 242,315 "ķ": 243,316 "ĸ": 244,317 "Ĺ": 245,318 "ĺ": 246,319 "Ļ": 247,320 "ļ": 248,321 "Ľ": 249,322 "ľ": 250,323 "Ŀ": 251,324 "ŀ": 252,325 "Ł": 253,326 "ł": 254,327 "Ń": 255,328 "à¤": 256,329 "à¥": 257,330 "Ġà¤": 258,331 "ा": 259,332 "à¥į": 260,333 "र": 261,334 "न": 262,335 "ि": 263,336 "त": 264,337 "à¥ĩ": 265,338 "à¤ķ": 266,339 "म": 267,340 "à¥ĭ": 268,341 "य": 269,342 "à¥ģ": 270,343 "à¥Ģ": 271,344 "ल": 272,345 "व": 273,346 "स": 274,347 "ह": 275,348 "द": 276,349 "प": 277,350 "Ġस": 278,351 "Ġà¤ķ": 279,352 "à¤Ĥ": 280,353 "Ġप": 281,354 "à¤Ĺ": 282,355 "à¥Ī": 283,356 "Ġम": 284,357 "à¤ľ": 285,358 "ष": 286,359 "Ġह": 287,360 "à¥Ĥ": 288,361 "Ġà¤Ń": 289,362 "ण": 290,363 "श": 291,364 "Ġà¤Ĺ": 292,365 "Ġत": 293,366 "à¤Ń": 294,367 "Ġव": 295,368 "ÂŃ": 296,369 "Ġब": 297,370 "à¤Ľ": 298,371 "Ġद": 299,372 "ध": 300,373 "Ġà¤ħ": 301,374 "ब": 302,375 "थ": 303,376 "Ġश": 304,377 "à¤ļ": 305,378 "Ġà¤ľ": 306,379 "Ġन": 307,380 "à¤Ł": 308,381 "Ġर": 309,382 "à¤Ī": 310,383 "Ġà¥": 311,384 "Ġà¤Ĺर": 312,385 "à¤ı": 313,386 "à¤ģ": 314,387 "à¥ĥ": 315,388 "Ġà¤Ĩ": 316,389 "Ġà¤ī": 317,390 "Ġय": 318,391 "à¤ĸ": 319,392 "Ġल": 320,393 "हर": 321,394 "ड": 322,395 "à¤ĥ": 323,396 "à¤ī": 324,397 "Ġà¤Ńन": 325,398 "à¤ĩ": 326,399 "Ġà¤ļ": 327,400 "à¥Į": 328,401 "रम": 329,402 "Ġà¤ķर": 330,403 "à¤Ĺव": 331,404 "रà¤ķ": 332,405 "Ġ।": 333,406 "फ": 334,407 "à¤ħ": 335,408 "à¤ł": 336,409 "âĢ": 337,410 "Ġसम": 338,411 "à¤ıà¤ķ": 339,412 "यस": 340,413 "ाà¤ģ": 341,414 "Ġà¤ĩ": 342,415 "नà¤ķ": 343,416 "à¥ĩà¤Ĥ": 344,417 "रह": 345,418 "à¤ŀ": 346,419 "Ġà¤Ľ": 347,420 "à¥ĭà¤Ĥ": 348,421 "Ġà¤Ńà¤Ĺव": 349,422 "ाà¤Ĥ": 350,423 "रत": 351,424 "रण": 352,425 "Ġपन": 353,426 "Ġथ": 354,427 "Ġपर": 355,428 "ाÂŃ": 356,429 "à¤Ĩ": 357,430 "Ġध": 358,431 "पन": 359,432 "वर": 360,433 "à¤¦à¤Ľ": 361,434 "à¤ķर": 362,435 "वत": 363,436 "।": 364,437 "à¤Ĺर": 365,438 "Ġà¤ħन": 366,439 "à¥Īà¤Ĥ": 367,440 "à¤ĺ": 368,441 "·": 369,442 "Ġमन": 370,443 "पर": 371,444 "यà¤ķ": 372,445 "à¥įâĢ": 373,446 "Ķर": 374,447 "सà¤ķ": 375,448 "ढ": 376,449 "Ġमह": 377,450 "Ġà¤ı": 378,451 "Ġà¤ĸ": 379,452 "à¥įâĢĮ": 380,453 "Ġà¤Ķर": 381,454 "सम": 382,455 "Ġ॥": 383,456 "पत": 384,457 "यम": 385,458 "िà¤Ĥ": 386,459 "à¤īन": 387,460 "à¤ľà¤¨": 388,461 "रस": 389,462 "à¤Ļ": 390,463 "Ġफ": 391,464 "Ġ।।": 392,465 "वन": 393,466 "Ġरह": 394,467 "à¤Ńन": 395,468 "Ġà¤ĺ": 396,469 "à¤Ńय": 397,470 "मन": 398,471 "Ġà¤ħर": 399,472 "Ġà¤Ĩफ": 400,473 "सल": 401,474 "Ġसà¤ķ": 402,475 "तम": 403,476 "यत": 404,477 "Ġबन": 405,478 "सर": 406,479 "âĢ¢": 407,480 "Ġसब": 408,481 "नम": 409,482 "रद": 410,483 "Ġà¤Ńय": 411,484 "िÂŃ": 412,485 "नन": 413,486 "वस": 414,487 "§à¥": 415,488 "यन": 416,489 "à¥ĭÂŃ": 417,490 "लà¤ķ": 418,491 "मह": 419,492 "तर": 420,493 "à¤Ľà¤¨": 421,494 "वà¤ķ": 422,495 "वल": 423,496 "मà¤ķ": 424,497 "à¤ªà¤Ľ": 425,498 "Ġà¤īन": 426,499 "Ġà¤ł": 427,500 "लम": 428,501 "०": 429,502 "सन": 430,503 "à¤Ŀ": 431,504 "Ġयह": 432,505 "à¥Ģà¤Ĥ": 433,506 "à¤ıर": 434,507 "रव": 435,508 "़": 436,509 "मण": 437,510 "दर": 438,511 "Ġà¤ķह": 439,512 "रल": 440,513 "तà¤ķ": 441,514 "Ġà¤īत": 442,515 "Ġवर": 443,516 "रन": 444,517 "à¤ķल": 445,518 "तन": 446,519 "Ġà¤ķरन": 447,520 "Ġà¤ľà¤¨": 448,521 "à¤Ĺत": 449,522 "Ġà¤Ńà¤ıà¤ķ": 450,523 "à¥Įà¤Ĥ": 451,524 "रà¤Ń": 452,525 "Ġà¤ıà¤ķ": 453,526 "Ġतथ": 454,527 "à¤ĩन": 455,528 "ÂŃâĢ¢": 456,529 "Ġसत": 457,530 "à¤ħन": 458,531 "णà¤ķ": 459,532 "पम": 460,533 "Ġपरम": 461,534 "Ġà¤Ńà¤ķ": 462,535 "Ġयस": 463,536 "à¤ļन": 464,537 "à¥ģà¤ģ": 465,538 "पद": 466,539 "Ġतत": 467,540 "Ġसर": 468,541 "à¤ķम": 469,542 "Ġ१à¥": 470,543 "मस": 471,544 "पस": 472,545 "नल": 473,546 "यह": 474,547 "Ġà¤ħध": 475,548 "à¤ķन": 476,549 "मद": 477,550 "Ġवह": 478,551 "Ġà¤ĩस": 479,552 "ा,": 480,553 "हन": 481,554 "पà¤ķ": 482,555 "Ġà¤ħत": 483,556 "Ġलà¤Ĺ": 484,557 "à¥ĩÂŃ": 485,558 "Ġधर": 486,559 "तस": 487,560 "िà¤ĥ": 488,561 "Ġनह": 489,562 "Ġवस": 490,563 "Ġà¤ĩत": 491,564 "ाà¤ĥ": 492,565 "तत": 493,566 "Ġà¤ħपन": 494,567 "मल": 495,568 "Ġपत": 496,569 "Ġà¤Ľà¤¨": 497,570 "Ġà¤Ĩत": 498,571 "तव": 499572 },573 "merges": [574 [575 "à",576 "¤"577 ],578 [579 "à",580 "¥"581 ],582 [583 "Ġ",584 "à¤"585 ],586 [587 "à¤",588 "¾"589 ],590 [591 "à¥",592 "į"593 ],594 [595 "à¤",596 "°"597 ],598 [599 "à¤",600 "¨"601 ],602 [603 "à¤",604 "¿"605 ],606 [607 "à¤",608 "¤"609 ],610 [611 "à¥",612 "ĩ"613 ],614 [615 "à¤",616 "ķ"617 ],618 [619 "à¤",620 "®"621 ],622 [623 "à¥",624 "ĭ"625 ],626 [627 "à¤",628 "¯"629 ],630 [631 "à¥",632 "ģ"633 ],634 [635 "à¥",636 "Ģ"637 ],638 [639 "à¤",640 "²"641 ],642 [643 "à¤",644 "µ"645 ],646 [647 "à¤",648 "¸"649 ],650 [651 "à¤",652 "¹"653 ],654 [655 "à¤",656 "¦"657 ],658 [659 "à¤",660 "ª"661 ],662 [663 "Ġà¤",664 "¸"665 ],666 [667 "Ġà¤",668 "ķ"669 ],670 [671 "à¤",672 "Ĥ"673 ],674 [675 "Ġà¤",676 "ª"677 ],678 [679 "à¤",680 "Ĺ"681 ],682 [683 "à¥",684 "Ī"685 ],686 [687 "Ġà¤",688 "®"689 ],690 [691 "à¤",692 "ľ"693 ],694 [695 "à¤",696 "·"697 ],698 [699 "Ġà¤",700 "¹"701 ],702 [703 "à¥",704 "Ĥ"705 ],706 [707 "Ġà¤",708 "Ń"709 ],710 [711 "à¤",712 "£"713 ],714 [715 "à¤",716 "¶"717 ],718 [719 "Ġà¤",720 "Ĺ"721 ],722 [723 "Ġà¤",724 "¤"725 ],726 [727 "à¤",728 "Ń"729 ],730 [731 "Ġà¤",732 "µ"733 ],734 [735 "Â",736 "Ń"737 ],738 [739 "Ġà¤",740 "¬"741 ],742 [743 "à¤",744 "Ľ"745 ],746 [747 "Ġà¤",748 "¦"749 ],750 [751 "à¤",752 "§"753 ],754 [755 "Ġà¤",756 "ħ"757 ],758 [759 "à¤",760 "¬"761 ],762 [763 "à¤",764 "¥"765 ],766 [767 "Ġà¤",768 "¶"769 ],770 [771 "à¤",772 "ļ"773 ],774 [775 "Ġà¤",776 "ľ"777 ],778 [779 "Ġà¤",780 "¨"781 ],782 [783 "à¤",784 "Ł"785 ],786 [787 "Ġà¤",788 "°"789 ],790 [791 "à¤",792 "Ī"793 ],794 [795 "Ġ",796 "à¥"797 ],798 [799 "Ġà¤Ĺ",800 "र"801 ],802 [803 "à¤",804 "ı"805 ],806 [807 "à¤",808 "ģ"809 ],810 [811 "à¥",812 "ĥ"813 ],814 [815 "Ġà¤",816 "Ĩ"817 ],818 [819 "Ġà¤",820 "ī"821 ],822 [823 "Ġà¤",824 "¯"825 ],826 [827 "à¤",828 "ĸ"829 ],830 [831 "Ġà¤",832 "²"833 ],834 [835 "ह",836 "र"837 ],838 [839 "à¤",840 "¡"841 ],842 [843 "à¤",844 "ĥ"845 ],846 [847 "à¤",848 "ī"849 ],850 [851 "Ġà¤Ń",852 "न"853 ],854 [855 "à¤",856 "ĩ"857 ],858 [859 "Ġà¤",860 "ļ"861 ],862 [863 "à¥",864 "Į"865 ],866 [867 "र",868 "म"869 ],870 [871 "Ġà¤ķ",872 "र"873 ],874 [875 "à¤Ĺ",876 "व"877 ],878 [879 "र",880 "à¤ķ"881 ],882 [883 "Ġà¥",884 "¤"885 ],886 [887 "à¤",888 "«"889 ],890 [891 "à¤",892 "ħ"893 ],894 [895 "à¤",896 "ł"897 ],898 [899 "â",900 "Ģ"901 ],902 [903 "Ġस",904 "म"905 ],906 [907 "à¤ı",908 "à¤ķ"909 ],910 [911 "य",912 "स"913 ],914 [915 "ा",916 "à¤ģ"917 ],918 [919 "Ġà¤",920 "ĩ"921 ],922 [923 "न",924 "à¤ķ"925 ],926 [927 "à¥ĩ",928 "à¤Ĥ"929 ],930 [931 "र",932 "ह"933 ],934 [935 "à¤",936 "ŀ"937 ],938 [939 "Ġà¤",940 "Ľ"941 ],942 [943 "à¥ĭ",944 "à¤Ĥ"945 ],946 [947 "Ġà¤Ń",948 "à¤Ĺव"949 ],950 [951 "ा",952 "à¤Ĥ"953 ],954 [955 "र",956 "त"957 ],958 [959 "र",960 "ण"961 ],962 [963 "Ġप",964 "न"965 ],966 [967 "Ġà¤",968 "¥"969 ],970 [971 "Ġप",972 "र"973 ],974 [975 "ा",976 "ÂŃ"977 ],978 [979 "à¤",980 "Ĩ"981 ],982 [983 "Ġà¤",984 "§"985 ],986 [987 "प",988 "न"989 ],990 [991 "व",992 "र"993 ],994 [995 "द",996 "à¤Ľ"997 ],998 [999 "à¤ķ",1000 "र"1001 ],1002 [1003 "व",1004 "त"1005 ],1006 [1007 "à¥",1008 "¤"1009 ],1010 [1011 "à¤Ĺ",1012 "र"1013 ],1014 [1015 "Ġà¤ħ",1016 "न"1017 ],1018 [1019 "à¥Ī",1020 "à¤Ĥ"1021 ],1022 [1023 "à¤",1024 "ĺ"1025 ],1026 [1027 "Â",1028 "·"1029 ],1030 [1031 "Ġम",1032 "न"1033 ],1034 [1035 "प",1036 "र"1037 ],1038 [1039 "य",1040 "à¤ķ"1041 ],1042 [1043 "à¥į",1044 "âĢ"1045 ],1046 [1047 "Ķ",1048 "र"1049 ],1050 [1051 "स",1052 "à¤ķ"1053 ],1054 [1055 "à¤",1056 "¢"1057 ],1058 [1059 "Ġम",1060 "ह"1061 ],1062 [1063 "Ġà¤",1064 "ı"1065 ],1066 [1067 "Ġà¤",1068 "ĸ"1069 ],1070 [1071 "à¥įâĢ",1072 "Į"1073 ],1074 [1075 "Ġà¤",1076 "Ķर"1077 ],1078 [1079 "स",1080 "म"1081 ],1082 [1083 "Ġà¥",1084 "¥"1085 ],1086 [1087 "प",1088 "त"1089 ],1090 [1091 "य",1092 "म"1093 ],1094 [1095 "ि",1096 "à¤Ĥ"1097 ],1098 [1099 "à¤ī",1100 "न"1101 ],1102 [1103 "à¤ľ",1104 "न"1105 ],1106 [1107 "र",1108 "स"1109 ],1110 [1111 "à¤",1112 "Ļ"1113 ],1114 [1115 "Ġà¤",1116 "«"1117 ],1118 [1119 "Ġ।",1120 "।"1121 ],1122 [1123 "व",1124 "न"1125 ],1126 [1127 "Ġर",1128 "ह"1129 ],1130 [1131 "à¤Ń",1132 "न"1133 ],1134 [1135 "Ġà¤",1136 "ĺ"1137 ],1138 [1139 "à¤Ń",1140 "य"1141 ],1142 [1143 "म",1144 "न"1145 ],1146 [1147 "Ġà¤ħ",1148 "र"1149 ],1150 [1151 "Ġà¤Ĩ",1152 "फ"1153 ],1154 [1155 "स",1156 "ल"1157 ],1158 [1159 "Ġस",1160 "à¤ķ"1161 ],1162 [1163 "त",1164 "म"1165 ],1166 [1167 "य",1168 "त"1169 ],1170 [1171 "Ġब",1172 "न"1173 ],1174 [1175 "स",1176 "र"1177 ],1178 [1179 "âĢ",1180 "¢"1181 ],1182 [1183 "Ġस",1184 "ब"1185 ],1186 [1187 "न",1188 "म"1189 ],1190 [1191 "र",1192 "द"1193 ],1194 [1195 "Ġà¤Ń",1196 "य"1197 ],1198 [1199 "ि",1200 "ÂŃ"