kumarsushant36/multiLingual_Toxic_Text_Classification
07
1{2 "added_tokens_decoder": {3 "0": {4 "content": "<s>",5 "lstrip": false,6 "normalized": false,7 "rstrip": false,8 "single_word": false,9 "special": true10 },11 "1": {12 "content": "<pad>",13 "lstrip": false,14 "normalized": false,15 "rstrip": false,16 "single_word": false,17 "special": true18 },19 "2": {20 "content": "</s>",21 "lstrip": false,22 "normalized": false,23 "rstrip": false,24 "single_word": false,25 "special": true26 },27 "3": {28 "content": "<unk>",29 "lstrip": false,30 "normalized": false,31 "rstrip": false,32 "single_word": false,33 "special": true34 },35 "250001": {36 "content": "ar_AR",37 "lstrip": false,38 "normalized": false,39 "rstrip": false,40 "single_word": false,41 "special": true42 },43 "250002": {44 "content": "cs_CZ",45 "lstrip": false,46 "normalized": false,47 "rstrip": false,48 "single_word": false,49 "special": true50 },51 "250003": {52 "content": "de_DE",53 "lstrip": false,54 "normalized": false,55 "rstrip": false,56 "single_word": false,57 "special": true58 },59 "250004": {60 "content": "en_XX",61 "lstrip": false,62 "normalized": false,63 "rstrip": false,64 "single_word": false,65 "special": true66 },67 "250005": {68 "content": "es_XX",69 "lstrip": false,70 "normalized": false,71 "rstrip": false,72 "single_word": false,73 "special": true74 },75 "250006": {76 "content": "et_EE",77 "lstrip": false,78 "normalized": false,79 "rstrip": false,80 "single_word": false,81 "special": true82 },83 "250007": {84 "content": "fi_FI",85 "lstrip": false,86 "normalized": false,87 "rstrip": false,88 "single_word": false,89 "special": true90 },91 "250008": {92 "content": "fr_XX",93 "lstrip": false,94 "normalized": false,95 "rstrip": false,96 "single_word": false,97 "special": true98 },99 "250009": {100 "content": "gu_IN",101 "lstrip": false,102 "normalized": false,103 "rstrip": false,104 "single_word": false,105 "special": true106 },107 "250010": {108 "content": "hi_IN",109 "lstrip": false,110 "normalized": false,111 "rstrip": false,112 "single_word": false,113 "special": true114 },115 "250011": {116 "content": "it_IT",117 "lstrip": false,118 "normalized": false,119 "rstrip": false,120 "single_word": false,121 "special": true122 },123 "250012": {124 "content": "ja_XX",125 "lstrip": false,126 "normalized": false,127 "rstrip": false,128 "single_word": false,129 "special": true130 },131 "250013": {132 "content": "kk_KZ",133 "lstrip": false,134 "normalized": false,135 "rstrip": false,136 "single_word": false,137 "special": true138 },139 "250014": {140 "content": "ko_KR",141 "lstrip": false,142 "normalized": false,143 "rstrip": false,144 "single_word": false,145 "special": true146 },147 "250015": {148 "content": "lt_LT",149 "lstrip": false,150 "normalized": false,151 "rstrip": false,152 "single_word": false,153 "special": true154 },155 "250016": {156 "content": "lv_LV",157 "lstrip": false,158 "normalized": false,159 "rstrip": false,160 "single_word": false,161 "special": true162 },163 "250017": {164 "content": "my_MM",165 "lstrip": false,166 "normalized": false,167 "rstrip": false,168 "single_word": false,169 "special": true170 },171 "250018": {172 "content": "ne_NP",173 "lstrip": false,174 "normalized": false,175 "rstrip": false,176 "single_word": false,177 "special": true178 },179 "250019": {180 "content": "nl_XX",181 "lstrip": false,182 "normalized": false,183 "rstrip": false,184 "single_word": false,185 "special": true186 },187 "250020": {188 "content": "ro_RO",189 "lstrip": false,190 "normalized": false,191 "rstrip": false,192 "single_word": false,193 "special": true194 },195 "250021": {196 "content": "ru_RU",197 "lstrip": false,198 "normalized": false,199 "rstrip": false,200 "single_word": false,201 "special": true202 },203 "250022": {204 "content": "si_LK",205 "lstrip": false,206 "normalized": false,207 "rstrip": false,208 "single_word": false,209 "special": true210 },211 "250023": {212 "content": "tr_TR",213 "lstrip": false,214 "normalized": false,215 "rstrip": false,216 "single_word": false,217 "special": true218 },219 "250024": {220 "content": "vi_VN",221 "lstrip": false,222 "normalized": false,223 "rstrip": false,224 "single_word": false,225 "special": true226 },227 "250025": {228 "content": "zh_CN",229 "lstrip": false,230 "normalized": false,231 "rstrip": false,232 "single_word": false,233 "special": true234 },235 "250026": {236 "content": "af_ZA",237 "lstrip": false,238 "normalized": false,239 "rstrip": false,240 "single_word": false,241 "special": true242 },243 "250027": {244 "content": "az_AZ",245 "lstrip": false,246 "normalized": false,247 "rstrip": false,248 "single_word": false,249 "special": true250 },251 "250028": {252 "content": "bn_IN",253 "lstrip": false,254 "normalized": false,255 "rstrip": false,256 "single_word": false,257 "special": true258 },259 "250029": {260 "content": "fa_IR",261 "lstrip": false,262 "normalized": false,263 "rstrip": false,264 "single_word": false,265 "special": true266 },267 "250030": {268 "content": "he_IL",269 "lstrip": false,270 "normalized": false,271 "rstrip": false,272 "single_word": false,273 "special": true274 },275 "250031": {276 "content": "hr_HR",277 "lstrip": false,278 "normalized": false,279 "rstrip": false,280 "single_word": false,281 "special": true282 },283 "250032": {284 "content": "id_ID",285 "lstrip": false,286 "normalized": false,287 "rstrip": false,288 "single_word": false,289 "special": true290 },291 "250033": {292 "content": "ka_GE",293 "lstrip": false,294 "normalized": false,295 "rstrip": false,296 "single_word": false,297 "special": true298 },299 "250034": {300 "content": "km_KH",301 "lstrip": false,302 "normalized": false,303 "rstrip": false,304 "single_word": false,305 "special": true306 },307 "250035": {308 "content": "mk_MK",309 "lstrip": false,310 "normalized": false,311 "rstrip": false,312 "single_word": false,313 "special": true314 },315 "250036": {316 "content": "ml_IN",317 "lstrip": false,318 "normalized": false,319 "rstrip": false,320 "single_word": false,321 "special": true322 },323 "250037": {324 "content": "mn_MN",325 "lstrip": false,326 "normalized": false,327 "rstrip": false,328 "single_word": false,329 "special": true330 },331 "250038": {332 "content": "mr_IN",333 "lstrip": false,334 "normalized": false,335 "rstrip": false,336 "single_word": false,337 "special": true338 },339 "250039": {340 "content": "pl_PL",341 "lstrip": false,342 "normalized": false,343 "rstrip": false,344 "single_word": false,345 "special": true346 },347 "250040": {348 "content": "ps_AF",349 "lstrip": false,350 "normalized": false,351 "rstrip": false,352 "single_word": false,353 "special": true354 },355 "250041": {356 "content": "pt_XX",357 "lstrip": false,358 "normalized": false,359 "rstrip": false,360 "single_word": false,361 "special": true362 },363 "250042": {364 "content": "sv_SE",365 "lstrip": false,366 "normalized": false,367 "rstrip": false,368 "single_word": false,369 "special": true370 },371 "250043": {372 "content": "sw_KE",373 "lstrip": false,374 "normalized": false,375 "rstrip": false,376 "single_word": false,377 "special": true378 },379 "250044": {380 "content": "ta_IN",381 "lstrip": false,382 "normalized": false,383 "rstrip": false,384 "single_word": false,385 "special": true386 },387 "250045": {388 "content": "te_IN",389 "lstrip": false,390 "normalized": false,391 "rstrip": false,392 "single_word": false,393 "special": true394 },395 "250046": {396 "content": "th_TH",397 "lstrip": false,398 "normalized": false,399 "rstrip": false,400 "single_word": false,401 "special": true402 },403 "250047": {404 "content": "tl_XX",405 "lstrip": false,406 "normalized": false,407 "rstrip": false,408 "single_word": false,409 "special": true410 },411 "250048": {412 "content": "uk_UA",413 "lstrip": false,414 "normalized": false,415 "rstrip": false,416 "single_word": false,417 "special": true418 },419 "250049": {420 "content": "ur_PK",421 "lstrip": false,422 "normalized": false,423 "rstrip": false,424 "single_word": false,425 "special": true426 },427 "250050": {428 "content": "xh_ZA",429 "lstrip": false,430 "normalized": false,431 "rstrip": false,432 "single_word": false,433 "special": true434 },435 "250051": {436 "content": "gl_ES",437 "lstrip": false,438 "normalized": false,439 "rstrip": false,440 "single_word": false,441 "special": true442 },443 "250052": {444 "content": "sl_SI",445 "lstrip": false,446 "normalized": false,447 "rstrip": false,448 "single_word": false,449 "special": true450 },451 "250053": {452 "content": "<mask>",453 "lstrip": true,454 "normalized": true,455 "rstrip": false,456 "single_word": false,457 "special": true458 }459 },460 "additional_special_tokens": [461 "ar_AR",462 "cs_CZ",463 "de_DE",464 "en_XX",465 "es_XX",466 "et_EE",467 "fi_FI",468 "fr_XX",469 "gu_IN",470 "hi_IN",471 "it_IT",472 "ja_XX",473 "kk_KZ",474 "ko_KR",475 "lt_LT",476 "lv_LV",477 "my_MM",478 "ne_NP",479 "nl_XX",480 "ro_RO",481 "ru_RU",482 "si_LK",483 "tr_TR",484 "vi_VN",485 "zh_CN",486 "af_ZA",487 "az_AZ",488 "bn_IN",489 "fa_IR",490 "he_IL",491 "hr_HR",492 "id_ID",493 "ka_GE",494 "km_KH",495 "mk_MK",496 "ml_IN",497 "mn_MN",498 "mr_IN",499 "pl_PL",500 "ps_AF",501 "pt_XX",502 "sv_SE",503 "sw_KE",504 "ta_IN",505 "te_IN",506 "th_TH",507 "tl_XX",508 "uk_UA",509 "ur_PK",510 "xh_ZA",511 "gl_ES",512 "sl_SI"513 ],514 "bos_token": "<s>",515 "clean_up_tokenization_spaces": true,516 "cls_token": "<s>",517 "eos_token": "</s>",518 "language_codes": "ML50",519 "mask_token": "<mask>",520 "model_max_length": 1000000000000000019884624838656,521 "pad_token": "<pad>",522 "sep_token": "</s>",523 "sp_model_kwargs": {},524 "src_lang": "en_XX",525 "tgt_lang": null,526 "tokenizer_class": "MBart50Tokenizer",527 "unk_token": "<unk>"528}529