LumenAI/trainer_test2
09
1{2 "added_tokens_decoder": {3 "0": {4 "content": "<s>",5 "lstrip": false,6 "normalized": false,7 "rstrip": false,8 "single_word": false,9 "special": true10 },11 "1": {12 "content": "<pad>",13 "lstrip": false,14 "normalized": false,15 "rstrip": false,16 "single_word": false,17 "special": true18 },19 "2": {20 "content": "</s>",21 "lstrip": false,22 "normalized": false,23 "rstrip": false,24 "single_word": false,25 "special": true26 },27 "3": {28 "content": "<unk>",29 "lstrip": false,30 "normalized": false,31 "rstrip": false,32 "single_word": false,33 "special": true34 },35 "57521": {36 "content": "<mask>",37 "lstrip": true,38 "normalized": true,39 "rstrip": false,40 "single_word": false,41 "special": true42 },43 "57522": {44 "content": "<sep/>",45 "lstrip": false,46 "normalized": true,47 "rstrip": false,48 "single_word": false,49 "special": false50 },51 "57523": {52 "content": "<s_iitcdip>",53 "lstrip": false,54 "normalized": false,55 "rstrip": false,56 "single_word": false,57 "special": true58 },59 "57524": {60 "content": "<s_synthdog>",61 "lstrip": false,62 "normalized": false,63 "rstrip": false,64 "single_word": false,65 "special": true66 },67 "57525": {68 "content": "<s_header>",69 "lstrip": false,70 "normalized": true,71 "rstrip": false,72 "single_word": false,73 "special": false74 },75 "57526": {76 "content": "</s_header>",77 "lstrip": false,78 "normalized": true,79 "rstrip": false,80 "single_word": false,81 "special": false82 },83 "57527": {84 "content": "<s_invoice_no>",85 "lstrip": false,86 "normalized": true,87 "rstrip": false,88 "single_word": false,89 "special": false90 },91 "57528": {92 "content": "</s_invoice_no>",93 "lstrip": false,94 "normalized": true,95 "rstrip": false,96 "single_word": false,97 "special": false98 },99 "57529": {100 "content": "<s_invoice_date>",101 "lstrip": false,102 "normalized": true,103 "rstrip": false,104 "single_word": false,105 "special": false106 },107 "57530": {108 "content": "</s_invoice_date>",109 "lstrip": false,110 "normalized": true,111 "rstrip": false,112 "single_word": false,113 "special": false114 },115 "57531": {116 "content": "<s_seller>",117 "lstrip": false,118 "normalized": true,119 "rstrip": false,120 "single_word": false,121 "special": false122 },123 "57532": {124 "content": "</s_seller>",125 "lstrip": false,126 "normalized": true,127 "rstrip": false,128 "single_word": false,129 "special": false130 },131 "57533": {132 "content": "<s_client>",133 "lstrip": false,134 "normalized": true,135 "rstrip": false,136 "single_word": false,137 "special": false138 },139 "57534": {140 "content": "</s_client>",141 "lstrip": false,142 "normalized": true,143 "rstrip": false,144 "single_word": false,145 "special": false146 },147 "57535": {148 "content": "<s_seller_tax_id>",149 "lstrip": false,150 "normalized": true,151 "rstrip": false,152 "single_word": false,153 "special": false154 },155 "57536": {156 "content": "</s_seller_tax_id>",157 "lstrip": false,158 "normalized": true,159 "rstrip": false,160 "single_word": false,161 "special": false162 },163 "57537": {164 "content": "<s_client_tax_id>",165 "lstrip": false,166 "normalized": true,167 "rstrip": false,168 "single_word": false,169 "special": false170 },171 "57538": {172 "content": "</s_client_tax_id>",173 "lstrip": false,174 "normalized": true,175 "rstrip": false,176 "single_word": false,177 "special": false178 },179 "57539": {180 "content": "<s_iban>",181 "lstrip": false,182 "normalized": true,183 "rstrip": false,184 "single_word": false,185 "special": false186 },187 "57540": {188 "content": "</s_iban>",189 "lstrip": false,190 "normalized": true,191 "rstrip": false,192 "single_word": false,193 "special": false194 },195 "57541": {196 "content": "<s_items>",197 "lstrip": false,198 "normalized": true,199 "rstrip": false,200 "single_word": false,201 "special": false202 },203 "57542": {204 "content": "</s_items>",205 "lstrip": false,206 "normalized": true,207 "rstrip": false,208 "single_word": false,209 "special": false210 },211 "57543": {212 "content": "<s_item_desc>",213 "lstrip": false,214 "normalized": true,215 "rstrip": false,216 "single_word": false,217 "special": false218 },219 "57544": {220 "content": "</s_item_desc>",221 "lstrip": false,222 "normalized": true,223 "rstrip": false,224 "single_word": false,225 "special": false226 },227 "57545": {228 "content": "<s_item_qty>",229 "lstrip": false,230 "normalized": true,231 "rstrip": false,232 "single_word": false,233 "special": false234 },235 "57546": {236 "content": "</s_item_qty>",237 "lstrip": false,238 "normalized": true,239 "rstrip": false,240 "single_word": false,241 "special": false242 },243 "57547": {244 "content": "<s_item_net_price>",245 "lstrip": false,246 "normalized": true,247 "rstrip": false,248 "single_word": false,249 "special": false250 },251 "57548": {252 "content": "</s_item_net_price>",253 "lstrip": false,254 "normalized": true,255 "rstrip": false,256 "single_word": false,257 "special": false258 },259 "57549": {260 "content": "<s_item_net_worth>",261 "lstrip": false,262 "normalized": true,263 "rstrip": false,264 "single_word": false,265 "special": false266 },267 "57550": {268 "content": "</s_item_net_worth>",269 "lstrip": false,270 "normalized": true,271 "rstrip": false,272 "single_word": false,273 "special": false274 },275 "57551": {276 "content": "<s_item_vat>",277 "lstrip": false,278 "normalized": true,279 "rstrip": false,280 "single_word": false,281 "special": false282 },283 "57552": {284 "content": "</s_item_vat>",285 "lstrip": false,286 "normalized": true,287 "rstrip": false,288 "single_word": false,289 "special": false290 },291 "57553": {292 "content": "<s_item_gross_worth>",293 "lstrip": false,294 "normalized": true,295 "rstrip": false,296 "single_word": false,297 "special": false298 },299 "57554": {300 "content": "</s_item_gross_worth>",301 "lstrip": false,302 "normalized": true,303 "rstrip": false,304 "single_word": false,305 "special": false306 },307 "57555": {308 "content": "<s_summary>",309 "lstrip": false,310 "normalized": true,311 "rstrip": false,312 "single_word": false,313 "special": false314 },315 "57556": {316 "content": "</s_summary>",317 "lstrip": false,318 "normalized": true,319 "rstrip": false,320 "single_word": false,321 "special": false322 },323 "57557": {324 "content": "<s_total_net_worth>",325 "lstrip": false,326 "normalized": true,327 "rstrip": false,328 "single_word": false,329 "special": false330 },331 "57558": {332 "content": "</s_total_net_worth>",333 "lstrip": false,334 "normalized": true,335 "rstrip": false,336 "single_word": false,337 "special": false338 },339 "57559": {340 "content": "<s_total_vat>",341 "lstrip": false,342 "normalized": true,343 "rstrip": false,344 "single_word": false,345 "special": false346 },347 "57560": {348 "content": "</s_total_vat>",349 "lstrip": false,350 "normalized": true,351 "rstrip": false,352 "single_word": false,353 "special": false354 },355 "57561": {356 "content": "<s_total_gross_worth>",357 "lstrip": false,358 "normalized": true,359 "rstrip": false,360 "single_word": false,361 "special": false362 },363 "57562": {364 "content": "</s_total_gross_worth>",365 "lstrip": false,366 "normalized": true,367 "rstrip": false,368 "single_word": false,369 "special": false370 },371 "57563": {372 "content": "<s_None>",373 "lstrip": false,374 "normalized": true,375 "rstrip": false,376 "single_word": false,377 "special": false378 },379 "57564": {380 "content": "</s_None>",381 "lstrip": false,382 "normalized": true,383 "rstrip": false,384 "single_word": false,385 "special": false386 },387 "57565": {388 "content": "<s_cord-v2>",389 "lstrip": false,390 "normalized": true,391 "rstrip": false,392 "single_word": false,393 "special": false394 },395 "57566": {396 "content": "<s_DocType>",397 "lstrip": false,398 "normalized": true,399 "rstrip": false,400 "single_word": false,401 "special": false402 },403 "57567": {404 "content": "</s_DocType>",405 "lstrip": false,406 "normalized": true,407 "rstrip": false,408 "single_word": false,409 "special": false410 },411 "57568": {412 "content": "<s_PolicyNumber>",413 "lstrip": false,414 "normalized": true,415 "rstrip": false,416 "single_word": false,417 "special": false418 },419 "57569": {420 "content": "</s_PolicyNumber>",421 "lstrip": false,422 "normalized": true,423 "rstrip": false,424 "single_word": false,425 "special": false426 },427 "57570": {428 "content": "<s_EffectiveDate>",429 "lstrip": false,430 "normalized": true,431 "rstrip": false,432 "single_word": false,433 "special": false434 },435 "57571": {436 "content": "</s_EffectiveDate>",437 "lstrip": false,438 "normalized": true,439 "rstrip": false,440 "single_word": false,441 "special": false442 },443 "57572": {444 "content": "<s_ExpirationDate>",445 "lstrip": false,446 "normalized": true,447 "rstrip": false,448 "single_word": false,449 "special": false450 },451 "57573": {452 "content": "</s_ExpirationDate>",453 "lstrip": false,454 "normalized": true,455 "rstrip": false,456 "single_word": false,457 "special": false458 },459 "57574": {460 "content": "<s_BuildingCoverage>",461 "lstrip": false,462 "normalized": true,463 "rstrip": false,464 "single_word": false,465 "special": false466 },467 "57575": {468 "content": "</s_BuildingCoverage>",469 "lstrip": false,470 "normalized": true,471 "rstrip": false,472 "single_word": false,473 "special": false474 },475 "57576": {476 "content": "<s_BuildingDeductibles>",477 "lstrip": false,478 "normalized": true,479 "rstrip": false,480 "single_word": false,481 "special": false482 },483 "57577": {484 "content": "</s_BuildingDeductibles>",485 "lstrip": false,486 "normalized": true,487 "rstrip": false,488 "single_word": false,489 "special": false490 },491 "57578": {492 "content": "<s_ContentCoverage>",493 "lstrip": false,494 "normalized": true,495 "rstrip": false,496 "single_word": false,497 "special": false498 },499 "57579": {500 "content": "</s_ContentCoverage>",501 "lstrip": false,502 "normalized": true,503 "rstrip": false,504 "single_word": false,505 "special": false506 },507 "57580": {508 "content": "<s_ContentDeductibles>",509 "lstrip": false,510 "normalized": true,511 "rstrip": false,512 "single_word": false,513 "special": false514 },515 "57581": {516 "content": "</s_ContentDeductibles>",517 "lstrip": false,518 "normalized": true,519 "rstrip": false,520 "single_word": false,521 "special": false522 },523 "57582": {524 "content": "<s_AnnualPremium>",525 "lstrip": false,526 "normalized": true,527 "rstrip": false,528 "single_word": false,529 "special": false530 },531 "57583": {532 "content": "</s_AnnualPremium>",533 "lstrip": false,534 "normalized": true,535 "rstrip": false,536 "single_word": false,537 "special": false538 },539 "57584": {540 "content": "<s_Floors>",541 "lstrip": false,542 "normalized": true,543 "rstrip": false,544 "single_word": false,545 "special": false546 },547 "57585": {548 "content": "</s_Floors>",549 "lstrip": false,550 "normalized": true,551 "rstrip": false,552 "single_word": false,553 "special": false554 },555 "57586": {556 "content": "<s_PropertyLocation>",557 "lstrip": false,558 "normalized": true,559 "rstrip": false,560 "single_word": false,561 "special": false562 },563 "57587": {564 "content": "</s_PropertyLocation>",565 "lstrip": false,566 "normalized": true,567 "rstrip": false,568 "single_word": false,569 "special": false570 }571 },572 "additional_special_tokens": [573 "<s_iitcdip>",574 "<s_synthdog>"575 ],576 "bos_token": "<s>",577 "clean_up_tokenization_spaces": true,578 "cls_token": "<s>",579 "eos_token": "</s>",580 "mask_token": "<mask>",581 "max_length": 768,582 "model_max_length": 1000000000000000019884624838656,583 "pad_to_multiple_of": null,584 "pad_token": "<pad>",585 "pad_token_type_id": 0,586 "padding_side": "right",587 "processor_class": "DonutProcessor",588 "sep_token": "</s>",589 "sp_model_kwargs": {},590 "stride": 0,591 "tokenizer_class": "XLMRobertaTokenizer",592 "truncation_side": "right",593 "truncation_strategy": "longest_first",594 "unk_token": "<unk>"595}596