Aamod37/Tokenizer-Testset
0196
1---2dataset_info:3- config_name: Code4 features:5 - name: text6 dtype: string7 splits:8 - name: assembly9 num_bytes: 40460410 num_examples: 10011 - name: c_sharp12 num_bytes: 59984913 num_examples: 10014 - name: c15 num_bytes: 75160016 num_examples: 10017 - name: cmake18 num_bytes: 48469719 num_examples: 10020 - name: cpp21 num_bytes: 59466322 num_examples: 10023 - name: css24 num_bytes: 38507125 num_examples: 10026 - name: cuda27 num_bytes: 87814328 num_examples: 10029 - name: dockerfile30 num_bytes: 8463031 num_examples: 10032 - name: fortran33 num_bytes: 93631134 num_examples: 10035 - name: go36 num_bytes: 36346737 num_examples: 10038 - name: html39 num_bytes: 106677840 num_examples: 10041 - name: java42 num_bytes: 39242843 num_examples: 10044 - name: javascript45 num_bytes: 94701746 num_examples: 10047 - name: makefile48 num_bytes: 33293649 num_examples: 10050 - name: markdown51 num_bytes: 32041752 num_examples: 10053 - name: mathematica54 num_bytes: 407740855 num_examples: 10056 - name: matlab57 num_bytes: 19198258 num_examples: 9359 - name: php60 num_bytes: 39089561 num_examples: 10062 - name: powershell63 num_bytes: 59092264 num_examples: 10065 - name: python66 num_bytes: 31389967 num_examples: 10068 - name: ruby69 num_bytes: 14504670 num_examples: 10071 - name: rust72 num_bytes: 57513573 num_examples: 10074 - name: shell75 num_bytes: 10762176 num_examples: 10077 - name: sql78 num_bytes: 232903579 num_examples: 10080 - name: systemverilog81 num_bytes: 67173582 num_examples: 10083 - name: tex84 num_bytes: 92230585 num_examples: 10086 - name: verilog87 num_bytes: 49878988 num_examples: 7589 - name: vhdl90 num_bytes: 163348491 num_examples: 10092 - name: yaml93 num_bytes: 10484794 num_examples: 10095 download_size: 666795096 dataset_size: 2109571497- config_name: Foregin_Language98 features:99 - name: text100 dtype: string101 splits:102 - name: Albanian103 num_bytes: 4142517.79841362104 num_examples: 1500105 - name: Amharic106 num_bytes: 7846096107 num_examples: 1000108 - name: Arabic109 num_bytes: 8255873.824221047110 num_examples: 1500111 - name: Armenian112 num_bytes: 7688387.368421053113 num_examples: 1500114 - name: Azerbaijani115 num_bytes: 2836257116 num_examples: 1000117 - name: Basque118 num_bytes: 2559309119 num_examples: 1000120 - name: Belarusian121 num_bytes: 4481692122 num_examples: 1000123 - name: Berber124 num_bytes: 78565.15775034293125 num_examples: 1500126 - name: Bulgarian127 num_bytes: 9499175.348810613128 num_examples: 1500129 - name: Catalan130 num_bytes: 3282141131 num_examples: 1000132 - name: Chinese133 num_bytes: 3033763134 num_examples: 1000135 - name: Croatian136 num_bytes: 2473535137 num_examples: 1000138 - name: Czech139 num_bytes: 5062562.850018395140 num_examples: 1500141 - name: Danish142 num_bytes: 5471278.512202754143 num_examples: 1500144 - name: Dhivehi145 num_bytes: 8278164.118705036146 num_examples: 1500147 - name: Dutch148 num_bytes: 4197950.456559082149 num_examples: 1500150 - name: Dzongkha151 num_bytes: 408372.7891156463152 num_examples: 1500153 - name: English154 num_bytes: 2770554155 num_examples: 681156 - name: Estonian157 num_bytes: 3530019158 num_examples: 1000159 - name: Filipino160 num_bytes: 3486086161 num_examples: 1000162 - name: Finnish163 num_bytes: 5783641.208189401164 num_examples: 1500165 - name: French166 num_bytes: 5293525.469871744167 num_examples: 1500168 - name: Georgian169 num_bytes: 6197840170 num_examples: 1000171 - name: German172 num_bytes: 5394139.67836137173 num_examples: 1500174 - name: Greek175 num_bytes: 4995578176 num_examples: 1000177 - name: Haitian_Creole178 num_bytes: 1835098179 num_examples: 1000180 - name: Hausa181 num_bytes: 2419131182 num_examples: 1000183 - name: Hebrew184 num_bytes: 13590094.915400552185 num_examples: 1500186 - name: Hungarian187 num_bytes: 4155556188 num_examples: 1000189 - name: Igbo190 num_bytes: 2586715191 num_examples: 1000192 - name: Indonesian193 num_bytes: 3893007.923930269194 num_examples: 1500195 - name: Italian196 num_bytes: 4950986.317262716197 num_examples: 1500198 - name: Japanese199 num_bytes: 9294558200 num_examples: 1000201 - name: Kazakh202 num_bytes: 11251520.032258065203 num_examples: 1500204 - name: Korean205 num_bytes: 6510924206 num_examples: 1000207 - name: Kyrgyz208 num_bytes: 3400152209 num_examples: 1000210 - name: Latvian211 num_bytes: 3853008212 num_examples: 1000213 - name: Lithuanian214 num_bytes: 3258722215 num_examples: 1000216 - name: Macedonian217 num_bytes: 4973146218 num_examples: 1000219 - name: Malagasy220 num_bytes: 1338541221 num_examples: 1000222 - name: Malay223 num_bytes: 604873.9922741015224 num_examples: 1500225 - name: Math226 num_bytes: 643161227 num_examples: 400228 - name: Norwegian229 num_bytes: 6876122.746696793230 num_examples: 1500231 - name: Nyanja232 num_bytes: 2221045233 num_examples: 1000234 - name: Persian235 num_bytes: 7591673.145441257236 num_examples: 1500237 - name: Polish238 num_bytes: 4917534.387895461239 num_examples: 1500240 - name: Portuguese241 num_bytes: 4557059.842941844242 num_examples: 1500243 - name: Quechua244 num_bytes: 662024245 num_examples: 1000246 - name: Romanian247 num_bytes: 2857160248 num_examples: 1000249 - name: Russian250 num_bytes: 9496321.702838063251 num_examples: 1500252 - name: Serbian253 num_bytes: 6856426254 num_examples: 1000255 - name: Shona256 num_bytes: 1608957257 num_examples: 1000258 - name: Sinhala259 num_bytes: 498122.2748815166260 num_examples: 1500261 - name: Slovak262 num_bytes: 3007295263 num_examples: 1000264 - name: Slovenian265 num_bytes: 3840123266 num_examples: 1000267 - name: Somali268 num_bytes: 4391859269 num_examples: 1000270 - name: Spanish271 num_bytes: 5254020.613401489272 num_examples: 1500273 - name: Swahili274 num_bytes: 2723949.939867709275 num_examples: 1500276 - name: Swedish277 num_bytes: 3344762278 num_examples: 1000279 - name: Tagalog280 num_bytes: 4042945.2734890664281 num_examples: 1500282 - name: Thai283 num_bytes: 11051167.548696967284 num_examples: 1500285 - name: Tibetan286 num_bytes: 6214621287 num_examples: 518288 - name: Turkish289 num_bytes: 4532041.874108134290 num_examples: 1500291 - name: Ukrainian292 num_bytes: 8489847.117670365293 num_examples: 1500294 - name: Uzbek295 num_bytes: 2368014296 num_examples: 1000297 - name: Vietnamese298 num_bytes: 6844524.656016253299 num_examples: 1500300 - name: Xhosa301 num_bytes: 280765.39678481256302 num_examples: 1500303 - name: Yoruba304 num_bytes: 2199289305 num_examples: 1000306 - name: Zulu307 num_bytes: 220632.51111429802308 num_examples: 1500309 download_size: 164297534310 dataset_size: 312554523.7936099311- config_name: Indian_Language312 features:313 - name: text314 dtype: string315 splits:316 - name: Ahirani317 num_bytes: 5305453318 num_examples: 181319 - name: Assamese320 num_bytes: 771552321 num_examples: 100322 - name: Awadhi323 num_bytes: 197703324 num_examples: 1248325 - name: Bagheli326 num_bytes: 5528768327 num_examples: 174328 - name: Bengali329 num_bytes: 500673330 num_examples: 100331 - name: Bhili332 num_bytes: 57847333 num_examples: 202334 - name: Bhojpuri335 num_bytes: 17698336 num_examples: 100337 - name: Bodo338 num_bytes: 1602076339 num_examples: 100340 - name: Braj341 num_bytes: 282696.7296882984342 num_examples: 1500343 - name: Bundeli344 num_bytes: 12807345 num_examples: 44346 - name: Chhattisgarhi347 num_bytes: 223223348 num_examples: 1177349 - name: Dogri350 num_bytes: 1543626351 num_examples: 100352 - name: Gujarati353 num_bytes: 617289354 num_examples: 100355 - name: Haryanvi356 num_bytes: 28727357 num_examples: 109358 - name: Hindi359 num_bytes: 592189360 num_examples: 100361 - name: Kannada362 num_bytes: 551612363 num_examples: 100364 - name: Kashmiri365 num_bytes: 1392333366 num_examples: 100367 - name: Khortha368 num_bytes: 6473369 num_examples: 29370 - name: Konkani371 num_bytes: 4261214372 num_examples: 285373 - name: Kumaoni374 num_bytes: 275443.3085501859375 num_examples: 1500376 - name: Magahi377 num_bytes: 260570.00883392227378 num_examples: 1500379 - name: Maithili380 num_bytes: 599526381 num_examples: 100382 - name: Malayalam383 num_bytes: 543830384 num_examples: 100385 - name: Manipuri386 num_bytes: 1616903387 num_examples: 100388 - name: Marathi389 num_bytes: 555549390 num_examples: 100391 - name: Marwari392 num_bytes: 117300393 num_examples: 478394 - name: Nagamese395 num_bytes: 161579.14338919925396 num_examples: 1500397 - name: Nepali398 num_bytes: 535806399 num_examples: 100400 - name: Oriya401 num_bytes: 447394402 num_examples: 100403 - name: Punjabi404 num_bytes: 591076405 num_examples: 100406 - name: Rajasthani407 num_bytes: 68948408 num_examples: 294409 - name: Sanskrit410 num_bytes: 1173968411 num_examples: 100412 - name: Santali413 num_bytes: 1770227414 num_examples: 100415 - name: Sindhi416 num_bytes: 549046417 num_examples: 100418 - name: Tamil419 num_bytes: 598620420 num_examples: 100421 - name: Telugu422 num_bytes: 463042423 num_examples: 100424 - name: Tulu425 num_bytes: 1298936426 num_examples: 426427 - name: Urdu428 num_bytes: 1017579429 num_examples: 100430 download_size: 12877133431 dataset_size: 36139302.190461606432- config_name: Latin433 features:434 - name: text435 dtype: string436 splits:437 - name: Assamese_Latin438 num_bytes: 280523439 num_examples: 100440 - name: Bengali_Latin441 num_bytes: 438872442 num_examples: 100443 - name: Gujarati_Latin444 num_bytes: 447843445 num_examples: 100446 - name: Hindi_Latin447 num_bytes: 387746448 num_examples: 100449 - name: Kannada_Latin450 num_bytes: 373804451 num_examples: 100452 - name: Malayalam_Latin453 num_bytes: 18571454 num_examples: 100455 - name: Marathi_Latin456 num_bytes: 398120457 num_examples: 100458 - name: Mizo_Latin459 num_bytes: 102792460 num_examples: 1000461 - name: Nepali_Latin462 num_bytes: 419220463 num_examples: 100464 - name: Oriya_Latin465 num_bytes: 318512466 num_examples: 100467 - name: Punjabi_Latin468 num_bytes: 604995469 num_examples: 100470 - name: Sanskrit_Latin471 num_bytes: 210584472 num_examples: 100473 - name: Sanskrit_Transliterated474 num_bytes: 217804.60544034475475 num_examples: 1500476 - name: Tamil_Latin477 num_bytes: 407949478 num_examples: 100479 - name: Telugu_Latin480 num_bytes: 1754990481 num_examples: 100482 download_size: 3573718483 dataset_size: 6382325.605440345484configs:485- config_name: Code486 data_files:487 - split: assembly488 path: Code/assembly-*489 - split: c_sharp490 path: Code/c_sharp-*491 - split: c492 path: Code/c-*493 - split: cmake494 path: Code/cmake-*495 - split: cpp496 path: Code/cpp-*497 - split: css498 path: Code/css-*499 - split: cuda500 path: Code/cuda-*501 - split: dockerfile502 path: Code/dockerfile-*503 - split: fortran504 path: Code/fortran-*505 - split: go506 path: Code/go-*507 - split: html508 path: Code/html-*509 - split: java510 path: Code/java-*511 - split: javascript512 path: Code/javascript-*513 - split: makefile514 path: Code/makefile-*515 - split: markdown516 path: Code/markdown-*517 - split: mathematica518 path: Code/mathematica-*519 - split: matlab520 path: Code/matlab-*521 - split: php522 path: Code/php-*523 - split: powershell524 path: Code/powershell-*525 - split: python526 path: Code/python-*527 - split: ruby528 path: Code/ruby-*529 - split: rust530 path: Code/rust-*531 - split: shell532 path: Code/shell-*533 - split: sql534 path: Code/sql-*535 - split: systemverilog536 path: Code/systemverilog-*537 - split: tex538 path: Code/tex-*539 - split: verilog540 path: Code/verilog-*541 - split: vhdl542 path: Code/vhdl-*543 - split: yaml544 path: Code/yaml-*545- config_name: Foregin_Language546 data_files:547 - split: Albanian548 path: Foregin_Language/Albanian-*549 - split: Amharic550 path: Foregin_Language/Amharic-*551 - split: Arabic552 path: Foregin_Language/Arabic-*553 - split: Armenian554 path: Foregin_Language/Armenian-*555 - split: Azerbaijani556 path: Foregin_Language/Azerbaijani-*557 - split: Basque558 path: Foregin_Language/Basque-*559 - split: Belarusian560 path: Foregin_Language/Belarusian-*561 - split: Berber562 path: Foregin_Language/Berber-*563 - split: Bulgarian564 path: Foregin_Language/Bulgarian-*565 - split: Catalan566 path: Foregin_Language/Catalan-*567 - split: Chinese568 path: Foregin_Language/Chinese-*569 - split: Croatian570 path: Foregin_Language/Croatian-*571 - split: Czech572 path: Foregin_Language/Czech-*573 - split: Danish574 path: Foregin_Language/Danish-*575 - split: Dhivehi576 path: Foregin_Language/Dhivehi-*577 - split: Dutch578 path: Foregin_Language/Dutch-*579 - split: Dzongkha580 path: Foregin_Language/Dzongkha-*581 - split: English582 path: Foregin_Language/English-*583 - split: Estonian584 path: Foregin_Language/Estonian-*585 - split: Filipino586 path: Foregin_Language/Filipino-*587 - split: Finnish588 path: Foregin_Language/Finnish-*589 - split: French590 path: Foregin_Language/French-*591 - split: Georgian592 path: Foregin_Language/Georgian-*593 - split: German594 path: Foregin_Language/German-*595 - split: Greek596 path: Foregin_Language/Greek-*597 - split: Haitian_Creole598 path: Foregin_Language/Haitian_Creole-*599 - split: Hausa600 path: Foregin_Language/Hausa-*601 - split: Hebrew602 path: Foregin_Language/Hebrew-*603 - split: Hungarian604 path: Foregin_Language/Hungarian-*605 - split: Igbo606 path: Foregin_Language/Igbo-*607 - split: Indonesian608 path: Foregin_Language/Indonesian-*609 - split: Italian610 path: Foregin_Language/Italian-*611 - split: Japanese612 path: Foregin_Language/Japanese-*613 - split: Kazakh614 path: Foregin_Language/Kazakh-*615 - split: Korean616 path: Foregin_Language/Korean-*617 - split: Kyrgyz618 path: Foregin_Language/Kyrgyz-*619 - split: Latvian620 path: Foregin_Language/Latvian-*621 - split: Lithuanian622 path: Foregin_Language/Lithuanian-*623 - split: Macedonian624 path: Foregin_Language/Macedonian-*625 - split: Malagasy626 path: Foregin_Language/Malagasy-*627 - split: Malay628 path: Foregin_Language/Malay-*629 - split: Math630 path: Foregin_Language/Math-*631 - split: Norwegian632 path: Foregin_Language/Norwegian-*633 - split: Nyanja634 path: Foregin_Language/Nyanja-*635 - split: Persian636 path: Foregin_Language/Persian-*637 - split: Polish638 path: Foregin_Language/Polish-*639 - split: Portuguese640 path: Foregin_Language/Portuguese-*641 - split: Quechua642 path: Foregin_Language/Quechua-*643 - split: Romanian644 path: Foregin_Language/Romanian-*645 - split: Russian646 path: Foregin_Language/Russian-*647 - split: Serbian648 path: Foregin_Language/Serbian-*649 - split: Shona650 path: Foregin_Language/Shona-*651 - split: Sinhala652 path: Foregin_Language/Sinhala-*653 - split: Slovak654 path: Foregin_Language/Slovak-*655 - split: Slovenian656 path: Foregin_Language/Slovenian-*657 - split: Somali658 path: Foregin_Language/Somali-*659 - split: Spanish660 path: Foregin_Language/Spanish-*661 - split: Swahili662 path: Foregin_Language/Swahili-*663 - split: Swedish664 path: Foregin_Language/Swedish-*665 - split: Tagalog666 path: Foregin_Language/Tagalog-*667 - split: Thai668 path: Foregin_Language/Thai-*669 - split: Tibetan670 path: Foregin_Language/Tibetan-*671 - split: Turkish672 path: Foregin_Language/Turkish-*673 - split: Ukrainian674 path: Foregin_Language/Ukrainian-*675 - split: Uzbek676 path: Foregin_Language/Uzbek-*677 - split: Vietnamese678 path: Foregin_Language/Vietnamese-*679 - split: Xhosa680 path: Foregin_Language/Xhosa-*681 - split: Yoruba682 path: Foregin_Language/Yoruba-*683 - split: Zulu684 path: Foregin_Language/Zulu-*685- config_name: Indian_Language686 data_files:687 - split: Ahirani688 path: Indian_Language/Ahirani-*689 - split: Assamese690 path: Indian_Language/Assamese-*691 - split: Awadhi692 path: Indian_Language/Awadhi-*693 - split: Bagheli694 path: Indian_Language/Bagheli-*695 - split: Bengali696 path: Indian_Language/Bengali-*697 - split: Bhili698 path: Indian_Language/Bhili-*699 - split: Bhojpuri700 path: Indian_Language/Bhojpuri-*701 - split: Bodo702 path: Indian_Language/Bodo-*703 - split: Braj704 path: Indian_Language/Braj-*705 - split: Bundeli706 path: Indian_Language/Bundeli-*707 - split: Chhattisgarhi708 path: Indian_Language/Chhattisgarhi-*709 - split: Dogri710 path: Indian_Language/Dogri-*711 - split: Gujarati712 path: Indian_Language/Gujarati-*713 - split: Haryanvi714 path: Indian_Language/Haryanvi-*715 - split: Hindi716 path: Indian_Language/Hindi-*717 - split: Kannada718 path: Indian_Language/Kannada-*719 - split: Kashmiri720 path: Indian_Language/Kashmiri-*721 - split: Khortha722 path: Indian_Language/Khortha-*723 - split: Konkani724 path: Indian_Language/Konkani-*725 - split: Kumaoni726 path: Indian_Language/Kumaoni-*727 - split: Magahi728 path: Indian_Language/Magahi-*729 - split: Maithili730 path: Indian_Language/Maithili-*731 - split: Malayalam732 path: Indian_Language/Malayalam-*733 - split: Manipuri734 path: Indian_Language/Manipuri-*735 - split: Marathi736 path: Indian_Language/Marathi-*737 - split: Marwari738 path: Indian_Language/Marwari-*739 - split: Nagamese740 path: Indian_Language/Nagamese-*741 - split: Nepali742 path: Indian_Language/Nepali-*743 - split: Oriya744 path: Indian_Language/Oriya-*745 - split: Punjabi746 path: Indian_Language/Punjabi-*747 - split: Rajasthani748 path: Indian_Language/Rajasthani-*749 - split: Sanskrit750 path: Indian_Language/Sanskrit-*751 - split: Santali752 path: Indian_Language/Santali-*753 - split: Sindhi754 path: Indian_Language/Sindhi-*755 - split: Tamil756 path: Indian_Language/Tamil-*757 - split: Telugu758 path: Indian_Language/Telugu-*759 - split: Tulu760 path: Indian_Language/Tulu-*761 - split: Urdu762 path: Indian_Language/Urdu-*763- config_name: Latin764 data_files:765 - split: Assamese_Latin766 path: Latin/Assamese_Latin-*767 - split: Bengali_Latin768 path: Latin/Bengali_Latin-*769 - split: Gujarati_Latin770 path: Latin/Gujarati_Latin-*771 - split: Hindi_Latin772 path: Latin/Hindi_Latin-*773 - split: Kannada_Latin774 path: Latin/Kannada_Latin-*775 - split: Malayalam_Latin776 path: Latin/Malayalam_Latin-*777 - split: Marathi_Latin778 path: Latin/Marathi_Latin-*779 - split: Mizo_Latin780 path: Latin/Mizo_Latin-*781 - split: Nepali_Latin782 path: Latin/Nepali_Latin-*783 - split: Oriya_Latin784 path: Latin/Oriya_Latin-*785 - split: Punjabi_Latin786 path: Latin/Punjabi_Latin-*787 - split: Sanskrit_Latin788 path: Latin/Sanskrit_Latin-*789 - split: Sanskrit_Transliterated790 path: Latin/Sanskrit_Transliterated-*791 - split: Tamil_Latin792 path: Latin/Tamil_Latin-*793 - split: Telugu_Latin794 path: Latin/Telugu_Latin-*795---796