Jkatzy/code-comments-small
Comment Dataset Opening comments extracted from code datasets with CommentMiner and ML4SE-toolkit. Files are grouped as <dataset>/<language>/part-*.parquet. The Hugging Face dataset card declares one config per source dataset and one split-safe language name per language. Each row contains dataset, record_id, opening_comment, language, path, repo, extracted_at, and metadata. For Parquet exports, metadata is stored as a JSON string so every source dataset shares one stable… See the full description on the dataset page: https://huggingface.co/datasets/Jkatzy/code-comments-small.
0702
1{2 "created_at": "2026-07-09T18:26:56+00:00",3 "source_repo": "Jkatzy/code-comments",4 "source_roots": [5 "var/code-comments-source",6 "var/comment-dataset-the-stack-v2-dedup"7 ],8 "records_per_setting": 1000,9 "records_written": 1096000,10 "format": "parquet",11 "layout": "<dataset>/<language>/part-*.parquet",12 "dataset_card_layout": "language-splits",13 "sampling": "first_rows_per_setting_repeat_short_settings",14 "groups": [15 {16 "dataset": "redpajama-github",17 "language": "1C Enterprise",18 "records": 1000,19 "source_records": 2030,20 "sampling_with_replacement": false,21 "source_shard_count": 17,22 "shards": [23 "redpajama-github/1C Enterprise/part-00000.parquet"24 ]25 },26 {27 "dataset": "redpajama-github",28 "language": "ABAP",29 "records": 1000,30 "source_records": 688,31 "sampling_with_replacement": true,32 "source_shard_count": 66,33 "shards": [34 "redpajama-github/ABAP/part-00000.parquet"35 ]36 },37 {38 "dataset": "redpajama-github",39 "language": "AGS Script",40 "records": 1000,41 "source_records": 10122,42 "sampling_with_replacement": false,43 "source_shard_count": 1,44 "shards": [45 "redpajama-github/AGS Script/part-00000.parquet"46 ]47 },48 {49 "dataset": "redpajama-github",50 "language": "AIDL",51 "records": 1000,52 "source_records": 7043,53 "sampling_with_replacement": false,54 "source_shard_count": 1,55 "shards": [56 "redpajama-github/AIDL/part-00000.parquet"57 ]58 },59 {60 "dataset": "redpajama-github",61 "language": "AL",62 "records": 1000,63 "source_records": 156,64 "sampling_with_replacement": true,65 "source_shard_count": 58,66 "shards": [67 "redpajama-github/AL/part-00000.parquet"68 ]69 },70 {71 "dataset": "redpajama-github",72 "language": "AMPL",73 "records": 1000,74 "source_records": 5271,75 "sampling_with_replacement": false,76 "source_shard_count": 1,77 "shards": [78 "redpajama-github/AMPL/part-00000.parquet"79 ]80 },81 {82 "dataset": "redpajama-github",83 "language": "ANTLR",84 "records": 1000,85 "source_records": 52367,86 "sampling_with_replacement": false,87 "source_shard_count": 1,88 "shards": [89 "redpajama-github/ANTLR/part-00000.parquet"90 ]91 },92 {93 "dataset": "redpajama-github",94 "language": "API Blueprint",95 "records": 1000,96 "source_records": 95,97 "sampling_with_replacement": true,98 "source_shard_count": 62,99 "shards": [100 "redpajama-github/API Blueprint/part-00000.parquet"101 ]102 },103 {104 "dataset": "redpajama-github",105 "language": "ASL",106 "records": 1000,107 "source_records": 6,108 "sampling_with_replacement": true,109 "source_shard_count": 6,110 "shards": [111 "redpajama-github/ASL/part-00000.parquet"112 ]113 },114 {115 "dataset": "redpajama-github",116 "language": "ASP",117 "records": 1000,118 "source_records": 2270,119 "sampling_with_replacement": false,120 "source_shard_count": 4,121 "shards": [122 "redpajama-github/ASP/part-00000.parquet"123 ]124 },125 {126 "dataset": "redpajama-github",127 "language": "ATS",128 "records": 1000,129 "source_records": 6,130 "sampling_with_replacement": true,131 "source_shard_count": 6,132 "shards": [133 "redpajama-github/ATS/part-00000.parquet"134 ]135 },136 {137 "dataset": "redpajama-github",138 "language": "ActionScript",139 "records": 1000,140 "source_records": 21359,141 "sampling_with_replacement": false,142 "source_shard_count": 1,143 "shards": [144 "redpajama-github/ActionScript/part-00000.parquet"145 ]146 },147 {148 "dataset": "redpajama-github",149 "language": "Ada",150 "records": 1000,151 "source_records": 17,152 "sampling_with_replacement": true,153 "source_shard_count": 17,154 "shards": [155 "redpajama-github/Ada/part-00000.parquet"156 ]157 },158 {159 "dataset": "redpajama-github",160 "language": "Agda",161 "records": 1000,162 "source_records": 127,163 "sampling_with_replacement": true,164 "source_shard_count": 90,165 "shards": [166 "redpajama-github/Agda/part-00000.parquet"167 ]168 },169 {170 "dataset": "redpajama-github",171 "language": "Alloy",172 "records": 1000,173 "source_records": 249,174 "sampling_with_replacement": true,175 "source_shard_count": 105,176 "shards": [177 "redpajama-github/Alloy/part-00000.parquet"178 ]179 },180 {181 "dataset": "redpajama-github",182 "language": "AngelScript",183 "records": 1000,184 "source_records": 2530,185 "sampling_with_replacement": false,186 "source_shard_count": 4,187 "shards": [188 "redpajama-github/AngelScript/part-00000.parquet"189 ]190 },191 {192 "dataset": "redpajama-github",193 "language": "ApacheConf",194 "records": 1000,195 "source_records": 12618,196 "sampling_with_replacement": false,197 "source_shard_count": 1,198 "shards": [199 "redpajama-github/ApacheConf/part-00000.parquet"200 ]201 },202 {203 "dataset": "redpajama-github",204 "language": "Apex",205 "records": 1000,206 "source_records": 3038,207 "sampling_with_replacement": false,208 "source_shard_count": 5,209 "shards": [210 "redpajama-github/Apex/part-00000.parquet"211 ]212 },213 {214 "dataset": "redpajama-github",215 "language": "AppleScript",216 "records": 1000,217 "source_records": 12073,218 "sampling_with_replacement": false,219 "source_shard_count": 1,220 "shards": [221 "redpajama-github/AppleScript/part-00000.parquet"222 ]223 },224 {225 "dataset": "redpajama-github",226 "language": "Arduino",227 "records": 1000,228 "source_records": 12118,229 "sampling_with_replacement": false,230 "source_shard_count": 1,231 "shards": [232 "redpajama-github/Arduino/part-00000.parquet"233 ]234 },235 {236 "dataset": "redpajama-github",237 "language": "AspectJ",238 "records": 1000,239 "source_records": 3960,240 "sampling_with_replacement": false,241 "source_shard_count": 2,242 "shards": [243 "redpajama-github/AspectJ/part-00000.parquet"244 ]245 },246 {247 "dataset": "redpajama-github",248 "language": "Assembly",249 "records": 1000,250 "source_records": 1906,251 "sampling_with_replacement": false,252 "source_shard_count": 7,253 "shards": [254 "redpajama-github/Assembly/part-00000.parquet"255 ]256 },257 {258 "dataset": "redpajama-github",259 "language": "Astro",260 "records": 1000,261 "source_records": 3,262 "sampling_with_replacement": true,263 "source_shard_count": 3,264 "shards": [265 "redpajama-github/Astro/part-00000.parquet"266 ]267 },268 {269 "dataset": "redpajama-github",270 "language": "Asymptote",271 "records": 1000,272 "source_records": 35,273 "sampling_with_replacement": true,274 "source_shard_count": 31,275 "shards": [276 "redpajama-github/Asymptote/part-00000.parquet"277 ]278 },279 {280 "dataset": "redpajama-github",281 "language": "AutoHotkey",282 "records": 1000,283 "source_records": 149,284 "sampling_with_replacement": true,285 "source_shard_count": 99,286 "shards": [287 "redpajama-github/AutoHotkey/part-00000.parquet"288 ]289 },290 {291 "dataset": "redpajama-github",292 "language": "AutoIt",293 "records": 1000,294 "source_records": 1,295 "sampling_with_replacement": true,296 "source_shard_count": 1,297 "shards": [298 "redpajama-github/AutoIt/part-00000.parquet"299 ]300 },301 {302 "dataset": "redpajama-github",303 "language": "Awk",304 "records": 1000,305 "source_records": 20748,306 "sampling_with_replacement": false,307 "source_shard_count": 1,308 "shards": [309 "redpajama-github/Awk/part-00000.parquet"310 ]311 },312 {313 "dataset": "redpajama-github",314 "language": "BASIC",315 "records": 1000,316 "source_records": 3,317 "sampling_with_replacement": true,318 "source_shard_count": 3,319 "shards": [320 "redpajama-github/BASIC/part-00000.parquet"321 ]322 },323 {324 "dataset": "redpajama-github",325 "language": "Ballerina",326 "records": 1000,327 "source_records": 56,328 "sampling_with_replacement": true,329 "source_shard_count": 45,330 "shards": [331 "redpajama-github/Ballerina/part-00000.parquet"332 ]333 },334 {335 "dataset": "redpajama-github",336 "language": "Batchfile",337 "records": 1000,338 "source_records": 1908,339 "sampling_with_replacement": false,340 "source_shard_count": 5,341 "shards": [342 "redpajama-github/Batchfile/part-00000.parquet"343 ]344 },345 {346 "dataset": "redpajama-github",347 "language": "Beef",348 "records": 1000,349 "source_records": 71,350 "sampling_with_replacement": true,351 "source_shard_count": 56,352 "shards": [353 "redpajama-github/Beef/part-00000.parquet"354 ]355 },356 {357 "dataset": "redpajama-github",358 "language": "Bison",359 "records": 1000,360 "source_records": 1865,361 "sampling_with_replacement": false,362 "source_shard_count": 5,363 "shards": [364 "redpajama-github/Bison/part-00000.parquet"365 ]366 },367 {368 "dataset": "redpajama-github",369 "language": "BitBake",370 "records": 1000,371 "source_records": 211,372 "sampling_with_replacement": true,373 "source_shard_count": 104,374 "shards": [375 "redpajama-github/BitBake/part-00000.parquet"376 ]377 },378 {379 "dataset": "redpajama-github",380 "language": "Blade",381 "records": 1000,382 "source_records": 38,383 "sampling_with_replacement": true,384 "source_shard_count": 33,385 "shards": [386 "redpajama-github/Blade/part-00000.parquet"387 ]388 },389 {390 "dataset": "redpajama-github",391 "language": "Bluespec",392 "records": 1000,393 "source_records": 176,394 "sampling_with_replacement": true,395 "source_shard_count": 95,396 "shards": [397 "redpajama-github/Bluespec/part-00000.parquet"398 ]399 },400 {401 "dataset": "redpajama-github",402 "language": "Boo",403 "records": 1000,404 "source_records": 324,405 "sampling_with_replacement": true,406 "source_shard_count": 105,407 "shards": [408 "redpajama-github/Boo/part-00000.parquet"409 ]410 },411 {412 "dataset": "redpajama-github",413 "language": "Boogie",414 "records": 1000,415 "source_records": 159,416 "sampling_with_replacement": true,417 "source_shard_count": 80,418 "shards": [419 "redpajama-github/Boogie/part-00000.parquet"420 ]421 },422 {423 "dataset": "redpajama-github",424 "language": "Brainfuck",425 "records": 1000,426 "source_records": 2219,427 "sampling_with_replacement": false,428 "source_shard_count": 6,429 "shards": [430 "redpajama-github/Brainfuck/part-00000.parquet"431 ]432 },433 {434 "dataset": "redpajama-github",435 "language": "Bro",436 "records": 1000,437 "source_records": 23,438 "sampling_with_replacement": true,439 "source_shard_count": 22,440 "shards": [441 "redpajama-github/Bro/part-00000.parquet"442 ]443 },444 {445 "dataset": "redpajama-github",446 "language": "C",447 "records": 1000,448 "source_records": 364291,449 "sampling_with_replacement": false,450 "source_shard_count": 6,451 "shards": [452 "redpajama-github/C/part-00000.parquet"453 ]454 },455 {456 "dataset": "redpajama-github",457 "language": "C#",458 "records": 1000,459 "source_records": 32429,460 "sampling_with_replacement": false,461 "source_shard_count": 1,462 "shards": [463 "redpajama-github/C#/part-00000.parquet"464 ]465 },466 {467 "dataset": "redpajama-github",468 "language": "C++",469 "records": 1000,470 "source_records": 58574,471 "sampling_with_replacement": false,472 "source_shard_count": 1,473 "shards": [474 "redpajama-github/C++/part-00000.parquet"475 ]476 },477 {478 "dataset": "redpajama-github",479 "language": "CAP CDS",480 "records": 1000,481 "source_records": 3,482 "sampling_with_replacement": true,483 "source_shard_count": 3,484 "shards": [485 "redpajama-github/CAP CDS/part-00000.parquet"486 ]487 },488 {489 "dataset": "redpajama-github",490 "language": "CMake",491 "records": 1000,492 "source_records": 305,493 "sampling_with_replacement": true,494 "source_shard_count": 85,495 "shards": [496 "redpajama-github/CMake/part-00000.parquet"497 ]498 },499 {500 "dataset": "redpajama-github",501 "language": "CSS",502 "records": 1000,503 "source_records": 361596,504 "sampling_with_replacement": false,505 "source_shard_count": 5,506 "shards": [507 "redpajama-github/CSS/part-00000.parquet"508 ]509 },510 {511 "dataset": "redpajama-github",512 "language": "CUE",513 "records": 1000,514 "source_records": 2,515 "sampling_with_replacement": true,516 "source_shard_count": 2,517 "shards": [518 "redpajama-github/CUE/part-00000.parquet"519 ]520 },521 {522 "dataset": "redpajama-github",523 "language": "CartoCSS",524 "records": 1000,525 "source_records": 3,526 "sampling_with_replacement": true,527 "source_shard_count": 3,528 "shards": [529 "redpajama-github/CartoCSS/part-00000.parquet"530 ]531 },532 {533 "dataset": "redpajama-github",534 "language": "Ceylon",535 "records": 1000,536 "source_records": 28,537 "sampling_with_replacement": true,538 "source_shard_count": 26,539 "shards": [540 "redpajama-github/Ceylon/part-00000.parquet"541 ]542 },543 {544 "dataset": "redpajama-github",545 "language": "Clean",546 "records": 1000,547 "source_records": 35,548 "sampling_with_replacement": true,549 "source_shard_count": 29,550 "shards": [551 "redpajama-github/Clean/part-00000.parquet"552 ]553 },554 {555 "dataset": "redpajama-github",556 "language": "Clojure",557 "records": 1000,558 "source_records": 1,559 "sampling_with_replacement": true,560 "source_shard_count": 1,561 "shards": [562 "redpajama-github/Clojure/part-00000.parquet"563 ]564 },565 {566 "dataset": "redpajama-github",567 "language": "Closure Templates",568 "records": 1000,569 "source_records": 39,570 "sampling_with_replacement": true,571 "source_shard_count": 34,572 "shards": [573 "redpajama-github/Closure Templates/part-00000.parquet"574 ]575 },576 {577 "dataset": "redpajama-github",578 "language": "CoffeeScript",579 "records": 1000,580 "source_records": 650,581 "sampling_with_replacement": true,582 "source_shard_count": 76,583 "shards": [584 "redpajama-github/CoffeeScript/part-00000.parquet"585 ]586 },587 {588 "dataset": "redpajama-github",589 "language": "ColdFusion",590 "records": 1000,591 "source_records": 36,592 "sampling_with_replacement": true,593 "source_shard_count": 26,594 "shards": [595 "redpajama-github/ColdFusion/part-00000.parquet"596 ]597 },598 {599 "dataset": "redpajama-github",600 "language": "Common Lisp",601 "records": 1000,602 "source_records": 2,603 "sampling_with_replacement": true,604 "source_shard_count": 2,605 "shards": [606 "redpajama-github/Common Lisp/part-00000.parquet"607 ]608 },609 {610 "dataset": "redpajama-github",611 "language": "Common Workflow Language",612 "records": 1000,613 "source_records": 1,614 "sampling_with_replacement": true,615 "source_shard_count": 1,616 "shards": [617 "redpajama-github/Common Workflow Language/part-00000.parquet"618 ]619 },620 {621 "dataset": "redpajama-github",622 "language": "Crystal",623 "records": 1000,624 "source_records": 6,625 "sampling_with_replacement": true,626 "source_shard_count": 6,627 "shards": [628 "redpajama-github/Crystal/part-00000.parquet"629 ]630 },631 {632 "dataset": "redpajama-github",633 "language": "Csound Document",634 "records": 1000,635 "source_records": 1,636 "sampling_with_replacement": true,637 "source_shard_count": 1,638 "shards": [639 "redpajama-github/Csound Document/part-00000.parquet"640 ]641 },642 {643 "dataset": "redpajama-github",644 "language": "Cucumber",645 "records": 1000,646 "source_records": 91,647 "sampling_with_replacement": true,648 "source_shard_count": 63,649 "shards": [650 "redpajama-github/Cucumber/part-00000.parquet"651 ]652 },653 {654 "dataset": "redpajama-github",655 "language": "Cuda",656 "records": 1000,657 "source_records": 2,658 "sampling_with_replacement": true,659 "source_shard_count": 2,660 "shards": [661 "redpajama-github/Cuda/part-00000.parquet"662 ]663 },664 {665 "dataset": "redpajama-github",666 "language": "Cycript",667 "records": 1000,668 "source_records": 4,669 "sampling_with_replacement": true,670 "source_shard_count": 4,671 "shards": [672 "redpajama-github/Cycript/part-00000.parquet"673 ]674 },675 {676 "dataset": "redpajama-github",677 "language": "Cython",678 "records": 1000,679 "source_records": 10,680 "sampling_with_replacement": true,681 "source_shard_count": 10,682 "shards": [683 "redpajama-github/Cython/part-00000.parquet"684 ]685 },686 {687 "dataset": "redpajama-github",688 "language": "D",689 "records": 1000,690 "source_records": 113,691 "sampling_with_replacement": true,692 "source_shard_count": 81,693 "shards": [694 "redpajama-github/D/part-00000.parquet"695 ]696 },697 {698 "dataset": "redpajama-github",699 "language": "DIGITAL Command Language",700 "records": 1000,701 "source_records": 1,702 "sampling_with_replacement": true,703 "source_shard_count": 1,704 "shards": [705 "redpajama-github/DIGITAL Command Language/part-00000.parquet"706 ]707 },708 {709 "dataset": "redpajama-github",710 "language": "DTrace",711 "records": 1000,712 "source_records": 17,713 "sampling_with_replacement": true,714 "source_shard_count": 16,715 "shards": [716 "redpajama-github/DTrace/part-00000.parquet"717 ]718 },719 {720 "dataset": "redpajama-github",721 "language": "Dart",722 "records": 1000,723 "source_records": 937,724 "sampling_with_replacement": true,725 "source_shard_count": 32,726 "shards": [727 "redpajama-github/Dart/part-00000.parquet"728 ]729 },730 {731 "dataset": "redpajama-github",732 "language": "Dhall",733 "records": 1000,734 "source_records": 75,735 "sampling_with_replacement": true,736 "source_shard_count": 61,737 "shards": [738 "redpajama-github/Dhall/part-00000.parquet"739 ]740 },741 {742 "dataset": "redpajama-github",743 "language": "Dockerfile",744 "records": 1000,745 "source_records": 1399,746 "sampling_with_replacement": false,747 "source_shard_count": 17,748 "shards": [749 "redpajama-github/Dockerfile/part-00000.parquet"750 ]751 },752 {753 "dataset": "redpajama-github",754 "language": "Dylan",755 "records": 1000,756 "source_records": 2,757 "sampling_with_replacement": true,758 "source_shard_count": 2,759 "shards": [760 "redpajama-github/Dylan/part-00000.parquet"761 ]762 },763 {764 "dataset": "redpajama-github",765 "language": "Eagle",766 "records": 1000,767 "source_records": 1,768 "sampling_with_replacement": true,769 "source_shard_count": 1,770 "shards": [771 "redpajama-github/Eagle/part-00000.parquet"772 ]773 },774 {775 "dataset": "redpajama-github",776 "language": "Elixir",777 "records": 1000,778 "source_records": 102,779 "sampling_with_replacement": true,780 "source_shard_count": 71,781 "shards": [782 "redpajama-github/Elixir/part-00000.parquet"783 ]784 },785 {786 "dataset": "redpajama-github",787 "language": "Elm",788 "records": 1000,789 "source_records": 214,790 "sampling_with_replacement": true,791 "source_shard_count": 99,792 "shards": [793 "redpajama-github/Elm/part-00000.parquet"794 ]795 },796 {797 "dataset": "redpajama-github",798 "language": "Erlang",799 "records": 1000,800 "source_records": 52,801 "sampling_with_replacement": true,802 "source_shard_count": 46,803 "shards": [804 "redpajama-github/Erlang/part-00000.parquet"805 ]806 },807 {808 "dataset": "redpajama-github",809 "language": "F#",810 "records": 1000,811 "source_records": 19,812 "sampling_with_replacement": true,813 "source_shard_count": 17,814 "shards": [815 "redpajama-github/F#/part-00000.parquet"816 ]817 },818 {819 "dataset": "redpajama-github",820 "language": "FLUX",821 "records": 1000,822 "source_records": 98,823 "sampling_with_replacement": true,824 "source_shard_count": 72,825 "shards": [826 "redpajama-github/FLUX/part-00000.parquet"827 ]828 },829 {830 "dataset": "redpajama-github",831 "language": "FORTRAN",832 "records": 1000,833 "source_records": 188,834 "sampling_with_replacement": true,835 "source_shard_count": 101,836 "shards": [837 "redpajama-github/FORTRAN/part-00000.parquet"838 ]839 },840 {841 "dataset": "redpajama-github",842 "language": "Forth",843 "records": 1000,844 "source_records": 25,845 "sampling_with_replacement": true,846 "source_shard_count": 25,847 "shards": [848 "redpajama-github/Forth/part-00000.parquet"849 ]850 },851 {852 "dataset": "redpajama-github",853 "language": "Fortran",854 "records": 1000,855 "source_records": 209,856 "sampling_with_replacement": true,857 "source_shard_count": 102,858 "shards": [859 "redpajama-github/Fortran/part-00000.parquet"860 ]861 },862 {863 "dataset": "redpajama-github",864 "language": "GAMS",865 "records": 1000,866 "source_records": 1,867 "sampling_with_replacement": true,868 "source_shard_count": 1,869 "shards": [870 "redpajama-github/GAMS/part-00000.parquet"871 ]872 },873 {874 "dataset": "redpajama-github",875 "language": "GAP",876 "records": 1000,877 "source_records": 10,878 "sampling_with_replacement": true,879 "source_shard_count": 9,880 "shards": [881 "redpajama-github/GAP/part-00000.parquet"882 ]883 },884 {885 "dataset": "redpajama-github",886 "language": "GDB",887 "records": 1000,888 "source_records": 8,889 "sampling_with_replacement": true,890 "source_shard_count": 8,891 "shards": [892 "redpajama-github/GDB/part-00000.parquet"893 ]894 },895 {896 "dataset": "redpajama-github",897 "language": "GDScript",898 "records": 1000,899 "source_records": 1,900 "sampling_with_replacement": true,901 "source_shard_count": 1,902 "shards": [903 "redpajama-github/GDScript/part-00000.parquet"904 ]905 },906 {907 "dataset": "redpajama-github",908 "language": "GLSL",909 "records": 1000,910 "source_records": 3214,911 "sampling_with_replacement": false,912 "source_shard_count": 2,913 "shards": [914 "redpajama-github/GLSL/part-00000.parquet"915 ]916 },917 {918 "dataset": "redpajama-github",919 "language": "Game Maker Language",920 "records": 1000,921 "source_records": 2,922 "sampling_with_replacement": true,923 "source_shard_count": 2,924 "shards": [925 "redpajama-github/Game Maker Language/part-00000.parquet"926 ]927 },928 {929 "dataset": "redpajama-github",930 "language": "Gherkin",931 "records": 1000,932 "source_records": 126,933 "sampling_with_replacement": true,934 "source_shard_count": 82,935 "shards": [936 "redpajama-github/Gherkin/part-00000.parquet"937 ]938 },939 {940 "dataset": "redpajama-github",941 "language": "Gnuplot",942 "records": 1000,943 "source_records": 13,944 "sampling_with_replacement": true,945 "source_shard_count": 12,946 "shards": [947 "redpajama-github/Gnuplot/part-00000.parquet"948 ]949 },950 {951 "dataset": "redpajama-github",952 "language": "Go",953 "records": 1000,954 "source_records": 19783,955 "sampling_with_replacement": false,956 "source_shard_count": 1,957 "shards": [958 "redpajama-github/Go/part-00000.parquet"959 ]960 },961 {962 "dataset": "redpajama-github",963 "language": "Golo",964 "records": 1000,965 "source_records": 1,966 "sampling_with_replacement": true,967 "source_shard_count": 1,968 "shards": [969 "redpajama-github/Golo/part-00000.parquet"970 ]971 },972 {973 "dataset": "redpajama-github",974 "language": "Gosu",975 "records": 1000,976 "source_records": 64,977 "sampling_with_replacement": true,978 "source_shard_count": 56,979 "shards": [980 "redpajama-github/Gosu/part-00000.parquet"981 ]982 },983 {984 "dataset": "redpajama-github",985 "language": "Groovy",986 "records": 1000,987 "source_records": 8367,988 "sampling_with_replacement": false,989 "source_shard_count": 1,990 "shards": [991 "redpajama-github/Groovy/part-00000.parquet"992 ]993 },994 {995 "dataset": "redpajama-github",996 "language": "HCL",997 "records": 1000,998 "source_records": 112,999 "sampling_with_replacement": true,1000 "source_shard_count": 78,1001 "shards": [1002 "redpajama-github/HCL/part-00000.parquet"1003 ]1004 },1005 {1006 "dataset": "redpajama-github",1007 "language": "HLSL",1008 "records": 1000,1009 "source_records": 304,1010 "sampling_with_replacement": true,1011 "source_shard_count": 104,1012 "shards": [1013 "redpajama-github/HLSL/part-00000.parquet"1014 ]1015 },1016 {1017 "dataset": "redpajama-github",1018 "language": "HTML",1019 "records": 1000,1020 "source_records": 8545,1021 "sampling_with_replacement": false,1022 "source_shard_count": 1,1023 "shards": [1024 "redpajama-github/HTML/part-00000.parquet"1025 ]1026 },1027 {1028 "dataset": "redpajama-github",1029 "language": "Hack",1030 "records": 1000,1031 "source_records": 22,1032 "sampling_with_replacement": true,1033 "source_shard_count": 19,1034 "shards": [1035 "redpajama-github/Hack/part-00000.parquet"1036 ]1037 },1038 {1039 "dataset": "redpajama-github",1040 "language": "Haml",1041 "records": 1000,1042 "source_records": 18,1043 "sampling_with_replacement": true,1044 "source_shard_count": 16,1045 "shards": [1046 "redpajama-github/Haml/part-00000.parquet"1047 ]1048 },1049 {1050 "dataset": "redpajama-github",1051 "language": "Haskell",1052 "records": 1000,1053 "source_records": 27400,1054 "sampling_with_replacement": false,1055 "source_shard_count": 1,1056 "shards": [1057 "redpajama-github/Haskell/part-00000.parquet"1058 ]1059 },1060 {1061 "dataset": "redpajama-github",1062 "language": "Haxe",1063 "records": 1000,1064 "source_records": 111,1065 "sampling_with_replacement": true,1066 "source_shard_count": 75,1067 "shards": [1068 "redpajama-github/Haxe/part-00000.parquet"1069 ]1070 },1071 {1072 "dataset": "redpajama-github",1073 "language": "HolyC",1074 "records": 1000,1075 "source_records": 1,1076 "sampling_with_replacement": true,1077 "source_shard_count": 1,1078 "shards": [1079 "redpajama-github/HolyC/part-00000.parquet"1080 ]1081 },1082 {1083 "dataset": "redpajama-github",1084 "language": "IDL",1085 "records": 1000,1086 "source_records": 84,1087 "sampling_with_replacement": true,1088 "source_shard_count": 59,1089 "shards": [1090 "redpajama-github/IDL/part-00000.parquet"1091 ]1092 },1093 {1094 "dataset": "redpajama-github",1095 "language": "IGOR Pro",1096 "records": 1000,1097 "source_records": 1,1098 "sampling_with_replacement": true,1099 "source_shard_count": 1,1100 "shards": [1101 "redpajama-github/IGOR Pro/part-00000.parquet"1102 ]1103 },1104 {1105 "dataset": "redpajama-github",1106 "language": "ImageJ Macro",1107 "records": 1000,1108 "source_records": 38,1109 "sampling_with_replacement": true,1110 "source_shard_count": 32,1111 "shards": [1112 "redpajama-github/ImageJ Macro/part-00000.parquet"1113 ]1114 },1115 {1116 "dataset": "redpajama-github",1117 "language": "Inform 7",1118 "records": 1000,1119 "source_records": 1,1120 "sampling_with_replacement": true,1121 "source_shard_count": 1,1122 "shards": [1123 "redpajama-github/Inform 7/part-00000.parquet"1124 ]1125 },1126 {1127 "dataset": "redpajama-github",1128 "language": "Inno Setup",1129 "records": 1000,1130 "source_records": 4,1131 "sampling_with_replacement": true,1132 "source_shard_count": 4,1133 "shards": [1134 "redpajama-github/Inno Setup/part-00000.parquet"1135 ]1136 },1137 {1138 "dataset": "redpajama-github",1139 "language": "Io",1140 "records": 1000,1141 "source_records": 13,1142 "sampling_with_replacement": true,1143 "source_shard_count": 11,1144 "shards": [1145 "redpajama-github/Io/part-00000.parquet"1146 ]1147 },1148 {1149 "dataset": "redpajama-github",1150 "language": "Java",1151 "records": 1000,1152 "source_records": 172136,1153 "sampling_with_replacement": false,1154 "source_shard_count": 2,1155 "shards": [1156 "redpajama-github/Java/part-00000.parquet"1157 ]1158 },1159 {1160 "dataset": "redpajama-github",1161 "language": "JavaScript",1162 "records": 1000,1163 "source_records": 106385,1164 "sampling_with_replacement": false,1165 "source_shard_count": 2,1166 "shards": [1167 "redpajama-github/JavaScript/part-00000.parquet"1168 ]1169 },1170 {1171 "dataset": "redpajama-github",1172 "language": "Jinja",1173 "records": 1000,1174 "source_records": 2,1175 "sampling_with_replacement": true,1176 "source_shard_count": 2,1177 "shards": [1178 "redpajama-github/Jinja/part-00000.parquet"1179 ]1180 },1181 {1182 "dataset": "redpajama-github",1183 "language": "Julia",1184 "records": 1000,1185 "source_records": 131,1186 "sampling_with_replacement": true,1187 "source_shard_count": 79,1188 "shards": [1189 "redpajama-github/Julia/part-00000.parquet"1190 ]1191 },1192 {1193 "dataset": "redpajama-github",1194 "language": "Kotlin",1195 "records": 1000,1196 "source_records": 16,1197 "sampling_with_replacement": true,1198 "source_shard_count": 16,1199 "shards": [1200 "redpajama-github/Kotlin/part-00000.parquet"