FineEnvs/SmolDataEnvs-harbor-eval
📊 SmolDataEnvs: Harbor (eval) 5.5K+ RL tasks for hill-climbing small models in code and data science. A 2B model on these tasks. Left: what it optimises. Right: 144 held-out tasks it never trains on. Two runs over the same 5,000 tasks: shuffled against a curriculum ordered easiest to hardest. The validation suite: 144 tasks, small enough to run every few hundred training steps without the eval becoming the expensive part of the loop. Packaged in Harbor format… See the full description on the dataset page: https://huggingface.co/datasets/FineEnvs/SmolDataEnvs-harbor-eval.
0
1[2 {3 "name": "smoldataenvs-harbor-eval",4 "version": "2.0",5 "description": "144 deterministic data-analysis tasks (no LLM judge).",6 "tasks": [7 {8 "name": "0000_886_886039_qa_5",9 "path": "tasks/0000_886_886039_qa_5"10 },11 {12 "name": "0000_901_901543_qa_2",13 "path": "tasks/0000_901_901543_qa_2"14 },15 {16 "name": "0000_901_901543_qa_3",17 "path": "tasks/0000_901_901543_qa_3"18 },19 {20 "name": "0000_903_903504_qa_1",21 "path": "tasks/0000_903_903504_qa_1"22 },23 {24 "name": "0000_939_939639_qa_2",25 "path": "tasks/0000_939_939639_qa_2"26 },27 {28 "name": "0000_939_939639_qa_3",29 "path": "tasks/0000_939_939639_qa_3"30 },31 {32 "name": "0000_949_949403_qa_3",33 "path": "tasks/0000_949_949403_qa_3"34 },35 {36 "name": "0000_975_975906_qa_1",37 "path": "tasks/0000_975_975906_qa_1"38 },39 {40 "name": "0000_986_986420_qa_2",41 "path": "tasks/0000_986_986420_qa_2"42 },43 {44 "name": "0000_986_986420_qa_4",45 "path": "tasks/0000_986_986420_qa_4"46 },47 {48 "name": "0000_988_988342_qa_4",49 "path": "tasks/0000_988_988342_qa_4"50 },51 {52 "name": "0000_988_988657_qa_4",53 "path": "tasks/0000_988_988657_qa_4"54 },55 {56 "name": "0000_991_991010_qa_1",57 "path": "tasks/0000_991_991010_qa_1"58 },59 {60 "name": "0000_991_991010_qa_2",61 "path": "tasks/0000_991_991010_qa_2"62 },63 {64 "name": "0000_991_991010_qa_3",65 "path": "tasks/0000_991_991010_qa_3"66 },67 {68 "name": "0000_994_994384_qa_5",69 "path": "tasks/0000_994_994384_qa_5"70 },71 {72 "name": "0000_994_994443_qa_4",73 "path": "tasks/0000_994_994443_qa_4"74 },75 {76 "name": "0001_004_1004716_qa_2",77 "path": "tasks/0001_004_1004716_qa_2"78 },79 {80 "name": "0001_024_1024990_qa_2",81 "path": "tasks/0001_024_1024990_qa_2"82 },83 {84 "name": "0001_027_1027372_qa_2",85 "path": "tasks/0001_027_1027372_qa_2"86 },87 {88 "name": "0001_029_1029150_qa_2",89 "path": "tasks/0001_029_1029150_qa_2"90 },91 {92 "name": "0001_036_1036744_qa_1",93 "path": "tasks/0001_036_1036744_qa_1"94 },95 {96 "name": "0001_042_1042725_qa_2",97 "path": "tasks/0001_042_1042725_qa_2"98 },99 {100 "name": "0001_044_1044175_qa_3",101 "path": "tasks/0001_044_1044175_qa_3"102 },103 {104 "name": "0001_062_1062946_qa_2",105 "path": "tasks/0001_062_1062946_qa_2"106 },107 {108 "name": "0001_065_1065379_qa_3",109 "path": "tasks/0001_065_1065379_qa_3"110 },111 {112 "name": "0001_067_1067596_qa_5",113 "path": "tasks/0001_067_1067596_qa_5"114 },115 {116 "name": "0001_068_1068916_qa_1",117 "path": "tasks/0001_068_1068916_qa_1"118 },119 {120 "name": "0001_068_1068916_qa_5",121 "path": "tasks/0001_068_1068916_qa_5"122 },123 {124 "name": "0001_078_1078204_qa_2",125 "path": "tasks/0001_078_1078204_qa_2"126 },127 {128 "name": "0001_079_1079149_qa_2",129 "path": "tasks/0001_079_1079149_qa_2"130 },131 {132 "name": "0001_082_1082305_qa_1",133 "path": "tasks/0001_082_1082305_qa_1"134 },135 {136 "name": "0001_083_1083006_qa_5",137 "path": "tasks/0001_083_1083006_qa_5"138 },139 {140 "name": "0001_091_1091047_qa_4",141 "path": "tasks/0001_091_1091047_qa_4"142 },143 {144 "name": "0001_091_1091047_qa_5",145 "path": "tasks/0001_091_1091047_qa_5"146 },147 {148 "name": "0001_095_1095486_qa_2",149 "path": "tasks/0001_095_1095486_qa_2"150 },151 {152 "name": "0001_102_1102505_qa_2",153 "path": "tasks/0001_102_1102505_qa_2"154 },155 {156 "name": "0001_104_1104901_qa_2",157 "path": "tasks/0001_104_1104901_qa_2"158 },159 {160 "name": "0001_104_1104901_qa_3",161 "path": "tasks/0001_104_1104901_qa_3"162 },163 {164 "name": "0001_104_1104959_qa_2",165 "path": "tasks/0001_104_1104959_qa_2"166 },167 {168 "name": "0001_105_1105529_qa_5",169 "path": "tasks/0001_105_1105529_qa_5"170 },171 {172 "name": "0001_105_1105892_qa_3",173 "path": "tasks/0001_105_1105892_qa_3"174 },175 {176 "name": "0001_105_1105892_qa_5",177 "path": "tasks/0001_105_1105892_qa_5"178 },179 {180 "name": "0001_116_1116342_qa_3",181 "path": "tasks/0001_116_1116342_qa_3"182 },183 {184 "name": "0001_116_1116342_qa_5",185 "path": "tasks/0001_116_1116342_qa_5"186 },187 {188 "name": "0001_123_1123075_qa_4",189 "path": "tasks/0001_123_1123075_qa_4"190 },191 {192 "name": "0001_124_1124736_qa_1",193 "path": "tasks/0001_124_1124736_qa_1"194 },195 {196 "name": "0001_124_1124736_qa_2",197 "path": "tasks/0001_124_1124736_qa_2"198 },199 {200 "name": "0001_133_1133224_qa_4",201 "path": "tasks/0001_133_1133224_qa_4"202 },203 {204 "name": "0001_133_1133625_qa_3",205 "path": "tasks/0001_133_1133625_qa_3"206 },207 {208 "name": "0001_134_1134780_qa_3",209 "path": "tasks/0001_134_1134780_qa_3"210 },211 {212 "name": "0001_134_1134780_qa_5",213 "path": "tasks/0001_134_1134780_qa_5"214 },215 {216 "name": "0001_548_1548552_qa_1",217 "path": "tasks/0001_548_1548552_qa_1"218 },219 {220 "name": "0001_551_1551019_qa_3",221 "path": "tasks/0001_551_1551019_qa_3"222 },223 {224 "name": "0001_573_1573460_qa_3",225 "path": "tasks/0001_573_1573460_qa_3"226 },227 {228 "name": "0001_574_1574525_qa_2",229 "path": "tasks/0001_574_1574525_qa_2"230 },231 {232 "name": "0001_583_1583897_qa_5",233 "path": "tasks/0001_583_1583897_qa_5"234 },235 {236 "name": "0001_586_1586397_qa_4",237 "path": "tasks/0001_586_1586397_qa_4"238 },239 {240 "name": "0001_594_1594994_qa_1",241 "path": "tasks/0001_594_1594994_qa_1"242 },243 {244 "name": "0001_598_1598064_qa_2",245 "path": "tasks/0001_598_1598064_qa_2"246 },247 {248 "name": "0001_613_1613977_qa_3",249 "path": "tasks/0001_613_1613977_qa_3"250 },251 {252 "name": "0001_649_1649151_qa_1",253 "path": "tasks/0001_649_1649151_qa_1"254 },255 {256 "name": "0001_666_1666101_qa_2",257 "path": "tasks/0001_666_1666101_qa_2"258 },259 {260 "name": "0001_672_1672543_qa_4",261 "path": "tasks/0001_672_1672543_qa_4"262 },263 {264 "name": "0001_683_1683832_qa_5",265 "path": "tasks/0001_683_1683832_qa_5"266 },267 {268 "name": "0001_689_1689474_qa_2",269 "path": "tasks/0001_689_1689474_qa_2"270 },271 {272 "name": "0001_736_1736876_qa_1",273 "path": "tasks/0001_736_1736876_qa_1"274 },275 {276 "name": "0001_756_1756499_qa_1",277 "path": "tasks/0001_756_1756499_qa_1"278 },279 {280 "name": "0001_830_1830390_qa_5",281 "path": "tasks/0001_830_1830390_qa_5"282 },283 {284 "name": "0001_850_1850034_qa_4",285 "path": "tasks/0001_850_1850034_qa_4"286 },287 {288 "name": "0001_860_1860651_qa_2",289 "path": "tasks/0001_860_1860651_qa_2"290 },291 {292 "name": "0001_868_1868953_qa_3",293 "path": "tasks/0001_868_1868953_qa_3"294 },295 {296 "name": "0001_875_1875604_qa_4",297 "path": "tasks/0001_875_1875604_qa_4"298 },299 {300 "name": "0001_955_1955907_qa_1",301 "path": "tasks/0001_955_1955907_qa_1"302 },303 {304 "name": "0002_094_2094844_qa_2",305 "path": "tasks/0002_094_2094844_qa_2"306 },307 {308 "name": "0002_152_2152188_qa_1",309 "path": "tasks/0002_152_2152188_qa_1"310 },311 {312 "name": "0002_157_2157761_qa_4",313 "path": "tasks/0002_157_2157761_qa_4"314 },315 {316 "name": "0002_219_2219893_qa_1",317 "path": "tasks/0002_219_2219893_qa_1"318 },319 {320 "name": "0002_238_2238090_qa_4",321 "path": "tasks/0002_238_2238090_qa_4"322 },323 {324 "name": "0002_248_2248246_qa_1",325 "path": "tasks/0002_248_2248246_qa_1"326 },327 {328 "name": "0002_264_2264461_qa_2",329 "path": "tasks/0002_264_2264461_qa_2"330 },331 {332 "name": "0002_335_2335686_qa_2",333 "path": "tasks/0002_335_2335686_qa_2"334 },335 {336 "name": "0002_350_2350400_qa_1",337 "path": "tasks/0002_350_2350400_qa_1"338 },339 {340 "name": "0010_256_10256782_qa_1",341 "path": "tasks/0010_256_10256782_qa_1"342 },343 {344 "name": "0010_306_10306280_qa_2",345 "path": "tasks/0010_306_10306280_qa_2"346 },347 {348 "name": "0010_307_10307949_qa_1",349 "path": "tasks/0010_307_10307949_qa_1"350 },351 {352 "name": "0010_397_10397139_qa_1",353 "path": "tasks/0010_397_10397139_qa_1"354 },355 {356 "name": "0010_397_10397139_qa_4",357 "path": "tasks/0010_397_10397139_qa_4"358 },359 {360 "name": "0010_908_10908139_qa_1",361 "path": "tasks/0010_908_10908139_qa_1"362 },363 {364 "name": "0011_000_11000770_qa_4",365 "path": "tasks/0011_000_11000770_qa_4"366 },367 {368 "name": "0011_000_11000770_qa_5",369 "path": "tasks/0011_000_11000770_qa_5"370 },371 {372 "name": "0011_045_11045524_qa_1",373 "path": "tasks/0011_045_11045524_qa_1"374 },375 {376 "name": "0011_045_11045524_qa_3",377 "path": "tasks/0011_045_11045524_qa_3"378 },379 {380 "name": "0011_045_11045524_qa_5",381 "path": "tasks/0011_045_11045524_qa_5"382 },383 {384 "name": "0011_081_11081921_qa_1",385 "path": "tasks/0011_081_11081921_qa_1"386 },387 {388 "name": "0011_088_11088784_qa_5",389 "path": "tasks/0011_088_11088784_qa_5"390 },391 {392 "name": "0011_089_11089643_qa_3",393 "path": "tasks/0011_089_11089643_qa_3"394 },395 {396 "name": "0011_121_11121152_qa_3",397 "path": "tasks/0011_121_11121152_qa_3"398 },399 {400 "name": "0011_222_11222401_qa_5",401 "path": "tasks/0011_222_11222401_qa_5"402 },403 {404 "name": "0011_355_11355842_qa_1",405 "path": "tasks/0011_355_11355842_qa_1"406 },407 {408 "name": "0011_412_11412178_qa_1",409 "path": "tasks/0011_412_11412178_qa_1"410 },411 {412 "name": "0011_508_11508075_qa_5",413 "path": "tasks/0011_508_11508075_qa_5"414 },415 {416 "name": "0011_587_11587142_qa_2",417 "path": "tasks/0011_587_11587142_qa_2"418 },419 {420 "name": "0011_587_11587142_qa_5",421 "path": "tasks/0011_587_11587142_qa_5"422 },423 {424 "name": "0011_723_11723635_qa_4",425 "path": "tasks/0011_723_11723635_qa_4"426 },427 {428 "name": "0011_754_11754124_qa_4",429 "path": "tasks/0011_754_11754124_qa_4"430 },431 {432 "name": "0011_820_11820848_qa_3",433 "path": "tasks/0011_820_11820848_qa_3"434 },435 {436 "name": "0011_843_11843765_qa_5",437 "path": "tasks/0011_843_11843765_qa_5"438 },439 {440 "name": "0011_869_11869966_qa_4",441 "path": "tasks/0011_869_11869966_qa_4"442 },443 {444 "name": "0012_222_12222108_qa_5",445 "path": "tasks/0012_222_12222108_qa_5"446 },447 {448 "name": "0012_330_12330544_qa_2",449 "path": "tasks/0012_330_12330544_qa_2"450 },451 {452 "name": "0012_337_12337856_qa_2",453 "path": "tasks/0012_337_12337856_qa_2"454 },455 {456 "name": "0012_556_12556896_qa_3",457 "path": "tasks/0012_556_12556896_qa_3"458 },459 {460 "name": "0012_692_12692192_qa_3",461 "path": "tasks/0012_692_12692192_qa_3"462 },463 {464 "name": "0012_953_12953382_qa_3",465 "path": "tasks/0012_953_12953382_qa_3"466 },467 {468 "name": "0013_243_13243838_qa_3",469 "path": "tasks/0013_243_13243838_qa_3"470 },471 {472 "name": "0013_355_13355821_qa_3",473 "path": "tasks/0013_355_13355821_qa_3"474 },475 {476 "name": "0013_695_13695761_qa_1",477 "path": "tasks/0013_695_13695761_qa_1"478 },479 {480 "name": "0013_706_13706047_qa_5",481 "path": "tasks/0013_706_13706047_qa_5"482 },483 {484 "name": "0013_718_13718057_qa_3",485 "path": "tasks/0013_718_13718057_qa_3"486 },487 {488 "name": "0013_791_13791869_qa_4",489 "path": "tasks/0013_791_13791869_qa_4"490 },491 {492 "name": "0013_884_13884693_qa_3",493 "path": "tasks/0013_884_13884693_qa_3"494 },495 {496 "name": "0013_884_13884693_qa_4",497 "path": "tasks/0013_884_13884693_qa_4"498 },499 {500 "name": "0014_516_14516985_qa_2",501 "path": "tasks/0014_516_14516985_qa_2"502 },503 {504 "name": "0014_516_14516985_qa_5",505 "path": "tasks/0014_516_14516985_qa_5"506 },507 {508 "name": "0014_594_14594724_qa_2",509 "path": "tasks/0014_594_14594724_qa_2"510 },511 {512 "name": "0014_594_14594724_qa_5",513 "path": "tasks/0014_594_14594724_qa_5"514 },515 {516 "name": "0014_681_14681789_qa_3",517 "path": "tasks/0014_681_14681789_qa_3"518 },519 {520 "name": "0014_938_14938015_qa_3",521 "path": "tasks/0014_938_14938015_qa_3"522 },523 {524 "name": "0015_124_15124276_qa_2",525 "path": "tasks/0015_124_15124276_qa_2"526 },527 {528 "name": "0015_168_15168316_qa_2",529 "path": "tasks/0015_168_15168316_qa_2"530 },531 {532 "name": "0015_168_15168316_qa_4",533 "path": "tasks/0015_168_15168316_qa_4"534 },535 {536 "name": "0015_288_15288932_qa_4",537 "path": "tasks/0015_288_15288932_qa_4"538 },539 {540 "name": "0015_525_15525489_qa_2",541 "path": "tasks/0015_525_15525489_qa_2"542 },543 {544 "name": "0015_628_15628139_qa_5",545 "path": "tasks/0015_628_15628139_qa_5"546 },547 {548 "name": "0015_628_15628586_qa_3",549 "path": "tasks/0015_628_15628586_qa_3"550 },551 {552 "name": "0015_693_15693594_qa_5",553 "path": "tasks/0015_693_15693594_qa_5"554 },555 {556 "name": "0016_039_16039611_qa_3",557 "path": "tasks/0016_039_16039611_qa_3"558 },559 {560 "name": "0016_567_16567793_qa_4",561 "path": "tasks/0016_567_16567793_qa_4"562 },563 {564 "name": "0016_567_16567793_qa_5",565 "path": "tasks/0016_567_16567793_qa_5"566 },567 {568 "name": "0016_613_16613084_qa_1",569 "path": "tasks/0016_613_16613084_qa_1"570 },571 {572 "name": "0016_734_16734421_qa_3",573 "path": "tasks/0016_734_16734421_qa_3"574 },575 {576 "name": "0016_971_16971389_qa_2",577 "path": "tasks/0016_971_16971389_qa_2"578 },579 {580 "name": "0017_436_17436469_qa_4",581 "path": "tasks/0017_436_17436469_qa_4"582 }583 ]584 }585]586