SlayerLab/pollock-mini-lm-125m
32.1k
1{2 "results": {3 "blimp_adjunct_island": {4 "name": "blimp_adjunct_island",5 "alias": "blimp_adjunct_island",6 "sample_len": 1000,7 "acc,none": 0.85,8 "acc_stderr,none": 0.0112972398234094189 },10 "blimp_anaphor_gender_agreement": {11 "name": "blimp_anaphor_gender_agreement",12 "alias": "blimp_anaphor_gender_agreement",13 "sample_len": 1000,14 "acc,none": 0.979,15 "acc_stderr,none": 0.004536472151306516516 },17 "blimp_anaphor_number_agreement": {18 "name": "blimp_anaphor_number_agreement",19 "alias": "blimp_anaphor_number_agreement",20 "sample_len": 1000,21 "acc,none": 0.992,22 "acc_stderr,none": 0.00281850030050449823 },24 "blimp_animate_subject_passive": {25 "name": "blimp_animate_subject_passive",26 "alias": "blimp_animate_subject_passive",27 "sample_len": 1000,28 "acc,none": 0.803,29 "acc_stderr,none": 0.01258369378796819630 },31 "blimp_animate_subject_trans": {32 "name": "blimp_animate_subject_trans",33 "alias": "blimp_animate_subject_trans",34 "sample_len": 1000,35 "acc,none": 0.866,36 "acc_stderr,none": 0.01077776229836973837 },38 "blimp_causative": {39 "name": "blimp_causative",40 "alias": "blimp_causative",41 "sample_len": 1000,42 "acc,none": 0.719,43 "acc_stderr,none": 0.0142211547084349444 },45 "blimp_complex_NP_island": {46 "name": "blimp_complex_NP_island",47 "alias": "blimp_complex_NP_island",48 "sample_len": 1000,49 "acc,none": 0.533,50 "acc_stderr,none": 0.01578480789113887651 },52 "blimp_coordinate_structure_constraint_complex_left_branch": {53 "name": "blimp_coordinate_structure_constraint_complex_left_branch",54 "alias": "blimp_coordinate_structure_constraint_complex_left_branch",55 "sample_len": 1000,56 "acc,none": 0.632,57 "acc_stderr,none": 0.01525807356152174358 },59 "blimp_coordinate_structure_constraint_object_extraction": {60 "name": "blimp_coordinate_structure_constraint_object_extraction",61 "alias": "blimp_coordinate_structure_constraint_object_extraction",62 "sample_len": 1000,63 "acc,none": 0.796,64 "acc_stderr,none": 0.01274937435902431165 },66 "blimp_determiner_noun_agreement_1": {67 "name": "blimp_determiner_noun_agreement_1",68 "alias": "blimp_determiner_noun_agreement_1",69 "sample_len": 1000,70 "acc,none": 0.985,71 "acc_stderr,none": 0.00384574957450297272 },73 "blimp_determiner_noun_agreement_2": {74 "name": "blimp_determiner_noun_agreement_2",75 "alias": "blimp_determiner_noun_agreement_2",76 "sample_len": 1000,77 "acc,none": 0.963,78 "acc_stderr,none": 0.005972157622389639579 },80 "blimp_determiner_noun_agreement_irregular_1": {81 "name": "blimp_determiner_noun_agreement_irregular_1",82 "alias": "blimp_determiner_noun_agreement_irregular_1",83 "sample_len": 1000,84 "acc,none": 0.94,85 "acc_stderr,none": 0.00751375115747497186 },87 "blimp_determiner_noun_agreement_irregular_2": {88 "name": "blimp_determiner_noun_agreement_irregular_2",89 "alias": "blimp_determiner_noun_agreement_irregular_2",90 "sample_len": 1000,91 "acc,none": 0.947,92 "acc_stderr,none": 0.0070881056172464993 },94 "blimp_determiner_noun_agreement_with_adj_2": {95 "name": "blimp_determiner_noun_agreement_with_adj_2",96 "alias": "blimp_determiner_noun_agreement_with_adj_2",97 "sample_len": 1000,98 "acc,none": 0.925,99 "acc_stderr,none": 0.008333333333333333100 },101 "blimp_determiner_noun_agreement_with_adj_irregular_1": {102 "name": "blimp_determiner_noun_agreement_with_adj_irregular_1",103 "alias": "blimp_determiner_noun_agreement_with_adj_irregular_1",104 "sample_len": 1000,105 "acc,none": 0.858,106 "acc_stderr,none": 0.011043457699378305107 },108 "blimp_determiner_noun_agreement_with_adj_irregular_2": {109 "name": "blimp_determiner_noun_agreement_with_adj_irregular_2",110 "alias": "blimp_determiner_noun_agreement_with_adj_irregular_2",111 "sample_len": 1000,112 "acc,none": 0.921,113 "acc_stderr,none": 0.00853415677333337114 },115 "blimp_determiner_noun_agreement_with_adjective_1": {116 "name": "blimp_determiner_noun_agreement_with_adjective_1",117 "alias": "blimp_determiner_noun_agreement_with_adjective_1",118 "sample_len": 1000,119 "acc,none": 0.95,120 "acc_stderr,none": 0.006895472974897918121 },122 "blimp_distractor_agreement_relational_noun": {123 "name": "blimp_distractor_agreement_relational_noun",124 "alias": "blimp_distractor_agreement_relational_noun",125 "sample_len": 1000,126 "acc,none": 0.849,127 "acc_stderr,none": 0.011328165223341657128 },129 "blimp_distractor_agreement_relative_clause": {130 "name": "blimp_distractor_agreement_relative_clause",131 "alias": "blimp_distractor_agreement_relative_clause",132 "sample_len": 1000,133 "acc,none": 0.762,134 "acc_stderr,none": 0.013473586661967093135 },136 "blimp_drop_argument": {137 "name": "blimp_drop_argument",138 "alias": "blimp_drop_argument",139 "sample_len": 1000,140 "acc,none": 0.811,141 "acc_stderr,none": 0.01238678458811779142 },143 "blimp_ellipsis_n_bar_1": {144 "name": "blimp_ellipsis_n_bar_1",145 "alias": "blimp_ellipsis_n_bar_1",146 "sample_len": 1000,147 "acc,none": 0.821,148 "acc_stderr,none": 0.012128730605719056149 },150 "blimp_ellipsis_n_bar_2": {151 "name": "blimp_ellipsis_n_bar_2",152 "alias": "blimp_ellipsis_n_bar_2",153 "sample_len": 1000,154 "acc,none": 0.903,155 "acc_stderr,none": 0.009363689373248062156 },157 "blimp_existential_there_object_raising": {158 "name": "blimp_existential_there_object_raising",159 "alias": "blimp_existential_there_object_raising",160 "sample_len": 1000,161 "acc,none": 0.809,162 "acc_stderr,none": 0.012436787112179482163 },164 "blimp_existential_there_quantifiers_1": {165 "name": "blimp_existential_there_quantifiers_1",166 "alias": "blimp_existential_there_quantifiers_1",167 "sample_len": 1000,168 "acc,none": 0.994,169 "acc_stderr,none": 0.002443352199329841170 },171 "blimp_existential_there_quantifiers_2": {172 "name": "blimp_existential_there_quantifiers_2",173 "alias": "blimp_existential_there_quantifiers_2",174 "sample_len": 1000,175 "acc,none": 0.334,176 "acc_stderr,none": 0.014922019523732928177 },178 "blimp_existential_there_subject_raising": {179 "name": "blimp_existential_there_subject_raising",180 "alias": "blimp_existential_there_subject_raising",181 "sample_len": 1000,182 "acc,none": 0.887,183 "acc_stderr,none": 0.010016552866696773184 },185 "blimp_expletive_it_object_raising": {186 "name": "blimp_expletive_it_object_raising",187 "alias": "blimp_expletive_it_object_raising",188 "sample_len": 1000,189 "acc,none": 0.781,190 "acc_stderr,none": 0.013084731950262116191 },192 "blimp_inchoative": {193 "name": "blimp_inchoative",194 "alias": "blimp_inchoative",195 "sample_len": 1000,196 "acc,none": 0.673,197 "acc_stderr,none": 0.014842213153411162198 },199 "blimp_intransitive": {200 "name": "blimp_intransitive",201 "alias": "blimp_intransitive",202 "sample_len": 1000,203 "acc,none": 0.814,204 "acc_stderr,none": 0.012310790208412926205 },206 "blimp_irregular_past_participle_adjectives": {207 "name": "blimp_irregular_past_participle_adjectives",208 "alias": "blimp_irregular_past_participle_adjectives",209 "sample_len": 1000,210 "acc,none": 0.896,211 "acc_stderr,none": 0.009658016218524242212 },213 "blimp_irregular_past_participle_verbs": {214 "name": "blimp_irregular_past_participle_verbs",215 "alias": "blimp_irregular_past_participle_verbs",216 "sample_len": 1000,217 "acc,none": 0.868,218 "acc_stderr,none": 0.010709373963527965219 },220 "blimp_irregular_plural_subject_verb_agreement_1": {221 "name": "blimp_irregular_plural_subject_verb_agreement_1",222 "alias": "blimp_irregular_plural_subject_verb_agreement_1",223 "sample_len": 1000,224 "acc,none": 0.907,225 "acc_stderr,none": 0.009188875634996669226 },227 "blimp_irregular_plural_subject_verb_agreement_2": {228 "name": "blimp_irregular_plural_subject_verb_agreement_2",229 "alias": "blimp_irregular_plural_subject_verb_agreement_2",230 "sample_len": 1000,231 "acc,none": 0.925,232 "acc_stderr,none": 0.008333333333333333233 },234 "blimp_left_branch_island_echo_question": {235 "name": "blimp_left_branch_island_echo_question",236 "alias": "blimp_left_branch_island_echo_question",237 "sample_len": 1000,238 "acc,none": 0.205,239 "acc_stderr,none": 0.012772554096113201240 },241 "blimp_left_branch_island_simple_question": {242 "name": "blimp_left_branch_island_simple_question",243 "alias": "blimp_left_branch_island_simple_question",244 "sample_len": 1000,245 "acc,none": 0.628,246 "acc_stderr,none": 0.01529214994204052247 },248 "blimp_matrix_question_npi_licensor_present": {249 "name": "blimp_matrix_question_npi_licensor_present",250 "alias": "blimp_matrix_question_npi_licensor_present",251 "sample_len": 1000,252 "acc,none": 0.533,253 "acc_stderr,none": 0.015784807891138876254 },255 "blimp_npi_present_1": {256 "name": "blimp_npi_present_1",257 "alias": "blimp_npi_present_1",258 "sample_len": 1000,259 "acc,none": 0.421,260 "acc_stderr,none": 0.015620595475301285261 },262 "blimp_npi_present_2": {263 "name": "blimp_npi_present_2",264 "alias": "blimp_npi_present_2",265 "sample_len": 1000,266 "acc,none": 0.525,267 "acc_stderr,none": 0.015799513429996023268 },269 "blimp_only_npi_licensor_present": {270 "name": "blimp_only_npi_licensor_present",271 "alias": "blimp_only_npi_licensor_present",272 "sample_len": 1000,273 "acc,none": 0.739,274 "acc_stderr,none": 0.013895037677965183275 },276 "blimp_only_npi_scope": {277 "name": "blimp_only_npi_scope",278 "alias": "blimp_only_npi_scope",279 "sample_len": 1000,280 "acc,none": 0.453,281 "acc_stderr,none": 0.015749255189977683282 },283 "blimp_passive_1": {284 "name": "blimp_passive_1",285 "alias": "blimp_passive_1",286 "sample_len": 1000,287 "acc,none": 0.889,288 "acc_stderr,none": 0.009938701010583716289 },290 "blimp_passive_2": {291 "name": "blimp_passive_2",292 "alias": "blimp_passive_2",293 "sample_len": 1000,294 "acc,none": 0.882,295 "acc_stderr,none": 0.010206869264381718296 },297 "blimp_principle_A_c_command": {298 "name": "blimp_principle_A_c_command",299 "alias": "blimp_principle_A_c_command",300 "sample_len": 1000,301 "acc,none": 0.745,302 "acc_stderr,none": 0.013790038620872863303 },304 "blimp_principle_A_case_1": {305 "name": "blimp_principle_A_case_1",306 "alias": "blimp_principle_A_case_1",307 "sample_len": 1000,308 "acc,none": 1.0,309 "acc_stderr,none": 0.0310 },311 "blimp_principle_A_case_2": {312 "name": "blimp_principle_A_case_2",313 "alias": "blimp_principle_A_case_2",314 "sample_len": 1000,315 "acc,none": 0.979,316 "acc_stderr,none": 0.0045364721513065165317 },318 "blimp_principle_A_domain_1": {319 "name": "blimp_principle_A_domain_1",320 "alias": "blimp_principle_A_domain_1",321 "sample_len": 1000,322 "acc,none": 0.982,323 "acc_stderr,none": 0.00420638724961144324 },325 "blimp_principle_A_domain_2": {326 "name": "blimp_principle_A_domain_2",327 "alias": "blimp_principle_A_domain_2",328 "sample_len": 1000,329 "acc,none": 0.774,330 "acc_stderr,none": 0.013232501619085452331 },332 "blimp_principle_A_domain_3": {333 "name": "blimp_principle_A_domain_3",334 "alias": "blimp_principle_A_domain_3",335 "sample_len": 1000,336 "acc,none": 0.572,337 "acc_stderr,none": 0.015654426245029267338 },339 "blimp_principle_A_reconstruction": {340 "name": "blimp_principle_A_reconstruction",341 "alias": "blimp_principle_A_reconstruction",342 "sample_len": 1000,343 "acc,none": 0.272,344 "acc_stderr,none": 0.014078856992462642345 },346 "blimp_regular_plural_subject_verb_agreement_1": {347 "name": "blimp_regular_plural_subject_verb_agreement_1",348 "alias": "blimp_regular_plural_subject_verb_agreement_1",349 "sample_len": 1000,350 "acc,none": 0.906,351 "acc_stderr,none": 0.009233052000787672352 },353 "blimp_regular_plural_subject_verb_agreement_2": {354 "name": "blimp_regular_plural_subject_verb_agreement_2",355 "alias": "blimp_regular_plural_subject_verb_agreement_2",356 "sample_len": 1000,357 "acc,none": 0.873,358 "acc_stderr,none": 0.010534798620855644359 },360 "blimp_sentential_negation_npi_licensor_present": {361 "name": "blimp_sentential_negation_npi_licensor_present",362 "alias": "blimp_sentential_negation_npi_licensor_present",363 "sample_len": 1000,364 "acc,none": 0.994,365 "acc_stderr,none": 0.002443352199329841366 },367 "blimp_sentential_negation_npi_scope": {368 "name": "blimp_sentential_negation_npi_scope",369 "alias": "blimp_sentential_negation_npi_scope",370 "sample_len": 1000,371 "acc,none": 0.567,372 "acc_stderr,none": 0.01567663091218125373 },374 "blimp_sentential_subject_island": {375 "name": "blimp_sentential_subject_island",376 "alias": "blimp_sentential_subject_island",377 "sample_len": 1000,378 "acc,none": 0.413,379 "acc_stderr,none": 0.015577986829936455380 },381 "blimp_superlative_quantifiers_1": {382 "name": "blimp_superlative_quantifiers_1",383 "alias": "blimp_superlative_quantifiers_1",384 "sample_len": 1000,385 "acc,none": 0.837,386 "acc_stderr,none": 0.011686212712746913387 },388 "blimp_superlative_quantifiers_2": {389 "name": "blimp_superlative_quantifiers_2",390 "alias": "blimp_superlative_quantifiers_2",391 "sample_len": 1000,392 "acc,none": 0.631,393 "acc_stderr,none": 0.015266698139154782394 },395 "blimp_tough_vs_raising_1": {396 "name": "blimp_tough_vs_raising_1",397 "alias": "blimp_tough_vs_raising_1",398 "sample_len": 1000,399 "acc,none": 0.642,400 "acc_stderr,none": 0.015167928865407633401 },402 "blimp_tough_vs_raising_2": {403 "name": "blimp_tough_vs_raising_2",404 "alias": "blimp_tough_vs_raising_2",405 "sample_len": 1000,406 "acc,none": 0.846,407 "acc_stderr,none": 0.011419913065098805408 },409 "blimp_transitive": {410 "name": "blimp_transitive",411 "alias": "blimp_transitive",412 "sample_len": 1000,413 "acc,none": 0.842,414 "acc_stderr,none": 0.011539894677559635415 },416 "blimp_wh_island": {417 "name": "blimp_wh_island",418 "alias": "blimp_wh_island",419 "sample_len": 1000,420 "acc,none": 0.907,421 "acc_stderr,none": 0.009188875634996669422 },423 "blimp_wh_questions_object_gap": {424 "name": "blimp_wh_questions_object_gap",425 "alias": "blimp_wh_questions_object_gap",426 "sample_len": 1000,427 "acc,none": 0.808,428 "acc_stderr,none": 0.012461592646660028429 },430 "blimp_wh_questions_subject_gap": {431 "name": "blimp_wh_questions_subject_gap",432 "alias": "blimp_wh_questions_subject_gap",433 "sample_len": 1000,434 "acc,none": 0.948,435 "acc_stderr,none": 0.007024624213817088436 },437 "blimp_wh_questions_subject_gap_long_distance": {438 "name": "blimp_wh_questions_subject_gap_long_distance",439 "alias": "blimp_wh_questions_subject_gap_long_distance",440 "sample_len": 1000,441 "acc,none": 0.925,442 "acc_stderr,none": 0.008333333333333333443 },444 "blimp_wh_vs_that_no_gap": {445 "name": "blimp_wh_vs_that_no_gap",446 "alias": "blimp_wh_vs_that_no_gap",447 "sample_len": 1000,448 "acc,none": 0.966,449 "acc_stderr,none": 0.0057338361396953854450 },451 "blimp_wh_vs_that_no_gap_long_distance": {452 "name": "blimp_wh_vs_that_no_gap_long_distance",453 "alias": "blimp_wh_vs_that_no_gap_long_distance",454 "sample_len": 1000,455 "acc,none": 0.978,456 "acc_stderr,none": 0.004640855259274689457 },458 "blimp_wh_vs_that_with_gap": {459 "name": "blimp_wh_vs_that_with_gap",460 "alias": "blimp_wh_vs_that_with_gap",461 "sample_len": 1000,462 "acc,none": 0.571,463 "acc_stderr,none": 0.015658997547870177464 },465 "blimp_wh_vs_that_with_gap_long_distance": {466 "name": "blimp_wh_vs_that_with_gap_long_distance",467 "alias": "blimp_wh_vs_that_with_gap_long_distance",468 "sample_len": 1000,469 "acc,none": 0.324,470 "acc_stderr,none": 0.014806864733738984471 },472 "lambada_openai": {473 "name": "lambada_openai",474 "alias": "lambada_openai",475 "sample_len": 5153,476 "perplexity,none": 49.9994288883284,477 "perplexity_stderr,none": 1.8531805865076105,478 "acc,none": 0.27498544537162817,479 "acc_stderr,none": 0.006220714778345952480 },481 "hellaswag": {482 "name": "hellaswag",483 "alias": "hellaswag",484 "sample_len": 10042,485 "acc,none": 0.28360884285998805,486 "acc_stderr,none": 0.004498280244494868,487 "acc_norm,none": 0.3014339772953595,488 "acc_norm_stderr,none": 0.004579429184835642489 },490 "piqa": {491 "name": "piqa",492 "alias": "piqa",493 "sample_len": 1838,494 "acc,none": 0.602829162132753,495 "acc_stderr,none": 0.011416453840790413,496 "acc_norm,none": 0.5914036996735582,497 "acc_norm_stderr,none": 0.011469240387245224498 },499 "sciq": {500 "name": "sciq",501 "alias": "sciq",502 "sample_len": 1000,503 "acc,none": 0.76,504 "acc_stderr,none": 0.013512312258920847,505 "acc_norm,none": 0.663,506 "acc_norm_stderr,none": 0.01495508791865346507 },508 "arc_easy": {509 "name": "arc_easy",510 "alias": "arc_easy",511 "sample_len": 2376,512 "acc,none": 0.48569023569023567,513 "acc_stderr,none": 0.010255580881603702,514 "acc_norm,none": 0.4313973063973064,515 "acc_norm_stderr,none": 0.010162752847747605516 },517 "arc_challenge": {518 "name": "arc_challenge",519 "alias": "arc_challenge",520 "sample_len": 1172,521 "acc,none": 0.21160409556313994,522 "acc_stderr,none": 0.011935916358632793,523 "acc_norm,none": 0.2440273037542662,524 "acc_norm_stderr,none": 0.012551447627856151525 },526 "blimp": {527 "alias": "blimp",528 "name": "blimp",529 "sample_len": 67000,530 "acc,none": 0.7801492537313434,531 "acc_stderr,none": 0.0014076564766588326,532 "sample_count": {533 "acc,none": 67000534 }535 }536 },537 "groups": {538 "blimp": {539 "alias": "blimp",540 "name": "blimp",541 "sample_len": 67000,542 "acc,none": 0.7801492537313434,543 "acc_stderr,none": 0.0014076564766588326,544 "sample_count": {545 "acc,none": 67000546 }547 }548 },549 "group_subtasks": {550 "blimp": [551 "blimp_adjunct_island",552 "blimp_anaphor_gender_agreement",553 "blimp_anaphor_number_agreement",554 "blimp_animate_subject_passive",555 "blimp_animate_subject_trans",556 "blimp_causative",557 "blimp_complex_NP_island",558 "blimp_coordinate_structure_constraint_complex_left_branch",559 "blimp_coordinate_structure_constraint_object_extraction",560 "blimp_determiner_noun_agreement_1",561 "blimp_determiner_noun_agreement_2",562 "blimp_determiner_noun_agreement_irregular_1",563 "blimp_determiner_noun_agreement_irregular_2",564 "blimp_determiner_noun_agreement_with_adj_2",565 "blimp_determiner_noun_agreement_with_adj_irregular_1",566 "blimp_determiner_noun_agreement_with_adj_irregular_2",567 "blimp_determiner_noun_agreement_with_adjective_1",568 "blimp_distractor_agreement_relational_noun",569 "blimp_distractor_agreement_relative_clause",570 "blimp_drop_argument",571 "blimp_ellipsis_n_bar_1",572 "blimp_ellipsis_n_bar_2",573 "blimp_existential_there_object_raising",574 "blimp_existential_there_quantifiers_1",575 "blimp_existential_there_quantifiers_2",576 "blimp_existential_there_subject_raising",577 "blimp_expletive_it_object_raising",578 "blimp_inchoative",579 "blimp_intransitive",580 "blimp_irregular_past_participle_adjectives",581 "blimp_irregular_past_participle_verbs",582 "blimp_irregular_plural_subject_verb_agreement_1",583 "blimp_irregular_plural_subject_verb_agreement_2",584 "blimp_left_branch_island_echo_question",585 "blimp_left_branch_island_simple_question",586 "blimp_matrix_question_npi_licensor_present",587 "blimp_npi_present_1",588 "blimp_npi_present_2",589 "blimp_only_npi_licensor_present",590 "blimp_only_npi_scope",591 "blimp_passive_1",592 "blimp_passive_2",593 "blimp_principle_A_c_command",594 "blimp_principle_A_case_1",595 "blimp_principle_A_case_2",596 "blimp_principle_A_domain_1",597 "blimp_principle_A_domain_2",598 "blimp_principle_A_domain_3",599 "blimp_principle_A_reconstruction",600 "blimp_regular_plural_subject_verb_agreement_1",601 "blimp_regular_plural_subject_verb_agreement_2",602 "blimp_sentential_negation_npi_licensor_present",603 "blimp_sentential_negation_npi_scope",604 "blimp_sentential_subject_island",605 "blimp_superlative_quantifiers_1",606 "blimp_superlative_quantifiers_2",607 "blimp_tough_vs_raising_1",608 "blimp_tough_vs_raising_2",609 "blimp_transitive",610 "blimp_wh_island",611 "blimp_wh_questions_object_gap",612 "blimp_wh_questions_subject_gap",613 "blimp_wh_questions_subject_gap_long_distance",614 "blimp_wh_vs_that_no_gap",615 "blimp_wh_vs_that_no_gap_long_distance",616 "blimp_wh_vs_that_with_gap",617 "blimp_wh_vs_that_with_gap_long_distance"618 ]619 },620 "configs": {621 "arc_challenge": {622 "task": "arc_challenge",623 "dataset_path": "allenai/ai2_arc",624 "dataset_name": "ARC-Challenge",625 "training_split": "train",626 "validation_split": "validation",627 "test_split": "test",628 "doc_to_text": "Question: {{question}}\nAnswer:",629 "doc_to_target": "{{choices.label.index(answerKey)}}",630 "unsafe_code": false,631 "doc_to_choice": "{{choices.text}}",632 "description": "",633 "target_delimiter": " ",634 "fewshot_delimiter": "\n\n",635 "fewshot_config": {636 "sampler": "default",637 "split": null,638 "process_docs": null,639 "fewshot_indices": null,640 "samples": null,641 "doc_to_text": "Question: {{question}}\nAnswer:",642 "doc_to_choice": "{{choices.text}}",643 "doc_to_target": "{{choices.label.index(answerKey)}}",644 "gen_prefix": null,645 "fewshot_delimiter": "\n\n",646 "target_delimiter": " "647 },648 "num_fewshot": 0,649 "metric_list": [650 {651 "metric": "acc",652 "aggregation": "mean",653 "higher_is_better": true654 },655 {656 "metric": "acc_norm",657 "aggregation": "mean",658 "higher_is_better": true659 }660 ],661 "output_type": "multiple_choice",662 "repeats": 1,663 "should_decontaminate": true,664 "doc_to_decontamination_query": "Question: {{question}}\nAnswer:",665 "metadata": {666 "version": 1.0,667 "pretrained": "SlayerLab/pollock-mini-lm-125m",668 "revision": "a2e53d9f1b690ebfbea375cf8bd3b5bdf69dece9",669 "dtype": "bfloat16",670 "config_source": "/root/rocm-torch/lib/python3.12/site-packages/lm_eval/tasks/arc/arc_challenge.yaml"671 }672 },673 "arc_easy": {674 "task": "arc_easy",675 "dataset_path": "allenai/ai2_arc",676 "dataset_name": "ARC-Easy",677 "training_split": "train",678 "validation_split": "validation",679 "test_split": "test",680 "doc_to_text": "Question: {{question}}\nAnswer:",681 "doc_to_target": "{{choices.label.index(answerKey)}}",682 "unsafe_code": false,683 "doc_to_choice": "{{choices.text}}",684 "description": "",685 "target_delimiter": " ",686 "fewshot_delimiter": "\n\n",687 "fewshot_config": {688 "sampler": "default",689 "split": null,690 "process_docs": null,691 "fewshot_indices": null,692 "samples": null,693 "doc_to_text": "Question: {{question}}\nAnswer:",694 "doc_to_choice": "{{choices.text}}",695 "doc_to_target": "{{choices.label.index(answerKey)}}",696 "gen_prefix": null,697 "fewshot_delimiter": "\n\n",698 "target_delimiter": " "699 },700 "num_fewshot": 0,701 "metric_list": [702 {703 "metric": "acc",704 "aggregation": "mean",705 "higher_is_better": true706 },707 {708 "metric": "acc_norm",709 "aggregation": "mean",710 "higher_is_better": true711 }712 ],713 "output_type": "multiple_choice",714 "repeats": 1,715 "should_decontaminate": true,716 "doc_to_decontamination_query": "Question: {{question}}\nAnswer:",717 "metadata": {718 "version": 1.0,719 "pretrained": "SlayerLab/pollock-mini-lm-125m",720 "revision": "a2e53d9f1b690ebfbea375cf8bd3b5bdf69dece9",721 "dtype": "bfloat16",722 "config_source": "/root/rocm-torch/lib/python3.12/site-packages/lm_eval/tasks/arc/arc_easy.yaml"723 }724 },725 "blimp_adjunct_island": {726 "task": "blimp_adjunct_island",727 "dataset_path": "nyu-mll/blimp",728 "dataset_name": "adjunct_island",729 "validation_split": "train",730 "doc_to_text": "",731 "doc_to_target": 0,732 "unsafe_code": false,733 "doc_to_choice": "{{[sentence_good, sentence_bad]}}",734 "description": "",735 "target_delimiter": " ",736 "fewshot_delimiter": "\n\n",737 "fewshot_config": {738 "sampler": "default",739 "split": null,740 "process_docs": null,741 "fewshot_indices": null,742 "samples": null,743 "doc_to_text": "",744 "doc_to_choice": "{{[sentence_good, sentence_bad]}}",745 "doc_to_target": 0,746 "gen_prefix": null,747 "fewshot_delimiter": "\n\n",748 "target_delimiter": " "749 },750 "num_fewshot": 0,751 "metric_list": [752 {753 "metric": "acc",754 "aggregation": "mean",755 "higher_is_better": true756 }757 ],758 "output_type": "multiple_choice",759 "repeats": 1,760 "should_decontaminate": true,761 "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",762 "metadata": {763 "version": 1.0,764 "pretrained": "SlayerLab/pollock-mini-lm-125m",765 "revision": "a2e53d9f1b690ebfbea375cf8bd3b5bdf69dece9",766 "dtype": "bfloat16",767 "config_source": "/root/rocm-torch/lib/python3.12/site-packages/lm_eval/tasks/blimp/adjunct_island.yaml"768 }769 },770 "blimp_anaphor_gender_agreement": {771 "task": "blimp_anaphor_gender_agreement",772 "dataset_path": "nyu-mll/blimp",773 "dataset_name": "anaphor_gender_agreement",774 "validation_split": "train",775 "doc_to_text": "",776 "doc_to_target": 0,777 "unsafe_code": false,778 "doc_to_choice": "{{[sentence_good, sentence_bad]}}",779 "description": "",780 "target_delimiter": " ",781 "fewshot_delimiter": "\n\n",782 "fewshot_config": {783 "sampler": "default",784 "split": null,785 "process_docs": null,786 "fewshot_indices": null,787 "samples": null,788 "doc_to_text": "",789 "doc_to_choice": "{{[sentence_good, sentence_bad]}}",790 "doc_to_target": 0,791 "gen_prefix": null,792 "fewshot_delimiter": "\n\n",793 "target_delimiter": " "794 },795 "num_fewshot": 0,796 "metric_list": [797 {798 "metric": "acc",799 "aggregation": "mean",800 "higher_is_better": true801 }802 ],803 "output_type": "multiple_choice",804 "repeats": 1,805 "should_decontaminate": true,806 "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",807 "metadata": {808 "version": 1.0,809 "pretrained": "SlayerLab/pollock-mini-lm-125m",810 "revision": "a2e53d9f1b690ebfbea375cf8bd3b5bdf69dece9",811 "dtype": "bfloat16",812 "config_source": "/root/rocm-torch/lib/python3.12/site-packages/lm_eval/tasks/blimp/anaphor_gender_agreement.yaml"813 }814 },815 "blimp_anaphor_number_agreement": {816 "task": "blimp_anaphor_number_agreement",817 "dataset_path": "nyu-mll/blimp",818 "dataset_name": "anaphor_number_agreement",819 "validation_split": "train",820 "doc_to_text": "",821 "doc_to_target": 0,822 "unsafe_code": false,823 "doc_to_choice": "{{[sentence_good, sentence_bad]}}",824 "description": "",825 "target_delimiter": " ",826 "fewshot_delimiter": "\n\n",827 "fewshot_config": {828 "sampler": "default",829 "split": null,830 "process_docs": null,831 "fewshot_indices": null,832 "samples": null,833 "doc_to_text": "",834 "doc_to_choice": "{{[sentence_good, sentence_bad]}}",835 "doc_to_target": 0,836 "gen_prefix": null,837 "fewshot_delimiter": "\n\n",838 "target_delimiter": " "839 },840 "num_fewshot": 0,841 "metric_list": [842 {843 "metric": "acc",844 "aggregation": "mean",845 "higher_is_better": true846 }847 ],848 "output_type": "multiple_choice",849 "repeats": 1,850 "should_decontaminate": true,851 "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",852 "metadata": {853 "version": 1.0,854 "pretrained": "SlayerLab/pollock-mini-lm-125m",855 "revision": "a2e53d9f1b690ebfbea375cf8bd3b5bdf69dece9",856 "dtype": "bfloat16",857 "config_source": "/root/rocm-torch/lib/python3.12/site-packages/lm_eval/tasks/blimp/anaphor_number_agreement.yaml"858 }859 },860 "blimp_animate_subject_passive": {861 "task": "blimp_animate_subject_passive",862 "dataset_path": "nyu-mll/blimp",863 "dataset_name": "animate_subject_passive",864 "validation_split": "train",865 "doc_to_text": "",866 "doc_to_target": 0,867 "unsafe_code": false,868 "doc_to_choice": "{{[sentence_good, sentence_bad]}}",869 "description": "",870 "target_delimiter": " ",871 "fewshot_delimiter": "\n\n",872 "fewshot_config": {873 "sampler": "default",874 "split": null,875 "process_docs": null,876 "fewshot_indices": null,877 "samples": null,878 "doc_to_text": "",879 "doc_to_choice": "{{[sentence_good, sentence_bad]}}",880 "doc_to_target": 0,881 "gen_prefix": null,882 "fewshot_delimiter": "\n\n",883 "target_delimiter": " "884 },885 "num_fewshot": 0,886 "metric_list": [887 {888 "metric": "acc",889 "aggregation": "mean",890 "higher_is_better": true891 }892 ],893 "output_type": "multiple_choice",894 "repeats": 1,895 "should_decontaminate": true,896 "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",897 "metadata": {898 "version": 1.0,899 "pretrained": "SlayerLab/pollock-mini-lm-125m",900 "revision": "a2e53d9f1b690ebfbea375cf8bd3b5bdf69dece9",901 "dtype": "bfloat16",902 "config_source": "/root/rocm-torch/lib/python3.12/site-packages/lm_eval/tasks/blimp/animate_subject_passive.yaml"903 }904 },905 "blimp_animate_subject_trans": {906 "task": "blimp_animate_subject_trans",907 "dataset_path": "nyu-mll/blimp",908 "dataset_name": "animate_subject_trans",909 "validation_split": "train",910 "doc_to_text": "",911 "doc_to_target": 0,912 "unsafe_code": false,913 "doc_to_choice": "{{[sentence_good, sentence_bad]}}",914 "description": "",915 "target_delimiter": " ",916 "fewshot_delimiter": "\n\n",917 "fewshot_config": {918 "sampler": "default",919 "split": null,920 "process_docs": null,921 "fewshot_indices": null,922 "samples": null,923 "doc_to_text": "",924 "doc_to_choice": "{{[sentence_good, sentence_bad]}}",925 "doc_to_target": 0,926 "gen_prefix": null,927 "fewshot_delimiter": "\n\n",928 "target_delimiter": " "929 },930 "num_fewshot": 0,931 "metric_list": [932 {933 "metric": "acc",934 "aggregation": "mean",935 "higher_is_better": true936 }937 ],938 "output_type": "multiple_choice",939 "repeats": 1,940 "should_decontaminate": true,941 "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",942 "metadata": {943 "version": 1.0,944 "pretrained": "SlayerLab/pollock-mini-lm-125m",945 "revision": "a2e53d9f1b690ebfbea375cf8bd3b5bdf69dece9",946 "dtype": "bfloat16",947 "config_source": "/root/rocm-torch/lib/python3.12/site-packages/lm_eval/tasks/blimp/animate_subject_trans.yaml"948 }949 },950 "blimp_causative": {951 "task": "blimp_causative",952 "dataset_path": "nyu-mll/blimp",953 "dataset_name": "causative",954 "validation_split": "train",955 "doc_to_text": "",956 "doc_to_target": 0,957 "unsafe_code": false,958 "doc_to_choice": "{{[sentence_good, sentence_bad]}}",959 "description": "",960 "target_delimiter": " ",961 "fewshot_delimiter": "\n\n",962 "fewshot_config": {963 "sampler": "default",964 "split": null,965 "process_docs": null,966 "fewshot_indices": null,967 "samples": null,968 "doc_to_text": "",969 "doc_to_choice": "{{[sentence_good, sentence_bad]}}",970 "doc_to_target": 0,971 "gen_prefix": null,972 "fewshot_delimiter": "\n\n",973 "target_delimiter": " "974 },975 "num_fewshot": 0,976 "metric_list": [977 {978 "metric": "acc",979 "aggregation": "mean",980 "higher_is_better": true981 }982 ],983 "output_type": "multiple_choice",984 "repeats": 1,985 "should_decontaminate": true,986 "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",987 "metadata": {988 "version": 1.0,989 "pretrained": "SlayerLab/pollock-mini-lm-125m",990 "revision": "a2e53d9f1b690ebfbea375cf8bd3b5bdf69dece9",991 "dtype": "bfloat16",992 "config_source": "/root/rocm-torch/lib/python3.12/site-packages/lm_eval/tasks/blimp/causative.yaml"993 }994 },995 "blimp_complex_NP_island": {996 "task": "blimp_complex_NP_island",997 "dataset_path": "nyu-mll/blimp",998 "dataset_name": "complex_NP_island",999 "validation_split": "train",1000 "doc_to_text": "",1001 "doc_to_target": 0,1002 "unsafe_code": false,1003 "doc_to_choice": "{{[sentence_good, sentence_bad]}}",1004 "description": "",1005 "target_delimiter": " ",1006 "fewshot_delimiter": "\n\n",1007 "fewshot_config": {1008 "sampler": "default",1009 "split": null,1010 "process_docs": null,1011 "fewshot_indices": null,1012 "samples": null,1013 "doc_to_text": "",1014 "doc_to_choice": "{{[sentence_good, sentence_bad]}}",1015 "doc_to_target": 0,1016 "gen_prefix": null,1017 "fewshot_delimiter": "\n\n",1018 "target_delimiter": " "1019 },1020 "num_fewshot": 0,1021 "metric_list": [1022 {1023 "metric": "acc",1024 "aggregation": "mean",1025 "higher_is_better": true1026 }1027 ],1028 "output_type": "multiple_choice",1029 "repeats": 1,1030 "should_decontaminate": true,1031 "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",1032 "metadata": {1033 "version": 1.0,1034 "pretrained": "SlayerLab/pollock-mini-lm-125m",1035 "revision": "a2e53d9f1b690ebfbea375cf8bd3b5bdf69dece9",1036 "dtype": "bfloat16",1037 "config_source": "/root/rocm-torch/lib/python3.12/site-packages/lm_eval/tasks/blimp/complex_NP_island.yaml"1038 }1039 },1040 "blimp_coordinate_structure_constraint_complex_left_branch": {1041 "task": "blimp_coordinate_structure_constraint_complex_left_branch",1042 "dataset_path": "nyu-mll/blimp",1043 "dataset_name": "coordinate_structure_constraint_complex_left_branch",1044 "validation_split": "train",1045 "doc_to_text": "",1046 "doc_to_target": 0,1047 "unsafe_code": false,1048 "doc_to_choice": "{{[sentence_good, sentence_bad]}}",1049 "description": "",1050 "target_delimiter": " ",1051 "fewshot_delimiter": "\n\n",1052 "fewshot_config": {1053 "sampler": "default",1054 "split": null,1055 "process_docs": null,1056 "fewshot_indices": null,1057 "samples": null,1058 "doc_to_text": "",1059 "doc_to_choice": "{{[sentence_good, sentence_bad]}}",1060 "doc_to_target": 0,1061 "gen_prefix": null,1062 "fewshot_delimiter": "\n\n",1063 "target_delimiter": " "1064 },1065 "num_fewshot": 0,1066 "metric_list": [1067 {1068 "metric": "acc",1069 "aggregation": "mean",1070 "higher_is_better": true1071 }1072 ],1073 "output_type": "multiple_choice",1074 "repeats": 1,1075 "should_decontaminate": true,1076 "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",1077 "metadata": {1078 "version": 1.0,1079 "pretrained": "SlayerLab/pollock-mini-lm-125m",1080 "revision": "a2e53d9f1b690ebfbea375cf8bd3b5bdf69dece9",1081 "dtype": "bfloat16",1082 "config_source": "/root/rocm-torch/lib/python3.12/site-packages/lm_eval/tasks/blimp/coordinate_structure_constraint_complex_left_branch.yaml"1083 }1084 },1085 "blimp_coordinate_structure_constraint_object_extraction": {1086 "task": "blimp_coordinate_structure_constraint_object_extraction",1087 "dataset_path": "nyu-mll/blimp",1088 "dataset_name": "coordinate_structure_constraint_object_extraction",1089 "validation_split": "train",1090 "doc_to_text": "",1091 "doc_to_target": 0,1092 "unsafe_code": false,1093 "doc_to_choice": "{{[sentence_good, sentence_bad]}}",1094 "description": "",1095 "target_delimiter": " ",1096 "fewshot_delimiter": "\n\n",1097 "fewshot_config": {1098 "sampler": "default",1099 "split": null,1100 "process_docs": null,1101 "fewshot_indices": null,1102 "samples": null,1103 "doc_to_text": "",1104 "doc_to_choice": "{{[sentence_good, sentence_bad]}}",1105 "doc_to_target": 0,1106 "gen_prefix": null,1107 "fewshot_delimiter": "\n\n",1108 "target_delimiter": " "1109 },1110 "num_fewshot": 0,1111 "metric_list": [1112 {1113 "metric": "acc",1114 "aggregation": "mean",1115 "higher_is_better": true1116 }1117 ],1118 "output_type": "multiple_choice",1119 "repeats": 1,1120 "should_decontaminate": true,1121 "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",1122 "metadata": {1123 "version": 1.0,1124 "pretrained": "SlayerLab/pollock-mini-lm-125m",1125 "revision": "a2e53d9f1b690ebfbea375cf8bd3b5bdf69dece9",1126 "dtype": "bfloat16",1127 "config_source": "/root/rocm-torch/lib/python3.12/site-packages/lm_eval/tasks/blimp/coordinate_structure_constraint_object_extraction.yaml"1128 }1129 },1130 "blimp_determiner_noun_agreement_1": {1131 "task": "blimp_determiner_noun_agreement_1",1132 "dataset_path": "nyu-mll/blimp",1133 "dataset_name": "determiner_noun_agreement_1",1134 "validation_split": "train",1135 "doc_to_text": "",1136 "doc_to_target": 0,1137 "unsafe_code": false,1138 "doc_to_choice": "{{[sentence_good, sentence_bad]}}",1139 "description": "",1140 "target_delimiter": " ",1141 "fewshot_delimiter": "\n\n",1142 "fewshot_config": {1143 "sampler": "default",1144 "split": null,1145 "process_docs": null,1146 "fewshot_indices": null,1147 "samples": null,1148 "doc_to_text": "",1149 "doc_to_choice": "{{[sentence_good, sentence_bad]}}",1150 "doc_to_target": 0,1151 "gen_prefix": null,1152 "fewshot_delimiter": "\n\n",1153 "target_delimiter": " "1154 },1155 "num_fewshot": 0,1156 "metric_list": [1157 {1158 "metric": "acc",1159 "aggregation": "mean",1160 "higher_is_better": true1161 }1162 ],1163 "output_type": "multiple_choice",1164 "repeats": 1,1165 "should_decontaminate": true,1166 "doc_to_decontamination_query": "{{sentence_good}} {{sentence_bad}}",1167 "metadata": {1168 "version": 1.0,1169 "pretrained": "SlayerLab/pollock-mini-lm-125m",1170 "revision": "a2e53d9f1b690ebfbea375cf8bd3b5bdf69dece9",1171 "dtype": "bfloat16",1172 "config_source": "/root/rocm-torch/lib/python3.12/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_1.yaml"1173 }1174 },1175 "blimp_determiner_noun_agreement_2": {1176 "task": "blimp_determiner_noun_agreement_2",1177 "dataset_path": "nyu-mll/blimp",1178 "dataset_name": "determiner_noun_agreement_2",1179 "validation_split": "train",1180 "doc_to_text": "",1181 "doc_to_target": 0,1182 "unsafe_code": false,1183 "doc_to_choice": "{{[sentence_good, sentence_bad]}}",1184 "description": "",1185 "target_delimiter": " ",1186 "fewshot_delimiter": "\n\n",1187 "fewshot_config": {1188 "sampler": "default",1189 "split": null,1190 "process_docs": null,1191 "fewshot_indices": null,1192 "samples": null,1193 "doc_to_text": "",1194 "doc_to_choice": "{{[sentence_good, sentence_bad]}}",1195 "doc_to_target": 0,1196 "gen_prefix": null,1197 "fewshot_delimiter": "\n\n",1198 "target_delimiter": " "1199 },1200 "num_fewshot": 0,