Mapika/decider-2b
83131k
1{2 "results": {3 "clinc_oos": {4 "n": 1500,5 "acc": 0.9773333333333334,6 "nll": 0.09173629432916641,7 "brier": 0.03913933411240578,8 "ece": 0.013027977565924286,9 "aurc": 0.002160483016578994,10 "acc_at_80": 0.9975,11 "acc_at_50": 0.9986666666666667,12 "chance": 0.1,13 "mean_conf": 0.9865841269493103,14 "sec": 2.4,15 "heldout": false16 },17 "banking77": {18 "n": 1500,19 "acc": 0.9666666666666667,20 "nll": 0.10046543180942535,21 "brier": 0.04899560660123825,22 "ece": 0.01317060301701228,23 "aurc": 0.0015889115300367546,24 "acc_at_80": 0.9991666666666666,25 "acc_at_50": 1.0,26 "chance": 0.1,27 "mean_conf": 0.9798372387886047,28 "sec": 1.3,29 "heldout": false30 },31 "massive_intent": {32 "n": 1500,33 "acc": 0.9573333333333334,34 "nll": 0.14932970702648163,35 "brier": 0.06778250634670258,36 "ece": 0.016796776016553244,37 "aurc": 0.004776378371526423,38 "acc_at_80": 0.9941666666666666,39 "acc_at_50": 0.9986666666666667,40 "chance": 0.1,41 "mean_conf": 0.9656777381896973,42 "sec": 1.3,43 "heldout": false44 },45 "massive_scenario": {46 "n": 1500,47 "acc": 0.756,48 "nll": 0.7479000687599182,49 "brier": 0.35114040970802307,50 "ece": 0.10180614854892094,51 "aurc": 0.08514369896051291,52 "acc_at_80": 0.8433333333333334,53 "acc_at_50": 0.9333333333333333,54 "chance": 0.1,55 "mean_conf": 0.8578061461448669,56 "sec": 1.3,57 "heldout": true58 },59 "bitext_support": {60 "n": 3000,61 "acc": 0.997,62 "nll": 0.011109774000942707,63 "brier": 0.004990722052752972,64 "ece": 0.0024439626336097706,65 "aurc": 1.969131620660601e-05,66 "acc_at_80": 1.0,67 "acc_at_50": 1.0,68 "chance": 0.09999999999999998,69 "mean_conf": 0.9992172718048096,70 "sec": 1.8,71 "heldout": false,72 "per_q": [73 0.9993333333333333,74 0.994666666666666775 ]76 },77 "support_tickets": {78 "n": 4500,79 "acc": 0.5708888888888889,80 "nll": 0.9939086437225342,81 "brier": 0.5205920934677124,82 "ece": 0.025038347750902176,83 "aurc": 0.21663716050861523,84 "acc_at_80": 0.6386111111111111,85 "acc_at_50": 0.76,86 "chance": 0.22777777777777777,87 "mean_conf": 0.5925066471099854,88 "sec": 2.4,89 "heldout": false,90 "per_q": [91 0.82,92 0.4106666666666667,93 0.48294 ]95 },96 "ag_news": {97 "n": 1500,98 "acc": 0.9106666666666666,99 "nll": 0.24641166627407074,100 "brier": 0.13118577003479004,101 "ece": 0.015849134624004363,102 "aurc": 0.01962706163947694,103 "acc_at_80": 0.9658333333333333,104 "acc_at_50": 0.9853333333333333,105 "chance": 0.25,106 "mean_conf": 0.9186857342720032,107 "sec": 1.3,108 "heldout": false109 },110 "dbpedia": {111 "n": 1500,112 "acc": 0.9913333333333333,113 "nll": 0.031197283416986465,114 "brier": 0.01437467709183693,115 "ece": 0.004729895810286192,116 "aurc": 0.00012132173934791126,117 "acc_at_80": 1.0,118 "acc_at_50": 1.0,119 "chance": 0.1,120 "mean_conf": 0.994809091091156,121 "sec": 1.6,122 "heldout": false123 },124 "yahoo_topics": {125 "n": 1500,126 "acc": 0.748,127 "nll": 0.7660647630691528,128 "brier": 0.35162588953971863,129 "ece": 0.03356954632202783,130 "aurc": 0.08826358611475389,131 "acc_at_80": 0.8416666666666667,132 "acc_at_50": 0.9293333333333333,133 "chance": 0.1,134 "mean_conf": 0.7593891024589539,135 "sec": 1.5,136 "heldout": false137 },138 "newsgroups": {139 "n": 1462,140 "acc": 0.8077975376196991,141 "nll": 0.5773629546165466,142 "brier": 0.26579606533050537,143 "ece": 0.06540349922900977,144 "aurc": 0.04088176394142939,145 "acc_at_80": 0.911965811965812,146 "acc_at_50": 0.9849521203830369,147 "chance": 0.09999999999999999,148 "mean_conf": 0.8732010126113892,149 "sec": 2.7,150 "heldout": false151 },152 "bbc_news": {153 "n": 1000,154 "acc": 0.927,155 "nll": 0.21117442846298218,156 "brier": 0.11654213815927505,157 "ece": 0.026313644617795985,158 "aurc": 0.012799123011482493,159 "acc_at_80": 0.9725,160 "acc_at_50": 0.998,161 "chance": 0.20000000000000004,162 "mean_conf": 0.9459534883499146,163 "sec": 2.6,164 "heldout": true165 },166 "trec": {167 "n": 500,168 "acc": 0.784,169 "nll": 0.6409680247306824,170 "brier": 0.32721808552742004,171 "ece": 0.07267213892936707,172 "aurc": 0.10896437878729759,173 "acc_at_80": 0.825,174 "acc_at_50": 0.92,175 "chance": 0.16666666666666663,176 "mean_conf": 0.8298088312149048,177 "sec": 0.4,178 "heldout": true179 },180 "student_questions": {181 "n": 1500,182 "acc": 0.9246666666666666,183 "nll": 0.21436315774917603,184 "brier": 0.11058846116065979,185 "ece": 0.017640525956948608,186 "aurc": 0.01580899073511152,187 "acc_at_80": 0.975,188 "acc_at_50": 0.992,189 "chance": 0.25,190 "mean_conf": 0.928837239742279,191 "sec": 1.5,192 "heldout": true193 },194 "dolly_category": {195 "n": 1500,196 "acc": 0.2986666666666667,197 "nll": 2.0224030017852783,198 "brier": 0.8629860281944275,199 "ece": 0.27236494100093844,200 "aurc": 0.5039295041873185,201 "acc_at_80": 0.32666666666666666,202 "acc_at_50": 0.416,203 "chance": 0.125,204 "mean_conf": 0.5710316300392151,205 "sec": 1.8,206 "heldout": true207 },208 "imdb": {209 "n": 1500,210 "acc": 0.9613333333333334,211 "nll": 0.1142592579126358,212 "brier": 0.06406151503324509,213 "ece": 0.024656122922897324,214 "aurc": 0.002928303347136234,215 "acc_at_80": 0.9975,216 "acc_at_50": 1.0,217 "chance": 0.5,218 "mean_conf": 0.9826299548149109,219 "sec": 2.8,220 "heldout": false221 },222 "sst2": {223 "n": 872,224 "acc": 0.9380733944954128,225 "nll": 0.17604376375675201,226 "brier": 0.09775912761688232,227 "ece": 0.030137871725296785,228 "aurc": 0.01111562757046914,229 "acc_at_80": 0.9842406876790831,230 "acc_at_50": 0.9954128440366973,231 "chance": 0.5,232 "mean_conf": 0.9652942419052124,233 "sec": 0.6,234 "heldout": false235 },236 "sst5": {237 "n": 1500,238 "acc": 0.604,239 "nll": 0.9462839961051941,240 "brier": 0.5338310599327087,241 "ece": 0.039909611364205684,242 "aurc": 0.30754727422711775,243 "acc_at_80": 0.635,244 "acc_at_50": 0.6866666666666666,245 "chance": 0.2,246 "mean_conf": 0.642160177230835,247 "sec": 1.1,248 "heldout": false249 },250 "yelp": {251 "n": 1500,252 "acc": 0.6826666666666666,253 "nll": 0.7469708919525146,254 "brier": 0.42934244871139526,255 "ece": 0.08389960529406866,256 "aurc": 0.1661291012599122,257 "acc_at_80": 0.7408333333333333,258 "acc_at_50": 0.816,259 "chance": 0.2,260 "mean_conf": 0.765559732913971,261 "sec": 2.3,262 "heldout": false263 },264 "amazon_stars": {265 "n": 1500,266 "acc": 0.6033333333333334,267 "nll": 0.9380196332931519,268 "brier": 0.5161817669868469,269 "ece": 0.07665159477790198,270 "aurc": 0.23412008065702158,271 "acc_at_80": 0.6541666666666667,272 "acc_at_50": 0.748,273 "chance": 0.2,274 "mean_conf": 0.6799849271774292,275 "sec": 1.3,276 "heldout": false277 },278 "emotion": {279 "n": 1500,280 "acc": 0.8693333333333333,281 "nll": 0.375094473361969,282 "brier": 0.1965477466583252,283 "ece": 0.047137337585290234,284 "aurc": 0.031846251103281674,285 "acc_at_80": 0.9383333333333334,286 "acc_at_50": 0.98,287 "chance": 0.16666666666666666,288 "mean_conf": 0.9126279950141907,289 "sec": 1.0,290 "heldout": false291 },292 "go_emotions": {293 "n": 1500,294 "acc": 0.7793333333333333,295 "nll": 0.6534133553504944,296 "brier": 0.3117011487483978,297 "ece": 0.04330594710508986,298 "aurc": 0.07619826478481186,299 "acc_at_80": 0.8633333333333333,300 "acc_at_50": 0.936,301 "chance": 0.1,302 "mean_conf": 0.8226393461227417,303 "sec": 1.2,304 "heldout": false305 },306 "tweet_sentiment": {307 "n": 1500,308 "acc": 0.7206666666666667,309 "nll": 0.6113293170928955,310 "brier": 0.37921401858329773,311 "ece": 0.04055946866671242,312 "aurc": 0.1466257866538856,313 "acc_at_80": 0.765,314 "acc_at_50": 0.8346666666666667,315 "chance": 0.3333333333333333,316 "mean_conf": 0.7430600523948669,317 "sec": 0.9,318 "heldout": false319 },320 "tweet_emotion": {321 "n": 1421,322 "acc": 0.8367346938775511,323 "nll": 0.45104628801345825,324 "brier": 0.2379906326532364,325 "ece": 0.025843723171785783,326 "aurc": 0.04399644830028519,327 "acc_at_80": 0.9076517150395779,328 "acc_at_50": 0.971830985915493,329 "chance": 0.25,330 "mean_conf": 0.8583319187164307,331 "sec": 1.0,332 "heldout": false333 },334 "tweet_irony": {335 "n": 784,336 "acc": 0.7946428571428571,337 "nll": 0.4676361083984375,338 "brier": 0.300514280796051,339 "ece": 0.035634773954445,340 "aurc": 0.1096303514372389,341 "acc_at_80": 0.8437001594896332,342 "acc_at_50": 0.8647959183673469,343 "chance": 0.5,344 "mean_conf": 0.8008275032043457,345 "sec": 0.5,346 "heldout": true347 },348 "fin_sentiment": {349 "n": 1500,350 "acc": 0.7933333333333333,351 "nll": 0.4863547086715698,352 "brier": 0.28904733061790466,353 "ece": 0.019916596571604408,354 "aurc": 0.08617910313882864,355 "acc_at_80": 0.8575,356 "acc_at_50": 0.9146666666666666,357 "chance": 0.3333333333333333,358 "mean_conf": 0.7970937490463257,359 "sec": 1.0,360 "heldout": true361 },362 "cr_reviews": {363 "n": 753,364 "acc": 0.9030544488711819,365 "nll": 0.25531020760536194,366 "brier": 0.14807458221912384,367 "ece": 0.05294444131977842,368 "aurc": 0.018211001734510564,369 "acc_at_80": 0.9651162790697675,370 "acc_at_50": 0.9946808510638298,371 "chance": 0.5,372 "mean_conf": 0.9507516026496887,373 "sec": 0.5,374 "heldout": true375 },376 "counterfactual": {377 "n": 1500,378 "acc": 0.954,379 "nll": 0.14455512166023254,380 "brier": 0.0772322341799736,381 "ece": 0.017266486525535557,382 "aurc": 0.007466418318042988,383 "acc_at_80": 0.9866666666666667,384 "acc_at_50": 0.9973333333333333,385 "chance": 0.5,386 "mean_conf": 0.9662280082702637,387 "sec": 1.0,388 "heldout": false389 },390 "subjectivity": {391 "n": 1500,392 "acc": 0.9593333333333334,393 "nll": 0.11742740124464035,394 "brier": 0.0635886937379837,395 "ece": 0.018429159760475204,396 "aurc": 0.004805976587222085,397 "acc_at_80": 0.9916666666666667,398 "acc_at_50": 0.9986666666666667,399 "chance": 0.5,400 "mean_conf": 0.9714123010635376,401 "sec": 1.0,402 "heldout": false403 },404 "tweet_offensive": {405 "n": 860,406 "acc": 0.8569767441860465,407 "nll": 0.321306049823761,408 "brier": 0.19854530692100525,409 "ece": 0.03273247341777005,410 "aurc": 0.04094359780914508,411 "acc_at_80": 0.9215116279069767,412 "acc_at_50": 0.9674418604651163,413 "chance": 0.5,414 "mean_conf": 0.8376211524009705,415 "sec": 0.6,416 "heldout": false417 },418 "tweet_hate": {419 "n": 1500,420 "acc": 0.5613333333333334,421 "nll": 0.984972357749939,422 "brier": 0.643694281578064,423 "ece": 0.2259776959021886,424 "aurc": 0.43112132729844205,425 "acc_at_80": 0.5641666666666667,426 "acc_at_50": 0.5213333333333333,427 "chance": 0.5,428 "mean_conf": 0.7813499569892883,429 "sec": 1.0,430 "heldout": false431 },432 "hate_offensive": {433 "n": 1500,434 "acc": 0.912,435 "nll": 0.24573880434036255,436 "brier": 0.13170845806598663,437 "ece": 0.0208393304944038,438 "aurc": 0.018957146656246305,439 "acc_at_80": 0.9683333333333334,440 "acc_at_50": 0.9866666666666667,441 "chance": 0.3333333333333333,442 "mean_conf": 0.9264606237411499,443 "sec": 1.0,444 "heldout": false445 },446 "civil_comments": {447 "n": 7500,448 "acc": 0.9330666666666667,449 "nll": 0.16645771265029907,450 "brier": 0.09957150369882584,451 "ece": 0.022440280524889583,452 "aurc": 0.008493115215878448,453 "acc_at_80": 0.9856666666666667,454 "acc_at_50": 1.0,455 "chance": 0.5,456 "mean_conf": 0.955498218536377,457 "sec": 2.1,458 "heldout": false,459 "per_q": [460 0.8733333333333333,461 0.9766666666666667,462 0.9906666666666667,463 0.85,464 0.9746666666666667465 ]466 },467 "toxic_chat": {468 "n": 3000,469 "acc": 0.9753333333333334,470 "nll": 0.0650152713060379,471 "brier": 0.035060759633779526,472 "ece": 0.011077121953169521,473 "aurc": 0.0014917009140776134,474 "acc_at_80": 0.99875,475 "acc_at_50": 0.9993333333333333,476 "chance": 0.5,477 "mean_conf": 0.985869288444519,478 "sec": 1.4,479 "heldout": false,480 "per_q": [481 0.958,482 0.9926666666666667483 ]484 },485 "sms_spam": {486 "n": 1000,487 "acc": 0.985,488 "nll": 0.06413730978965759,489 "brier": 0.02651730179786682,490 "ece": 0.013071102797985047,491 "aurc": 0.0017184124196473523,492 "acc_at_80": 0.9975,493 "acc_at_50": 0.998,494 "chance": 0.5,495 "mean_conf": 0.9963533878326416,496 "sec": 0.7,497 "heldout": false498 },499 "enron_spam": {500 "n": 1500,501 "acc": 0.9853333333333333,502 "nll": 0.05048864707350731,503 "brier": 0.023362739011645317,504 "ece": 0.009452791094779985,505 "aurc": 0.0010555480417750408,506 "acc_at_80": 0.9991666666666666,507 "acc_at_50": 0.9986666666666667,508 "chance": 0.5,509 "mean_conf": 0.9947861433029175,510 "sec": 2.9,511 "heldout": false512 },513 "insincere_questions": {514 "n": 1500,515 "acc": 0.952,516 "nll": 0.11656896770000458,517 "brier": 0.06938926875591278,518 "ece": 0.02098088498910269,519 "aurc": 0.004221551290811616,520 "acc_at_80": 0.9983333333333333,521 "acc_at_50": 0.9986666666666667,522 "chance": 0.5,523 "mean_conf": 0.9717850089073181,524 "sec": 0.9,525 "heldout": false526 },527 "ade": {528 "n": 1500,529 "acc": 0.8173333333333334,530 "nll": 0.4560953378677368,531 "brier": 0.2800695300102234,532 "ece": 0.0687519727150599,533 "aurc": 0.07776987989583131,534 "acc_at_80": 0.8625,535 "acc_at_50": 0.9173333333333333,536 "chance": 0.5,537 "mean_conf": 0.8821967840194702,538 "sec": 1.0,539 "heldout": true540 },541 "snli": {542 "n": 1476,543 "acc": 0.9214092140921409,544 "nll": 0.2496514618396759,545 "brier": 0.12413201481103897,546 "ece": 0.02373483920404258,547 "aurc": 0.02304695057683426,548 "acc_at_80": 0.9669771380186283,549 "acc_at_50": 0.981029810298103,550 "chance": 0.3333333333333333,551 "mean_conf": 0.9317938089370728,552 "sec": 1.3,553 "heldout": false554 },555 "mnli": {556 "n": 1500,557 "acc": 0.8846666666666667,558 "nll": 0.3279857933521271,559 "brier": 0.17651496827602386,560 "ece": 0.02796298484007516,561 "aurc": 0.02839381927398111,562 "acc_at_80": 0.9441666666666667,563 "acc_at_50": 0.9853333333333333,564 "chance": 0.3333333333333333,565 "mean_conf": 0.9101898074150085,566 "sec": 1.3,567 "heldout": false568 },569 "rte": {570 "n": 277,571 "acc": 0.8664259927797834,572 "nll": 0.3891109526157379,573 "brier": 0.2202744483947754,574 "ece": 0.06889465591107032,575 "aurc": 0.05497760440143546,576 "acc_at_80": 0.9144144144144144,577 "acc_at_50": 0.9565217391304348,578 "chance": 0.5,579 "mean_conf": 0.9213083386421204,580 "sec": 0.3,581 "heldout": false582 },583 "qnli": {584 "n": 1500,585 "acc": 0.938,586 "nll": 0.1725793182849884,587 "brier": 0.09701354801654816,588 "ece": 0.018119730949401894,589 "aurc": 0.01105178177734522,590 "acc_at_80": 0.9825,591 "acc_at_50": 0.9946666666666667,592 "chance": 0.5,593 "mean_conf": 0.9547027945518494,594 "sec": 1.2,595 "heldout": false596 },597 "qqp": {598 "n": 1500,599 "acc": 0.8486666666666667,600 "nll": 0.33667847514152527,601 "brier": 0.21694913506507874,602 "ece": 0.027132281541824343,603 "aurc": 0.04492686265704937,604 "acc_at_80": 0.9008333333333334,605 "acc_at_50": 0.9706666666666667,606 "chance": 0.5,607 "mean_conf": 0.8645197153091431,608 "sec": 1.1,609 "heldout": false610 },611 "mrpc": {612 "n": 408,613 "acc": 0.8480392156862745,614 "nll": 0.3339563310146332,615 "brier": 0.2105294018983841,616 "ece": 0.04274974734175441,617 "aurc": 0.04511716527430815,618 "acc_at_80": 0.9079754601226994,619 "acc_at_50": 0.9754901960784313,620 "chance": 0.5,621 "mean_conf": 0.8559635877609253,622 "sec": 0.4,623 "heldout": false624 },625 "paws": {626 "n": 1500,627 "acc": 0.724,628 "nll": 0.694682776927948,629 "brier": 0.4271811544895172,630 "ece": 0.17940687636534372,631 "aurc": 0.12480813768846086,632 "acc_at_80": 0.78,633 "acc_at_50": 0.8853333333333333,634 "chance": 0.5,635 "mean_conf": 0.9034069180488586,636 "sec": 1.3,637 "heldout": true638 },639 "cola": {640 "n": 1043,641 "acc": 0.8015340364333653,642 "nll": 0.4175913631916046,643 "brier": 0.2705059051513672,644 "ece": 0.027572413876095537,645 "aurc": 0.07668882275831405,646 "acc_at_80": 0.8657074340527577,647 "acc_at_50": 0.9329501915708812,648 "chance": 0.5,649 "mean_conf": 0.8236285448074341,650 "sec": 0.6,651 "heldout": false652 },653 "boolq": {654 "n": 1500,655 "acc": 0.876,656 "nll": 0.357710599899292,657 "brier": 0.19683362543582916,658 "ece": 0.059815298040707926,659 "aurc": 0.03911365860036364,660 "acc_at_80": 0.9241666666666667,661 "acc_at_50": 0.9693333333333334,662 "chance": 0.5,663 "mean_conf": 0.935267984867096,664 "sec": 1.9,665 "heldout": false666 },667 "strategyqa": {668 "n": 687,669 "acc": 0.5516739446870451,670 "nll": 0.7452422380447388,671 "brier": 0.5299192070960999,672 "ece": 0.17586313932049571,673 "aurc": 0.32907532533270495,674 "acc_at_80": 0.5745454545454546,675 "acc_at_50": 0.6366279069767442,676 "chance": 0.5,677 "mean_conf": 0.7275370359420776,678 "sec": 0.4,679 "heldout": true680 },681 "pubmedqa": {682 "n": 500,683 "acc": 0.756,684 "nll": 0.7593856453895569,685 "brier": 0.3763442933559418,686 "ece": 0.13125784403085708,687 "aurc": 0.11776619019690367,688 "acc_at_80": 0.8375,689 "acc_at_50": 0.908,690 "chance": 0.33333333333333326,691 "mean_conf": 0.8847917318344116,692 "sec": 1.2,693 "heldout": true694 },695 "arc": {696 "n": 1172,697 "acc": 0.8609215017064846,698 "nll": 0.3946582078933716,699 "brier": 0.20159626007080078,700 "ece": 0.03699784692847281,701 "aurc": 0.029178906832842126,702 "acc_at_80": 0.9402985074626866,703 "acc_at_50": 0.9863481228668942,704 "chance": 0.25015642775881686,705 "mean_conf": 0.8952547311782837,706 "sec": 1.0,707 "heldout": false708 },709 "commonsense_qa": {710 "n": 1221,711 "acc": 0.7461097461097461,712 "nll": 0.657385528087616,713 "brier": 0.3438973128795624,714 "ece": 0.06063837679461509,715 "aurc": 0.08669029470544302,716 "acc_at_80": 0.8311156601842374,717 "acc_at_50": 0.9311475409836065,718 "chance": 0.19999999999999998,719 "mean_conf": 0.8057844042778015,720 "sec": 0.8,721 "heldout": false722 },723 "qasc": {724 "n": 926,725 "acc": 0.8012958963282938,726 "nll": 0.612847626209259,727 "brier": 0.2819903790950775,728 "ece": 0.02709557122468436,729 "aurc": 0.05141996432246003,730 "acc_at_80": 0.8866396761133604,731 "acc_at_50": 0.9827213822894169,732 "chance": 0.125,733 "mean_conf": 0.7910774946212769,734 "sec": 0.8,735 "heldout": false736 },737 "openbookqa": {738 "n": 500,739 "acc": 0.816,740 "nll": 0.5107881426811218,741 "brier": 0.2631987929344177,742 "ece": 0.033833789587020886,743 "aurc": 0.05619724629150928,744 "acc_at_80": 0.8925,745 "acc_at_50": 0.976,746 "chance": 0.25,747 "mean_conf": 0.8462465405464172,748 "sec": 0.3,749 "heldout": false750 },751 "sciq": {752 "n": 1000,753 "acc": 0.982,754 "nll": 0.04754561558365822,755 "brier": 0.023483367636799812,756 "ece": 0.014726210862398123,757 "aurc": 0.000321545900436324,758 "acc_at_80": 1.0,759 "acc_at_50": 1.0,760 "chance": 0.25,761 "mean_conf": 0.9794425368309021,762 "sec": 1.1,763 "heldout": true764 },765 "hellaswag": {766 "n": 1500,767 "acc": 0.866,768 "nll": 0.3764917254447937,769 "brier": 0.19079357385635376,770 "ece": 0.06253247086207067,771 "aurc": 0.02407728816052661,772 "acc_at_80": 0.9525,773 "acc_at_50": 0.9946666666666667,774 "chance": 0.25,775 "mean_conf": 0.9285325407981873,776 "sec": 2.2,777 "heldout": false778 },779 "piqa": {780 "n": 1500,781 "acc": 0.83,782 "nll": 0.3894118666648865,783 "brier": 0.24019156396389008,784 "ece": 0.04349609518051145,785 "aurc": 0.0600857158889656,786 "acc_at_80": 0.8941666666666667,787 "acc_at_50": 0.944,788 "chance": 0.5,789 "mean_conf": 0.8716477751731873,790 "sec": 1.2,791 "heldout": false792 },793 "social_iqa": {794 "n": 1500,795 "acc": 0.708,796 "nll": 0.7983387112617493,797 "brier": 0.4352244734764099,798 "ece": 0.11886229503154755,799 "aurc": 0.15761526845761895,800 "acc_at_80": 0.7558333333333334,801 "acc_at_50": 0.844,802 "chance": 0.3333333333333333,803 "mean_conf": 0.8267069458961487,804 "sec": 1.1,805 "heldout": true806 },807 "winogrande": {808 "n": 1267,809 "acc": 0.6921862667719021,810 "nll": 0.8599734902381897,811 "brier": 0.48017874360084534,812 "ece": 0.18726083422234047,813 "aurc": 0.20496797159884195,814 "acc_at_80": 0.7268244575936884,815 "acc_at_50": 0.7823343848580442,816 "chance": 0.5,817 "mean_conf": 0.879352331161499,818 "sec": 0.8,819 "heldout": false820 },821 "race": {822 "n": 1500,823 "acc": 0.8286666666666667,824 "nll": 0.5294487476348877,825 "brier": 0.2535547614097595,826 "ece": 0.08820948191483813,827 "aurc": 0.0441451468912063,828 "acc_at_80": 0.9191666666666667,829 "acc_at_50": 0.976,830 "chance": 0.25,831 "mean_conf": 0.9168761372566223,832 "sec": 3.8,833 "heldout": false834 },835 "mmlu": {836 "n": 1500,837 "acc": 0.6293333333333333,838 "nll": 0.9297420382499695,839 "brier": 0.47942283749580383,840 "ece": 0.1056801940202713,841 "aurc": 0.15805034675139773,842 "acc_at_80": 0.7116666666666667,843 "acc_at_50": 0.8626666666666667,844 "chance": 0.25,845 "mean_conf": 0.735013484954834,846 "sec": 1.5,847 "heldout": false848 },849 "medqa": {850 "n": 1273,851 "acc": 0.5443833464257659,852 "nll": 1.2023534774780273,853 "brier": 0.6267179250717163,854 "ece": 0.18061440218553143,855 "aurc": 0.2882548640060218,856 "acc_at_80": 0.5943025540275049,857 "acc_at_50": 0.6808176100628931,858 "chance": 0.25,859 "mean_conf": 0.7214598655700684,860 "sec": 2.0,861 "heldout": false862 },863 "truthfulqa": {864 "n": 817,865 "acc": 0.5373317013463892,866 "nll": 1.5506761074066162,867 "brier": 0.6525235176086426,868 "ece": 0.16036296565646493,869 "aurc": 0.2660050026273411,870 "acc_at_80": 0.599388379204893,871 "acc_at_50": 0.7279411764705882,872 "chance": 0.22622350449767833,873 "mean_conf": 0.6951266527175903,874 "sec": 0.7,875 "heldout": true876 },877 "fin_phrasebank": {878 "n": 970,879 "acc": 0.6938144329896907,880 "nll": 0.710655927658081,881 "brier": 0.432155579328537,882 "ece": 0.09275854675425697,883 "aurc": 0.1946467193061562,884 "acc_at_80": 0.7332474226804123,885 "acc_at_50": 0.8061855670103093,886 "chance": 0.33333333333333326,887 "mean_conf": 0.7770669460296631,888 "sec": 2.0,889 "heldout": true890 },891 "bias_in_bios": {892 "n": 3000,893 "acc": 0.9513333333333334,894 "nll": 0.1453825682401657,895 "brier": 0.0709514245390892,896 "ece": 0.013315296888351444,897 "aurc": 0.0035279360929807786,898 "acc_at_80": 0.99625,899 "acc_at_50": 1.0,900 "chance": 0.30000000000000004,901 "mean_conf": 0.9636430740356445,902 "sec": 2.0,903 "heldout": false,904 "per_q": [905 0.9053333333333333,906 0.9973333333333333907 ]908 },909 "helpsteer2": {910 "n": 5190,911 "acc": 0.5895953757225434,912 "nll": 0.9746355414390564,913 "brier": 0.5325403809547424,914 "ece": 0.057795672655564984,915 "aurc": 0.2721544202844174,916 "acc_at_80": 0.6536608863198459,917 "acc_at_50": 0.735645472061657,918 "chance": 0.2,919 "mean_conf": 0.6461822986602783,920 "sec": 4.8,921 "heldout": false,922 "per_q": [923 0.4325626204238921,924 0.48554913294797686,925 0.7186897880539499,926 0.6204238921001927,927 0.6907514450867052928 ]929 },930 "helpsteer3_pref": {931 "n": 1176,932 "acc": 0.407312925170068,933 "nll": 1.518996000289917,934 "brier": 0.7332671284675598,935 "ece": 0.1154663114726138,936 "aurc": 0.5171907165068561,937 "acc_at_80": 0.4303931987247609,938 "acc_at_50": 0.46258503401360546,939 "chance": 0.14285714285714282,940 "mean_conf": 0.5227791666984558,941 "sec": 6.4,942 "heldout": false943 },944 "hate_speech_scales": {945 "n": 7500,946 "acc": 0.554,947 "nll": 1.0020463466644287,948 "brier": 0.5518749952316284,949 "ece": 0.05263813356558482,950 "aurc": 0.2725105506447112,951 "acc_at_80": 0.6043333333333333,952 "acc_at_50": 0.7026666666666667,953 "chance": 0.22666666666666674,954 "mean_conf": 0.6066381335258484,955 "sec": 3.0,956 "heldout": false,957 "per_q": [958 0.7033333333333334,959 0.472,960 0.43866666666666665,961 0.5686666666666667,962 0.5873333333333334963 ]964 },965 "liar2": {966 "n": 1500,967 "acc": 0.37066666666666664,968 "nll": 1.487428069114685,969 "brier": 0.7230392098426819,970 "ece": 0.042378827313582085,971 "aurc": 0.48226130915305326,972 "acc_at_80": 0.4008333333333333,973 "acc_at_50": 0.472,974 "chance": 0.16666666666666666,975 "mean_conf": 0.4129585921764374,976 "sec": 1.9,977 "heldout": false978 },979 "prosocial_safety": {980 "n": 1500,981 "acc": 0.5126666666666667,982 "nll": 1.1911370754241943,983 "brier": 0.601366400718689,984 "ece": 0.04166399759054183,985 "aurc": 0.28211418515491415,986 "acc_at_80": 0.5758333333333333,987 "acc_at_50": 0.6933333333333334,988 "chance": 0.2,989 "mean_conf": 0.541668713092804,990 "sec": 1.3,991 "heldout": false992 },993 "ultrafeedback_pref": {994 "n": 1500,995 "acc": 0.7506666666666667,996 "nll": 0.5978830456733704,997 "brier": 0.3642973303794861,998 "ece": 0.12037857890129089,999 "aurc": 0.12885069203247962,1000 "acc_at_80": 0.8033333333333333,1001 "acc_at_50": 0.864,1002 "chance": 0.5,1003 "mean_conf": 0.8620939254760742,1004 "sec": 5.2,1005 "heldout": false1006 },1007 "shp": {1008 "n": 1500,1009 "acc": 0.7433333333333333,1010 "nll": 0.5542265176773071,1011 "brier": 0.3617444932460785,1012 "ece": 0.08052628513177235,1013 "aurc": 0.1415611995055057,1014 "acc_at_80": 0.7875,1015 "acc_at_50": 0.8613333333333333,1016 "chance": 0.5,1017 "mean_conf": 0.8218792080879211,1018 "sec": 3.8,1019 "heldout": false1020 },1021 "hh_rlhf": {1022 "n": 1488,1023 "acc": 0.6283602150537635,1024 "nll": 0.7063506841659546,1025 "brier": 0.48729172348976135,1026 "ece": 0.1238586102682416,1027 "aurc": 0.2775212031705402,1028 "acc_at_80": 0.6470588235294118,1029 "acc_at_50": 0.6975806451612904,1030 "chance": 0.5,1031 "mean_conf": 0.745833694934845,1032 "sec": 3.1,1033 "heldout": false1034 },1035 "arena_pref": {1036 "n": 1500,1037 "acc": 0.4826666666666667,1038 "nll": 1.3242307901382446,1039 "brier": 0.7265019416809082,1040 "ece": 0.24603747620185218,1041 "aurc": 0.42174124750905745,1042 "acc_at_80": 0.5166666666666667,1043 "acc_at_50": 0.564,1044 "chance": 0.3333333333333333,1045 "mean_conf": 0.7287040948867798,1046 "sec": 4.9,1047 "heldout": true1048 },1049 "reward_bench": {1050 "n": 1500,1051 "acc": 0.8186666666666667,1052 "nll": 0.4543558359146118,1053 "brier": 0.274769127368927,1054 "ece": 0.07598515292008719,1055 "aurc": 0.07768967728506293,1056 "acc_at_80": 0.8666666666666667,1057 "acc_at_50": 0.9333333333333333,1058 "chance": 0.5,1059 "mean_conf": 0.8845229744911194,1060 "sec": 3.9,1061 "heldout": true1062 },1063 "glaive_tools": {1064 "n": 1348,1065 "acc": 0.9413946587537092,1066 "nll": 0.19328154623508453,1067 "brier": 0.10143861919641495,1068 "ece": 0.022093097250963853,1069 "aurc": 0.01690886528489672,1070 "acc_at_80": 0.9693877551020408,1071 "acc_at_50": 0.983679525222552,1072 "chance": 0.16666666666666666,1073 "mean_conf": 0.955422580242157,1074 "sec": 1.6,1075 "heldout": false1076 },1077 "toolace": {1078 "n": 1000,1079 "acc": 0.931,1080 "nll": 0.2512577474117279,1081 "brier": 0.11597812175750732,1082 "ece": 0.031337061107158684,1083 "aurc": 0.02448370972130845,1084 "acc_at_80": 0.97,1085 "acc_at_50": 0.982,1086 "chance": 0.12470555555555554,1087 "mean_conf": 0.9608047604560852,1088 "sec": 2.0,1089 "heldout": false1090 },1091 "hermes_tools": {1092 "n": 1500,1093 "acc": 0.7226666666666667,1094 "nll": 0.8366944789886475,1095 "brier": 0.464679479598999,1096 "ece": 0.23630806771914162,1097 "aurc": 0.06836947758114767,1098 "acc_at_80": 0.8566666666666667,1099 "acc_at_50": 0.984,1100 "chance": 0.125,1101 "mean_conf": 0.9589747786521912,1102 "sec": 4.7,1103 "heldout": true1104 },1105 "copa": {1106 "n": 100,1107 "acc": 0.93,1108 "nll": 0.20605678856372833,1109 "brier": 0.11243906617164612,1110 "ece": 0.028781158924102764,1111 "aurc": 0.019294681675211874,1112 "acc_at_80": 0.975,1113 "acc_at_50": 0.98,1114 "chance": 0.5,1115 "mean_conf": 0.9292109608650208,1116 "sec": 0.1,1117 "heldout": false1118 },1119 "wic": {1120 "n": 638,1121 "acc": 0.6943573667711599,1122 "nll": 0.6192224621772766,1123 "brier": 0.41049328446388245,1124 "ece": 0.10207162110783091,1125 "aurc": 0.2013770101215354,1126 "acc_at_80": 0.7490196078431373,1127 "acc_at_50": 0.8056426332288401,1128 "chance": 0.5,1129 "mean_conf": 0.7964289784431458,1130 "sec": 0.4,1131 "heldout": false1132 },1133 "multirc": {1134 "n": 1500,1135 "acc": 0.8873333333333333,1136 "nll": 0.353663831949234,1137 "brier": 0.18488313257694244,1138 "ece": 0.057831449985504116,1139 "aurc": 0.04080578362832766,1140 "acc_at_80": 0.9341666666666667,1141 "acc_at_50": 0.972,1142 "chance": 0.5,1143 "mean_conf": 0.9418337345123291,1144 "sec": 3.5,1145 "heldout": false1146 },1147 "cb": {1148 "n": 56,1149 "acc": 0.8571428571428571,1150 "nll": 0.39587274193763733,1151 "brier": 0.1826748102903366,1152 "ece": 0.061279319226741805,1153 "aurc": 0.024518910173493138,1154 "acc_at_80": 0.9555555555555556,1155 "acc_at_50": 1.0,1156 "chance": 0.3333333333333333,1157 "mean_conf": 0.8873938322067261,1158 "sec": 0.1,1159 "heldout": true1160 },1161 "fever": {1162 "n": 1500,1163 "acc": 0.888,1164 "nll": 0.3543616235256195,1165 "brier": 0.18396615982055664,1166 "ece": 0.05364456375439957,1167 "aurc": 0.035409527613622344,1168 "acc_at_80": 0.9416666666666667,1169 "acc_at_50": 0.9706666666666667,1170 "chance": 0.3333333333333333,1171 "mean_conf": 0.9349583387374878,1172 "sec": 1.4,1173 "heldout": false1174 },1175 "wiki_qa": {1176 "n": 879,1177 "acc": 0.8953356086461889,1178 "nll": 0.2812374532222748,1179 "brier": 0.15986107289791107,1180 "ece": 0.03158905461379691,1181 "aurc": 0.029564777940039345,1182 "acc_at_80": 0.9473684210526315,1183 "acc_at_50": 0.9772727272727273,1184 "chance": 0.5,1185 "mean_conf": 0.9265106320381165,1186 "sec": 0.7,1187 "heldout": false1188 },1189 "msmarco_rel": {1190 "n": 1446,1191 "acc": 0.6535269709543569,1192 "nll": 0.6131556034088135,1193 "brier": 0.42479565739631653,1194 "ece": 0.03628980136511236,1195 "aurc": 0.23044128645847725,1196 "acc_at_80": 0.6871218668971478,1197 "acc_at_50": 0.7565698478561549,1198 "chance": 0.5,1199 "mean_conf": 0.6898168325424194,1200 "sec": 1.5,