inference-optimization/MiniMax-M2.5-BF16
022
1{2 "schema_version": "0.2.2",3 "evaluation_id": "mmlu_pro_chat/inference-optimization/MiniMax-M2.5-BF16/1777979462.9421",4 "evaluation_timestamp": "1777560618",5 "retrieved_timestamp": "1777979462.9421",6 "source_metadata": {7 "source_name": "lm-evaluation-harness",8 "source_type": "evaluation_run",9 "source_organization_name": "RedHatAI",10 "evaluator_relationship": "third_party"11 },12 "eval_library": {13 "name": "lm_eval",14 "version": "0.4.12.dev0"15 },16 "model_info": {17 "name": "inference-optimization/MiniMax-M2.5-BF16",18 "id": "inference-optimization/MiniMax-M2.5-BF16",19 "developer": "inference-optimization",20 "additional_details": {21 "model_args": "{'model': 'inference-optimization/MiniMax-M2.5-BF16', 'max_length': 196608, 'base_url': 'http://0.0.0.0:8000/v1/chat/completions', 'num_concurrent': 28, 'max_retries': 3, 'tokenized_requests': False, 'tokenizer_backend': None, 'timeout': 2400}",22 "seed": "1234",23 "num_seeds_merged": "3"24 }25 },26 "evaluation_results": [27 {28 "evaluation_name": "mmlu_pro_chat_biology/custom-extract",29 "source_data": {30 "dataset_name": "mmlu_pro_chat",31 "source_type": "hf_dataset",32 "hf_repo": "TIGER-Lab/MMLU-Pro",33 "hf_split": "test"34 },35 "evaluation_timestamp": "1777601539",36 "metric_config": {37 "evaluation_description": "exact_match (filter: custom-extract)",38 "lower_is_better": false,39 "score_type": "continuous",40 "min_score": 0.0,41 "max_score": 1.042 },43 "score_details": {44 "score": 0.9112040911204091,45 "details": {46 "seed_scores": "[0.9121338912133892, 0.9107391910739191, 0.9107391910739191]",47 "seed_values": "[1234, 4158, 42]"48 },49 "uncertainty": {50 "standard_error": {51 "value": 0.0004649000464900244,52 "method": "across_seeds"53 },54 "num_samples": 355 }56 },57 "generation_config": {58 "generation_args": {59 "temperature": 1.0,60 "top_p": 0.95,61 "top_k": 40.0,62 "max_tokens": 64000,63 "max_attempts": 164 },65 "additional_details": {66 "until": "[]",67 "do_sample": "true",68 "min_p": "0.0",69 "presence_penalty": "1.5",70 "repetition_penalty": "1.0",71 "seed": "1234",72 "num_fewshot": "0"73 }74 }75 },76 {77 "evaluation_name": "mmlu_pro_chat_business/custom-extract",78 "source_data": {79 "dataset_name": "mmlu_pro_chat",80 "source_type": "hf_dataset",81 "hf_repo": "TIGER-Lab/MMLU-Pro",82 "hf_split": "test"83 },84 "evaluation_timestamp": "1777601539",85 "metric_config": {86 "evaluation_description": "exact_match (filter: custom-extract)",87 "lower_is_better": false,88 "score_type": "continuous",89 "min_score": 0.0,90 "max_score": 1.091 },92 "score_details": {93 "score": 0.8635403464300803,94 "details": {95 "seed_scores": "[0.8631178707224335, 0.8593155893536122, 0.8681875792141952]",96 "seed_values": "[1234, 4158, 42]"97 },98 "uncertainty": {99 "standard_error": {100 "value": 0.0025698194044352423,101 "method": "across_seeds"102 },103 "num_samples": 3104 }105 },106 "generation_config": {107 "generation_args": {108 "temperature": 1.0,109 "top_p": 0.95,110 "top_k": 40.0,111 "max_tokens": 64000,112 "max_attempts": 1113 },114 "additional_details": {115 "until": "[]",116 "do_sample": "true",117 "min_p": "0.0",118 "presence_penalty": "1.5",119 "repetition_penalty": "1.0",120 "seed": "1234",121 "num_fewshot": "0"122 }123 }124 },125 {126 "evaluation_name": "mmlu_pro_chat_chemistry/custom-extract",127 "source_data": {128 "dataset_name": "mmlu_pro_chat",129 "source_type": "hf_dataset",130 "hf_repo": "TIGER-Lab/MMLU-Pro",131 "hf_split": "test"132 },133 "evaluation_timestamp": "1777601539",134 "metric_config": {135 "evaluation_description": "exact_match (filter: custom-extract)",136 "lower_is_better": false,137 "score_type": "continuous",138 "min_score": 0.0,139 "max_score": 1.0140 },141 "score_details": {142 "score": 0.8539458186101295,143 "details": {144 "seed_scores": "[0.8533568904593639, 0.8524734982332155, 0.8560070671378092]",145 "seed_values": "[1234, 4158, 42]"146 },147 "uncertainty": {148 "standard_error": {149 "value": 0.0010617053225747983,150 "method": "across_seeds"151 },152 "num_samples": 3153 }154 },155 "generation_config": {156 "generation_args": {157 "temperature": 1.0,158 "top_p": 0.95,159 "top_k": 40.0,160 "max_tokens": 64000,161 "max_attempts": 1162 },163 "additional_details": {164 "until": "[]",165 "do_sample": "true",166 "min_p": "0.0",167 "presence_penalty": "1.5",168 "repetition_penalty": "1.0",169 "seed": "1234",170 "num_fewshot": "0"171 }172 }173 },174 {175 "evaluation_name": "mmlu_pro_chat_computer_science/custom-extract",176 "source_data": {177 "dataset_name": "mmlu_pro_chat",178 "source_type": "hf_dataset",179 "hf_repo": "TIGER-Lab/MMLU-Pro",180 "hf_split": "test"181 },182 "evaluation_timestamp": "1777601539",183 "metric_config": {184 "evaluation_description": "exact_match (filter: custom-extract)",185 "lower_is_better": false,186 "score_type": "continuous",187 "min_score": 0.0,188 "max_score": 1.0189 },190 "score_details": {191 "score": 0.8585365853658536,192 "details": {193 "seed_scores": "[0.8560975609756097, 0.8658536585365854, 0.8536585365853658]",194 "seed_values": "[1234, 4158, 42]"195 },196 "uncertainty": {197 "standard_error": {198 "value": 0.003725671296712069,199 "method": "across_seeds"200 },201 "num_samples": 3202 }203 },204 "generation_config": {205 "generation_args": {206 "temperature": 1.0,207 "top_p": 0.95,208 "top_k": 40.0,209 "max_tokens": 64000,210 "max_attempts": 1211 },212 "additional_details": {213 "until": "[]",214 "do_sample": "true",215 "min_p": "0.0",216 "presence_penalty": "1.5",217 "repetition_penalty": "1.0",218 "seed": "1234",219 "num_fewshot": "0"220 }221 }222 },223 {224 "evaluation_name": "mmlu_pro_chat_economics/custom-extract",225 "source_data": {226 "dataset_name": "mmlu_pro_chat",227 "source_type": "hf_dataset",228 "hf_repo": "TIGER-Lab/MMLU-Pro",229 "hf_split": "test"230 },231 "evaluation_timestamp": "1777601539",232 "metric_config": {233 "evaluation_description": "exact_match (filter: custom-extract)",234 "lower_is_better": false,235 "score_type": "continuous",236 "min_score": 0.0,237 "max_score": 1.0238 },239 "score_details": {240 "score": 0.8574249605055293,241 "details": {242 "seed_scores": "[0.8566350710900474, 0.8590047393364929, 0.8566350710900474]",243 "seed_values": "[1234, 4158, 42]"244 },245 "uncertainty": {246 "standard_error": {247 "value": 0.0007898894154818222,248 "method": "across_seeds"249 },250 "num_samples": 3251 }252 },253 "generation_config": {254 "generation_args": {255 "temperature": 1.0,256 "top_p": 0.95,257 "top_k": 40.0,258 "max_tokens": 64000,259 "max_attempts": 1260 },261 "additional_details": {262 "until": "[]",263 "do_sample": "true",264 "min_p": "0.0",265 "presence_penalty": "1.5",266 "repetition_penalty": "1.0",267 "seed": "1234",268 "num_fewshot": "0"269 }270 }271 },272 {273 "evaluation_name": "mmlu_pro_chat_engineering/custom-extract",274 "source_data": {275 "dataset_name": "mmlu_pro_chat",276 "source_type": "hf_dataset",277 "hf_repo": "TIGER-Lab/MMLU-Pro",278 "hf_split": "test"279 },280 "evaluation_timestamp": "1777601539",281 "metric_config": {282 "evaluation_description": "exact_match (filter: custom-extract)",283 "lower_is_better": false,284 "score_type": "continuous",285 "min_score": 0.0,286 "max_score": 1.0287 },288 "score_details": {289 "score": 0.7027863777089783,290 "details": {291 "seed_scores": "[0.717234262125903, 0.6893704850361198, 0.7017543859649122]",292 "seed_values": "[1234, 4158, 42]"293 },294 "uncertainty": {295 "standard_error": {296 "value": 0.008060113184630189,297 "method": "across_seeds"298 },299 "num_samples": 3300 }301 },302 "generation_config": {303 "generation_args": {304 "temperature": 1.0,305 "top_p": 0.95,306 "top_k": 40.0,307 "max_tokens": 64000,308 "max_attempts": 1309 },310 "additional_details": {311 "until": "[]",312 "do_sample": "true",313 "min_p": "0.0",314 "presence_penalty": "1.5",315 "repetition_penalty": "1.0",316 "seed": "1234",317 "num_fewshot": "0"318 }319 }320 },321 {322 "evaluation_name": "mmlu_pro_chat_health/custom-extract",323 "source_data": {324 "dataset_name": "mmlu_pro_chat",325 "source_type": "hf_dataset",326 "hf_repo": "TIGER-Lab/MMLU-Pro",327 "hf_split": "test"328 },329 "evaluation_timestamp": "1777601539",330 "metric_config": {331 "evaluation_description": "exact_match (filter: custom-extract)",332 "lower_is_better": false,333 "score_type": "continuous",334 "min_score": 0.0,335 "max_score": 1.0336 },337 "score_details": {338 "score": 0.7889160554197229,339 "details": {340 "seed_scores": "[0.7872860635696821, 0.78239608801956, 0.7970660146699267]",341 "seed_values": "[1234, 4158, 42]"342 },343 "uncertainty": {344 "standard_error": {345 "value": 0.004312553074269904,346 "method": "across_seeds"347 },348 "num_samples": 3349 }350 },351 "generation_config": {352 "generation_args": {353 "temperature": 1.0,354 "top_p": 0.95,355 "top_k": 40.0,356 "max_tokens": 64000,357 "max_attempts": 1358 },359 "additional_details": {360 "until": "[]",361 "do_sample": "true",362 "min_p": "0.0",363 "presence_penalty": "1.5",364 "repetition_penalty": "1.0",365 "seed": "1234",366 "num_fewshot": "0"367 }368 }369 },370 {371 "evaluation_name": "mmlu_pro_chat_history/custom-extract",372 "source_data": {373 "dataset_name": "mmlu_pro_chat",374 "source_type": "hf_dataset",375 "hf_repo": "TIGER-Lab/MMLU-Pro",376 "hf_split": "test"377 },378 "evaluation_timestamp": "1777601539",379 "metric_config": {380 "evaluation_description": "exact_match (filter: custom-extract)",381 "lower_is_better": false,382 "score_type": "continuous",383 "min_score": 0.0,384 "max_score": 1.0385 },386 "score_details": {387 "score": 0.6885389326334208,388 "details": {389 "seed_scores": "[0.6929133858267716, 0.6876640419947506, 0.6850393700787402]",390 "seed_values": "[1234, 4158, 42]"391 },392 "uncertainty": {393 "standard_error": {394 "value": 0.0023147430542997277,395 "method": "across_seeds"396 },397 "num_samples": 3398 }399 },400 "generation_config": {401 "generation_args": {402 "temperature": 1.0,403 "top_p": 0.95,404 "top_k": 40.0,405 "max_tokens": 64000,406 "max_attempts": 1407 },408 "additional_details": {409 "until": "[]",410 "do_sample": "true",411 "min_p": "0.0",412 "presence_penalty": "1.5",413 "repetition_penalty": "1.0",414 "seed": "1234",415 "num_fewshot": "0"416 }417 }418 },419 {420 "evaluation_name": "mmlu_pro_chat_law/custom-extract",421 "source_data": {422 "dataset_name": "mmlu_pro_chat",423 "source_type": "hf_dataset",424 "hf_repo": "TIGER-Lab/MMLU-Pro",425 "hf_split": "test"426 },427 "evaluation_timestamp": "1777601539",428 "metric_config": {429 "evaluation_description": "exact_match (filter: custom-extract)",430 "lower_is_better": false,431 "score_type": "continuous",432 "min_score": 0.0,433 "max_score": 1.0434 },435 "score_details": {436 "score": 0.5924916742355434,437 "details": {438 "seed_scores": "[0.5940054495912807, 0.5912806539509536, 0.592188919164396]",439 "seed_values": "[1234, 4158, 42]"440 },441 "uncertainty": {442 "standard_error": {443 "value": 0.0008010146264198092,444 "method": "across_seeds"445 },446 "num_samples": 3447 }448 },449 "generation_config": {450 "generation_args": {451 "temperature": 1.0,452 "top_p": 0.95,453 "top_k": 40.0,454 "max_tokens": 64000,455 "max_attempts": 1456 },457 "additional_details": {458 "until": "[]",459 "do_sample": "true",460 "min_p": "0.0",461 "presence_penalty": "1.5",462 "repetition_penalty": "1.0",463 "seed": "1234",464 "num_fewshot": "0"465 }466 }467 },468 {469 "evaluation_name": "mmlu_pro_chat_math/custom-extract",470 "source_data": {471 "dataset_name": "mmlu_pro_chat",472 "source_type": "hf_dataset",473 "hf_repo": "TIGER-Lab/MMLU-Pro",474 "hf_split": "test"475 },476 "evaluation_timestamp": "1777601539",477 "metric_config": {478 "evaluation_description": "exact_match (filter: custom-extract)",479 "lower_is_better": false,480 "score_type": "continuous",481 "min_score": 0.0,482 "max_score": 1.0483 },484 "score_details": {485 "score": 0.9286947939797681,486 "details": {487 "seed_scores": "[0.9289415247964471, 0.9333826794966691, 0.923760177646188]",488 "seed_values": "[1234, 4158, 42]"489 },490 "uncertainty": {491 "standard_error": {492 "value": 0.0027805150924215634,493 "method": "across_seeds"494 },495 "num_samples": 3496 }497 },498 "generation_config": {499 "generation_args": {500 "temperature": 1.0,501 "top_p": 0.95,502 "top_k": 40.0,503 "max_tokens": 64000,504 "max_attempts": 1505 },506 "additional_details": {507 "until": "[]",508 "do_sample": "true",509 "min_p": "0.0",510 "presence_penalty": "1.5",511 "repetition_penalty": "1.0",512 "seed": "1234",513 "num_fewshot": "0"514 }515 }516 },517 {518 "evaluation_name": "mmlu_pro_chat_other/custom-extract",519 "source_data": {520 "dataset_name": "mmlu_pro_chat",521 "source_type": "hf_dataset",522 "hf_repo": "TIGER-Lab/MMLU-Pro",523 "hf_split": "test"524 },525 "evaluation_timestamp": "1777601539",526 "metric_config": {527 "evaluation_description": "exact_match (filter: custom-extract)",528 "lower_is_better": false,529 "score_type": "continuous",530 "min_score": 0.0,531 "max_score": 1.0532 },533 "score_details": {534 "score": 0.7846320346320347,535 "details": {536 "seed_scores": "[0.7867965367965368, 0.7813852813852814, 0.7857142857142857]",537 "seed_values": "[1234, 4158, 42]"538 },539 "uncertainty": {540 "standard_error": {541 "value": 0.001653165835121153,542 "method": "across_seeds"543 },544 "num_samples": 3545 }546 },547 "generation_config": {548 "generation_args": {549 "temperature": 1.0,550 "top_p": 0.95,551 "top_k": 40.0,552 "max_tokens": 64000,553 "max_attempts": 1554 },555 "additional_details": {556 "until": "[]",557 "do_sample": "true",558 "min_p": "0.0",559 "presence_penalty": "1.5",560 "repetition_penalty": "1.0",561 "seed": "1234",562 "num_fewshot": "0"563 }564 }565 },566 {567 "evaluation_name": "mmlu_pro_chat_philosophy/custom-extract",568 "source_data": {569 "dataset_name": "mmlu_pro_chat",570 "source_type": "hf_dataset",571 "hf_repo": "TIGER-Lab/MMLU-Pro",572 "hf_split": "test"573 },574 "evaluation_timestamp": "1777601539",575 "metric_config": {576 "evaluation_description": "exact_match (filter: custom-extract)",577 "lower_is_better": false,578 "score_type": "continuous",579 "min_score": 0.0,580 "max_score": 1.0581 },582 "score_details": {583 "score": 0.7348029392117568,584 "details": {585 "seed_scores": "[0.7254509018036072, 0.7354709418837675, 0.7434869739478958]",586 "seed_values": "[1234, 4158, 42]"587 },588 "uncertainty": {589 "standard_error": {590 "value": 0.005217267652576261,591 "method": "across_seeds"592 },593 "num_samples": 3594 }595 },596 "generation_config": {597 "generation_args": {598 "temperature": 1.0,599 "top_p": 0.95,600 "top_k": 40.0,601 "max_tokens": 64000,602 "max_attempts": 1603 },604 "additional_details": {605 "until": "[]",606 "do_sample": "true",607 "min_p": "0.0",608 "presence_penalty": "1.5",609 "repetition_penalty": "1.0",610 "seed": "1234",611 "num_fewshot": "0"612 }613 }614 },615 {616 "evaluation_name": "mmlu_pro_chat_physics/custom-extract",617 "source_data": {618 "dataset_name": "mmlu_pro_chat",619 "source_type": "hf_dataset",620 "hf_repo": "TIGER-Lab/MMLU-Pro",621 "hf_split": "test"622 },623 "evaluation_timestamp": "1777601539",624 "metric_config": {625 "evaluation_description": "exact_match (filter: custom-extract)",626 "lower_is_better": false,627 "score_type": "continuous",628 "min_score": 0.0,629 "max_score": 1.0630 },631 "score_details": {632 "score": 0.8604054400821145,633 "details": {634 "seed_scores": "[0.867590454195535, 0.8568129330254042, 0.8568129330254042]",635 "seed_values": "[1234, 4158, 42]"636 },637 "uncertainty": {638 "standard_error": {639 "value": 0.0035925070567102892,640 "method": "across_seeds"641 },642 "num_samples": 3643 }644 },645 "generation_config": {646 "generation_args": {647 "temperature": 1.0,648 "top_p": 0.95,649 "top_k": 40.0,650 "max_tokens": 64000,651 "max_attempts": 1652 },653 "additional_details": {654 "until": "[]",655 "do_sample": "true",656 "min_p": "0.0",657 "presence_penalty": "1.5",658 "repetition_penalty": "1.0",659 "seed": "1234",660 "num_fewshot": "0"661 }662 }663 },664 {665 "evaluation_name": "mmlu_pro_chat_psychology/custom-extract",666 "source_data": {667 "dataset_name": "mmlu_pro_chat",668 "source_type": "hf_dataset",669 "hf_repo": "TIGER-Lab/MMLU-Pro",670 "hf_split": "test"671 },672 "evaluation_timestamp": "1777601539",673 "metric_config": {674 "evaluation_description": "exact_match (filter: custom-extract)",675 "lower_is_better": false,676 "score_type": "continuous",677 "min_score": 0.0,678 "max_score": 1.0679 },680 "score_details": {681 "score": 0.8070175438596491,682 "details": {683 "seed_scores": "[0.8020050125313283, 0.8120300751879699, 0.8070175438596491]",684 "seed_values": "[1234, 4158, 42]"685 },686 "uncertainty": {687 "standard_error": {688 "value": 0.0028939863117274475,689 "method": "across_seeds"690 },691 "num_samples": 3692 }693 },694 "generation_config": {695 "generation_args": {696 "temperature": 1.0,697 "top_p": 0.95,698 "top_k": 40.0,699 "max_tokens": 64000,700 "max_attempts": 1701 },702 "additional_details": {703 "until": "[]",704 "do_sample": "true",705 "min_p": "0.0",706 "presence_penalty": "1.5",707 "repetition_penalty": "1.0",708 "seed": "1234",709 "num_fewshot": "0"710 }711 }712 },713 {714 "evaluation_name": "mmlu_pro_chat/custom-extract",715 "source_data": {716 "dataset_name": "mmlu_pro_chat",717 "source_type": "other"718 },719 "evaluation_timestamp": "1777601539",720 "metric_config": {721 "evaluation_description": "exact_match (filter: custom-extract)",722 "lower_is_better": false,723 "score_type": "continuous",724 "min_score": 0.0,725 "max_score": 1.0726 },727 "score_details": {728 "score": 0.808344414893617,729 "details": {730 "seed_scores": "[0.8097573138297872, 0.8068484042553191, 0.8084275265957447]",731 "seed_values": "[1234, 4158, 42]"732 },733 "uncertainty": {734 "standard_error": {735 "value": 0.0008407574748167996,736 "method": "across_seeds"737 },738 "num_samples": 3739 }740 }741 }742 ]743}