Multi-Domain-Expert-Learning/given-mpt-7b
053
1---2license: apache-2.03inference: false4---5# Given-MPT-7B6 7This is a merge of the following MPT-7B models:8 9- **g**orilla-llm/gorilla-mpt-7b-hf-v010- **i**bm/mpt-7b-instruct211- Teh**V**enom/MPT-7b-WizardLM_Uncensored-Storywriter-Merge12- **e**mozilla/mpt-7b-storysummarizer13- **n**omic-ai/gpt4all-mpt14 15 16## Model License17 18Apache 2.019 20 21## Purpose22 23This model is for experimenting with merging and routing to expert layers.24 25# Test eval on only 10% of eval set26 27hf-causal (pretrained=Multi-Domain-Expert-Layers/given-mpt-7b,dtype=bfloat16,trust_remote_code=True), limit: 0.1, provide_description: False, num_fewshot: 0, batch_size: None28| Task |Version| Metric | Value | |Stderr|29|-------------------------------------------------|------:|-----------|------:|---|-----:|30|arc_challenge | 0|acc | 0.4274|± |0.0459|31| | |acc_norm | 0.3846|± |0.0452|32|arc_easy | 0|acc | 0.7863|± |0.0381|33| | |acc_norm | 0.7350|± |0.0410|34|hellaswag | 0|acc | 0.5556|± |0.0461|35| | |acc_norm | 0.8120|± |0.0363|36|hendrycksTest-college_chemistry | 0|acc | 0.3600|± |0.0482|37| | |acc_norm | 0.3700|± |0.0485|38|hendrycksTest-college_computer_science | 0|acc | 0.3400|± |0.0476|39| | |acc_norm | 0.3600|± |0.0482|40|hendrycksTest-college_mathematics | 0|acc | 0.2500|± |0.0435|41| | |acc_norm | 0.2900|± |0.0456|42|hendrycksTest-college_medicine | 0|acc | 0.3675|± |0.0448|43| | |acc_norm | 0.3162|± |0.0432|44|hendrycksTest-college_physics | 0|acc | 0.2451|± |0.0428|45| | |acc_norm | 0.2941|± |0.0453|46|hendrycksTest-computer_security | 0|acc | 0.4800|± |0.0502|47| | |acc_norm | 0.4400|± |0.0499|48|hendrycksTest-conceptual_physics | 0|acc | 0.2051|± |0.0375|49| | |acc_norm | 0.1709|± |0.0350|50|hendrycksTest-econometrics | 0|acc | 0.2982|± |0.0430|51| | |acc_norm | 0.2368|± |0.0400|52|hendrycksTest-electrical_engineering | 0|acc | 0.3248|± |0.0435|53| | |acc_norm | 0.3590|± |0.0445|54|hendrycksTest-elementary_mathematics | 0|acc | 0.3333|± |0.0438|55| | |acc_norm | 0.3162|± |0.0432|56|hendrycksTest-formal_logic | 0|acc | 0.3077|± |0.0429|57| | |acc_norm | 0.3248|± |0.0435|58|hendrycksTest-global_facts | 0|acc | 0.3000|± |0.0461|59| | |acc_norm | 0.2700|± |0.0446|60|hendrycksTest-high_school_biology | 0|acc | 0.3675|± |0.0448|61| | |acc_norm | 0.3077|± |0.0429|62|hendrycksTest-high_school_chemistry | 0|acc | 0.2564|± |0.0405|63| | |acc_norm | 0.2906|± |0.0422|64|hendrycksTest-high_school_computer_science | 0|acc | 0.4100|± |0.0494|65| | |acc_norm | 0.4400|± |0.0499|66|hendrycksTest-high_school_european_history | 0|acc | 0.4359|± |0.0460|67| | |acc_norm | 0.3590|± |0.0445|68|hendrycksTest-high_school_geography | 0|acc | 0.3248|± |0.0435|69| | |acc_norm | 0.3675|± |0.0448|70|hendrycksTest-high_school_government_and_politics| 0|acc | 0.3932|± |0.0454|71| | |acc_norm | 0.3932|± |0.0454|72|hendrycksTest-high_school_macroeconomics | 0|acc | 0.3333|± |0.0438|73| | |acc_norm | 0.3248|± |0.0435|74|hendrycksTest-high_school_mathematics | 0|acc | 0.2051|± |0.0375|75| | |acc_norm | 0.2564|± |0.0405|76|hendrycksTest-high_school_microeconomics | 0|acc | 0.3504|± |0.0443|77| | |acc_norm | 0.4188|± |0.0458|78|hendrycksTest-high_school_physics | 0|acc | 0.2650|± |0.0410|79| | |acc_norm | 0.2906|± |0.0422|80|hendrycksTest-high_school_psychology | 0|acc | 0.3761|± |0.0450|81| | |acc_norm | 0.3419|± |0.0440|82|hendrycksTest-high_school_statistics | 0|acc | 0.3077|± |0.0429|83| | |acc_norm | 0.3504|± |0.0443|84|hendrycksTest-high_school_us_history | 0|acc | 0.3333|± |0.0438|85| | |acc_norm | 0.3333|± |0.0438|86|hendrycksTest-high_school_world_history | 0|acc | 0.3333|± |0.0438|87| | |acc_norm | 0.3419|± |0.0440|88|hendrycksTest-human_aging | 0|acc | 0.3761|± |0.0450|89| | |acc_norm | 0.3162|± |0.0432|90|hendrycksTest-human_sexuality | 0|acc | 0.4274|± |0.0459|91| | |acc_norm | 0.3761|± |0.0450|92|hendrycksTest-international_law | 0|acc | 0.4188|± |0.0458|93| | |acc_norm | 0.4957|± |0.0464|94|hendrycksTest-jurisprudence | 0|acc | 0.3148|± |0.0449|95| | |acc_norm | 0.4815|± |0.0483|96|hendrycksTest-logical_fallacies | 0|acc | 0.3504|± |0.0443|97| | |acc_norm | 0.3675|± |0.0448|98|hendrycksTest-machine_learning | 0|acc | 0.3214|± |0.0443|99| | |acc_norm | 0.2946|± |0.0433|100|hendrycksTest-management | 0|acc | 0.3786|± |0.0480|101| | |acc_norm | 0.3495|± |0.0472|102|hendrycksTest-marketing | 0|acc | 0.5043|± |0.0464|103| | |acc_norm | 0.4188|± |0.0458|104|hendrycksTest-medical_genetics | 0|acc | 0.3200|± |0.0469|105| | |acc_norm | 0.4100|± |0.0494|106|hendrycksTest-miscellaneous | 0|acc | 0.5299|± |0.0463|107| | |acc_norm | 0.4872|± |0.0464|108|hendrycksTest-moral_disputes | 0|acc | 0.3248|± |0.0435|109| | |acc_norm | 0.3162|± |0.0432|110|hendrycksTest-moral_scenarios | 0|acc | 0.3248|± |0.0435|111| | |acc_norm | 0.2479|± |0.0401|112|hendrycksTest-nutrition | 0|acc | 0.3675|± |0.0448|113| | |acc_norm | 0.3932|± |0.0454|114|hendrycksTest-philosophy | 0|acc | 0.2991|± |0.0425|115| | |acc_norm | 0.3504|± |0.0443|116|hendrycksTest-prehistory | 0|acc | 0.2821|± |0.0418|117| | |acc_norm | 0.3248|± |0.0435|118|hendrycksTest-professional_accounting | 0|acc | 0.2137|± |0.0381|119| | |acc_norm | 0.2222|± |0.0386|120|hendrycksTest-professional_law | 0|acc | 0.3077|± |0.0429|121| | |acc_norm | 0.2735|± |0.0414|122|hendrycksTest-professional_medicine | 0|acc | 0.2991|± |0.0425|123| | |acc_norm | 0.2650|± |0.0410|124|hendrycksTest-professional_psychology | 0|acc | 0.3248|± |0.0435|125| | |acc_norm | 0.3419|± |0.0440|126|hendrycksTest-public_relations | 0|acc | 0.3909|± |0.0467|127| | |acc_norm | 0.3545|± |0.0458|128|hendrycksTest-security_studies | 0|acc | 0.3419|± |0.0440|129| | |acc_norm | 0.2906|± |0.0422|130|hendrycksTest-sociology | 0|acc | 0.3761|± |0.0450|131| | |acc_norm | 0.3162|± |0.0432|132|hendrycksTest-us_foreign_policy | 0|acc | 0.5000|± |0.0503|133| | |acc_norm | 0.4100|± |0.0494|134|hendrycksTest-virology | 0|acc | 0.3932|± |0.0454|135| | |acc_norm | 0.3248|± |0.0435|136|hendrycksTest-world_religions | 0|acc | 0.5299|± |0.0463|137| | |acc_norm | 0.5128|± |0.0464|138|truthfulqa_gen | 1|bleurt_max |-0.8551|± |0.0501|139| | |bleurt_acc | 0.3590|± |0.0445|140| | |bleurt_diff|-0.1292|± |0.0483|141| | |bleu_max |19.3738|± |1.8461|142| | |bleu_acc | 0.3932|± |0.0454|143| | |bleu_diff |-4.3883|± |2.1748|144| | |rouge1_max |41.8428|± |2.6156|145| | |rouge1_acc | 0.3162|± |0.0432|146| | |rouge1_diff|-8.8583|± |2.7745|147| | |rouge2_max |26.3956|± |2.8311|148| | |rouge2_acc | 0.2137|± |0.0381|149| | |rouge2_diff|-9.5287|± |3.3258|150| | |rougeL_max |39.5215|± |2.5620|151| | |rougeL_acc | 0.3162|± |0.0432|152| | |rougeL_diff|-8.5753|± |2.8259|153 154 155