HuggingFaceFW/fineweb

1# ruff: noqa: F405, F403, F4012"""3Custom evaluation tasks for lighteval4 5Do note that we ran the evals with `max_samples=1000` to speed up large evals.6Most custom prompt changes were in an attempt to improve signal for small models in general.7 8This file generally creates just a TASKS_TABLE and TASKS_GROUPS which are then imported by LightEval.9 10Example usage (lighteval_tasks.py is the path to this file):11===================12accelerate launch --num_processes=1 lighteval/run_evals_accelerate.py --model_args="pretrained=HuggingFaceFW/ablation-model-fineweb-edu" \13    --custom_tasks "lighteval_tasks.py" --output_dir [OUTPUTPATH] --max_samples 1000 \ 14    --tasks "custom|hellaswag|0|1,custom|winogrande|0|1,custom|piqa|0|1,custom|siqa|0|1,custom|openbookqa|0|1,custom|arc:easy|0|1,custom|arc:challenge|0|1,custom|commonsense_qa|0|1,custom|mmlu:abstract_algebra|0|1,custom|mmlu:anatomy|0|1,custom|mmlu:astronomy|0|1,custom|mmlu:business_ethics|0|1,custom|mmlu:clinical_knowledge|0|1,custom|mmlu:college_biology|0|1,custom|mmlu:college_chemistry|0|1,custom|mmlu:college_computer_science|0|1,custom|mmlu:college_mathematics|0|1,custom|mmlu:college_medicine|0|1,custom|mmlu:college_physics|0|1,custom|mmlu:computer_security|0|1,custom|mmlu:conceptual_physics|0|1,custom|mmlu:econometrics|0|1,custom|mmlu:electrical_engineering|0|1,custom|mmlu:elementary_mathematics|0|1,custom|mmlu:formal_logic|0|1,custom|mmlu:global_facts|0|1,custom|mmlu:high_school_biology|0|1,custom|mmlu:high_school_chemistry|0|1,custom|mmlu:high_school_computer_science|0|1,custom|mmlu:high_school_european_history|0|1,custom|mmlu:high_school_geography|0|1,custom|mmlu:high_school_government_and_politics|0|1,custom|mmlu:high_school_macroeconomics|0|1,custom|mmlu:high_school_mathematics|0|1,custom|mmlu:high_school_microeconomics|0|1,custom|mmlu:high_school_physics|0|1,custom|mmlu:high_school_psychology|0|1,custom|mmlu:high_school_statistics|0|1,custom|mmlu:high_school_us_history|0|1,custom|mmlu:high_school_world_history|0|1,custom|mmlu:human_aging|0|1,custom|mmlu:human_sexuality|0|1,custom|mmlu:international_law|0|1,custom|mmlu:jurisprudence|0|1,custom|mmlu:logical_fallacies|0|1,custom|mmlu:machine_learning|0|1,custom|mmlu:management|0|1,custom|mmlu:marketing|0|1,custom|mmlu:medical_genetics|0|1,custom|mmlu:miscellaneous|0|1,custom|mmlu:moral_disputes|0|1,custom|mmlu:moral_scenarios|0|1,custom|mmlu:nutrition|0|1,custom|mmlu:philosophy|0|1,custom|mmlu:prehistory|0|1,custom|mmlu:professional_accounting|0|1,custom|mmlu:professional_law|0|1,custom|mmlu:professional_medicine|0|1,custom|mmlu:professional_psychology|0|1,custom|mmlu:public_relations|0|1,custom|mmlu:security_studies|0|1,custom|mmlu:sociology|0|1,custom|mmlu:us_foreign_policy|0|1,custom|mmlu:virology|0|1,custom|mmlu:world_religions|0|1"15===================16 17More info here: https://github.com/huggingface/lighteval?tab=readme-ov-file#evaluate-a-model-on-extended-community-or-custom-tasks18For more info on differences between MMLU implementations: https://huggingface.co/blog/open-llm-leaderboard-mmlu#1001-flavors-of-mmlu19In particular, the default leaderboard MMLU implementation (which uses "A", "B", etc as answer targets) gives generally random results on small/non instruction tuned models.20Instead, we use the full MMLU answer as the target.21"""22import re23from typing import List, Tuple24 25from lighteval.metrics import Metrics26from lighteval.tasks.lighteval_task import LightevalTaskConfig27from lighteval.tasks.requests import Doc28from lighteval.tasks.tasks_prompt_formatting import LETTER_INDICES29 30_TASKS_STRINGS: List[Tuple[LightevalTaskConfig, str]] = []31_TASKS: List[LightevalTaskConfig] = []32 33## COMMON_SENSE_REASONING_TASKS ##34COMMON_SENSE_REASONING_TASKS = [35    LightevalTaskConfig(36        name="hellaswag",37        prompt_function="hellaswag_prompt",38        hf_repo="hellaswag",39        hf_subset="default",40        metric=["loglikelihood_acc", "loglikelihood_acc_norm_nospace"],41    ),42    LightevalTaskConfig(43        name="winogrande",44        prompt_function="winogrande",45        hf_repo="winogrande",46        hf_subset="winogrande_xl",47        metric=["loglikelihood_acc", "loglikelihood_acc_norm_nospace"],48    ),49    LightevalTaskConfig(50        name="piqa",51        prompt_function="piqa_harness",52        hf_repo="piqa",53        hf_subset="plain_text",54        metric=["loglikelihood_acc", "loglikelihood_acc_norm_nospace"],55    ),56    LightevalTaskConfig(57        name="siqa",58        prompt_function="siqa_prompt",59        hf_repo="lighteval/siqa",60        hf_subset="default",61        hf_avail_splits=["train", "validation"],62        metric=["loglikelihood_acc", "loglikelihood_acc_norm_nospace"],63    ),64    LightevalTaskConfig(65        name="openbookqa",66        prompt_function="openbookqa",67        hf_repo="openbookqa",68        hf_subset="main",69        metric=["loglikelihood_acc", "loglikelihood_acc_norm_nospace"],70    ),71    LightevalTaskConfig(72        name="arc:easy",73        prompt_function="arc",74        hf_repo="ai2_arc",75        hf_subset="ARC-Easy",76        evaluation_splits=["test"],77        generation_size=1,78        metric=["loglikelihood_acc", "loglikelihood_acc_norm_nospace"],79    ),80    LightevalTaskConfig(81        name="arc:challenge",82        prompt_function="arc",83        hf_repo="ai2_arc",84        hf_subset="ARC-Challenge",85        evaluation_splits=["test"],86        generation_size=1,87        metric=["loglikelihood_acc", "loglikelihood_acc_norm_nospace"],88    ),89    LightevalTaskConfig(90        name="commonsense_qa",91        prompt_function="commonsense_qa_prompt",92        hf_repo="commonsense_qa",93        hf_subset="default",94        metric=["loglikelihood_acc", "loglikelihood_acc_norm_nospace"],95    ),96]97 98 99def commonsense_qa_prompt(line, task_name: str = None):100    return Doc(101        task_name=task_name,102        query=line["question"],103        choices=[f" {c}" for c in line["choices"]["text"]],104        gold_index=LETTER_INDICES.index(line["answerKey"].strip()),105        instruction="",106    )107 108 109def siqa_prompt(line, task_name: str = None):110    return Doc(111        task_name=task_name,112        query=line["context"] + " " + line["question"],113        choices=[f" {c}" for c in [line["answerA"], line["answerB"], line["answerC"]]],114        gold_index=int(line["label"]) - 1,115        instruction="",116    )117 118 119def hellaswag_prompt(line, task_name: str = None):120    def preprocess(text):121        """Comes from AiHarness"""122        # text = text.strip()123        # NOTE: Brackets are artifacts of the WikiHow dataset portion of HellaSwag.124        text = text.replace(" [title]", ". ")125        text = re.sub("\\[.*?\\]", "", text)126        text = text.replace("  ", " ")127        return text128 129    ctx = f"{line['ctx_a']} {line['ctx_b'].capitalize()} "130    return Doc(131        task_name=task_name,132        query=preprocess(line["activity_label"] + ": " + ctx),133        choices=[" " + preprocess(ending) for ending in line["endings"]],134        gold_index=int(line["label"]) if line["label"] != "" else -1,  # -1 for test135        # "metric": "choices_loglikelihood",136    )137 138 139# 0 short for common sense140COMMON_SENSE_REASONING_STRING = [(t, f"custom|{t.name}|0|1") for t in COMMON_SENSE_REASONING_TASKS]141_TASKS_STRINGS.extend(COMMON_SENSE_REASONING_STRING)142_TASKS += COMMON_SENSE_REASONING_TASKS143 144## MMLU ##145class CustomMMLUEvaluationTask(LightevalTaskConfig):146    def __init__(147        self,148        name,149        prompt_function="mmlu_prompt",150        hf_repo="lighteval/mmlu",151        hf_subset=None,152        #  metric=[Metrics.loglikelihood_acc_single_token],153        metric=[Metrics.loglikelihood_acc, Metrics.loglikelihood_acc_norm_nospace],154        hf_avail_splits=None,155        evaluation_splits=["test"],156        few_shots_split="dev",157        few_shots_select=None,158        suite=None,159        generation_size=-1,160        stop_sequence=None,161        output_regex=None,162        frozen=False,163    ):164        super().__init__(165            name=name,166            prompt_function=prompt_function,167            hf_repo=hf_repo,168            hf_subset=hf_subset,169            metric=metric,170            hf_avail_splits=hf_avail_splits,171            evaluation_splits=evaluation_splits,172            few_shots_split=few_shots_split,173            few_shots_select=few_shots_select,174            suite=suite,175            generation_size=generation_size,176            stop_sequence=stop_sequence,177            output_regex=output_regex,178            frozen=frozen,179        )180 181 182MMLU_TASKS = [183    CustomMMLUEvaluationTask(name="mmlu:abstract_algebra", hf_subset="abstract_algebra"),184    CustomMMLUEvaluationTask(name="mmlu:anatomy", hf_subset="anatomy"),185    CustomMMLUEvaluationTask(name="mmlu:astronomy", hf_subset="astronomy"),186    CustomMMLUEvaluationTask(name="mmlu:business_ethics", hf_subset="business_ethics"),187    CustomMMLUEvaluationTask(name="mmlu:clinical_knowledge", hf_subset="clinical_knowledge"),188    CustomMMLUEvaluationTask(name="mmlu:college_biology", hf_subset="college_biology"),189    CustomMMLUEvaluationTask(name="mmlu:college_chemistry", hf_subset="college_chemistry"),190    CustomMMLUEvaluationTask(name="mmlu:college_computer_science", hf_subset="college_computer_science"),191    CustomMMLUEvaluationTask(name="mmlu:college_mathematics", hf_subset="college_mathematics"),192    CustomMMLUEvaluationTask(name="mmlu:college_medicine", hf_subset="college_medicine"),193    CustomMMLUEvaluationTask(name="mmlu:college_physics", hf_subset="college_physics"),194    CustomMMLUEvaluationTask(name="mmlu:computer_security", hf_subset="computer_security"),195    CustomMMLUEvaluationTask(name="mmlu:conceptual_physics", hf_subset="conceptual_physics"),196    CustomMMLUEvaluationTask(name="mmlu:econometrics", hf_subset="econometrics"),197    CustomMMLUEvaluationTask(name="mmlu:electrical_engineering", hf_subset="electrical_engineering"),198    CustomMMLUEvaluationTask(name="mmlu:elementary_mathematics", hf_subset="elementary_mathematics"),199    CustomMMLUEvaluationTask(name="mmlu:formal_logic", hf_subset="formal_logic"),200    CustomMMLUEvaluationTask(name="mmlu:global_facts", hf_subset="global_facts"),201    CustomMMLUEvaluationTask(name="mmlu:high_school_biology", hf_subset="high_school_biology"),202    CustomMMLUEvaluationTask(name="mmlu:high_school_chemistry", hf_subset="high_school_chemistry"),203    CustomMMLUEvaluationTask(name="mmlu:high_school_computer_science", hf_subset="high_school_computer_science"),204    CustomMMLUEvaluationTask(name="mmlu:high_school_european_history", hf_subset="high_school_european_history"),205    CustomMMLUEvaluationTask(name="mmlu:high_school_geography", hf_subset="high_school_geography"),206    CustomMMLUEvaluationTask(207        name="mmlu:high_school_government_and_politics", hf_subset="high_school_government_and_politics"208    ),209    CustomMMLUEvaluationTask(name="mmlu:high_school_macroeconomics", hf_subset="high_school_macroeconomics"),210    CustomMMLUEvaluationTask(name="mmlu:high_school_mathematics", hf_subset="high_school_mathematics"),211    CustomMMLUEvaluationTask(name="mmlu:high_school_microeconomics", hf_subset="high_school_microeconomics"),212    CustomMMLUEvaluationTask(name="mmlu:high_school_physics", hf_subset="high_school_physics"),213    CustomMMLUEvaluationTask(name="mmlu:high_school_psychology", hf_subset="high_school_psychology"),214    CustomMMLUEvaluationTask(name="mmlu:high_school_statistics", hf_subset="high_school_statistics"),215    CustomMMLUEvaluationTask(name="mmlu:high_school_us_history", hf_subset="high_school_us_history"),216    CustomMMLUEvaluationTask(name="mmlu:high_school_world_history", hf_subset="high_school_world_history"),217    CustomMMLUEvaluationTask(name="mmlu:human_aging", hf_subset="human_aging"),218    CustomMMLUEvaluationTask(name="mmlu:human_sexuality", hf_subset="human_sexuality"),219    CustomMMLUEvaluationTask(name="mmlu:international_law", hf_subset="international_law"),220    CustomMMLUEvaluationTask(name="mmlu:jurisprudence", hf_subset="jurisprudence"),221    CustomMMLUEvaluationTask(name="mmlu:logical_fallacies", hf_subset="logical_fallacies"),222    CustomMMLUEvaluationTask(name="mmlu:machine_learning", hf_subset="machine_learning"),223    CustomMMLUEvaluationTask(name="mmlu:management", hf_subset="management"),224    CustomMMLUEvaluationTask(name="mmlu:marketing", hf_subset="marketing"),225    CustomMMLUEvaluationTask(name="mmlu:medical_genetics", hf_subset="medical_genetics"),226    CustomMMLUEvaluationTask(name="mmlu:miscellaneous", hf_subset="miscellaneous"),227    CustomMMLUEvaluationTask(name="mmlu:moral_disputes", hf_subset="moral_disputes"),228    CustomMMLUEvaluationTask(name="mmlu:moral_scenarios", hf_subset="moral_scenarios"),229    CustomMMLUEvaluationTask(name="mmlu:nutrition", hf_subset="nutrition"),230    CustomMMLUEvaluationTask(name="mmlu:philosophy", hf_subset="philosophy"),231    CustomMMLUEvaluationTask(name="mmlu:prehistory", hf_subset="prehistory"),232    CustomMMLUEvaluationTask(name="mmlu:professional_accounting", hf_subset="professional_accounting"),233    CustomMMLUEvaluationTask(name="mmlu:professional_law", hf_subset="professional_law"),234    CustomMMLUEvaluationTask(name="mmlu:professional_medicine", hf_subset="professional_medicine"),235    CustomMMLUEvaluationTask(name="mmlu:professional_psychology", hf_subset="professional_psychology"),236    CustomMMLUEvaluationTask(name="mmlu:public_relations", hf_subset="public_relations"),237    CustomMMLUEvaluationTask(name="mmlu:security_studies", hf_subset="security_studies"),238    CustomMMLUEvaluationTask(name="mmlu:sociology", hf_subset="sociology"),239    CustomMMLUEvaluationTask(name="mmlu:us_foreign_policy", hf_subset="us_foreign_policy"),240    CustomMMLUEvaluationTask(name="mmlu:virology", hf_subset="virology"),241    CustomMMLUEvaluationTask(name="mmlu:world_religions", hf_subset="world_religions"),242]243 244 245def mmlu_prompt(line, task_name: str = None):246    """MMLU prompt without letters"""247    topic = line["subject"]248    prompt = f"The following are questions about {topic.replace('_', ' ')}.\nQuestion: "249    prompt += line["question"] + "\nAnswer:"250 251    return Doc(252        task_name=task_name,253        query=prompt,254        choices=[f" {c}" for c in line["choices"]],255        gold_index=line["answer"],256        instruction=f"The following are questions about {topic.replace('_', ' ')}.\n",257    )258 259 260MMLU_STRING = [(t, f"custom|{t.name}|0|1") for t in MMLU_TASKS]261_TASKS_STRINGS.extend(MMLU_STRING)262_TASKS += MMLU_TASKS263 264# common sense reasoning + mmlu265EARLY_SIGNAL_TASKS = ",".join([t[1] for t in COMMON_SENSE_REASONING_STRING] + [t[1] for t in MMLU_STRING])266 267# Convert to dict for lighteval268TASKS_TABLE = [task.as_dict() for task in _TASKS]269# You can have a few pre-organised groups of tasks270TASKS_GROUPS = {271    "early-signal": EARLY_SIGNAL_TASKS,272}273