EmbodiedAgentInterface/backend

1# ruff: noqa: F405, F403, F4012"""3Custom evaluation tasks for lighteval. Complete this task with your own configuration if you want to use a custom lighteval task.4 5This file generally create just a TASKS_TABLE and TASKS_GROUPS which are then imported by LightEval.6 7Author:8"""9from lighteval.tasks.lighteval_task import LightevalTaskConfig10from lighteval.tasks.requests import Doc11from lighteval.tasks.tasks_prompt_formatting import LETTER_INDICES12 13 14## EVAL WITH NO SUBSET ##15# This is how you create a simple tasks (like hellaswag) which has one single subset16# attached to it, and one evaluation possible.17task = LightevalTaskConfig(18    name="myothertask",19    prompt_function="prompt_fn",  # must be defined in the file or imported from src/lighteval/tasks/tasks_prompt_formatting.py20    suite=["community"],21    hf_repo="",22    hf_subset="default",23    hf_avail_splits=[],24    evaluation_splits=[],25    few_shots_split="",26    few_shots_select="",27    metric=[""],28)29 30## EVALS WITH SUBSET31# This is how you create a subset task (like MMLU), which has several subset32# each being its own evaluation task.33 34# fmt: off35SAMPLE_SUBSETS = [] # list of all the subsets to use for this eval36# fmt: on37 38 39class CustomSubsetTask(LightevalTaskConfig):40    def __init__(41        self,42        name,43        hf_subset,44    ):45        super().__init__(46            name=name,47            hf_subset=hf_subset,48            prompt_function="prompt_fn",  # must be defined in the file49            hf_repo="",50            metric=[""],51            hf_avail_splits=[],52            evaluation_splits=[],53            few_shots_split="",54            few_shots_select="",55            suite=["community"],56            generation_size=-1,57            stop_sequence=None,58            output_regex=None,59            frozen=False,60        )61 62 63## DEFINE YOUR PROMPT FUNCTIONS64# Define as many as you need for your different tasks65def prompt_fn(line, task_name: str = None):66    """Defines how to go from a dataset line to a doc object.67    Follow examples in src/lighteval/tasks/tasks_prompt_formatting.py, or get more info68    about what this function should do in the README.69    """70    return Doc(71        task_name=task_name,72        query="",73        choices="",74        gold_index=0,75        instruction="",76    )77 78 79## STORE YOUR EVALS80SUBSET_TASKS = [CustomSubsetTask(name=f"mytask:{subset}", hf_subset=subset) for subset in SAMPLE_SUBSETS]81_TASKS = SUBSET_TASKS + [task]82 83## MODULE LOGIC84# You should not need to touch this85# Convert to dict for lighteval86TASKS_TABLE = [task.as_dict() for task in _TASKS]87 88if __name__ == "__main__":89    print(t["name"] for t in TASKS_TABLE)90    print(len(TASKS_TABLE))91