EmbodiedAgentInterface/backend

1import json2import os3import logging4from datetime import datetime5 6from lm_eval import tasks, evaluator, utils7from lm_eval.tasks import TaskManager8 9from src.envs import RESULTS_REPO, API10from src.backend.manage_requests import EvalRequest11from src.logging import setup_logger12 13from typing import Union14 15logging.getLogger("openai").setLevel(logging.WARNING)16logger = setup_logger(__name__)17 18def run_evaluation(eval_request: EvalRequest, task_names: list, num_fewshot: int, batch_size: Union[int, str], device: str, local_dir: str, results_repo: str, no_cache: bool =True, limit: int =None):19    """Runs one evaluation for the current evaluation request file, then pushes the results to the hub.20 21    Args:22        eval_request (EvalRequest): Input evaluation request file representation23        task_names (list): Tasks to launch24        num_fewshot (int): Number of few shots to use25        batch_size (int or str): Selected batch size or 'auto'26        device (str): "cpu" or "cuda:0", depending on what you assigned to the space27        local_dir (str): Where to save the results locally28        results_repo (str): To which repository to upload the results29        no_cache (bool, optional): Whether to use a cache or not30        limit (int, optional): Whether to use a number of samples only for the evaluation - only for debugging31 32    Returns:33        _type_: _description_34    """35    if limit:36        logger.info(37            "WARNING: --limit SHOULD ONLY BE USED FOR TESTING. REAL METRICS SHOULD NOT BE COMPUTED USING LIMIT."38        )39 40    task_manager = TaskManager()41    all_tasks = task_manager.all_tasks42    task_names = utils.pattern_match(task_names, all_tasks)43 44    logger.info(f"Selected Tasks: {task_names}")45 46    results = evaluator.simple_evaluate(47        model="hf",48        model_args=eval_request.get_model_args(),49        tasks=task_names,50        num_fewshot=num_fewshot,51        batch_size=batch_size,52        device=device,53        limit=limit,54        write_out=True # Whether to write out an example document and model input, for checking task integrity55    )56 57    results["config"]["model_dtype"] = eval_request.precision58    results["config"]["model_name"] = eval_request.model59    results["config"]["model_sha"] = eval_request.revision60 61    dumped = json.dumps(results, indent=2)62    logger.info(dumped)63 64    output_path = os.path.join(local_dir, *eval_request.model.split("/"), f"results_{datetime.now()}.json")65    os.makedirs(os.path.dirname(output_path), exist_ok=True)66    with open(output_path, "w") as f:67        f.write(dumped)68 69    logger.info(evaluator.make_table(results))70 71    API.upload_file(72        path_or_fileobj=output_path,73        path_in_repo=f"{eval_request.model}/results_{datetime.now()}.json",74        repo_id=results_repo,75        repo_type="dataset",76    )77 78    return results79