EmbodiedAgentInterface/backend

1import json2import argparse3import logging4from datetime import datetime5 6from lighteval.main_accelerate import main, EnvConfig, create_model_config, load_model7 8from src.envs import RESULTS_REPO, CACHE_PATH, TOKEN9from src.backend.manage_requests import EvalRequest10from src.logging import setup_logger11 12logging.getLogger("openai").setLevel(logging.WARNING)13logger = setup_logger(__name__)14 15def run_evaluation(eval_request: EvalRequest, task_names: str, batch_size: int, local_dir: str, accelerator: str, region: str, vendor: str, instance_size: str, instance_type: str, limit=None):16    """Runs one evaluation for the current evaluation request file using lighteval, then pushes the results to the hub.17 18    Args:19        eval_request (EvalRequest): Input evaluation request file representation20        task_names (list): Tasks to launch21        batch_size (int): Selected batch size22        accelerator (str): Inference endpoint parameter for running the evaluation23        region (str):  Inference endpoint parameter for running the evaluation24        vendor (str):  Inference endpoint parameter for running the evaluation25        instance_size (str):  Inference endpoint parameter for running the evaluation26        instance_type (str):  Inference endpoint parameter for running the evaluation27        local_dir (str): Where to save the results locally28        no_cache (bool, optional): Whether to use a cache or not.29        limit (int, optional): Whether to use a number of samples only for the evaluation - only for debugging30    """    31 32    if limit:33        logger.info("WARNING: --limit SHOULD ONLY BE USED FOR TESTING. REAL METRICS SHOULD NOT BE COMPUTED USING LIMIT.")34 35    args_dict = {36            # Endpoint parameters37            "endpoint_model_name":eval_request.model,38            "accelerator": accelerator,39            "vendor": vendor,40            "region": region,41            "instance_size": instance_size,42            "instance_type": instance_type,43            "reuse_existing": False,44            "model_dtype": eval_request.precision,45            "revision": eval_request.revision,46            # Save parameters47            "push_results_to_hub": True,48            "save_details": True,49            "push_details_to_hub": True,50            "public_run": False,51            "cache_dir": CACHE_PATH,52            "results_org": RESULTS_REPO,53            "output_dir": local_dir,54            "job_id": str(datetime.now()),55            # Experiment parameters56            "override_batch_size": batch_size,57            "custom_tasks": "custom_tasks.py",58            "tasks": task_names,59            "max_samples": limit,60            "use_chat_template": False,61            "system_prompt": None,62            # Parameters which would be set to things by the kwargs if actually using argparse63            "inference_server_address": None,64            "model_args": None,65            "num_fewshot_seeds": None,66            "delta_weights": False,67            "adapter_weights": False68    }69    args = argparse.Namespace(**args_dict)70 71    try:72        results = main(args)73 74        results["config"]["model_dtype"] = eval_request.precision75        results["config"]["model_name"] = eval_request.model76        results["config"]["model_sha"] = eval_request.revision77 78        dumped = json.dumps(results, indent=2)79        logger.info(dumped)80    except Exception as e: # if eval failed, we force a cleanup81        env_config = EnvConfig(token=TOKEN, cache_dir=args.cache_dir)82 83        model_config = create_model_config(args=args, accelerator=accelerator)84        model, _ = load_model(config=model_config, env_config=env_config)85        model.cleanup()86 87 88    return results89