EmbodiedAgentInterface/backend

1import json2import os3import logging4from datetime import datetime5 6from lighteval.main_accelerate import main, EnvConfig, create_model_config, load_model7 8from src.envs import RESULTS_REPO, CACHE_PATH, TOKEN9from src.backend.manage_requests import EvalRequest10from src.logging import setup_logger11 12logging.getLogger("openai").setLevel(logging.WARNING)13logger = setup_logger(__name__)14 15def run_evaluation(eval_request: EvalRequest, task_names: str, batch_size: int, local_dir: str, accelerator: str, region: str, vendor: str, instance_size: str, instance_type: str, limit=None):16    if limit:17        logger.info("WARNING: --limit SHOULD ONLY BE USED FOR TESTING. REAL METRICS SHOULD NOT BE COMPUTED USING LIMIT.")18 19    args = {20            "endpoint_model_name":f"{eval_request.model}_{eval_request.precision}".lower(),21            "accelerator": accelerator,22            "vendor": vendor,23            "region": region,24            "instance_size": instance_size,25            "instance_type": instance_type,26            "max_samples": limit,27            "job_id": str(datetime.now()),28            "push_results_to_hub": True,29            "save_details": True,30            "push_details_to_hub": True,31            "public_run": False,32            "cache_dir": CACHE_PATH,33            "results_org": RESULTS_REPO,34            "output_dir": local_dir,35            "override_batch_size": batch_size,36            "custom_tasks": "custom_tasks.py",37            "tasks": task_names38    }39 40    try:41        results = main(args)42 43        results["config"]["model_dtype"] = eval_request.precision44        results["config"]["model_name"] = eval_request.model45        results["config"]["model_sha"] = eval_request.revision46 47        dumped = json.dumps(results, indent=2)48        logger.info(dumped)49    except Exception: # if eval failed, we force a cleanup50        env_config = EnvConfig(token=TOKEN, cache_dir=args.cache_dir)51 52        model_config = create_model_config(args=args, accelerator=accelerator)53        model, _ = load_model(config=model_config, env_config=env_config)54        model.cleanup()55 56 57    return results58