meg/backend

1import pprint2import re3 4from huggingface_hub import snapshot_download, delete_inference_endpoint5 6from src.backend.inference_endpoint import create_endpoint7from src.backend.manage_requests import check_completed_evals, \8    get_eval_requests, set_eval_request, PENDING_STATUS, FINISHED_STATUS, \9    FAILED_STATUS, RUNNING_STATUS10from src.backend.run_toxicity_eval import compute_results11from src.backend.sort_queue import sort_models_by_priority12from src.envs import (REQUESTS_REPO, EVAL_REQUESTS_PATH_BACKEND, RESULTS_REPO,13                      EVAL_RESULTS_PATH_BACKEND, API, TOKEN)14from src.logging import setup_logger15 16logger = setup_logger(__name__)17 18pp = pprint.PrettyPrinter(width=80)19 20 21snapshot_download(repo_id=RESULTS_REPO, revision="main",22                  local_dir=EVAL_RESULTS_PATH_BACKEND, repo_type="dataset",23                  max_workers=60, token=TOKEN)24snapshot_download(repo_id=REQUESTS_REPO, revision="main",25                  local_dir=EVAL_REQUESTS_PATH_BACKEND, repo_type="dataset",26                  max_workers=60, token=TOKEN)27 28 29def run_auto_eval():30    # pull the eval dataset from the hub and parse any eval requests31    # check completed evals and set them to finished32    check_completed_evals(33        api=API,34        completed_status=FINISHED_STATUS,35        failed_status=FAILED_STATUS,36        hf_repo=REQUESTS_REPO,37        local_dir=EVAL_REQUESTS_PATH_BACKEND,38        hf_repo_results=RESULTS_REPO,39        local_dir_results=EVAL_RESULTS_PATH_BACKEND40    )41 42    # Get all eval requests that are PENDING43    eval_requests = get_eval_requests(hf_repo=REQUESTS_REPO,44                                      local_dir=EVAL_REQUESTS_PATH_BACKEND)45    # Sort the evals by priority (first submitted, first run)46    eval_requests = sort_models_by_priority(api=API, models=eval_requests)47 48    logger.info(49        f"Found {len(eval_requests)} {PENDING_STATUS} eval requests")50 51    if len(eval_requests) == 0:52        return53 54    eval_request = eval_requests[0]55    logger.info(pp.pformat(eval_request))56 57    set_eval_request(58        api=API,59        eval_request=eval_request,60        set_to_status=RUNNING_STATUS,61        hf_repo=REQUESTS_REPO,62        local_dir=EVAL_REQUESTS_PATH_BACKEND,63    )64 65    logger.info(66        f'Starting Evaluation of {eval_request.json_filepath} on Inference endpoints')67    endpoint_name = _make_endpoint_name(eval_request)68    endpoint_url = create_endpoint(endpoint_name, eval_request.model)69    logger.info("Created an endpoint url at %s" % endpoint_url)70    results = compute_results(endpoint_url, eval_request)71    logger.info("FINISHED!")72    logger.info(results)73    logger.info(f'Completed Evaluation of {eval_request.json_filepath}')74    set_eval_request(api=API,75                     eval_request=eval_request,76                     set_to_status=FINISHED_STATUS,77                     hf_repo=REQUESTS_REPO,78                     local_dir=EVAL_REQUESTS_PATH_BACKEND,79                     )80    # Delete endpoint when we're done.81    delete_inference_endpoint(endpoint_name)82 83 84def _make_endpoint_name(eval_request):85    model_repository = eval_request.model86    # Naming convention for endpoints87    endpoint_name_tmp = re.sub("[/.]", "-",88                               model_repository.lower()) + "-toxicity-eval"89    # Endpoints apparently can't have more than 32 characters.90    endpoint_name = endpoint_name_tmp[:32]91    return endpoint_name92 93 94if __name__ == "__main__":95    run_auto_eval()96