rodrigomasini/data_only_hallucination_leaderboard

1#!/usr/bin/env python2 3from huggingface_hub import snapshot_download4 5from src.backend.manage_requests import get_eval_requests6from src.backend.sort_queue import sort_models_by_priority7from src.backend.envs import Tasks, EVAL_REQUESTS_PATH_BACKEND, EVAL_RESULTS_PATH_BACKEND8 9from src.backend.manage_requests import EvalRequest10from src.leaderboard.read_evals import EvalResult11 12from src.envs import QUEUE_REPO, RESULTS_REPO, API13 14import logging15import pprint16 17logging.getLogger("openai").setLevel(logging.WARNING)18 19logging.basicConfig(level=logging.ERROR)20pp = pprint.PrettyPrinter(width=80)21 22PENDING_STATUS = "PENDING"23RUNNING_STATUS = "RUNNING"24FINISHED_STATUS = "FINISHED"25FAILED_STATUS = "FAILED"26 27TASKS_HARNESS = [task.value for task in Tasks]28 29snapshot_download(repo_id=RESULTS_REPO, revision="main", local_dir=EVAL_RESULTS_PATH_BACKEND, repo_type="dataset", max_workers=60)30snapshot_download(repo_id=QUEUE_REPO, revision="main", local_dir=EVAL_REQUESTS_PATH_BACKEND, repo_type="dataset", max_workers=60)31 32 33def request_to_result_name(request: EvalRequest) -> str:34    org_and_model = request.model.split("/", 1)35    if len(org_and_model) == 1:36        model = org_and_model[0]37        res = f"{model}_{request.precision}"38    else:39        org = org_and_model[0]40        model = org_and_model[1]41        res = f"{org}_{model}_{request.precision}"42    return res43 44 45def process_finished_requests() -> bool:46    current_finished_status = [FINISHED_STATUS]47 48    if False:49        import os50        import dateutil51        model_result_filepaths = []52        results_path = f'{EVAL_RESULTS_PATH_BACKEND}/EleutherAI/gpt-neo-1.3B'53        requests_path = f'{EVAL_REQUESTS_PATH_BACKEND}/EleutherAI/gpt-neo-1.3B_eval_request_False_False_False.json'54 55        for root, _, files in os.walk(results_path):56            # We should only have json files in model results57            if len(files) == 0 or any([not f.endswith(".json") for f in files]):58                continue59 60            # Sort the files by date61            try:62                files.sort(key=lambda x: x.removesuffix(".json").removeprefix("results_")[:-7])63            except dateutil.parser._parser.ParserError:64                files = [files[-1]]65 66            for file in files:67                model_result_filepaths.append(os.path.join(root, file))68 69        eval_results = {}70        for model_result_filepath in model_result_filepaths:71            # Creation of result72            eval_result = EvalResult.init_from_json_file(model_result_filepath)73            eval_result.update_with_request_file(requests_path)74 75            print('XXX', eval_result)76 77            # Store results of same eval together78            eval_name = eval_result.eval_name79            if eval_name in eval_results.keys():80                eval_results[eval_name].results.update({k: v for k, v in eval_result.results.items() if v is not None})81            else:82                eval_results[eval_name] = eval_result83 84        print(eval_results)85 86        return True87 88    # Get all eval request that are FINISHED, if you want to run other evals, change this parameter89    eval_requests: list[EvalRequest] = get_eval_requests(job_status=current_finished_status, hf_repo=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH_BACKEND)90    # Sort the evals by priority (first submitted first run)91    eval_requests: list[EvalRequest] = sort_models_by_priority(api=API, models=eval_requests)92 93    # XXX94    # eval_requests = [r for r in eval_requests if 'neo-1.3B' in r.model]95 96    import random97    random.shuffle(eval_requests)98 99    from src.leaderboard.read_evals import get_raw_eval_results100    eval_results: list[EvalResult] = get_raw_eval_results(EVAL_RESULTS_PATH_BACKEND, EVAL_REQUESTS_PATH_BACKEND, True)101 102    result_name_to_request = {request_to_result_name(r): r for r in eval_requests}103    result_name_to_result = {r.eval_name: r for r in eval_results}104 105    for eval_request in eval_requests:106        result_name: str = request_to_result_name(eval_request)107 108        # Check the corresponding result109        from typing import Optional110        eval_result: Optional[EvalResult] = result_name_to_result[result_name] if result_name in result_name_to_result else None111 112        # Iterate over tasks and, if we do not have results for a task, run the relevant evaluations113        for task in TASKS_HARNESS:114            task_name = task.benchmark115 116            if eval_result is None or task_name not in eval_result.results:117                eval_request: EvalRequest = result_name_to_request[result_name]118 119                # print(eval_result)120                print(result_name, 'is incomplete -- missing task:', task_name)121 122 123if __name__ == "__main__":124    res = process_finished_requests()125