open-llm-leaderboard/open_llm_leaderboard

1import json2import os3from datetime import datetime, timezone4 5 6import gradio as gr7import numpy as np8import pandas as pd9from apscheduler.schedulers.background import BackgroundScheduler10from huggingface_hub import HfApi11from transformers import AutoConfig12 13from src.auto_leaderboard.get_model_metadata import apply_metadata14from src.assets.text_content import *15from src.auto_leaderboard.load_results import get_eval_results_dicts, make_clickable_model16from src.assets.hardcoded_evals import gpt4_values, gpt35_values, baseline17from src.assets.css_html_js import custom_css, get_window_url_params18from src.utils_display import AutoEvalColumn, EvalQueueColumn, fields, styled_error, styled_warning, styled_message19from src.init import get_all_requested_models, load_all_info_from_hub20 21# clone / pull the lmeh eval data22H4_TOKEN = os.environ.get("H4_TOKEN", None)23 24QUEUE_REPO = "open-llm-leaderboard/requests"25RESULTS_REPO = "open-llm-leaderboard/results"26 27PRIVATE_QUEUE_REPO = "open-llm-leaderboard/private-requests"28PRIVATE_RESULTS_REPO = "open-llm-leaderboard/private-results"29 30IS_PUBLIC = bool(os.environ.get("IS_PUBLIC", True))31ADD_PLOTS = False32 33EVAL_REQUESTS_PATH = "eval-queue"34EVAL_RESULTS_PATH = "eval-results"35 36EVAL_REQUESTS_PATH_PRIVATE = "eval-queue-private"37EVAL_RESULTS_PATH_PRIVATE = "eval-results-private"38 39api = HfApi()40 41def restart_space():42    api.restart_space(43        repo_id="HuggingFaceH4/open_llm_leaderboard", token=H4_TOKEN44    )45 46eval_queue, requested_models, eval_results = load_all_info_from_hub(QUEUE_REPO, RESULTS_REPO, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH)47 48if not IS_PUBLIC:49    eval_queue_private, requested_models_private, eval_results_private = load_all_info_from_hub(PRIVATE_QUEUE_REPO, PRIVATE_RESULTS_REPO, EVAL_REQUESTS_PATH_PRIVATE, EVAL_RESULTS_PATH_PRIVATE)50else:51    eval_queue_private, eval_results_private = None, None52 53COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden]54TYPES = [c.type for c in fields(AutoEvalColumn) if not c.hidden]55COLS_LITE = [c.name for c in fields(AutoEvalColumn) if c.displayed_by_default and not c.hidden]56TYPES_LITE = [c.type for c in fields(AutoEvalColumn) if c.displayed_by_default and not c.hidden]57 58if not IS_PUBLIC:59    COLS.insert(2, AutoEvalColumn.is_8bit.name)60    TYPES.insert(2, AutoEvalColumn.is_8bit.type)61 62EVAL_COLS = [c.name for c in fields(EvalQueueColumn)]63EVAL_TYPES = [c.type for c in fields(EvalQueueColumn)]64 65BENCHMARK_COLS = [c.name for c in [AutoEvalColumn.arc, AutoEvalColumn.hellaswag, AutoEvalColumn.mmlu, AutoEvalColumn.truthfulqa]]66 67 68def has_no_nan_values(df, columns):69    return df[columns].notna().all(axis=1)70 71 72def has_nan_values(df, columns):73    return df[columns].isna().any(axis=1)74 75 76def get_leaderboard_df():77    if eval_results:78        print("Pulling evaluation results for the leaderboard.")79        eval_results.git_pull()80    if eval_results_private:81        print("Pulling evaluation results for the leaderboard.")82        eval_results_private.git_pull()83 84    all_data = get_eval_results_dicts(IS_PUBLIC)85 86    if not IS_PUBLIC:87        all_data.append(gpt4_values)88        all_data.append(gpt35_values)89 90    all_data.append(baseline)91    apply_metadata(all_data)  # Populate model type based on known hardcoded values in `metadata.py`92 93    df = pd.DataFrame.from_records(all_data)94    df = df.sort_values(by=[AutoEvalColumn.average.name], ascending=False)95    df = df[COLS]96 97    # filter out if any of the benchmarks have not been produced98    df = df[has_no_nan_values(df, BENCHMARK_COLS)]99    return df100 101 102def get_evaluation_queue_df():103    # todo @saylortwift: replace the repo by the one you created for the eval queue104    if eval_queue:105        print("Pulling changes for the evaluation queue.")106        eval_queue.git_pull()107    if eval_queue_private:108        print("Pulling changes for the evaluation queue.")109        eval_queue_private.git_pull()110 111    entries = [112        entry113        for entry in os.listdir(EVAL_REQUESTS_PATH)114        if not entry.startswith(".")115    ]116    all_evals = []117 118    for entry in entries:119        if ".json" in entry:120            file_path = os.path.join(EVAL_REQUESTS_PATH, entry)121            with open(file_path) as fp:122                data = json.load(fp)123 124            data["# params"] = "unknown"125            data["model"] = make_clickable_model(data["model"])126            data["revision"] = data.get("revision", "main")127 128            all_evals.append(data)129        elif ".md" not in entry:130            # this is a folder131            sub_entries = [132                e133                for e in os.listdir(f"{EVAL_REQUESTS_PATH}/{entry}")134                if not e.startswith(".")135            ]136            for sub_entry in sub_entries:137                file_path = os.path.join(EVAL_REQUESTS_PATH, entry, sub_entry)138                with open(file_path) as fp:139                    data = json.load(fp)140 141                # data["# params"] = get_n_params(data["model"])142                data["model"] = make_clickable_model(data["model"])143                all_evals.append(data)144 145    pending_list = [e for e in all_evals if e["status"] == "PENDING"]146    running_list = [e for e in all_evals if e["status"] == "RUNNING"]147    finished_list = [e for e in all_evals if e["status"].startswith("FINISHED")]148    df_pending = pd.DataFrame.from_records(pending_list, columns=EVAL_COLS)149    df_running = pd.DataFrame.from_records(running_list, columns=EVAL_COLS)150    df_finished = pd.DataFrame.from_records(finished_list, columns=EVAL_COLS)151    return df_finished[EVAL_COLS], df_running[EVAL_COLS], df_pending[EVAL_COLS]152 153 154 155original_df = get_leaderboard_df()156leaderboard_df = original_df.copy()157(158    finished_eval_queue_df,159    running_eval_queue_df,160    pending_eval_queue_df,161) = get_evaluation_queue_df()162 163def is_model_on_hub(model_name, revision) -> bool:164    try:165        AutoConfig.from_pretrained(model_name, revision=revision)166        return True, None167    168    except ValueError as e:169        return False, "needs to be launched with `trust_remote_code=True`. For safety reason, we do not allow these models to be automatically submitted to the leaderboard."170 171    except Exception as e:172        print(f"Could not get the model config from the hub.: {e}")173        return False, "was not found on hub!"174 175 176def add_new_eval(177    model: str,178    base_model: str,179    revision: str,180    is_8_bit_eval: bool,181    private: bool,182    is_delta_weight: bool,183):184    current_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")185 186    # check the model actually exists before adding the eval187    if revision == "":188        revision = "main"189 190    if is_delta_weight: 191        base_model_on_hub, error = is_model_on_hub(base_model, revision)192        if not base_model_on_hub:193            return styled_error(f'Base model "{base_model}" {error}')194 195    model_on_hub, error = is_model_on_hub(model, revision)196    if not model_on_hub:197        return styled_error(f'Model "{model}" {error}')198 199    print("adding new eval")200 201    eval_entry = {202        "model": model,203        "base_model": base_model,204        "revision": revision,205        "private": private,206        "8bit_eval": is_8_bit_eval,207        "is_delta_weight": is_delta_weight,208        "status": "PENDING",209        "submitted_time": current_time,210    }211 212    user_name = ""213    model_path = model214    if "/" in model:215        user_name = model.split("/")[0]216        model_path = model.split("/")[1]217 218    OUT_DIR = f"{EVAL_REQUESTS_PATH}/{user_name}"219    os.makedirs(OUT_DIR, exist_ok=True)220    out_path = f"{OUT_DIR}/{model_path}_eval_request_{private}_{is_8_bit_eval}_{is_delta_weight}.json"221 222    # Check for duplicate submission223    if out_path.split("eval-queue/")[1].lower() in requested_models:224        return styled_warning("This model has been already submitted.")225 226    with open(out_path, "w") as f:227        f.write(json.dumps(eval_entry))228 229    api.upload_file(230        path_or_fileobj=out_path,231        path_in_repo=out_path.split("eval-queue/")[1],232        repo_id=QUEUE_REPO,233        token=H4_TOKEN,234        repo_type="dataset",235        commit_message=f"Add {model} to eval queue",236    )237 238    # remove the local file239    os.remove(out_path)240 241    return styled_message("Your request has been submitted to the evaluation queue!\nPlease wait for up to an hour for the model to show in the PENDING list.")242 243 244def refresh():245    leaderboard_df = get_leaderboard_df()246    (247        finished_eval_queue_df,248        running_eval_queue_df,249        pending_eval_queue_df,250    ) = get_evaluation_queue_df()251    return (252        leaderboard_df,253        finished_eval_queue_df,254        running_eval_queue_df,255        pending_eval_queue_df,256    )257 258 259def search_table(df, query):260    filtered_df = df[df[AutoEvalColumn.dummy.name].str.contains(query, case=False)]261    return filtered_df262 263 264def change_tab(query_param):265    query_param = query_param.replace("'", '"')266    query_param = json.loads(query_param)267 268    if (269        isinstance(query_param, dict)270        and "tab" in query_param271        and query_param["tab"] == "evaluation"272    ):273        return gr.Tabs.update(selected=1)274    else:275        return gr.Tabs.update(selected=0)276 277 278demo = gr.Blocks(css=custom_css)279with demo:280    gr.HTML(TITLE)281    gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")282    with gr.Row():283        with gr.Box(elem_id="search-bar-table-box"):284            search_bar = gr.Textbox(285                placeholder="🔍 Search your model and press ENTER...",286                show_label=False,287                elem_id="search-bar",288            )289 290    with gr.Tabs(elem_classes="tab-buttons") as tabs:291        with gr.TabItem("🏅 LLM Benchmark (lite)", elem_id="llm-benchmark-tab-table", id=0):292            leaderboard_table_lite = gr.components.Dataframe(293                value=leaderboard_df[COLS_LITE],294                headers=COLS_LITE,295                datatype=TYPES_LITE,296                max_rows=None,297                elem_id="leaderboard-table-lite",298            )299            # Dummy leaderboard for handling the case when the user uses backspace key300            hidden_leaderboard_table_for_search_lite = gr.components.Dataframe(301                value=original_df[COLS_LITE],302                headers=COLS_LITE,303                datatype=TYPES_LITE,304                max_rows=None,305                visible=False,306            )307            search_bar.submit(308                search_table,309                [hidden_leaderboard_table_for_search_lite, search_bar],310                leaderboard_table_lite,311            )312 313        with gr.TabItem("📊 Extended view", elem_id="llm-benchmark-tab-table", id=1):314            leaderboard_table = gr.components.Dataframe(315                value=leaderboard_df,316                headers=COLS,317                datatype=TYPES,318                max_rows=None,319                elem_id="leaderboard-table",320            )321 322            # Dummy leaderboard for handling the case when the user uses backspace key323            hidden_leaderboard_table_for_search = gr.components.Dataframe(324                value=original_df,325                headers=COLS,326                datatype=TYPES,327                max_rows=None,328                visible=False,329            )330            search_bar.submit(331                search_table,332                [hidden_leaderboard_table_for_search, search_bar],333                leaderboard_table,334            )335        with gr.TabItem("About", elem_id="llm-benchmark-tab-table", id=2):336            gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")337 338    with gr.Column():339        with gr.Row():340            gr.Markdown(EVALUATION_QUEUE_TEXT, elem_classes="markdown-text")341 342        with gr.Column():343            with gr.Accordion("✅ Finished Evaluations", open=False):344                with gr.Row():345                    finished_eval_table = gr.components.Dataframe(346                        value=finished_eval_queue_df,347                        headers=EVAL_COLS,348                        datatype=EVAL_TYPES,349                        max_rows=5,350                    )351            with gr.Accordion("🔄 Running Evaluation Queue", open=False):352                with gr.Row():353                    running_eval_table = gr.components.Dataframe(354                        value=running_eval_queue_df,355                        headers=EVAL_COLS,356                        datatype=EVAL_TYPES,357                        max_rows=5,358                    )359 360            with gr.Accordion("⏳ Pending Evaluation Queue", open=False):361                with gr.Row():362                    pending_eval_table = gr.components.Dataframe(363                        value=pending_eval_queue_df,364                        headers=EVAL_COLS,365                        datatype=EVAL_TYPES,366                        max_rows=5,367                    )368 369        with gr.Row():370            refresh_button = gr.Button("Refresh")371            refresh_button.click(372                refresh,373                inputs=[],374                outputs=[375                    leaderboard_table,376                    finished_eval_table,377                    running_eval_table,378                    pending_eval_table,379                ],380            )381        with gr.Accordion("Submit a new model for evaluation"):382            with gr.Row():383                with gr.Column():384                    model_name_textbox = gr.Textbox(label="Model name")385                    revision_name_textbox = gr.Textbox(386                        label="revision", placeholder="main"387                    )388 389                with gr.Column():390                    is_8bit_toggle = gr.Checkbox(391                        False, label="8 bit eval", visible=not IS_PUBLIC392                    )393                    private = gr.Checkbox(394                        False, label="Private", visible=not IS_PUBLIC395                    )396                    is_delta_weight = gr.Checkbox(False, label="Delta weights")397                    base_model_name_textbox = gr.Textbox(398                        label="base model (for delta)"399                    )400 401            submit_button = gr.Button("Submit Eval")402            submission_result = gr.Markdown()403            submit_button.click(404                add_new_eval,405                [406                    model_name_textbox,407                    base_model_name_textbox,408                    revision_name_textbox,409                    is_8bit_toggle,410                    private,411                    is_delta_weight,412                ],413                submission_result,414            )415 416    with gr.Row():417        with gr.Accordion("📙 Citation", open=False):418            citation_button = gr.Textbox(419                value=CITATION_BUTTON_TEXT,420                label=CITATION_BUTTON_LABEL,421                elem_id="citation-button",422            ).style(show_copy_button=True)423 424    dummy = gr.Textbox(visible=False)425    demo.load(426        change_tab,427        dummy,428        tabs,429        _js=get_window_url_params,430    )431 432scheduler = BackgroundScheduler()433scheduler.add_job(restart_space, "interval", seconds=3600)434scheduler.start()435demo.queue(concurrency_count=40).launch()436