bigcode/bigcode-models-leaderboard

1# some code blocks are taken from https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard/tree/main2import json3import os4from datetime import datetime, timezone5 6import gradio as gr7import pandas as pd8from huggingface_hub import HfApi9 10from src.css_html import custom_css11from src.text_content import ABOUT_TEXT, SUBMISSION_TEXT, SUBMISSION_TEXT_212from src.utils import (13    AutoEvalColumn,14    fields,15    is_model_on_hub,16    make_clickable_names,17    plot_throughput,18    styled_error,19    styled_message,20)21 22TOKEN = os.environ.get("HF_TOKEN", None)23api = HfApi(TOKEN)24df = pd.read_csv("data/code_eval_board.csv")25 26QUEUE_REPO = "bigcode/evaluation-requests"27EVAL_REQUESTS_PATH = "eval-queue"28COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden]29TYPES = [c.type for c in fields(AutoEvalColumn) if not c.hidden]30COLS_LITE = [31    c.name for c in fields(AutoEvalColumn) if c.displayed_by_default and not c.hidden32]33TYPES_LITE = [34    c.type for c in fields(AutoEvalColumn) if c.displayed_by_default and not c.hidden35]36 37 38def add_new_eval(39    model: str,40    revision: str,41    precision: str,42    model_type: str,43):44    precision = precision45    current_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")46 47    if model_type is None or model_type == "":48        return styled_error("Please select a model type.")49 50    # check the model actually exists before adding the eval51    if revision == "":52        revision = "main"53 54    model_on_hub, error = is_model_on_hub(model, revision)55    if not model_on_hub:56        return styled_error(f'Model "{model}" {error}')57 58    print("adding new eval")59 60    eval_entry = {61        "model": model,62        "revision": revision,63        "precision": precision,64        "status": "PENDING",65        "submitted_time": current_time,66        "model_type": model_type.split(" ")[1],67    }68 69    user_name = ""70    model_path = model71    if "/" in model:72        user_name = model.split("/")[0]73        model_path = model.split("/")[1]74 75    OUT_DIR = f"{EVAL_REQUESTS_PATH}/{user_name}"76    os.makedirs(OUT_DIR, exist_ok=True)77    out_path = f"{OUT_DIR}/{model_path}_eval_request_{precision}.json"78    print(f"Saving eval request to {out_path}")79 80    with open(out_path, "w") as f:81        f.write(json.dumps(eval_entry))82 83    api.upload_file(84        path_or_fileobj=out_path,85        path_in_repo=out_path.split("eval-queue/")[1],86        repo_id=QUEUE_REPO,87        repo_type="dataset",88        commit_message=f"Add {model} to eval queue",89    )90 91    # remove the local file92    os.remove(out_path)93 94    return styled_message("Your request has been submitted to the evaluation queue!\n")95 96 97def select_columns(df, columns):98    always_here_cols = [99        AutoEvalColumn.model_type_symbol.name,100        AutoEvalColumn.model.name,101    ]102    # We use COLS to maintain sorting103    filtered_df = df[104        always_here_cols + [c for c in COLS if c in df.columns and c in columns]105    ]106    return filtered_df107 108 109def filter_items(df, leaderboard_table, query):110    if query == "all":111        return df[leaderboard_table.columns]112    else:113        query = query[0]  # take only the emoji character114    filtered_df = df[(df["T"] == query)]115    return filtered_df[leaderboard_table.columns]116 117 118def search_table(df, leaderboard_table, query):119    filtered_df = df[(df["Models"].str.contains(query, case=False))]120    return filtered_df[leaderboard_table.columns]121 122 123df = make_clickable_names(df)124 125 126demo = gr.Blocks(css=custom_css)127with demo:128    with gr.Row():129        gr.Markdown(130            """<div style="text-align: center;"><h1> ⭐ Big <span style='color: #e6b800;'>Code</span> Models <span style='color: #e6b800;'>Leaderboard</span></h1></div>\131            <br>\132            <p>Inspired from the <a href="https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard">🤗 Open LLM Leaderboard</a> and <a href="https://huggingface.co/spaces/optimum/llm-perf-leaderboard">🤗 Open LLM-Perf Leaderboard 🏋️</a>, we compare performance of base multilingual code generation models on <a href="https://huggingface.co/datasets/openai_humaneval">HumanEval</a> benchmark and <a href="https://huggingface.co/datasets/nuprl/MultiPL-E">MultiPL-E</a>. We also measure throughput and provide\133            information about the models. We only compare open pre-trained multilingual code models, that people can start from as base models for their trainings.</p>""",134            elem_classes="markdown-text",135        )136 137    with gr.Tabs(elem_classes="tab-buttons") as tabs:138        with gr.Column():139            with gr.Tabs(elem_classes="A100-tabs") as A100_tabs:140                with gr.TabItem("🔍 Evaluation table", id=0):141                    with gr.Column():142                        with gr.Accordion("➡️ See All Columns", open=False):143                            shown_columns = gr.CheckboxGroup(144                                choices=[145                                    c146                                    for c in COLS147                                    if c148                                    not in [149                                        AutoEvalColumn.dummy.name,150                                        AutoEvalColumn.model.name,151                                        AutoEvalColumn.model_type_symbol.name,152                                    ]153                                ],154                                value=[155                                    c156                                    for c in COLS_LITE157                                    if c158                                    not in [159                                        AutoEvalColumn.dummy.name,160                                        AutoEvalColumn.model.name,161                                        AutoEvalColumn.model_type_symbol.name,162                                    ]163                                ],164                                label="",165                                elem_id="column-select",166                                interactive=True,167                            )168                        # with gr.Column(min_width=780):169                        with gr.Row():170                            search_bar = gr.Textbox(171                                placeholder="🔍 Search for your model and press ENTER...",172                                show_label=False,173                                elem_id="search-bar",174                            )175                            filter_columns = gr.Radio(176                                label="⏚ Filter model types",177                                choices=["all", "🟢 base", "🔶 instruction-tuned", "🔴 external-evaluation"],178                                value="all",179                                elem_id="filter-columns",180                            )181 182                    leaderboard_df = gr.components.Dataframe(183                        value=df[184                            [185                                AutoEvalColumn.model_type_symbol.name,186                                AutoEvalColumn.model.name,187                            ]188                            + shown_columns.value189                        ],190                        headers=[191                            AutoEvalColumn.model_type_symbol.name,192                            AutoEvalColumn.model.name,193                        ]194                        + shown_columns.value,195                        datatype=TYPES,196                        elem_id="leaderboard-table",197                        interactive=False,198                    )199 200                    hidden_leaderboard_df = gr.components.Dataframe(201                        value=df,202                        headers=COLS,203                        datatype=["str" for _ in range(len(COLS))],204                        visible=False,205                    )206                    search_bar.submit(207                        search_table,208                        [hidden_leaderboard_df, leaderboard_df, search_bar],209                        leaderboard_df,210                    )211                    filter_columns.change(212                        filter_items,213                        [hidden_leaderboard_df, leaderboard_df, filter_columns],214                        leaderboard_df,215                    )216                    shown_columns.change(217                        select_columns,218                        [hidden_leaderboard_df, shown_columns],219                        leaderboard_df,220                    )221                    gr.Markdown(222                        """223                    **Notes:**224                    - Win Rate represents how often a model outperforms other models in each language, averaged across all languages.225                    - The scores of instruction-tuned models might be significantly higher on humaneval-python than other languages. We use the instruction format of HumanEval. For other languages, we use base MultiPL-E prompts.226                    - For more details check the 📝 About section.227                    - Models with a 🔴 symbol represent external evaluation results submission, this means that we didn't verify the results, you can find the author's submission under `Submission PR` field.228                    """,229                        elem_classes="markdown-text",230                    )231 232                with gr.TabItem("📊 Performance Plot", id=1):233                    with gr.Row():234                        bs_1_plot = gr.components.Plot(235                            value=plot_throughput(df, bs=1),236                            elem_id="bs1-plot",237                            show_label=False,238                        )239                        bs_50_plt = gr.components.Plot(240                            value=plot_throughput(df, bs=50),241                            elem_id="bs50-plot",242                            show_label=False,243                        )244                    gr.Markdown(245                        "**Note:** Zero throughput on the right plot refers to OOM, for more details check the 📝 About section.",246                        elem_classes="markdown-text",247                    )248                with gr.TabItem("📝 About", id=2):249                    gr.Markdown(ABOUT_TEXT, elem_classes="markdown-text")250                with gr.TabItem("Submit results 🚀", id=3):251                    gr.Markdown(SUBMISSION_TEXT)252                    gr.Markdown(253                        "## 📤  Submit your model here:", elem_classes="markdown-text"254                    )255                    with gr.Column():256                        with gr.Row():257                            model_name = gr.Textbox(label="Model name")258                            revision_name = gr.Textbox(259                                label="revision", placeholder="main"260                            )261                        with gr.Row():262                            precision = gr.Dropdown(263                                choices=[264                                    "float16",265                                    "bfloat16",266                                    "8bit",267                                    "4bit",268                                ],269                                label="Precision",270                                multiselect=False,271                                value="float16",272                                interactive=True,273                            )274                            model_type = gr.Dropdown(275                                choices=["🟢 base", "🔶 instruction-tuned"],276                                label="Model type",277                                multiselect=False,278                                value=None,279                                interactive=True,280                            )281                        submit_button = gr.Button("Submit Eval")282                        submission_result = gr.Markdown()283                        submit_button.click(284                            add_new_eval,285                            inputs=[model_name, revision_name, precision, model_type],286                            outputs=[submission_result],287                        )288                        gr.Markdown(SUBMISSION_TEXT_2)289 290 291demo.launch()292