doubao-bench/WebCoderBench

1import gradio as gr2from gradio_leaderboard import Leaderboard, ColumnFilter, SelectColumns3import pandas as pd4from apscheduler.schedulers.background import BackgroundScheduler5from huggingface_hub import snapshot_download6 7from src.about import (8    CITATION_BUTTON_LABEL,9    CITATION_BUTTON_TEXT,10    EVALUATION_QUEUE_TEXT,11    INTRODUCTION_TEXT,12    LLM_BENCHMARKS_TEXT,13    TITLE,14)15from src.display.css_html_js import custom_css16from src.display.utils import (17    BENCHMARK_COLS,18    COLS,19    EVAL_COLS,20    EVAL_TYPES,21    AutoEvalColumn,22    ModelType,23    fields,24    WeightType,25    Precision26)27from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, QUEUE_REPO, REPO_ID, RESULTS_REPO, TOKEN28from src.populate import get_evaluation_queue_df, get_leaderboard_df29from src.submission.submit import add_new_eval30 31 32def restart_space():33    API.restart_space(repo_id=REPO_ID)34 35### Space initialisation36# try:37#     print(EVAL_REQUESTS_PATH)38#     snapshot_download(39#         repo_id=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN40#     )41# except Exception:42#     restart_space()43# try:44#     print(EVAL_RESULTS_PATH)45#     snapshot_download(46#         repo_id=RESULTS_REPO, local_dir=EVAL_RESULTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN47#     )48# except Exception:49#     restart_space()50 51 52LEADERBOARD_DF = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS, BENCHMARK_COLS)53 54(55    finished_eval_queue_df,56    running_eval_queue_df,57    pending_eval_queue_df,58) = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS)59 60def init_leaderboard(dataframe):61    if dataframe.empty:62        dataframe = pd.DataFrame(columns=[c.name for c in fields(AutoEvalColumn)])63 64    return Leaderboard(65        value=dataframe,66        datatype=[c.type for c in fields(AutoEvalColumn)],67        select_columns=SelectColumns(68            default_selection=[c.name for c in fields(AutoEvalColumn) if c.displayed_by_default],69            cant_deselect=[c.name for c in fields(AutoEvalColumn) if c.never_hidden],70            label="Select Columns to Display:",71        ),72        search_columns=[AutoEvalColumn.model.name],73        hide_columns=[c.name for c in fields(AutoEvalColumn) if c.hidden],74        filter_columns=[],75        bool_checkboxgroup_label="Hide models",76        interactive=False,77    )78 79 80demo = gr.Blocks(css=custom_css)81with demo:82    gr.HTML(TITLE)83    gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")84 85    with gr.Tabs(elem_classes="tab-buttons") as tabs:86        with gr.TabItem("🏅 LLM Benchmark", elem_id="llm-benchmark-tab-table", id=0):87            leaderboard = init_leaderboard(LEADERBOARD_DF)88 89        with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=2):90            gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")91 92        with gr.TabItem("🚀 Submit", elem_id="llm-benchmark-tab-table", id=3):93            gr.Markdown("""94We welcome community submissions of new model evaluation results. Those submissions will be listed as 'External', and authors must upload their generated outputs for peer review.95 96## Evaluation97Evaluation [Setup](https://huggingface.co/docs/hub/spaces-overview) and [Usage](https://huggingface.co/docs/hub/spaces-overview). This will generate a markdown report summarizing the results.98 99## Submission100To submit your results, create a Pull Request in the [Community Tab](https://huggingface.co/spaces/doubao-bench/web-bench-leaderboard/discussions) to add them to the `src/custom-eval-results` folder in this repository:101 102* Create a new folder named with your provider and model names (e.g., `ollama_mistral-small`, using underscores to separate parts).103* Each folder stores the evaluation results of only one model.104* Add a `base_meta.json` file with the following fields:105    * **Model**: the name of your model106    * **Model Link**: the link to the model page107    * **Provider**: the name of the provider108    * **Openness**: the openness of the model109    * **Agent**: the agent used for evaluation, `Web-Agent` or your custom agent name110* Put your generated reports (e.g. `eval-20258513-102235.zip`) in your folder.111* The title of the PR should be: `[Community Submission] Model: org/model, Username: your_username`.112* **Tips**: `gen_meta.json` will be created after our review.113 114We will promptly merge and review your submission. Once the review is complete, we will publish the results on the leaderboard.115""")116 117 118 119scheduler = BackgroundScheduler()120scheduler.add_job(restart_space, "interval", seconds=1800)121scheduler.start()122demo.queue(default_concurrency_limit=40).launch(share=True)