MLRS/MELABench
0
1import os2import traceback3from concurrent.futures import ThreadPoolExecutor4from threading import Lock5from pathlib import Path6 7import gradio as gr8from gradio_leaderboard import Leaderboard, ColumnFilter, SelectColumns9import pandas as pd10from apscheduler.schedulers.background import BackgroundScheduler11from huggingface_hub import hf_hub_download12from huggingface_hub.utils import disable_progress_bars13 14from src.about import (15 CITATION_BUTTON_LABEL,16 CITATION_BUTTON_TEXT,17 EVALUATION_QUEUE_TEXT,18 INTRODUCTION_TEXT,19 LLM_BENCHMARKS_TEXT,20 TITLE,21)22from src.display.css_html_js import custom_css23from src.display.utils import (24 BENCHMARK_COLS,25 COLS,26 EVAL_COLS,27 EVAL_TYPES,28 AutoEvalColumn,29 ModelTraining,30 fields,31 MalteseTraining32)33from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, QUEUE_REPO, REPO_ID, RESULTS_REPO, TOKEN34from src.populate import get_evaluation_queue_df, get_leaderboard_df35from src.submission.submit import add_new_eval, read_configuration36 37 38def restart_space():39 API.restart_space(repo_id=REPO_ID)40 41 42EMPTY_LEADERBOARD_DF = pd.DataFrame(columns=COLS)43EMPTY_QUEUE_DF = pd.DataFrame(columns=EVAL_COLS)44leaderboard_lock = Lock()45leaderboard_data = None46leaderboard_status = "⌛ Loading leaderboard data from the MLRS datasets…"47 48 49def snapshot_dataset(repo_id: str, local_dir: str):50 """Synchronize a private dataset without snapshot_download's empty-list bug."""51 files = API.list_repo_files(repo_id, repo_type="dataset", token=TOKEN)52 if not files:53 raise RuntimeError(f"No files were returned for {repo_id}.")54 55 Path(local_dir).mkdir(parents=True, exist_ok=True)56 57 def download_file(filename: str):58 return hf_hub_download(59 repo_id=repo_id,60 filename=filename,61 repo_type="dataset",62 local_dir=local_dir,63 token=TOKEN,64 )65 66 with ThreadPoolExecutor(max_workers=16) as executor:67 list(executor.map(download_file, files))68 return local_dir69 70 71def refresh_leaderboard_data():72 """Fetch and prepare leaderboard data after the web server is available."""73 global leaderboard_data, leaderboard_status74 75 try:76 snapshot_dataset(QUEUE_REPO, EVAL_REQUESTS_PATH)77 snapshot_dataset(RESULTS_REPO, EVAL_RESULTS_PATH)78 79 data = (80 get_leaderboard_df(81 os.path.join(EVAL_RESULTS_PATH, "zero-shot"),82 EVAL_REQUESTS_PATH,83 COLS,84 BENCHMARK_COLS,85 ),86 get_leaderboard_df(87 os.path.join(EVAL_RESULTS_PATH, "few-shot"),88 EVAL_REQUESTS_PATH,89 COLS,90 BENCHMARK_COLS,91 ),92 *get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS),93 )94 except Exception:95 message = "⚠️ Leaderboard data could not be loaded. Retrying automatically."96 print(traceback.format_exc())97 with leaderboard_lock:98 leaderboard_status = message99 return100 101 with leaderboard_lock:102 leaderboard_data = data103 leaderboard_status = "✅ Leaderboard data loaded."104 105 106def get_leaderboard_data():107 """Return the latest shared data without delaying a connected client."""108 with leaderboard_lock:109 if leaderboard_data is None:110 return (111 EMPTY_LEADERBOARD_DF,112 EMPTY_LEADERBOARD_DF,113 EMPTY_QUEUE_DF,114 EMPTY_QUEUE_DF,115 EMPTY_QUEUE_DF,116 leaderboard_status,117 )118 return (*leaderboard_data, leaderboard_status)119 120 121### Space initialisation122disable_progress_bars()123 124def init_leaderboard(dataframe, fewshot=True):125 return Leaderboard(126 value=dataframe,127 datatype=[c.type for c in fields(AutoEvalColumn)],128 select_columns=SelectColumns(129 default_selection=[c.name for c in fields(AutoEvalColumn) if c.displayed_by_default and not (fewshot and c.hidden_in_fewshot)],130 cant_deselect=[c.name for c in fields(AutoEvalColumn) if c.never_hidden],131 label="Select Columns to Display:",132 ),133 search_columns=[AutoEvalColumn.model.name],134 hide_columns=[c.name for c in fields(AutoEvalColumn) if c.hidden],135 filter_columns=[136 ColumnFilter(AutoEvalColumn.model_training.name, type="checkboxgroup", label="Model types"),137 ColumnFilter(AutoEvalColumn.maltese_training.name, type="checkboxgroup", label="Maltese training"),138 ColumnFilter(139 AutoEvalColumn.language_count.name,140 type="slider",141 min=1,142 max=1000,143 label="Number of languages during training",144 ),145 ColumnFilter(146 AutoEvalColumn.params.name,147 type="slider",148 min=0.01,149 max=150,150 label="Select the number of parameters (B)",151 ),152 ColumnFilter(AutoEvalColumn.prompt_version.name, type="checkboxgroup", label="Prompt Version"),153 ColumnFilter(AutoEvalColumn.n_shot.name, type="slider", min=0, max=100, label="Number of Shots"),154 ColumnFilter(155 AutoEvalColumn.still_on_hub.name, type="boolean", label="Deleted/incomplete", default=True156 ),157 ],158 bool_checkboxgroup_label="Hide models",159 interactive=False,160 )161 162 163demo = gr.Blocks(css=custom_css)164with demo:165 gr.HTML(TITLE)166 gr.HTML(INTRODUCTION_TEXT, elem_classes="markdown-text")167 leaderboard_loading_status = gr.Markdown(leaderboard_status)168 leaderboard_refresh_timer = gr.Timer(value=5)169 170 with gr.Tabs(elem_classes="tab-buttons") as tabs:171 with gr.TabItem("🏅 LLM Benchmark", elem_id="llm-benchmark-tab-table", id=0):172 with gr.TabItem("Zero-Shot", elem_id="zero-shot"):173 leaderboard = init_leaderboard(EMPTY_LEADERBOARD_DF, fewshot=False)174 with gr.TabItem("Few-Shot", elem_id="few-shot"):175 leaderboard_few_shot = init_leaderboard(EMPTY_LEADERBOARD_DF, fewshot=True)176 177 with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=2):178 gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")179 180 with gr.TabItem("🚀 Submit here! ", elem_id="llm-benchmark-tab-table", id=3):181 with gr.Column():182 with gr.Row():183 gr.Markdown(EVALUATION_QUEUE_TEXT, elem_classes="markdown-text")184 185 with gr.Column():186 with gr.Accordion(187 "✅ Finished Evaluations",188 open=False,189 ):190 with gr.Row():191 finished_eval_table = gr.components.Dataframe(192 value=EMPTY_QUEUE_DF,193 headers=EVAL_COLS,194 datatype=EVAL_TYPES,195 row_count=5,196 )197 with gr.Accordion(198 "🔄 Running Evaluation Queue",199 open=False,200 ):201 with gr.Row():202 running_eval_table = gr.components.Dataframe(203 value=EMPTY_QUEUE_DF,204 headers=EVAL_COLS,205 datatype=EVAL_TYPES,206 row_count=5,207 )208 209 with gr.Accordion(210 "⏳ Pending Evaluation Queue",211 open=False,212 ):213 with gr.Row():214 pending_eval_table = gr.components.Dataframe(215 value=EMPTY_QUEUE_DF,216 headers=EVAL_COLS,217 datatype=EVAL_TYPES,218 row_count=5,219 )220 with gr.Row():221 gr.Markdown("# ✉️✨ Submit your model here!", elem_classes="markdown-text")222 223 with gr.Row():224 files = gr.File(225 label="Files (Configuration File & Prediction Outputs)",226 file_count="directory",227 type="filepath",228 )229 230 with gr.Row(equal_height=True):231 with gr.Column():232 model_name = gr.Textbox(233 label="Model name",234 info="Read automatically from the results file.",235 interactive=False,236 )237 version = gr.Textbox(238 label="Prompt Version",239 info="Read automatically from the results file.",240 interactive=False,241 )242 n_shots = gr.Number(243 label="Number of Shots",244 info="Read automatically from the results file.",245 interactive=False,246 )247 248 with gr.Column():249 model_training = gr.Dropdown(250 choices=[t.to_str(": ") for t in ModelTraining if t != ModelTraining.NK],251 label="Model Training",252 info="How to model is trained.",253 multiselect=False,254 value=None,255 interactive=True,256 )257 maltese_training = gr.Dropdown(258 choices=[t.to_str(": ") for t in MalteseTraining if t != ModelTraining.NK],259 label="Maltese Training",260 info="The last stage of training in which Maltese was included.",261 multiselect=False,262 value=None,263 interactive=True,264 )265 language_count = gr.Number(266 label="Number of languages",267 info="Include languages for all training stages. Set to 0 if unknown.",268 minimum=0,269 interactive=True,270 )271 272 submit_button = gr.Button("Submit Eval")273 submission_result = gr.Markdown()274 275 configuration = gr.State()276 file_paths = gr.State()277 files.change(read_configuration,278 files,279 [configuration, file_paths, model_name, version, n_shots, submission_result])280 281 submit_button.click(282 add_new_eval,283 [284 model_training,285 maltese_training,286 language_count,287 configuration,288 file_paths289 ],290 submission_result,291 )292 293 with gr.Row():294 with gr.Accordion("📙 Citation", open=False):295 citation_button = gr.Textbox(296 value=CITATION_BUTTON_TEXT,297 label=CITATION_BUTTON_LABEL,298 lines=20,299 elem_id="citation-button",300 show_copy_button=True,301 )302 303 refresh_outputs = [304 leaderboard,305 leaderboard_few_shot,306 finished_eval_table,307 running_eval_table,308 pending_eval_table,309 leaderboard_loading_status,310 ]311 demo.load(get_leaderboard_data, outputs=refresh_outputs)312 leaderboard_refresh_timer.tick(get_leaderboard_data, outputs=refresh_outputs)313 314scheduler = BackgroundScheduler()315scheduler.add_job(restart_space, "interval", seconds=1800)316scheduler.start()317demo.queue(default_concurrency_limit=40).launch(prevent_thread_lock=True)318refresh_leaderboard_data()319demo.block_thread()320