open-llm-leaderboard/open_llm_leaderboard
14k
1import json2import os3from datetime import datetime, timezone4 5 6import gradio as gr7import numpy as np8import pandas as pd9from apscheduler.schedulers.background import BackgroundScheduler10from huggingface_hub import HfApi11from transformers import AutoConfig12 13from src.auto_leaderboard.get_model_metadata import apply_metadata14from src.assets.text_content import *15from src.auto_leaderboard.load_results import get_eval_results_dicts, make_clickable_model16from src.assets.hardcoded_evals import gpt4_values, gpt35_values, baseline17from src.assets.css_html_js import custom_css, get_window_url_params18from src.utils_display import AutoEvalColumn, EvalQueueColumn, fields, styled_error, styled_warning, styled_message19from src.init import get_all_requested_models, load_all_info_from_hub20 21# clone / pull the lmeh eval data22H4_TOKEN = os.environ.get("H4_TOKEN", None)23 24QUEUE_REPO = "open-llm-leaderboard/requests"25RESULTS_REPO = "open-llm-leaderboard/results"26 27PRIVATE_QUEUE_REPO = "open-llm-leaderboard/private-requests"28PRIVATE_RESULTS_REPO = "open-llm-leaderboard/private-results"29 30IS_PUBLIC = bool(os.environ.get("IS_PUBLIC", True))31 32EVAL_REQUESTS_PATH = "eval-queue"33EVAL_RESULTS_PATH = "eval-results"34 35EVAL_REQUESTS_PATH_PRIVATE = "eval-queue-private"36EVAL_RESULTS_PATH_PRIVATE = "eval-results-private"37 38api = HfApi()39 40def restart_space():41 api.restart_space(42 repo_id="HuggingFaceH4/open_llm_leaderboard", token=H4_TOKEN43 )44 45eval_queue, requested_models, eval_results = load_all_info_from_hub(QUEUE_REPO, RESULTS_REPO, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH)46 47if not IS_PUBLIC:48 eval_queue_private, requested_models_private, eval_results_private = load_all_info_from_hub(PRIVATE_QUEUE_REPO, PRIVATE_RESULTS_REPO, EVAL_REQUESTS_PATH_PRIVATE, EVAL_RESULTS_PATH_PRIVATE)49else:50 eval_queue_private, eval_results_private = None, None51 52COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden]53TYPES = [c.type for c in fields(AutoEvalColumn) if not c.hidden]54COLS_LITE = [c.name for c in fields(AutoEvalColumn) if c.displayed_by_default and not c.hidden]55TYPES_LITE = [c.type for c in fields(AutoEvalColumn) if c.displayed_by_default and not c.hidden]56 57if not IS_PUBLIC:58 COLS.insert(2, AutoEvalColumn.precision.name)59 TYPES.insert(2, AutoEvalColumn.precision.type)60 61EVAL_COLS = [c.name for c in fields(EvalQueueColumn)]62EVAL_TYPES = [c.type for c in fields(EvalQueueColumn)]63 64BENCHMARK_COLS = [c.name for c in [AutoEvalColumn.arc, AutoEvalColumn.hellaswag, AutoEvalColumn.mmlu, AutoEvalColumn.truthfulqa]]65 66 67def has_no_nan_values(df, columns):68 return df[columns].notna().all(axis=1)69 70 71def has_nan_values(df, columns):72 return df[columns].isna().any(axis=1)73 74 75def get_leaderboard_df():76 if eval_results:77 print("Pulling evaluation results for the leaderboard.")78 eval_results.git_pull()79 if eval_results_private:80 print("Pulling evaluation results for the leaderboard.")81 eval_results_private.git_pull()82 83 all_data = get_eval_results_dicts(IS_PUBLIC)84 85 if not IS_PUBLIC:86 all_data.append(gpt4_values)87 all_data.append(gpt35_values)88 89 all_data.append(baseline)90 apply_metadata(all_data) # Populate model type based on known hardcoded values in `metadata.py`91 92 df = pd.DataFrame.from_records(all_data)93 df = df.sort_values(by=[AutoEvalColumn.average.name], ascending=False)94 df = df[COLS]95 96 # filter out if any of the benchmarks have not been produced97 df = df[has_no_nan_values(df, BENCHMARK_COLS)]98 return df99 100 101def get_evaluation_queue_df():102 if eval_queue:103 print("Pulling changes for the evaluation queue.")104 eval_queue.git_pull()105 if eval_queue_private:106 print("Pulling changes for the evaluation queue.")107 eval_queue_private.git_pull()108 109 entries = [110 entry111 for entry in os.listdir(EVAL_REQUESTS_PATH)112 if not entry.startswith(".")113 ]114 all_evals = []115 116 for entry in entries:117 if ".json" in entry:118 file_path = os.path.join(EVAL_REQUESTS_PATH, entry)119 with open(file_path) as fp:120 data = json.load(fp)121 122 data["# params"] = "unknown"123 data["model"] = make_clickable_model(data["model"])124 data["revision"] = data.get("revision", "main")125 126 all_evals.append(data)127 elif ".md" not in entry:128 # this is a folder129 sub_entries = [130 e131 for e in os.listdir(f"{EVAL_REQUESTS_PATH}/{entry}")132 if not e.startswith(".")133 ]134 for sub_entry in sub_entries:135 file_path = os.path.join(EVAL_REQUESTS_PATH, entry, sub_entry)136 with open(file_path) as fp:137 data = json.load(fp)138 139 # data["# params"] = get_n_params(data["model"])140 data["model"] = make_clickable_model(data["model"])141 all_evals.append(data)142 143 pending_list = [e for e in all_evals if e["status"] in ["PENDING", "RERUN"]]144 running_list = [e for e in all_evals if e["status"] == "RUNNING"]145 finished_list = [e for e in all_evals if e["status"].startswith("FINISHED")]146 df_pending = pd.DataFrame.from_records(pending_list, columns=EVAL_COLS)147 df_running = pd.DataFrame.from_records(running_list, columns=EVAL_COLS)148 df_finished = pd.DataFrame.from_records(finished_list, columns=EVAL_COLS)149 return df_finished[EVAL_COLS], df_running[EVAL_COLS], df_pending[EVAL_COLS]150 151 152 153original_df = get_leaderboard_df()154leaderboard_df = original_df.copy()155(156 finished_eval_queue_df,157 running_eval_queue_df,158 pending_eval_queue_df,159) = get_evaluation_queue_df()160 161def is_model_on_hub(model_name, revision) -> bool:162 try:163 AutoConfig.from_pretrained(model_name, revision=revision)164 return True, None165 166 except ValueError as e:167 return False, "needs to be launched with `trust_remote_code=True`. For safety reason, we do not allow these models to be automatically submitted to the leaderboard."168 169 except Exception as e:170 print(f"Could not get the model config from the hub.: {e}")171 return False, "was not found on hub!"172 173 174def add_new_eval(175 model: str,176 base_model: str,177 revision: str,178 precision: str,179 private: bool,180 weight_type: str,181 model_type: str,182):183 precision = precision.split(" ")[0]184 current_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")185 186 if model_type is None or model_type == "":187 return styled_error("Please select a model type.")188 189 # check the model actually exists before adding the eval190 if revision == "":191 revision = "main"192 193 if weight_type in ["Delta", "Adapter"]:194 base_model_on_hub, error = is_model_on_hub(base_model, revision)195 if not base_model_on_hub:196 return styled_error(f'Base model "{base_model}" {error}')197 198 199 if not weight_type == "Adapter":200 model_on_hub, error = is_model_on_hub(model, revision)201 if not model_on_hub:202 return styled_error(f'Model "{model}" {error}')203 204 print("adding new eval")205 206 eval_entry = {207 "model": model,208 "base_model": base_model,209 "revision": revision,210 "private": private,211 "precision": precision,212 "weight_type": weight_type,213 "status": "PENDING",214 "submitted_time": current_time,215 "model_type": model_type,216 }217 218 user_name = ""219 model_path = model220 if "/" in model:221 user_name = model.split("/")[0]222 model_path = model.split("/")[1]223 224 OUT_DIR = f"{EVAL_REQUESTS_PATH}/{user_name}"225 os.makedirs(OUT_DIR, exist_ok=True)226 out_path = f"{OUT_DIR}/{model_path}_eval_request_{private}_{precision}_{weight_type}.json"227 228 # Check for duplicate submission229 if out_path.split("eval-queue/")[1].lower() in requested_models:230 return styled_warning("This model has been already submitted.")231 232 with open(out_path, "w") as f:233 f.write(json.dumps(eval_entry))234 235 api.upload_file(236 path_or_fileobj=out_path,237 path_in_repo=out_path.split("eval-queue/")[1],238 repo_id=QUEUE_REPO,239 token=H4_TOKEN,240 repo_type="dataset",241 commit_message=f"Add {model} to eval queue",242 )243 244 # remove the local file245 os.remove(out_path)246 247 return styled_message("Your request has been submitted to the evaluation queue!\nPlease wait for up to an hour for the model to show in the PENDING list.")248 249 250def refresh():251 leaderboard_df = get_leaderboard_df()252 (253 finished_eval_queue_df,254 running_eval_queue_df,255 pending_eval_queue_df,256 ) = get_evaluation_queue_df()257 return (258 leaderboard_df,259 finished_eval_queue_df,260 running_eval_queue_df,261 pending_eval_queue_df,262 )263 264 265def search_table(df, leaderboard_table, query):266 if AutoEvalColumn.model_type.name in leaderboard_table.columns:267 filtered_df = df[268 (df[AutoEvalColumn.dummy.name].str.contains(query, case=False))269 | (df[AutoEvalColumn.model_type.name].str.contains(query, case=False))270 ]271 else:272 filtered_df = df[(df[AutoEvalColumn.dummy.name].str.contains(query, case=False))]273 return filtered_df[leaderboard_table.columns]274 275 276def select_columns(df, columns):277 always_here_cols = [AutoEvalColumn.model_type_symbol.name, AutoEvalColumn.model.name]278 # We use COLS to maintain sorting 279 filtered_df = df[always_here_cols + [c for c in COLS if c in df.columns and c in columns] + [AutoEvalColumn.dummy.name]]280 return filtered_df281 282#TODO allow this to filter by values of any columns283def filter_items(df, leaderboard_table, query):284 if query == "all":285 return df[leaderboard_table.columns]286 else:287 query = query[0] #take only the emoji character288 if AutoEvalColumn.model_type_symbol.name in leaderboard_table.columns:289 filtered_df = df[(df[AutoEvalColumn.model_type_symbol.name] == query)]290 else:291 return leaderboard_table.columns292 return filtered_df[leaderboard_table.columns]293 294def change_tab(query_param):295 query_param = query_param.replace("'", '"')296 query_param = json.loads(query_param)297 298 if (299 isinstance(query_param, dict)300 and "tab" in query_param301 and query_param["tab"] == "evaluation"302 ):303 return gr.Tabs.update(selected=1)304 else:305 return gr.Tabs.update(selected=0)306 307 308demo = gr.Blocks(css=custom_css)309with demo:310 gr.HTML(TITLE)311 gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")312 313 with gr.Tabs(elem_classes="tab-buttons") as tabs:314 with gr.TabItem("🏅 LLM Benchmark", elem_id="llm-benchmark-tab-table", id=0):315 with gr.Row():316 shown_columns = gr.CheckboxGroup(317 choices = [c for c in COLS if c not in [AutoEvalColumn.dummy.name, AutoEvalColumn.model.name, AutoEvalColumn.model_type_symbol.name]], 318 value = [c for c in COLS_LITE if c not in [AutoEvalColumn.dummy.name, AutoEvalColumn.model.name, AutoEvalColumn.model_type_symbol.name]],319 label="Select columns to show", 320 elem_id="column-select", 321 interactive=True,322 )323 with gr.Column(min_width=320):324 search_bar = gr.Textbox(325 placeholder="🔍 Search for your model and press ENTER...",326 show_label=False,327 elem_id="search-bar",328 )329 filter_columns = gr.Radio(330 label="⏚ Filter model types",331 choices = ["all", "🟢 base", "🔶 fine-tuned", "🟦 RL-tuned"],332 value="all",333 elem_id="filter-columns"334 )335 leaderboard_table = gr.components.Dataframe(336 value=leaderboard_df[[AutoEvalColumn.model_type_symbol.name, AutoEvalColumn.model.name] + shown_columns.value+ [AutoEvalColumn.dummy.name]],337 headers=[AutoEvalColumn.model_type_symbol.name, AutoEvalColumn.model.name] + shown_columns.value + [AutoEvalColumn.dummy.name],338 datatype=TYPES,339 max_rows=None,340 elem_id="leaderboard-table",341 interactive=False,342 visible=True,343 )344 345 # Dummy leaderboard for handling the case when the user uses backspace key346 hidden_leaderboard_table_for_search = gr.components.Dataframe(347 value=original_df,348 headers=COLS,349 datatype=TYPES,350 max_rows=None,351 visible=False,352 )353 search_bar.submit(354 search_table,355 [hidden_leaderboard_table_for_search, leaderboard_table, search_bar],356 leaderboard_table,357 )358 shown_columns.change(select_columns, [hidden_leaderboard_table_for_search, shown_columns], leaderboard_table)359 filter_columns.change(filter_items, [hidden_leaderboard_table_for_search, leaderboard_table, filter_columns], leaderboard_table)360 with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=2):361 gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")362 363 with gr.TabItem("🚀 Submit here! ", elem_id="llm-benchmark-tab-table", id=3):364 with gr.Column():365 with gr.Row():366 gr.Markdown(EVALUATION_QUEUE_TEXT, elem_classes="markdown-text")367 368 with gr.Column():369 with gr.Accordion(f"✅ Finished Evaluations ({len(finished_eval_queue_df)})", open=False):370 with gr.Row():371 finished_eval_table = gr.components.Dataframe(372 value=finished_eval_queue_df,373 headers=EVAL_COLS,374 datatype=EVAL_TYPES,375 max_rows=5,376 )377 with gr.Accordion(f"🔄 Running Evaluation Queue ({len(running_eval_queue_df)})", open=False):378 with gr.Row():379 running_eval_table = gr.components.Dataframe(380 value=running_eval_queue_df,381 headers=EVAL_COLS,382 datatype=EVAL_TYPES,383 max_rows=5,384 )385 386 with gr.Accordion(f"⏳ Pending Evaluation Queue ({len(pending_eval_queue_df)})", open=False):387 with gr.Row():388 pending_eval_table = gr.components.Dataframe(389 value=pending_eval_queue_df,390 headers=EVAL_COLS,391 datatype=EVAL_TYPES,392 max_rows=5,393 )394 with gr.Row():395 gr.Markdown("# ✉️✨ Submit your model here!", elem_classes="markdown-text")396 397 with gr.Row():398 with gr.Column():399 model_name_textbox = gr.Textbox(label="Model name")400 revision_name_textbox = gr.Textbox(401 label="revision", placeholder="main"402 )403 private = gr.Checkbox(404 False, label="Private", visible=not IS_PUBLIC405 )406 model_type = gr.Dropdown(407 choices=["pretrained", "fine-tuned", "RL-tuned"], 408 label="Model type", 409 multiselect=False,410 value=None,411 interactive=True,412 )413 414 with gr.Column():415 precision = gr.Dropdown(416 choices=["float16", "bfloat16", "8bit (LLM.int8)", "4bit (QLoRA / FP4)"], 417 label="Precision", 418 multiselect=False,419 value="float16",420 interactive=True,421 )422 weight_type = gr.Dropdown(423 choices=["Original", "Delta", "Adapter"],424 label="Weights type", 425 multiselect=False,426 value="Original",427 interactive=True,428 )429 base_model_name_textbox = gr.Textbox(430 label="Base model (for delta or adapter weights)"431 )432 433 submit_button = gr.Button("Submit Eval")434 submission_result = gr.Markdown()435 submit_button.click(436 add_new_eval,437 [438 model_name_textbox,439 base_model_name_textbox,440 revision_name_textbox,441 precision,442 private,443 weight_type,444 model_type445 ],446 submission_result,447 )448 449 with gr.Row():450 refresh_button = gr.Button("Refresh")451 refresh_button.click(452 refresh,453 inputs=[],454 outputs=[455 leaderboard_table,456 finished_eval_table,457 running_eval_table,458 pending_eval_table,459 ],460 )461 462 with gr.Row():463 with gr.Accordion("📙 Citation", open=False):464 citation_button = gr.Textbox(465 value=CITATION_BUTTON_TEXT,466 label=CITATION_BUTTON_LABEL,467 elem_id="citation-button",468 ).style(show_copy_button=True)469 470 dummy = gr.Textbox(visible=False)471 demo.load(472 change_tab,473 dummy,474 tabs,475 _js=get_window_url_params,476 )477 478scheduler = BackgroundScheduler()479scheduler.add_job(restart_space, "interval", seconds=3600)480scheduler.start()481demo.queue(concurrency_count=40).launch()482