open-llm-leaderboard/open_llm_leaderboard
14k
1import gradio as gr2import pandas as pd3from apscheduler.schedulers.background import BackgroundScheduler4from huggingface_hub import snapshot_download5from gradio_space_ci import configure_space_ci # FOR CI6 7from src.display.about import (8 CITATION_BUTTON_LABEL,9 CITATION_BUTTON_TEXT,10 EVALUATION_QUEUE_TEXT,11 INTRODUCTION_TEXT,12 LLM_BENCHMARKS_TEXT,13 FAQ_TEXT,14 TITLE,15)16from src.display.css_html_js import custom_css17from src.display.utils import (18 BENCHMARK_COLS,19 COLS,20 EVAL_COLS,21 EVAL_TYPES,22 NUMERIC_INTERVALS,23 TYPES,24 AutoEvalColumn,25 ModelType,26 fields,27 WeightType,28 Precision29)30from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, H4_TOKEN, IS_PUBLIC, QUEUE_REPO, REPO_ID, RESULTS_REPO31from src.populate import get_evaluation_queue_df, get_leaderboard_df32from src.submission.submit import add_new_eval33from src.tools.collections import update_collections34from src.tools.plots import (35 create_metric_plot_obj,36 create_plot_df,37 create_scores_df,38)39 40 41def restart_space():42 API.restart_space(repo_id=REPO_ID, token=H4_TOKEN)43 44try:45 print(EVAL_REQUESTS_PATH)46 snapshot_download(47 repo_id=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=3048 )49except Exception:50 restart_space()51try:52 print(EVAL_RESULTS_PATH)53 snapshot_download(54 repo_id=RESULTS_REPO, local_dir=EVAL_RESULTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=3055 )56except Exception:57 restart_space()58 59 60raw_data, original_df = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS, BENCHMARK_COLS)61update_collections(original_df.copy())62leaderboard_df = original_df.copy()63 64plot_df = create_plot_df(create_scores_df(raw_data))65 66(67 finished_eval_queue_df,68 running_eval_queue_df,69 pending_eval_queue_df,70) = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS)71 72 73# Searching and filtering74def update_table(75 hidden_df: pd.DataFrame,76 columns: list,77 type_query: list,78 precision_query: str,79 size_query: list,80 show_deleted: bool,81 query: str,82):83 filtered_df = filter_models(hidden_df, type_query, size_query, precision_query, show_deleted)84 filtered_df = filter_queries(query, filtered_df)85 df = select_columns(filtered_df, columns)86 return df87 88 89def search_table(df: pd.DataFrame, query: str) -> pd.DataFrame:90 return df[(df[AutoEvalColumn.dummy.name].str.contains(query, case=False))]91 92 93def select_columns(df: pd.DataFrame, columns: list) -> pd.DataFrame:94 always_here_cols = [95 AutoEvalColumn.model_type_symbol.name,96 AutoEvalColumn.model.name,97 ]98 # We use COLS to maintain sorting99 filtered_df = df[100 always_here_cols + [c for c in COLS if c in df.columns and c in columns] + [AutoEvalColumn.dummy.name]101 ]102 return filtered_df103 104 105def filter_queries(query: str, filtered_df: pd.DataFrame):106 """Added by Abishek"""107 final_df = []108 if query != "":109 queries = [q.strip() for q in query.split(";")]110 for _q in queries:111 _q = _q.strip()112 if _q != "":113 temp_filtered_df = search_table(filtered_df, _q)114 if len(temp_filtered_df) > 0:115 final_df.append(temp_filtered_df)116 if len(final_df) > 0:117 filtered_df = pd.concat(final_df)118 filtered_df = filtered_df.drop_duplicates(119 subset=[AutoEvalColumn.model.name, AutoEvalColumn.precision.name, AutoEvalColumn.revision.name]120 )121 122 return filtered_df123 124 125def filter_models(126 df: pd.DataFrame, type_query: list, size_query: list, precision_query: list, show_deleted: bool127) -> pd.DataFrame:128 # Show all models129 if show_deleted:130 filtered_df = df131 else: # Show only still on the hub models132 filtered_df = df[df[AutoEvalColumn.still_on_hub.name] == True]133 134 type_emoji = [t[0] for t in type_query]135 filtered_df = filtered_df.loc[df[AutoEvalColumn.model_type_symbol.name].isin(type_emoji)]136 filtered_df = filtered_df.loc[df[AutoEvalColumn.precision.name].isin(precision_query + ["None"])]137 138 numeric_interval = pd.IntervalIndex(sorted([NUMERIC_INTERVALS[s] for s in size_query]))139 params_column = pd.to_numeric(df[AutoEvalColumn.params.name], errors="coerce")140 mask = params_column.apply(lambda x: any(numeric_interval.contains(x)))141 filtered_df = filtered_df.loc[mask]142 143 return filtered_df144 145 146demo = gr.Blocks(css=custom_css)147with demo:148 gr.HTML(TITLE)149 gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")150 151 with gr.Tabs(elem_classes="tab-buttons") as tabs:152 with gr.TabItem("🏅 LLM Benchmark", elem_id="llm-benchmark-tab-table", id=0):153 with gr.Row():154 with gr.Column():155 with gr.Row():156 search_bar = gr.Textbox(157 placeholder=" 🔍 Search for your model (separate multiple queries with `;`) and press ENTER...",158 show_label=False,159 elem_id="search-bar",160 )161 with gr.Row():162 shown_columns = gr.CheckboxGroup(163 choices=[164 c.name165 for c in fields(AutoEvalColumn)166 if not c.hidden and not c.never_hidden and not c.dummy167 ],168 value=[169 c.name170 for c in fields(AutoEvalColumn)171 if c.displayed_by_default and not c.hidden and not c.never_hidden172 ],173 label="Select columns to show",174 elem_id="column-select",175 interactive=True,176 )177 with gr.Row():178 deleted_models_visibility = gr.Checkbox(179 value=False, label="Show gated/private/deleted models", interactive=True180 )181 with gr.Column(min_width=320):182 #with gr.Box(elem_id="box-filter"):183 filter_columns_type = gr.CheckboxGroup(184 label="Model types",185 choices=[t.to_str() for t in ModelType],186 value=[t.to_str() for t in ModelType],187 interactive=True,188 elem_id="filter-columns-type",189 )190 filter_columns_precision = gr.CheckboxGroup(191 label="Precision",192 choices=[i.value.name for i in Precision],193 value=[i.value.name for i in Precision],194 interactive=True,195 elem_id="filter-columns-precision",196 )197 filter_columns_size = gr.CheckboxGroup(198 label="Model sizes (in billions of parameters)",199 choices=list(NUMERIC_INTERVALS.keys()),200 value=list(NUMERIC_INTERVALS.keys()),201 interactive=True,202 elem_id="filter-columns-size",203 )204 205 leaderboard_table = gr.components.Dataframe(206 value=leaderboard_df[207 [c.name for c in fields(AutoEvalColumn) if c.never_hidden]208 + shown_columns.value209 + [AutoEvalColumn.dummy.name]210 ],211 headers=[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value,212 datatype=TYPES,213 elem_id="leaderboard-table",214 interactive=False,215 visible=True,216 column_widths=["2%", "33%"] 217 )218 219 # Dummy leaderboard for handling the case when the user uses backspace key220 hidden_leaderboard_table_for_search = gr.components.Dataframe(221 value=original_df[COLS],222 headers=COLS,223 datatype=TYPES,224 visible=False,225 )226 search_bar.submit(227 update_table,228 [229 hidden_leaderboard_table_for_search,230 shown_columns,231 filter_columns_type,232 filter_columns_precision,233 filter_columns_size,234 deleted_models_visibility,235 search_bar,236 ],237 leaderboard_table,238 )239 for selector in [shown_columns, filter_columns_type, filter_columns_precision, filter_columns_size, deleted_models_visibility]:240 selector.change(241 update_table,242 [243 hidden_leaderboard_table_for_search,244 shown_columns,245 filter_columns_type,246 filter_columns_precision,247 filter_columns_size,248 deleted_models_visibility,249 search_bar,250 ],251 leaderboard_table,252 queue=True,253 )254 255 with gr.TabItem("📈 Metrics through time", elem_id="llm-benchmark-tab-table", id=4):256 with gr.Row():257 with gr.Column():258 chart = create_metric_plot_obj(259 plot_df,260 [AutoEvalColumn.average.name],261 title="Average of Top Scores and Human Baseline Over Time (from last update)",262 )263 gr.Plot(value=chart, min_width=500) 264 with gr.Column():265 chart = create_metric_plot_obj(266 plot_df,267 BENCHMARK_COLS,268 title="Top Scores and Human Baseline Over Time (from last update)",269 )270 gr.Plot(value=chart, min_width=500) 271 with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=2):272 gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")273 gr.Markdown(FAQ_TEXT, elem_classes="markdown-text")274 275 with gr.TabItem("🚀 Submit here! ", elem_id="llm-benchmark-tab-table", id=3):276 with gr.Column():277 with gr.Row():278 gr.Markdown(EVALUATION_QUEUE_TEXT, elem_classes="markdown-text")279 280 with gr.Column():281 with gr.Accordion(282 f"✅ Finished Evaluations ({len(finished_eval_queue_df)})",283 open=False,284 ):285 with gr.Row():286 finished_eval_table = gr.components.Dataframe(287 value=finished_eval_queue_df,288 headers=EVAL_COLS,289 datatype=EVAL_TYPES,290 row_count=5,291 )292 with gr.Accordion(293 f"🔄 Running Evaluation Queue ({len(running_eval_queue_df)})",294 open=False,295 ):296 with gr.Row():297 running_eval_table = gr.components.Dataframe(298 value=running_eval_queue_df,299 headers=EVAL_COLS,300 datatype=EVAL_TYPES,301 row_count=5,302 )303 304 with gr.Accordion(305 f"⏳ Pending Evaluation Queue ({len(pending_eval_queue_df)})",306 open=False,307 ):308 with gr.Row():309 pending_eval_table = gr.components.Dataframe(310 value=pending_eval_queue_df,311 headers=EVAL_COLS,312 datatype=EVAL_TYPES,313 row_count=5,314 )315 with gr.Row():316 gr.Markdown("# ✉️✨ Submit your model here!", elem_classes="markdown-text")317 318 with gr.Row():319 with gr.Column():320 model_name_textbox = gr.Textbox(label="Model name")321 revision_name_textbox = gr.Textbox(label="Revision commit", placeholder="main")322 private = gr.Checkbox(False, label="Private", visible=not IS_PUBLIC)323 model_type = gr.Dropdown(324 choices=[t.to_str(" : ") for t in ModelType if t != ModelType.Unknown],325 label="Model type",326 multiselect=False,327 value=None,328 interactive=True,329 )330 331 with gr.Column():332 precision = gr.Dropdown(333 choices=[i.value.name for i in Precision if i != Precision.Unknown],334 label="Precision",335 multiselect=False,336 value="float16",337 interactive=True,338 )339 weight_type = gr.Dropdown(340 choices=[i.value.name for i in WeightType],341 label="Weights type",342 multiselect=False,343 value="Original",344 interactive=True,345 )346 base_model_name_textbox = gr.Textbox(label="Base model (for delta or adapter weights)")347 348 submit_button = gr.Button("Submit Eval")349 submission_result = gr.Markdown()350 submit_button.click(351 add_new_eval,352 [353 model_name_textbox,354 base_model_name_textbox,355 revision_name_textbox,356 precision,357 private,358 weight_type,359 model_type,360 ],361 submission_result,362 )363 364 with gr.Row():365 with gr.Accordion("📙 Citation", open=False):366 citation_button = gr.Textbox(367 value=CITATION_BUTTON_TEXT,368 label=CITATION_BUTTON_LABEL,369 lines=20,370 elem_id="citation-button",371 show_copy_button=True,372 )373 374scheduler = BackgroundScheduler()375scheduler.add_job(restart_space, "interval", seconds=1800)376scheduler.start()377 378# Both launches the space and its CI379configure_space_ci(380 demo.queue(default_concurrency_limit=40),381 trusted_authors=[], # add manually trusted authors382 private="True", # ephemeral spaces will have same visibility as the main space. Otherwise, set to `True` or `False` explicitly.383 variables={}, # We overwrite HF_HOME as tmp CI spaces will have no cache 384 secrets=["HF_TOKEN", "H4_TOKEN"], # which secret do I want to copy from the main space? Can be a `List[str]`.385 hardware=None, # "cpu-basic" by default. Otherwise set to "auto" to have same hardware as the main space or any valid string value.386 storage=None, # no storage by default. Otherwise set to "auto" to have same storage as the main space or any valid string value.387).launch()