CompileError/WebCoderBench
0
1import html2import json3from pathlib import Path4 5import gradio as gr6import matplotlib.pyplot as plt7import pandas as pd8from apscheduler.schedulers.background import BackgroundScheduler9from gradio_leaderboard import ColumnFilter, Leaderboard, SelectColumns10from huggingface_hub import snapshot_download11 12from src.about import (13 INTRODUCTION_TEXT,14 LLM_BENCHMARKS_TEXT,15 TITLE,16)17from src.display.css_html_js import custom_css, custom_js18from src.display.utils import (19 BENCHMARK_COLS,20 COLS,21 EVAL_COLS,22 AutoEvalColumn,23 fields,24)25from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, QUEUE_REPO, REPO_ID, RESULTS_REPO, TOKEN26from src.populate import get_evaluation_queue_df, get_leaderboard_df27 28plt.switch_backend("Agg")29 30 31def restart_space():32 API.restart_space(repo_id=REPO_ID)33 34 35ENABLE_REMOTE_DATA_SYNC = False36 37if ENABLE_REMOTE_DATA_SYNC:38 try:39 print(EVAL_REQUESTS_PATH)40 snapshot_download(41 repo_id=QUEUE_REPO,42 local_dir=EVAL_REQUESTS_PATH,43 repo_type="dataset",44 tqdm_class=None,45 etag_timeout=30,46 token=TOKEN,47 )48 except Exception:49 restart_space()50 try:51 print(EVAL_RESULTS_PATH)52 snapshot_download(53 repo_id=RESULTS_REPO,54 local_dir=EVAL_RESULTS_PATH,55 repo_type="dataset",56 tqdm_class=None,57 etag_timeout=30,58 token=TOKEN,59 )60 except Exception:61 restart_space()62 63 LEADERBOARD_DF = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS, BENCHMARK_COLS)64 65 (66 finished_eval_queue_df,67 running_eval_queue_df,68 pending_eval_queue_df,69 ) = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS)70 71 72def init_leaderboard(dataframe):73 if dataframe is None or dataframe.empty:74 raise ValueError("Leaderboard DataFrame is empty or None.")75 return Leaderboard(76 value=dataframe,77 datatype=[c.type for c in fields(AutoEvalColumn)],78 select_columns=SelectColumns(79 default_selection=[c.name for c in fields(AutoEvalColumn) if c.displayed_by_default],80 cant_deselect=[c.name for c in fields(AutoEvalColumn) if c.never_hidden],81 label="Select Columns to Display:",82 ),83 search_columns=[AutoEvalColumn.model.name, AutoEvalColumn.license.name],84 hide_columns=[c.name for c in fields(AutoEvalColumn) if c.hidden],85 filter_columns=[86 ColumnFilter(AutoEvalColumn.model_type.name, type="checkboxgroup", label="Model types"),87 ColumnFilter(AutoEvalColumn.precision.name, type="checkboxgroup", label="Precision"),88 ColumnFilter(89 AutoEvalColumn.params.name,90 type="slider",91 min=0.01,92 max=150,93 label="Select the number of parameters (B)",94 ),95 ColumnFilter(AutoEvalColumn.still_on_hub.name, type="boolean", label="Deleted/incomplete", default=True),96 ],97 bool_checkboxgroup_label="Hide models",98 interactive=False,99 )100def create_score_df(json_path: Path, decimals: int):101 data = json.loads(json_path.read_text(encoding="utf-8"))102 103 if not data:104 return pd.DataFrame()105 106 records = []107 for item in data:108 flat_record = {"ID": item.get("ID"), "Model": item.get("Model")}109 for category, sub_items in item.items():110 if category in ("ID", "Model"):111 continue112 if isinstance(sub_items, dict):113 cleaned_category = str(category).replace("\n", " ")114 for sub_category, value in sub_items.items():115 header = f"{cleaned_category}\n{sub_category}"116 flat_record[header] = value117 records.append(flat_record)118 119 df = pd.DataFrame(records)120 score_cols = [c for c in df.columns if c not in ("ID", "Model")]121 for col in score_cols:122 df[col] = df[col].apply(123 lambda v: ("" if pd.isna(v) else (f"{float(v):.{decimals}f}" if isinstance(v, (int, float)) else v))124 )125 return df126 127 128def dataframe_height(df: pd.DataFrame):129 rows = 0 if df is None else int(getattr(df, "shape", (0, 0))[0])130 row_px = 40131 header_px = 44132 padding_px = 96133 height = header_px + (rows * row_px) + padding_px134 return max(320, min(1200, height))135 136 137def create_grouped_score_table_html(df: pd.DataFrame):138 if df is None or df.empty:139 return '<div class="score-table-scroll"><table class="score-table"></table></div>'140 141 fixed_cols = [c for c in ("ID", "Model") if c in df.columns]142 143 group_to_cols = {}144 group_order = []145 data_cols_in_order = []146 147 for col in df.columns:148 if col in fixed_cols:149 continue150 if isinstance(col, str) and "\n" in col:151 group, metric = col.split("\n", 1)152 else:153 group, metric = str(col), str(col)154 155 if group not in group_to_cols:156 group_to_cols[group] = []157 group_order.append(group)158 159 group_to_cols[group].append((metric, col))160 data_cols_in_order.append(col)161 162 parts = ['<div class="score-table-scroll"><table class="score-table">']163 164 parts.append("<colgroup>")165 if len(fixed_cols) >= 1:166 parts.append('<col style="width:40px">')167 if len(fixed_cols) >= 2:168 parts.append('<col style="width:220px">')169 for _ in range(len(data_cols_in_order)):170 parts.append('<col style="width:180px">')171 parts.append("</colgroup>")172 173 parts.append("<thead>")174 175 parts.append("<tr>")176 for col in fixed_cols:177 parts.append(f'<th rowspan="2"><div class="score-cell">{html.escape(str(col))}</div></th>')178 merged_groups = set()179 for group in group_order:180 metrics = group_to_cols[group]181 if len(metrics) == 1 and str(metrics[0][0]) == str(group):182 merged_groups.add(group)183 parts.append(f'<th rowspan="2"><div class="score-cell">{html.escape(str(group))}</div></th>')184 else:185 parts.append(186 f'<th colspan="{len(metrics)}"><div class="score-cell">{html.escape(str(group))}</div></th>'187 )188 parts.append("</tr>")189 190 parts.append("<tr>")191 for group in group_order:192 if group in merged_groups:193 continue194 for metric, _ in group_to_cols[group]:195 parts.append(f"<th><div class=\"score-cell\">{html.escape(str(metric))}</div></th>")196 parts.append("</tr>")197 198 parts.append("</thead>")199 parts.append("<tbody>")200 201 col_indices = [df.columns.get_loc(c) for c in (fixed_cols + data_cols_in_order)]202 for row in df.itertuples(index=False, name=None):203 parts.append("<tr>")204 for idx in col_indices:205 value = row[idx]206 text = "" if pd.isna(value) else str(value)207 parts.append(f"<td><div class=\"score-cell\">{html.escape(text)}</div></td>")208 parts.append("</tr>")209 210 parts.append("</tbody></table></div>")211 return "".join(parts)212 213 214def create_raw_score_df():215 raw_path = Path(__file__).resolve().parent / "src" / "raw_score.json"216 return create_score_df(raw_path, decimals=2)217 218 219def create_unweighted_z_score_df():220 z_path = Path(__file__).resolve().parent / "src" / "unweighted_z_score.json"221 return create_score_df(z_path, decimals=4)222 223 224def create_weighted_z_score_df():225 z_path = Path(__file__).resolve().parent / "src" / "weighted_z_score.json"226 data = json.loads(z_path.read_text(encoding="utf-8"))227 228 if not data:229 return pd.DataFrame()230 231 records = []232 for item in data:233 flat_record = {"ID": item.get("ID"), "Model": item.get("Model")}234 for category, sub_items in item.items():235 if category in ("ID", "Model"):236 continue237 if not isinstance(sub_items, dict):238 continue239 240 cleaned_category = str(category).replace("\n", " ")241 for sub_category, value in sub_items.items():242 if cleaned_category == "Overall Score":243 header = "Overall Score"244 else:245 header = f"{cleaned_category}\n{sub_category}"246 flat_record[header] = value247 records.append(flat_record)248 249 df = pd.DataFrame(records)250 251 overall_col = "Overall Score" if "Overall Score" in df.columns else None252 if overall_col is None:253 for c in df.columns:254 if isinstance(c, str) and c.endswith("\nOverall Score"):255 overall_col = c256 break257 258 if overall_col is not None:259 df[overall_col] = pd.to_numeric(df[overall_col], errors="coerce")260 df = df.sort_values(by=overall_col, ascending=False, kind="mergesort")261 262 cols = list(df.columns)263 fixed = [c for c in ("ID", "Model") if c in cols]264 rest = [c for c in cols if c not in set(fixed + [overall_col])]265 df = df[fixed + [overall_col] + rest]266 267 score_cols = [c for c in df.columns if c not in ("ID", "Model")]268 for col in score_cols:269 df[col] = pd.to_numeric(df[col], errors="coerce").apply(270 lambda v: "" if pd.isna(v) else f"{float(v) * 100:.2f}%"271 )272 273 if "ID" in df.columns:274 df["ID"] = pd.to_numeric(df["ID"], errors="coerce").astype("Int64")275 276 return df277 278 279def create_weights_table_html():280 weights_path = Path(__file__).resolve().parent / "src" / "weights.json"281 payload = json.loads(weights_path.read_text(encoding="utf-8"))282 bounds = payload["bounds"]283 min_row = bounds["min_row"]284 min_col = bounds["min_col"]285 rows = payload["rows"]286 287 covered = set()288 spans = {}289 for m in payload["merges"]:290 r1, c1, r2, c2 = m["r1"], m["c1"], m["r2"], m["c2"]291 spans[(r1, c1)] = {"rowspan": r2 - r1 + 1, "colspan": c2 - c1 + 1}292 for r in range(r1, r2 + 1):293 for c in range(c1, c2 + 1):294 if (r, c) != (r1, c1):295 covered.add((r, c))296 297 parts = ['<div class="weights-scroll"><table class="weights-table">']298 for r_index, row in enumerate(rows, start=min_row):299 parts.append("<tr>")300 for c_index, value in enumerate(row, start=min_col):301 if (r_index, c_index) in covered:302 continue303 span = spans.get((r_index, c_index))304 attrs = ""305 if span is not None:306 attrs = f" rowspan=\"{span['rowspan']}\" colspan=\"{span['colspan']}\""307 tag = "th" if r_index == min_row else "td"308 if r_index != min_row:309 if isinstance(value, (int, float)):310 value = f"{float(value) * 100:.2f}%"311 elif isinstance(value, str):312 stripped = value.strip()313 try:314 value = f"{float(stripped) * 100:.2f}%"315 except Exception:316 pass317 text = "" if value is None else html.escape(str(value))318 parts.append(f"<{tag}{attrs}>{text}</{tag}>")319 parts.append("</tr>")320 parts.append("</table></div>")321 return "".join(parts)322 323 324def parse_number(value):325 if value is None or (isinstance(value, float) and pd.isna(value)):326 return None327 if isinstance(value, (int, float)):328 return float(value)329 text = str(value).strip()330 if text == "":331 return None332 if text.endswith("%"):333 try:334 return float(text[:-1])335 except Exception:336 return None337 try:338 return float(text)339 except Exception:340 return None341 342 343def shorten_label(text: str, max_len: int = 18):344 text = "" if text is None else str(text)345 if len(text) <= max_len:346 return text347 return text[: max_len - 1] + "…"348 349 350def build_metric_choices(raw_df: pd.DataFrame):351 choices = [("Overall Score", "weighted::Overall Score")]352 for col in raw_df.columns:353 if col in ("ID", "Model"):354 continue355 if isinstance(col, str) and "\n" in col:356 group, metric = col.split("\n", 1)357 label = f"{group} / {metric}"358 else:359 label = str(col)360 choices.append((label, f"raw::{col}"))361 return choices362 363 364def build_rank_bar_plot(metric_key: str):365 if not isinstance(metric_key, str) or "::" not in metric_key:366 metric_key = "weighted::Overall Score"367 source, col = metric_key.split("::", 1)368 369 if source == "weighted":370 df = WEIGHTED_Z_DF371 else:372 df = RAW_SCORE_DF373 374 if df is None or df.empty or col not in df.columns:375 fig, ax = plt.subplots(figsize=(10, 4))376 ax.set_axis_off()377 return fig378 379 series = []380 for _, row in df.iterrows():381 model = row.get("Model", "")382 value = parse_number(row.get(col))383 if value is None:384 continue385 series.append((str(model), float(value)))386 387 series.sort(key=lambda x: x[1], reverse=True)388 389 labels = [shorten_label(m) for m, _ in series]390 values = [v for _, v in series]391 n = len(values)392 width = min(22, max(10, 0.55 * max(1, n)))393 fig, ax = plt.subplots(figsize=(width, 5))394 ax.bar(range(n), values)395 ax.set_xticks(range(n))396 ax.set_xticklabels(labels, rotation=35, ha="right")397 398 title = "Overall Score" if (source == "weighted" and col == "Overall Score") else str(col)399 ax.set_title(title)400 ax.margins(x=0.01)401 fig.tight_layout()402 return fig403 404 405RAW_SCORE_DF = create_raw_score_df()406UNWEIGHTED_Z_DF = create_unweighted_z_score_df()407WEIGHTED_Z_DF = create_weighted_z_score_df()408 409METRIC_CHOICES = build_metric_choices(RAW_SCORE_DF)410DEFAULT_METRIC = "weighted::Overall Score"411 412SCORE_TABLE_HEIGHT_CSS = f"""413#raw-score-table .score-table-scroll {{414 max-height: {dataframe_height(RAW_SCORE_DF)}px;415}}416 417#unweighted-z-table .score-table-scroll {{418 max-height: {dataframe_height(UNWEIGHTED_Z_DF)}px;419}}420 421#weighted-z-table .score-table-scroll {{422 max-height: {dataframe_height(WEIGHTED_Z_DF)}px;423}}424"""425 426demo = gr.Blocks(css=custom_css + SCORE_TABLE_HEIGHT_CSS)427 428 429with demo:430 gr.HTML(custom_js)431 gr.HTML(TITLE)432 gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")433 434 with gr.Tabs(elem_classes="tab-buttons") as tabs:435 with gr.TabItem("Result", elem_id="result-tab", id=0):436 with gr.Tabs(elem_classes="tab-buttons") as nested_tabs:437 with gr.TabItem("Chart"):438 metric = gr.Dropdown(choices=METRIC_CHOICES, value=DEFAULT_METRIC, label="Metric")439 chart = gr.Plot(value=build_rank_bar_plot(DEFAULT_METRIC))440 metric.change(build_rank_bar_plot, inputs=metric, outputs=chart)441 with gr.TabItem("Raw Score"):442 gr.HTML(create_grouped_score_table_html(RAW_SCORE_DF), elem_id="raw-score-table")443 with gr.TabItem("Weights"):444 gr.HTML(create_weights_table_html(), elem_id="weights-table")445 with gr.TabItem("Unweighted Z-score"):446 gr.HTML(create_grouped_score_table_html(UNWEIGHTED_Z_DF), elem_id="unweighted-z-table")447 with gr.TabItem("Weighted Z-score"):448 gr.HTML(create_grouped_score_table_html(WEIGHTED_Z_DF), elem_id="weighted-z-table")449 450 with gr.TabItem("About", elem_id="llm-benchmark-tab-table", id=2):451 gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")452 453 454scheduler = BackgroundScheduler()455scheduler.add_job(restart_space, "interval", seconds=1800)456scheduler.start()457demo.queue(default_concurrency_limit=40).launch()458