PrunaAI/P-Bench
24
1from html import escape2from math import ceil, floor, log103from pathlib import Path4import base645import random6 7import gradio as gr8import pandas as pd9import plotly.graph_objects as go10 11from model_display import display_model_name12 13_LOGO_PATH = Path(__file__).resolve().parent / "pruna-logo.png"14_LOGO_DATA_URI = (15 "data:image/png;base64,"16 + base64.b64encode(_LOGO_PATH.read_bytes()).decode("ascii")17 if _LOGO_PATH.exists()18 else "https://playground.pruna.ai/logo.svg"19)20 21 22MAX_COMPARE_MODELS = 423DEFAULT_COMPARE_PROMPTS = 324MAX_COMPARE_PROMPTS = 825MAX_PARETO_METRICS = 826_PARETO_SLOT_COUNT = 1 + MAX_PARETO_METRICS * 827_PARETO_PRICE_COLUMN = "Price / Image (USD)"28_PARETO_VIDEO_PRICE_COLUMN = "Price / Second of Video (USD)"29_PARETO_PRICE_COLUMNS = (_PARETO_PRICE_COLUMN, _PARETO_VIDEO_PRICE_COLUMN)30_PARETO_TIME_COLUMN = "Min Generation Time (s)"31_PARETO_VIDEO_TIME_COLUMN = "Pareto Time / Output Video Second (s)"32_PARETO_TIME_COLUMNS = (_PARETO_VIDEO_TIME_COLUMN, _PARETO_TIME_COLUMN)33_PARETO_PRICE_TITLES = {34 _PARETO_PRICE_COLUMN: "Price per image (USD)",35 _PARETO_VIDEO_PRICE_COLUMN: "Price per second of video (USD)",36}37_PARETO_TIME_TITLES = {38 _PARETO_TIME_COLUMN: "Min generation time (s)",39 _PARETO_VIDEO_TIME_COLUMN: "Generation time per second of video",40}41_PARETO_SCALE_CHOICES = [42 ("Log", "Logarithmic"),43 ("Linear", "Linear"),44]45_PARETO_SCALE_VALUES = {value for _, value in _PARETO_SCALE_CHOICES}46_PARETO_SCALE_DEFAULT = "Logarithmic"47_PARETO_PRUNA_COLOR = "#c084fc"48_PARETO_OTHER_COLOR = "#9aa3b5"49_PARETO_FRONTIER_OUTLINE = "#3fa87e"50 51TAB_LEADERBOARDS = "leaderboards"52TAB_PARETO = "pareto"53TAB_SAMPLES = "samples"54TAB_ABOUT = "about"55 56MODALITY_TEXT_TO_VIDEO = "text_to_video"57MODALITY_VIDEO_TO_VIDEO = "video_to_video"58MODALITY_TEXT_TO_IMAGE = "text_to_image"59MODALITY_CHOICES = [60 ("Text to Video", MODALITY_TEXT_TO_VIDEO),61 ("Video to Video", MODALITY_VIDEO_TO_VIDEO),62 ("Text to Image", MODALITY_TEXT_TO_IMAGE),63]64 65_MODEL_CHOICES_CACHE = {}66_VIEW_EVENTS = {67 "show_progress": "hidden",68 "trigger_mode": "always_last",69 "concurrency_id": "workspace-views",70 "concurrency_limit": 1,71}72 73ABOUT_OVERVIEW_CONTENT = """74# About P-Bench75 76P-Bench compares **text-to-video**, **video-to-video**, and **text-to-image**77models, including optimized or accelerated endpoints, on **quality, speed,78and price**. Each view is a **dataset** scored with a **metric**, written as79`Dataset | Metric`. There is no single score across P-Bench.80 81## How to read it82 831. Pick a **model type** (Text to Video, Video to Video, or Text to Image),84 then a **dataset** and a **metric**.852. **Leaderboards**: ranked by that metric. Price and generation time sit in86 the same table when the source publishes them.873. **Pareto plots**: mark models that are not beaten on both higher score88 and lower price (or time). Only datasets with price or generation time89 can open this tab (not Arena AI).904. **Samples**: the same prompts, side by side. Only for datasets we91 generated (VBench-2.0 Dataset, Qwen Image Dataset, OneIG Alignment92 Dataset, and the Pruna Internal Video-Edit Benchmark). Video-edit93 samples show the source clip first, then each model's edit.94 95## How a score is made96 97On **Qwen Image Dataset** and **OneIG Alignment Dataset**:98 991. Each endpoint is given the same prompt suite.1002. It generates one image per prompt when the run succeeds. Not every model101 has every prompt or every metric.1023. Quality is scored automatically (OneIG alignment, P-Judger) and, where103 available, by human preference (Datapoint Elo, Rapidata Elo).1044. Price per image and generation time are joined from the evaluation table.105 106**Artificial Analysis** and **Arena AI** are external leaderboards. We import107their published Elos (and Artificial Analysis price). We do not run their108prompt suites, so samples are not shown.109 110## Current datasets111 112### VBench-2.0 Dataset113VBench-2.0 prompts, comparing P-Video-2 variants with Fal-hosted models.114Quality is Datapoint Elo and Rapidata Elo from pairwise preference. Price115is USD per second of output video. Time per second of video is Fal wall116time, except Pruna models which use model execution time. Samples are117available.118 119### Pruna Internal Video-Edit Benchmark120Pruna's internal video-to-video editing benchmark, collected by our121research engineers. It combines prompts from public video-editing122benchmarks with use-case examples we gathered for advertisement,123e-commerce, real estate, concept art, and similar work. The suite also124covers camera-angle and movement changes, lighting, and text in video125(altering, adding, or removing it). Quality is Datapoint Elo from126pairwise preference. Price is USD per second of output video;127generation time is wall time per second of output video. Samples show128the source clip beside each model's edit.129 130### Qwen Image Dataset131100 prompts from the 1,000-prompt Qwen Image Bench set, sampled for coverage132across its fine-grained (L3) categories. Metrics include Datapoint Elo,133Rapidata Elo, and P-Judger. Samples are available.134 135### OneIG Alignment Dataset136Prompt-image **alignment** on anime / stylization (100), human / portrait137(100), and general object prompts (99). This is the alignment slice of138OneIG, not the full suite. Alignment Overall is the mean of the category139scores that exist for that row. Also includes Datapoint category Elo,140Rapidata Elo, and P-Judger. Samples are available.141 142### Artificial Analysis Dataset143External text-to-image Elo and price per image from Artificial Analysis.144Their prompt set is private, so samples are not shown. Pareto plots use145price vs score only.146 147### Arena AI Dataset148External text-to-image Elo (overall and category) from Arena AI. Their149prompt set is private, so samples are not shown. Price and generation time150are not in this export, so Pareto plots are unavailable.151"""152 153ABOUT_DETAILS_CONTENT = """154# Metrics & limits155 156## Metrics157 158- **Alignment**: how well the image matches the prompt (OneIG category scores).159- **P-Judger**: Pruna's automatic judger. A private scoring model that rates160 each (prompt, image) pair in one pass and returns an absolute quality score,161 not a pairwise Elo. It is not a text-to-image model on the leaderboard. Use162 it alongside Datapoint and Rapidata Elo, not instead of them.163- **Datapoint Elo**: human-preference Elo from Datapoint pairwise comparisons.164- **Rapidata Elo**: human-preference Elo from Rapidata pairwise comparisons.165 Rapidata rejects prompts over 400 characters, so this Elo is on a subset166 of each suite (see Setup). Rapidata is not a dataset.167- **Artificial Analysis Elo**: Elo published by Artificial Analysis on their168 own dataset.169- **Arena Elo**: Elo published by Arena AI on their own dataset, plus170 category Elos (branding, 3D, cartoon/anime, photorealistic, art, portraits,171 text rendering).172- **Generation time**: median and minimum generation time in seconds for173 images, as reported in the evaluation table. For video, generation time174 per second of output video is the more informative figure. On the175 text-to-video benchmark this is Fal wall time, except Pruna models which176 use model execution time. On video-edit it is end-to-end wall time. This177 is not a p95, and we do not state warm vs cold or concurrent load. Not178 available for Arena AI.179- **Price**: USD per image for text-to-image, or USD per second of output180 video for text-to-video and video-to-video. We do not state list price vs181 amount paid, or whether failed generations are included. Not available182 for Arena AI.183 184Scores from different datasets or metrics are **not interchangeable**. A high185OneIG alignment score is not the same quantity as a Datapoint Elo. Compare186models *within* a Dataset | Metric view.187 188## Setup189 190- **Evaluation window:** July–August 2026.191- **Update policy:** numbers come from evaluation snapshots in the tables,192 not a live API poll.193- **Prompt counts:** OneIG Alignment uses 100 anime, 100 human, and 99 object194 prompts (299 total). Qwen Image Dataset uses 100 prompts sampled from the195 1,000-prompt pool for roughly even coverage of its fine-grained (L3)196 categories. The VBench-2.0 Dataset uses about 90197 generations per model. The Pruna Internal Video-Edit Benchmark uses 78198 prompts across advertising, e-commerce, real estate, camera, lighting,199 text, and related categories. Artificial Analysis and Arena AI use their200 own private prompt sets.201- **Generation (Qwen and OneIG):** one image per prompt per endpoint when202 the run exists. Default resolution is 1024×1024. Exceptions: FLUX 1.1 Pro203 Ultra at 2K, FLUX 2 Flex at 1008×1008, and any endpoint labeled 2K. The204 seed is derived from the prompt, so every model gets the same seed for the205 same prompt. Steps, CFG, prompt rewrite, and safety filters follow each206 endpoint's default. This does not describe Artificial Analysis or Arena AI.207- **Generation (Text-to-Video):** one clip per prompt per endpoint when the208 run exists. About 90 generations per model.209- **Generation (Video-Edit):** one edited clip per prompt per endpoint when210 the run exists. Every model sees the same source video for a prompt.211- **Datapoint (Qwen and OneIG):** every model pair is compared on every212 prompt, with 10 votes per battle.213- **Rapidata (Qwen and OneIG):** prompts longer than 400 characters are214 dropped, leaving 212 OneIG prompts and 85 Qwen Image Dataset prompts. 4215 votes per pair; about 26,000 votes on OneIG and 35,000 on Qwen Image216 Dataset.217 218## Limits219 220- Empty cells mean that track was not run or not reported for that model.221- Artificial Analysis and Arena AI samples, prompts, and (for Arena) price222 or latency are not available to P-Bench.223- Rapidata Elo is not on the full prompt suite, so it is not directly224 comparable to Datapoint Elo even on the same dataset.225- Elo ratings can shift when the comparison pool changes: treat them as226 relative rankings for the snapshot, not absolute constants.227- Close scores can be a tie in practice; the table does not show confidence228 intervals.229- Optimized or accelerated endpoints (when labeled) may differ from the base230 model publisher's default serving stack.231- APIs and prices drift; a rank is for the snapshot, not a guarantee.232 233Built by [Pruna AI](https://www.pruna.ai/). If you have questions, reach out234to us on [Discord](https://discord.gg/JFQmtFKCjd).235"""236 237FOOTER_CONTENT = """238<div class="page-footer">239 <h3>Join the Pruna AI community</h3>240 <div class="community-footer-links">241 <a rel="nofollow" href="https://x.com/PrunaAI" target="_blank">X</a>242 <a rel="nofollow" href="https://www.linkedin.com/company/pruna-ai" target="_blank">LinkedIn</a>243 <a rel="nofollow" href="https://www.instagram.com/pruna.ai/" target="_blank">Instagram</a>244 <a rel="nofollow" href="https://discord.gg/JFQmtFKCjd" target="_blank">Discord</a>245 <a rel="nofollow" href="https://github.com/PrunaAI/pruna" target="_blank">GitHub</a>246 <a rel="nofollow" href="https://www.pruna.ai/" target="_blank">pruna.ai</a>247 </div>248</div>249"""250 251CITATION_CONTENT = """252```bibtex253@misc{P-Bench,254 title={P-Bench: A Leaderboard for Text-to-Image Models},255 author={PrunaAI},256 year={2026},257 howpublished={\\url{https://huggingface.co/spaces/PrunaAI/P-Bench}}258}259```260"""261 262 263def render_header():264 gr.HTML(265 f"""266 <header class="app-header">267 <div class="app-header-bar">268 <div class="app-header-brand">269 <img class="app-header-logo" src="{_LOGO_DATA_URI}" alt="" />270 <h1>P-Bench</h1>271 </div>272 <button type="button" class="theme-toggle" data-mode="dark" aria-label="Switch to light mode" title="Switch to light mode">273 <svg class="theme-icon-sun" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true">274 <circle cx="12" cy="12" r="4"></circle>275 <path d="M12 2v2M12 20v2M4.93 4.93l1.41 1.41M17.66 17.66l1.41 1.41M2 12h2M20 12h2M4.93 19.07l1.41-1.41M17.66 6.34l1.41-1.41"></path>276 </svg>277 <svg class="theme-icon-moon" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" aria-hidden="true">278 <path d="M21 14.5A8.5 8.5 0 1 1 9.5 3 7 7 0 0 0 21 14.5z"></path>279 </svg>280 </button>281 </div>282 <p class="app-header-tagline">Compare models on quality, speed, and price</p>283 </header>284 """,285 padding=False,286 elem_classes="app-header-host",287 )288 289 290def _item(items, item_id):291 for item in items:292 if item["id"] == item_id:293 return item294 return items[0] if items else None295 296 297def _dataset_modality(dataset):298 return (dataset or {}).get("modality") or MODALITY_TEXT_TO_IMAGE299 300 301def _datasets_for_modality(datasets, modality):302 if not modality:303 return list(datasets)304 scoped = [305 dataset306 for dataset in datasets307 if _dataset_modality(dataset) == modality308 ]309 return scoped or list(datasets)310 311 312def _modality_choices(datasets):313 present = {_dataset_modality(dataset) for dataset in datasets}314 return [315 (label, value) for label, value in MODALITY_CHOICES if value in present316 ]317 318 319def _default_dataset_id(datasets, modality, preferred=None):320 scoped = _datasets_for_modality(datasets, modality)321 if preferred and any(dataset["id"] == preferred for dataset in scoped):322 return preferred323 return scoped[0]["id"] if scoped else None324 325 326def _dataset_choices(327 datasets, *, modality=None, require_samples=False, require_pareto=False328):329 scoped = _datasets_for_modality(datasets, modality)330 return [331 (dataset["name"], dataset["id"])332 for dataset in scoped333 if (not require_samples or dataset.get("samples"))334 and (not require_pareto or _dataset_has_pareto(datasets, dataset["id"]))335 ]336 337 338def _dataset_has_samples(datasets, dataset_id):339 dataset = _item(datasets, dataset_id)340 return bool(dataset and dataset.get("samples"))341 342 343def _sample_model_ids(datasets, dataset_id):344 dataset = _item(datasets, dataset_id)345 samples = dataset.get("samples") if dataset else None346 if not samples:347 return set()348 models = set(samples.get("models") or [])349 return models | {display_model_name(model) for model in models}350 351 352def _sample_media_map(samples):353 return (samples or {}).get("images") or {}354 355 356def _resolve_sample_model(samples, model):357 media = _sample_media_map(samples)358 if model in media:359 return model360 wanted = {str(model or "").strip(), display_model_name(model)}361 wanted.discard("")362 for key in media:363 if key in wanted or display_model_name(key) in wanted:364 return key365 return None366 367 368def _default_sample_models(samples):369 models = list((samples or {}).get("models") or [])370 preferred = [model for model in models if _is_pruna_model(model)]371 preferred.sort(372 key=lambda model: (373 "draft" in str(model).casefold()374 or "draft" in display_model_name(model).casefold(),375 display_model_name(model).casefold(),376 )377 )378 return (preferred or models)[:2]379 380 381def _pareto_price_column(data):382 columns = getattr(data, "columns", []) if data is not None else []383 for column in _PARETO_PRICE_COLUMNS:384 if column in columns:385 return column386 return None387 388 389def _pareto_time_column(data):390 columns = getattr(data, "columns", []) if data is not None else []391 for column in _PARETO_TIME_COLUMNS:392 if column in columns:393 return column394 return None395 396 397def _dataset_has_pareto(datasets, dataset_id):398 dataset = _item(datasets, dataset_id)399 data = dataset.get("data") if dataset else None400 return (401 _pareto_price_column(data) is not None402 or _pareto_time_column(data) is not None403 )404 405 406def _dataset_dropdown_update(datasets, tab, dataset_id, modality=None):407 """Limit the dataset list to what the current tab can show."""408 if modality is None:409 modality = _dataset_modality(_item(datasets, dataset_id))410 return gr.update(411 choices=_dataset_choices(412 datasets,413 modality=modality,414 require_samples=tab == TAB_SAMPLES415 and _dataset_has_samples(datasets, dataset_id),416 require_pareto=tab == TAB_PARETO417 and _dataset_has_pareto(datasets, dataset_id),418 ),419 value=dataset_id,420 )421 422 423ALL_METRICS_ID = "__all__"424 425 426def _metric_choices(datasets, metrics, dataset_id):427 dataset = _item(datasets, dataset_id)428 if not dataset:429 return []430 allowed = set(dataset.get("metric_ids") or [])431 data = dataset.get("data")432 columns = getattr(data, "columns", [])433 return [434 (_display_label(metric["column"]), metric["id"])435 for metric in metrics436 if metric["id"] in allowed and metric["column"] in columns437 ]438 439 440def _metric_dropdown_choices(datasets, metrics, dataset_id):441 choices = _metric_choices(datasets, metrics, dataset_id)442 if not choices:443 return []444 return [("Select all", ALL_METRICS_ID)] + choices445 446 447def _normalize_metric_ids(metric_id):448 if metric_id is None or metric_id == "":449 return []450 if isinstance(metric_id, (list, tuple)):451 return [item for item in metric_id if item]452 return [metric_id]453 454 455def _coerce_metric(datasets, metrics, dataset_id, metric_id):456 """Valid metric ids for this dataset. Empty means all metrics."""457 wanted = _normalize_metric_ids(metric_id)458 valid_ids = [choice[1] for choice in _metric_choices(datasets, metrics, dataset_id)]459 valid = set(valid_ids)460 if ALL_METRICS_ID in wanted:461 return list(valid_ids)462 return [item for item in wanted if item in valid]463 464 465def _metric_dropdown_value(metric_id):466 return [467 item468 for item in _normalize_metric_ids(metric_id)469 if item != ALL_METRICS_ID470 ]471 472 473def _model_choices(datasets, dataset_id, *, require_samples=False):474 cached = _MODEL_CHOICES_CACHE.get(dataset_id)475 if cached is None:476 dataset = _item(datasets, dataset_id)477 data = dataset.get("data") if dataset else None478 if data is None or "Model" not in getattr(data, "columns", []):479 cached = []480 else:481 models = data["Model"].dropna().astype(str).unique().tolist()482 # (label, value) so the UI shows the shared name but filters on the raw id.483 cached = sorted(484 ((display_model_name(model), model) for model in models),485 key=lambda item: item[0].casefold(),486 )487 _MODEL_CHOICES_CACHE[dataset_id] = cached488 if not require_samples:489 return cached490 allowed = _sample_model_ids(datasets, dataset_id)491 return [choice for choice in cached if choice[1] in allowed]492 493 494def _model_choice_values(choices):495 return [choice[1] if isinstance(choice, tuple) else choice for choice in choices]496 497 498def _metric_columns(datasets, metrics, dataset_id):499 dataset = _item(datasets, dataset_id)500 if not dataset:501 return []502 data = dataset.get("data")503 columns = getattr(data, "columns", [])504 allowed = set(dataset.get("metric_ids") or [])505 return [506 metric["column"]507 for metric in metrics508 if metric["id"] in allowed and metric["column"] in columns509 ]510 511 512_LEADERBOARD_IDENTITY_COLUMNS = [513 "Model",514 "Platform",515 "Endpoint Owner",516 "Optimized",517]518_LEADERBOARD_META_COLUMNS = [519 "Time / Output Video Second (s)",520 "Median Generation Time (s)",521 "Min Generation Time (s)",522 "Price / Image (USD)",523 "Price / Second of Video (USD)",524 "Evaluation Date (UTC)",525 "Date",526]527 528 529def _columns_for_metric(dataset, score_columns):530 """When metrics are selected, show identity + those scores + time/price."""531 available = list(getattr(dataset.get("data"), "columns", [])) or list(532 dataset.get("columns") or []533 )534 identity = [column for column in _LEADERBOARD_IDENTITY_COLUMNS if column in available]535 meta = [column for column in _LEADERBOARD_META_COLUMNS if column in available]536 scores = [column for column in (score_columns or []) if column in available]537 if scores:538 return [*identity, *scores, *meta]539 return [column for column in (dataset.get("columns") or available) if column != "URL"]540 541 542def resolve_view(datasets, metrics, dataset_id, metric_id):543 dataset = _item(datasets, dataset_id)544 if not dataset:545 return None546 data_columns = getattr(dataset.get("data"), "columns", [])547 score_columns = []548 for metric_key in _coerce_metric(datasets, metrics, dataset_id, metric_id):549 metric = _item(metrics, metric_key)550 if metric and metric["column"] in data_columns:551 score_columns.append(metric["column"])552 if score_columns:553 columns = _columns_for_metric(dataset, score_columns)554 else:555 columns = [556 column557 for column in (dataset.get("columns") or [])558 if column != "URL"559 ]560 score_columns = _metric_columns(datasets, metrics, dataset_id)561 return {562 "data": dataset["data"],563 "columns": columns,564 "score_column": score_columns[0] if score_columns else None,565 "score_columns": score_columns,566 "samples": dataset.get("samples"),567 "note": dataset.get("note"),568 }569 570 571def _format_leaderboard_cell(column, value):572 if pd.isna(value) or value is None or value == "":573 return "-"574 label = str(column).lower()575 if label == "rank":576 return str(int(value))577 if "price" in label:578 return _format_price(value)579 if "time" in label or "generation" in label:580 try:581 return f"{float(value):.2f}"582 except (TypeError, ValueError):583 return escape(str(value))584 if label in {"model", "platform", "endpoint owner", "optimized"}:585 if label == "model":586 return escape(display_model_name(value))587 return escape(str(value))588 try:589 number = float(value)590 except (TypeError, ValueError):591 return escape(str(value))592 if abs(number) >= 100:593 return f"{number:.1f}"594 return f"{number:.4f}".rstrip("0").rstrip(".")595 596 597def _leaderboard_sort_value(column, value):598 """Raw value used by client-side column sorting."""599 if pd.isna(value) or value is None or value == "":600 return ""601 label = str(column).lower()602 if label in {"model", "platform", "endpoint owner", "optimized", "date"}:603 return str(value).casefold()604 try:605 return f"{float(value):.10g}"606 except (TypeError, ValueError):607 return str(value).casefold()608 609 610def _leaderboard_sort_type(column):611 label = str(column).lower()612 if label in {"model", "platform", "endpoint owner", "optimized", "date"}:613 return "text"614 return "number"615 616 617def _leaderboard_cell_class(column):618 if column == "Rank":619 return "rank"620 if column == "Model":621 return "model-cell"622 if _leaderboard_sort_type(column) == "text":623 return "text-cell"624 return "metric-score"625 626 627def _assign_leaderboard_ranks(data, overall_column):628 """Rank the full table by the selected metric. Filters keep these numbers."""629 if data is None:630 return data631 ranked = data.copy()632 if "Rank" in ranked.columns:633 ranked = ranked.drop(columns=["Rank"])634 if overall_column and overall_column in ranked.columns:635 ranked = ranked.sort_values(636 overall_column, ascending=False, na_position="last"637 )638 ranked = ranked.reset_index(drop=True)639 ranked.insert(0, "Rank", ranked.index + 1)640 return ranked641 642 643def _leaderboard_html(data, columns):644 leaderboard = _leaderboard_dataframe(data, columns)645 if leaderboard.empty:646 return (647 '<div class="ranking-table-scroll">'648 '<div class="ranking-table empty-state">No models match the current filters.</div>'649 "</div>"650 )651 652 header_cells = []653 for index, column in enumerate(leaderboard.columns):654 sort_type = _leaderboard_sort_type(column)655 cell_class = _leaderboard_cell_class(column)656 header_cells.append(657 f'<th class="sortable-col {cell_class}" data-col="{index}" '658 f'data-type="{sort_type}" title="Sort by {escape(str(column))}">'659 f"{escape(str(column))}</th>"660 )661 body_rows = []662 for _, row in leaderboard.iterrows():663 cells = []664 for column in leaderboard.columns:665 css = _leaderboard_cell_class(column)666 sort_value = escape(_leaderboard_sort_value(column, row[column]), quote=True)667 if column == "Model":668 cells.append(669 f'<td class="{css}" data-sort-value="{sort_value}">'670 f"<strong>{_format_leaderboard_cell(column, row[column])}</strong></td>"671 )672 else:673 cells.append(674 f'<td class="{css}" data-sort-value="{sort_value}">'675 f"{_format_leaderboard_cell(column, row[column])}</td>"676 )677 body_rows.append(f"<tr>{''.join(cells)}</tr>")678 679 return f"""680 <div class="ranking-table-scroll">681 <div class="ranking-table">682 <table>683 <thead><tr>{''.join(header_cells)}</tr></thead>684 <tbody>{''.join(body_rows)}</tbody>685 </table>686 </div>687 </div>688 """689 690 691def _filter_choices(data, column):692 if data is None or column not in data.columns:693 return []694 return sorted(data[column].dropna().astype(str).unique().tolist())695 696 697def _filter_leaderboard(data, platform, owner, optimized, models=None):698 filtered = data.copy()699 if models:700 if "Model" in filtered.columns:701 filtered = filtered[filtered["Model"].astype(str).isin(models)]702 703 for column, values in [704 ("Platform", platform),705 ("Endpoint Owner", owner),706 ("Optimized", optimized),707 ]:708 if values and column in filtered.columns:709 filtered = filtered[filtered[column].astype(str).isin(values)]710 return filtered711 712 713def _leaderboard_dataframe(data, columns):714 ordered = []715 if "Rank" in data.columns:716 ordered.append("Rank")717 for column in columns:718 if (719 column in data.columns720 and column not in {"URL", "Rank"}721 and column not in ordered722 ):723 ordered.append(column)724 return data[ordered].rename(columns=_display_label)725 726 727def _display_label(column):728 labels = {729 "OneIG Overall Score": "Overall",730 "OneIG (Anime Alignment)": "Anime",731 "OneIG (Human Alignment)": "Human",732 "OneIG (Object Alignment)": "Object",733 "OneIG Anime Elo": "Anime Elo (Datapoint)",734 "OneIG Human Elo": "Human Elo (Datapoint)",735 "OneIG Object Elo": "Object Elo (Datapoint)",736 "P-Judge Overall": "P-Judger (Pruna)",737 "Datapoint Elo": "Datapoint Elo",738 "Rapidata Elo": "Rapidata Elo",739 "Artificial Analysis Elo": "Artificial Analysis Elo",740 "Arena Elo": "Overall Elo",741 "Arena Branding / Commercial Elo": "Branding / Commercial",742 "Arena 3D Imaging Elo": "3D Imaging",743 "Arena Cartoon / Anime Elo": "Cartoon / Anime",744 "Arena Photorealistic Elo": "Photorealistic",745 "Arena Art Elo": "Art",746 "Arena Portraits Elo": "Portraits",747 "Arena Text Rendering Elo": "Text Rendering",748 "Median Generation Time (s)": "Median generation time",749 "Min Generation Time (s)": "Min generation time",750 "Time / Output Video Second (s)": "Generation time per second of video",751 "Price / Image (USD)": "Price per image",752 "Price / Second of Video (USD)": "Price per second of video",753 "Evaluation Date (UTC)": "Date",754 "Date": "Date",755 }756 return labels.get(column, column)757 758 759def _format_price(value):760 return "-" if pd.isna(value) or value is None else f"${float(value):.3f}"761 762 763def _pareto_frontier_mask(x_values, scores):764 """True for non-dominated points when maximizing score and minimizing x."""765 n = len(x_values)766 mask = [True] * n767 for i in range(n):768 for j in range(n):769 if i == j:770 continue771 better_or_equal = x_values[j] <= x_values[i] and scores[j] >= scores[i]772 strictly_better = x_values[j] < x_values[i] or scores[j] > scores[i]773 if better_or_equal and strictly_better:774 mask[i] = False775 break776 return mask777 778 779def _pareto_unavailable_html(message):780 return (781 "<p class='pareto-note-copy'>"782 f"{escape(message)}"783 "</p>"784 )785 786 787def _pareto_note_update(message):788 if message:789 return gr.update(790 value=_pareto_unavailable_html(message),791 visible=True,792 )793 return gr.update(value="", visible=False)794 795 796def _pareto_plot_update(fig):797 if fig is not None:798 return gr.update(value=fig, visible=True)799 return gr.update(value=None, visible=False)800 801 802def _skip_all(count):803 return tuple(gr.skip() for _ in range(count))804 805 806def _pareto_skip_updates():807 return _skip_all(_PARETO_SLOT_COUNT)808 809 810def _selection_key(dataset_id, metric_id, models):811 return (812 dataset_id,813 tuple(_normalize_metric_ids(metric_id)),814 tuple(models or ()),815 )816 817 818def _applied_key(view_state):819 view_state = view_state or {}820 return _selection_key(821 view_state.get("dataset_id"),822 view_state.get("metric_id"),823 view_state.get("models"),824 )825 826 827def _is_pruna_model(model_id) -> bool:828 raw = str(model_id or "").casefold()829 label = display_model_name(model_id).casefold()830 return any(831 value.startswith(prefix)832 for value in (raw, label)833 for prefix in ("p-image", "p_image", "p-video", "p_video")834 )835 836 837def _pareto_fill_colors(models):838 return [839 _PARETO_PRUNA_COLOR if _is_pruna_model(model) else _PARETO_OTHER_COLOR840 for model in models841 ]842 843 844def _build_pareto_figure(845 data,846 score_column,847 x_column,848 x_title,849 x_hover_prefix="",850 x_hover_suffix="",851 x_axis_type="linear",852):853 scatter = (854 data[["Model", score_column, x_column]]855 .dropna()856 .copy()857 .reset_index(drop=True)858 )859 if scatter.empty:860 return None861 862 x_values = scatter[x_column].astype(float).tolist()863 scores = scatter[score_column].astype(float).tolist()864 on_frontier = _pareto_frontier_mask(x_values, scores)865 866 dominated = scatter.loc[[not flag for flag in on_frontier]].copy()867 frontier = scatter.loc[on_frontier].sort_values(x_column).copy()868 dominated_colors = _pareto_fill_colors(dominated["Model"]) if not dominated.empty else []869 frontier_colors = _pareto_fill_colors(frontier["Model"]) if not frontier.empty else []870 if not dominated.empty:871 dominated["Model"] = dominated["Model"].map(display_model_name)872 if not frontier.empty:873 frontier["Model"] = frontier["Model"].map(display_model_name)874 hover = (875 "<b>%{text}</b><br>"876 f"{escape(x_title)}: {x_hover_prefix}%{{x:.4f}}{x_hover_suffix}"877 "<br>Score: %{y:.4f}<extra></extra>"878 )879 880 fig = go.Figure()881 if not dominated.empty:882 fig.add_trace(883 go.Scatter(884 x=dominated[x_column],885 y=dominated[score_column],886 mode="markers",887 name="Below frontier",888 text=dominated["Model"],889 hovertemplate=hover,890 showlegend=False,891 marker={892 "size": 9,893 "color": dominated_colors,894 "opacity": 0.85,895 "line": {"width": 0},896 },897 )898 )899 if not frontier.empty:900 fig.add_trace(901 go.Scatter(902 x=frontier[x_column],903 y=frontier[score_column],904 mode="lines+markers",905 name="On frontier",906 text=frontier["Model"],907 hovertemplate=hover,908 showlegend=False,909 line={"color": _PARETO_FRONTIER_OUTLINE, "width": 2.5},910 marker={911 "size": 12,912 "color": frontier_colors,913 "line": {"width": 2.5, "color": _PARETO_FRONTIER_OUTLINE},914 },915 )916 )917 for name, marker in (918 (919 "Pruna",920 {921 "size": 10,922 "color": _PARETO_PRUNA_COLOR,923 "line": {"width": 0},924 },925 ),926 (927 "Other models",928 {929 "size": 10,930 "color": _PARETO_OTHER_COLOR,931 "line": {"width": 0},932 },933 ),934 (935 "On frontier",936 {937 "size": 12,938 "color": "rgba(0,0,0,0)",939 "line": {"width": 2.5, "color": _PARETO_FRONTIER_OUTLINE},940 },941 ),942 ):943 fig.add_trace(944 go.Scatter(945 x=[None],946 y=[None],947 mode="markers",948 name=name,949 marker=marker,950 hoverinfo="skip",951 )952 )953 954 score_label = _display_label(score_column)955 fig.update_layout(956 title=None,957 xaxis_title=x_title,958 yaxis_title=score_label,959 autosize=True,960 height=420,961 margin={"l": 56, "r": 28, "t": 28, "b": 80},962 legend={963 "orientation": "h",964 "yanchor": "top",965 "y": -0.24,966 "xanchor": "center",967 "x": 0.5,968 "bgcolor": "rgba(0,0,0,0)",969 "font": {"color": "#d4d4d4", "size": 12},970 },971 plot_bgcolor="#1d1429",972 paper_bgcolor="#171021",973 font={"color": "#d4d4d4", "size": 13},974 )975 axis_font = {"color": "#fafafa", "size": 13}976 tick_font = {"color": "#a3a3a3", "size": 12}977 x_axis_ticks = {}978 if x_axis_type == "log":979 positive_x = scatter.loc[scatter[x_column] > 0, x_column].astype(float)980 if not positive_x.empty:981 minimum = positive_x.min()982 maximum = positive_x.max()983 tick_values = [984 factor * (10**exponent)985 for exponent in range(986 floor(log10(minimum)),987 ceil(log10(maximum)) + 1,988 )989 for factor in (1, 2, 5)990 if minimum * 0.8 <= factor * (10**exponent) <= maximum * 1.2991 ]992 x_axis_ticks = {993 "tickmode": "array",994 "tickvals": tick_values,995 "ticktext": [f"{value:g}" for value in tick_values],996 }997 fig.update_xaxes(998 type=x_axis_type,999 **x_axis_ticks,1000 showgrid=True,1001 gridcolor="rgba(74, 57, 98, 0.55)",1002 zeroline=False,1003 title_font=axis_font,1004 tickfont=tick_font,1005 color="#d4d4d4",1006 )1007 fig.update_yaxes(1008 showgrid=True,1009 gridcolor="rgba(74, 57, 98, 0.55)",1010 zeroline=False,1011 title_font=axis_font,1012 tickfont=tick_font,1013 color="#d4d4d4",1014 )1015 return fig1016 1017 1018def _is_log_scale(scale):1019 return scale == "Logarithmic"1020 1021 1022def _pareto_axis_type(scale):1023 return "log" if _is_log_scale(scale) else "linear"1024 1025 1026def _pareto_scale_radio(*extra_classes):1027 return gr.Radio(1028 choices=_PARETO_SCALE_CHOICES,1029 value=_PARETO_SCALE_DEFAULT,1030 show_label=False,1031 container=False,1032 elem_classes=["pareto-scale-toggle", *extra_classes],1033 )1034 1035 1036def _pareto_plot_heading(title):1037 with gr.Row(equal_height=False, elem_classes="pareto-heading-row"):1038 gr.Markdown(f"#### {title}", elem_classes="pareto-subhead")1039 with gr.Column(min_width=140, elem_classes="pareto-scale-control"):1040 return _pareto_scale_radio()1041 1042 1043def _default_pareto_scales():1044 return [_PARETO_SCALE_DEFAULT] * MAX_PARETO_METRICS1045 1046 1047def _normalize_pareto_scales(scales):1048 values = list(scales or [])1049 if len(values) < MAX_PARETO_METRICS:1050 values.extend(1051 [_PARETO_SCALE_DEFAULT] * (MAX_PARETO_METRICS - len(values))1052 )1053 return values[:MAX_PARETO_METRICS]1054 1055 1056def _uniform_pareto_scales(scale):1057 return [scale] * MAX_PARETO_METRICS1058 1059 1060def _pareto_master_scale_update(price_scales, time_scales):1061 values = list(price_scales) + list(time_scales)1062 if values and all(value == values[0] for value in values):1063 return gr.update(value=values[0])1064 return gr.update(value=None)1065 1066 1067def _pareto_axis(data, score_column, x_column, x_title, missing_message, empty_message, **hover):1068 if x_column not in data.columns:1069 return None, missing_message1070 fig = _build_pareto_figure(1071 data,1072 score_column,1073 x_column=x_column,1074 x_title=x_title,1075 **hover,1076 )1077 if fig is None:1078 return None, empty_message1079 return fig, None1080 1081 1082def _pareto_pair(1083 data,1084 score_column,1085 latency_scale=_PARETO_SCALE_DEFAULT,1086 price_scale=_PARETO_SCALE_DEFAULT,1087):1088 score_missing = "No score data is available for this metric."1089 if data is None or not score_column or score_column not in data.columns:1090 return None, score_missing, None, score_missing1091 1092 price_column = _pareto_price_column(data) or _PARETO_PRICE_COLUMN1093 price_title = _PARETO_PRICE_TITLES.get(price_column, "Price (USD)")1094 price_missing = (1095 "Price per second of video isn't available for this dataset."1096 if price_column == _PARETO_VIDEO_PRICE_COLUMN1097 else "Price per image isn't available for this dataset."1098 )1099 price_fig, price_message = _pareto_axis(1100 data,1101 score_column,1102 price_column,1103 price_title,1104 price_missing,1105 "No models have both a score and a price for this metric.",1106 x_hover_prefix="$",1107 x_axis_type=_pareto_axis_type(price_scale),1108 )1109 time_column = _pareto_time_column(data) or _PARETO_TIME_COLUMN1110 time_title = _PARETO_TIME_TITLES.get(time_column, "Generation time (s)")1111 time_missing = (1112 "Generation time per second of video isn't available for this dataset."1113 if time_column == _PARETO_VIDEO_TIME_COLUMN1114 else "Min generation time isn't available for this dataset."1115 )1116 time_empty = (1117 "No models have both a score and generation time per second of "1118 "video for this metric."1119 if time_column == _PARETO_VIDEO_TIME_COLUMN1120 else "No models have both a score and a min generation time for this metric."1121 )1122 time_fig, time_message = _pareto_axis(1123 data,1124 score_column,1125 time_column,1126 time_title,1127 time_missing,1128 time_empty,1129 x_hover_suffix="s",1130 x_axis_type=_pareto_axis_type(latency_scale),1131 )1132 return price_fig, price_message, time_fig, time_message1133 1134 1135def _pareto_dataset_message(data):1136 has_price = _pareto_price_column(data) is not None1137 has_time = _pareto_time_column(data) is not None1138 if has_price or has_time:1139 return None1140 return (1141 "Price and generation time aren't available for "1142 "this dataset, so these plots can't be drawn."1143 )1144 1145 1146def _pareto_slot_note(price_fig, price_message, time_fig, time_message, data):1147 has_price = _pareto_price_column(data) is not None1148 has_time = _pareto_time_column(data) is not None1149 notes = []1150 if has_price and not has_time:1151 notes.append(1152 "Generation time isn't available for this dataset, so only "1153 "price vs score is shown."1154 )1155 elif has_time and not has_price:1156 notes.append(1157 "Price isn't available for this dataset, so only "1158 "time vs score is shown."1159 )1160 if price_fig is None and has_price:1161 notes.append(price_message)1162 if time_fig is None and has_time:1163 notes.append(time_message)1164 if len(notes) == 2 and notes[0] == notes[1]:1165 notes = notes[:1]1166 return " ".join(notes)1167 1168 1169def _pareto_slot_updates(1170 data,1171 score_columns,1172 price_scales=None,1173 time_scales=None,1174):1175 """Updates for a fixed bank of Gradio Plot slots (visible/hidden)."""1176 score_columns = [column for column in (score_columns or []) if column]1177 price_scales = _normalize_pareto_scales(price_scales)1178 time_scales = _normalize_pareto_scales(time_scales)1179 has_price = _pareto_price_column(data) is not None1180 has_time = _pareto_time_column(data) is not None1181 dataset_note = _pareto_dataset_message(data)1182 updates = [_pareto_note_update(dataset_note)]1183 hide_all_slots = not has_price and not has_time1184 hidden_slot = (1185 gr.update(visible=False),1186 "",1187 _pareto_note_update(""),1188 gr.update(visible=False),1189 gr.update(visible=False),1190 _pareto_plot_update(None),1191 gr.update(visible=False),1192 _pareto_plot_update(None),1193 )1194 for index in range(MAX_PARETO_METRICS):1195 if hide_all_slots or index >= len(score_columns):1196 updates.extend(hidden_slot)1197 continue1198 score_column = score_columns[index]1199 price_fig, price_message, time_fig, time_message = _pareto_pair(1200 data,