lerobot/video-benchmark
2
1"""Canonical UI vocabulary for the Video Benchmark.2 3Single source of truth for everything the Gradio app needs to *describe*4a column, option, parameter group, leaderboard ranking, or About-page5narration. ``app.py`` imports these constants directly to wire up its6filter chips, dropdowns, table headers, and prose blocks; adding or7renaming a knob means editing exactly one file.8 9Each ``COLUMNS`` entry carries display metadata plus, for numeric10columns, a printf-style ``fmt_spec`` (e.g. ``"%.2f"``) so values render11consistently across the table and tooltips.12"""13from __future__ import annotations14 15from typing import Any16 17from dataclasses import dataclass18 19 20@dataclass(frozen=True)21class BenchmarkConfig:22 """Everything the UI needs to render one benchmark's tabs.23 24 A benchmark bundles its Hub datasets, column metadata, filter-chip25 vocabularies, leaderboard configuration, Submit-form options, and26 About/Parameters prose. ``app.py`` builds one tab set per benchmark27 from this config, so adding a benchmark means adding one instance.28 """29 key: str30 label: str31 title: str32 subtitle: str33 results_dataset: str34 submissions_dataset: str35 columns: list[dict[str, Any]]36 repos: list[str]37 filter_options: dict[str, list[str]] # chip key -> options (UI vocabulary)38 filter_order: list[str] # chip render order (keys of filter_options)39 leaderboard_axes: list[str]40 leaderboard_cats: dict[str, dict[str, Any]]41 leaderboard_group_keys: tuple[str, ...]42 submit_options: dict[str, list[str]]43 submit_defaults: dict[str, Any]44 full_sweep: dict[str, Any]45 param_groups: list[dict[str, Any]]46 param_notes: dict[str, dict[str, str]]47 about_html: str48 keep_keys: tuple[str, ...]49 scatter_x: str # Compare scatter x metric key50 scatter_y: str # Compare scatter y metric key51 codec_count: int # for the hero stats row52 backend_count: int53 54 55# --------------------------------------------------------------------------- #56# Datasets the UI talks to. Mirrored as constants so the frontend can render57# friendly links without duplicating strings.58# --------------------------------------------------------------------------- #59RESULTS_DATASET = "lerobot/video-benchmark-results"60SUBMISSIONS_DATASET = "lerobot/video-benchmark-submissions"61RESULTS_DATASET_DEPTH = "lerobot/depth-benchmark-results"62 63 64# --------------------------------------------------------------------------- #65# Column metadata66#67# Order matters: it drives table column order, the Column-picker grouping,68# Parameters-page ordering, and Compare-tab dropdowns. ``metric=True`` flags69# numeric columns that participate in composite ranking and color ramps;70# ``lower=True`` / ``higher=True`` set the polarity.71# --------------------------------------------------------------------------- #72COLUMNS: list[dict[str, Any]] = [73 # Config74 {"key": "repo_id", "label": "Dataset", "short": "Dataset", "group": "Config",75 "desc": "Hugging Face Hub dataset repo ID. One representative episode per dataset."},76 {"key": "vcodec", "label": "Codec", "short": "Codec", "group": "Config",77 "desc": "Video codec. Generic codecs (h264, hevc, av1) are supported, or you can pin a specific encoding library (e.g. libsvtav1)."},78 {"key": "encoder", "label": "Encoder", "short": "Encoder", "group": "Config", "default_hidden": True,79 "desc": "Encoding library used to produce the video."},80 {"key": "decoder", "label": "Decoder", "short": "Decoder", "group": "Config", "default_hidden": True,81 "desc": "Decoding library used to read the video."},82 {"key": "pix_fmt", "label": "Pixel format", "short": "Pixel format", "group": "Config",83 "desc": "Pixel format. RGB uses three-channel 8-bit (uint8) formats (e.g. yuv420p, yuv444p)."},84 {"key": "g", "label": "GOP", "short": "GOP", "group": "Config",85 "desc": "Group Of Pictures — the keyframe interval. A small value keeps seeks cheap but makes files bigger, while a large value compresses better at the cost of expensive random access."},86 {"key": "crf", "label": "CRF", "short": "CRF", "group": "Config",87 "desc": "Constant Rate Factor — the quality knob: lower gives bigger files and higher fidelity, higher gives smaller files and more loss."},88 {"key": "timestamps_mode", "label": "Access pattern", "short": "Access", "group": "Config",89 "desc": "How frames are requested. 1_frame: seek, decode one frame, done. 2_frames: two adjacent frames. 6_frames: contiguous window. 2_frames_4_space: two samples four frames apart — worst-case for GOP-heavy settings."},90 {"key": "backend", "label": "Backend", "short": "Backend", "group": "Config",91 "desc": "Video decoding library (pyav, torchcodec)."},92 93 # Compression94 {"key": "video_images_size_ratio", "label": "Video/Image\nsize ratio ↓", "short": "Video/Image size ratio",95 "group": "Compression", "metric": True, "lower": True, "fmt_spec": "%.4f",96 "desc": "Encoded video size ÷ sum of original PNG image sizes. Lower means better compression — the video takes less disk than the raw frames."},97 {"key": "video_images_load_time_ratio", "label": "Video/Image\nload ratio ↓", "short": "Video/Image load ratio",98 "group": "Compression", "metric": True, "lower": True, "fmt_spec": "%.4f",99 "desc": "Video decoding time ÷ PNG image load time for the same frames. Below 1 means the video decodes faster than reading the raw frames; above 1 means you pay for the compression at read time."},100 101 # Speed102 {"key": "median_load_time_video_ms", "label": "Decoding (ms) ↓", "short": "Decoding",103 "group": "Speed", "metric": True, "lower": True, "fmt_spec": "%.2f", "std_key": "std_load_time_video_ms",104 "desc": "Median wall-clock time to decode N frames from the compressed video."},105 {"key": "encoding_time_ms", "label": "Encoding (ms) ↓", "short": "Encoding",106 "group": "Speed", "metric": True, "lower": True, "fmt_spec": "%.2f",107 "desc": "Wall time to encode the whole episode once."},108 {"key": "encoding_fps", "label": "Encoding (fps) ↑", "short": "Encoding fps",109 "group": "Speed", "metric": True, "higher": True, "fmt_spec": "%.1f",110 "desc": "Encoding throughput — frames per second across the whole episode."},111 112 # Quality113 {"key": "median_psnr", "label": "PSNR (dB) ↑", "short": "PSNR",114 "group": "Quality", "metric": True, "higher": True, "fmt_spec": "%.2f", "std_key": "std_psnr",115 "desc": "Peak signal-to-noise ratio in dB. Higher means the decoded frame is closer to the source, i.e. less reconstruction error."},116 {"key": "median_ssim", "label": "SSIM (0–1) ↑", "short": "SSIM",117 "group": "Quality", "metric": True, "higher": True, "fmt_spec": "%.4f", "std_key": "std_ssim",118 "desc": "Structural similarity index in [0,1]. Higher means the decoded frame better preserves the source's structure (1 = identical)."},119 {"key": "median_mse", "label": "MSE (px²) ↓", "short": "MSE",120 "group": "Quality", "metric": True, "lower": True, "fmt_spec": "%.2f", "std_key": "std_mse",121 "desc": "Mean-squared error between decoded and original frames (px²). Lower means less pixel error (0 = identical)."},122]123 124# Qualitative palette for leaderboard rank badges + radar polygons.125# Muted on purpose — the cards overlay four to twelve colors on the126# same card surface (and the radar stacks the same count on one axis127# system), so a saturated HF-brand palette reads as visual noise.128# These are chromatic neighbours of the brand hues, pulled 20–30%129# toward gray, so they stay recognizable next to the brand palette130# without fighting it for attention.131PODIUM_COLORS: list[str] = [132 "#6B8EBF", # dusty blue133 "#D99B5A", # warm sand134 "#7FA97E", # sage135 "#A98FBF", # muted lavender136 "#C98595", # dusty rose137 "#6FA3A3", # dusty teal138 "#B8A773", # soft olive139 "#8A95A8", # slate140 "#D1A663", # honey141 "#7FA3B8", # powder blue142 "#B88A73", # terracotta143 "#8F9F8A", # moss144]145 146# Codec → brand-aligned but desaturated accent color. Shared by the147# leaderboard cards (codec pill), the leaderboard dataframe (codec148# column chip), and anywhere else codec identity is rendered. Keys149# are lowercased for case-insensitive lookup.150CODEC_COLORS: dict[str, str] = {151 "h264": "#6B8EBF", # dusty blue (mirrors HF blue)152 "hevc": "#D99B5A", # warm sand (mirrors HF orange)153 "libsvtav1": "#7FA97E", # sage (mirrors HF green)154}155CODEC_FALLBACK_COLOR: str = "#8A95A8"156 157STACK_COLORS: list[str] = [158 "#6B8EBF", "#D99B5A", "#7FA97E", "#A98FBF", "#C98595", "#6FA3A3",159]160 161 162# --------------------------------------------------------------------------- #163# Filter-bar option lists (used by the Results tab's chip filters).164# These define the *UI vocabulary*; actual row values come from the Hub165# dataset and may be a subset.166# --------------------------------------------------------------------------- #167REPOS: list[str] = [168 "lerobot/pusht_image",169 "lerobot/aloha_mobile_shrimp_image",170 "lerobot/paris_street",171 "lerobot/kitchen",172]173VCODECS: list[str] = ["h264", "hevc", "libsvtav1"]174PIX_FMTS: list[str] = ["yuv444p", "yuv420p"]175G_VALUES: list[int] = [ 2, 3, 4, 5, 6, 10, 15, 20, 40]176CRF_VALUES: list[int] = [0, 5, 10, 15, 20, 25, 30, 40, 50]177TS_MODES: list[str] = ["1_frame", "2_frames", "2_frames_4_space", "6_frames"]178BACKENDS: list[str] = ["pyav", "torchcodec"]179 180 181# --------------------------------------------------------------------------- #182# Submit-form option lists. Wider than the filter-bar lists because the183# Submit form lets users *queue* sweeps over knobs that aren't represented184# in current results.185# --------------------------------------------------------------------------- #186SUBMIT_OPTIONS: dict[str, list[str]] = {187 "repos": REPOS,188 "vcodecs": VCODECS,189 "pix_fmts": PIX_FMTS,190 "g": ["1", "2", "3", "4", "5", "6", "10", "15", "20", "40", "100"],191 "crf": ["0", "5", "10", "15", "20", "25", "30", "40", "50"],192 "timestamps_modes": TS_MODES,193 "backends": BACKENDS,194}195 196SUBMIT_DEFAULTS: dict[str, Any] = {197 "repos": REPOS[:2],198 "vcodecs": ["h264"],199 "pix_fmts": ["yuv420p"],200 "g": ["2", "10"],201 "crf": ["10", "30"],202 "timestamps_modes": ["1_frame", "2_frames"],203 "backends": ["pyav"],204 "samples_per_config": 50,205}206 207 208# --------------------------------------------------------------------------- #209# Maintainer "full sweep" — every curated knob, full Cartesian product. The210# Submit tab exposes this behind a collapsed accordion (no public link). The211# resulting submission size (~tens of thousands of configs) intentionally212# blows past ``MAX_CONFIGS_PER_SUBMISSION``; the corresponding handler in213# ``app.py`` bypasses that cap because this path is gated behind a confirm214# checkbox and is meant to be triggered only by maintainers re-baselining215# the leaderboard.216# --------------------------------------------------------------------------- #217FULL_SWEEP: dict[str, Any] = {218 "repos": list(REPOS),219 "vcodecs": list(VCODECS),220 "pix_fmts": list(PIX_FMTS),221 "g": [str(v) for v in G_VALUES],222 "crf": [str(v) for v in CRF_VALUES],223 "timestamps_modes": list(TS_MODES),224 "backends": list(BACKENDS),225 "samples_per_config": 50,226}227 228 229# --------------------------------------------------------------------------- #230# Leaderboards231# --------------------------------------------------------------------------- #232# All categories share the same six axes — what changes between tabs is233# the *weighting* used to rank configurations.234LEADERBOARD_AXES: list[str] = [235 "encoding_time_ms",236 "median_load_time_video_ms",237 "video_images_size_ratio",238 "median_mse",239 "median_psnr",240 "median_ssim",241]242 243LEADERBOARD_CATS: dict[str, dict[str, Any]] = {244 "Overall": {245 "desc": "Balanced across encoding, decoding, size, and quality.",246 "weights": {247 "encoding_time_ms": 1, "median_load_time_video_ms": 1,248 "video_images_size_ratio": 1, "median_mse": 1,249 "median_psnr": 1, "median_ssim": 1,250 },251 },252 "Quality": {253 "desc": "Pure reconstruction fidelity: MSE, PSNR, SSIM only.",254 "weights": {255 "encoding_time_ms": 0, "median_load_time_video_ms": 0,256 "video_images_size_ratio": 0, "median_mse": 1,257 "median_psnr": 1, "median_ssim": 1,258 },259 },260 "Encoding": {261 "desc": "Encoding throughput and output size, fidelity ignored.",262 "weights": {263 "encoding_time_ms": 1, "median_load_time_video_ms": 0,264 "video_images_size_ratio": 1, "median_mse": 0,265 "median_psnr": 0, "median_ssim": 0,266 },267 },268 "Decoding": {269 "desc": "Pure decoding latency at the selected access pattern.",270 "weights": {271 "encoding_time_ms": 0, "median_load_time_video_ms": 1,272 "video_images_size_ratio": 0, "median_mse": 0,273 "median_psnr": 0, "median_ssim": 0,274 },275 },276}277 278 279# --------------------------------------------------------------------------- #280# Parameters reference (drives the About tab's "Parameters reference" pane)281# --------------------------------------------------------------------------- #282PARAM_GROUPS: list[dict[str, Any]] = [283 {"t": "Inputs — what you benchmark",284 "desc": "The corpus. Each dataset is a LeRobot episode recording with a few minutes of RGB observations.",285 "keys": ["repo_id"]},286 {"t": "Encoding — how the video is compressed",287 "desc": "Passed to the FFmpeg/PyAV encoder. These are the knobs operators actually tune.",288 "keys": ["vcodec", "pix_fmt", "g", "crf", "encoder_threads"]},289 {"t": "Decoding — how you read it back",290 "desc": "The other half of the equation. The same MP4 can decode very differently depending on library and access pattern.",291 "keys": ["backend", "timestamps_mode"]},292 {"t": "Fidelity metrics — how faithful is decoded frame vs. source",293 "desc": "We decode the compressed video, re-read the original PNG frames, and compare pixel-for-pixel.",294 "keys": ["median_mse", "median_psnr", "median_ssim"]},295 {"t": "Performance metrics — how fast, how small",296 "desc": "The cost side. Run on a single CPU thread unless noted.",297 "keys": ["encoding_time_ms", "encoding_fps", "video_images_size_ratio", "median_load_time_video_ms", "video_images_load_time_ratio"]},298]299 300# Reference-only parameters: documented on the About page's parameter301# reference but not part of the results data, so they have no column,302# filter, or Compare entry.303PARAM_NOTES: dict[str, dict[str, str]] = {304 "encoder_threads": {305 "label": "Encoder threads",306 "desc": "CPU threads used to encode a clip. Local-only — the hosted benchmark pins encoding to one thread for comparable timings, so it is not submittable here.",307 },308}309 310 311# --------------------------------------------------------------------------- #312# About-page prose. Authored here so the React layer just renders it. The313# code snippet uses literal newlines; React renders it inside a <pre>.314# --------------------------------------------------------------------------- #315ABOUT_HTML: str = """\316<h3 id="what">What this is</h3>317<p>318A benchmark for video encoding and decoding in the context of robotics datasets. LeRobot stores episode observations as MP4 rather than PNG sequences — this page quantifies <i>how much</i> we gain in size and what we pay in decoding latency and pixel fidelity.319</p>320 321<p>322<h3 id="metrics">Metrics</h3>323<p>The size and speed metrics measure the cost of storing and reading frames as video; the fidelity metrics measure what you lose to lossy compression.</p>324<p><b>Size & speed</b></p>325<ul>326<li><b>Video/image size ratio</b> — encoded video ÷ sum of PNG frames. Lower means better compression — the video takes less disk than the raw frames.</li>327<li><b>Video/image load ratio</b> — video decoding time ÷ PNG image load time for the same frames. <1 means the video is faster to read than the PNGs; >1 means you pay for the compression at read time.</li>328<li><b>Decoding time</b> — median wall-clock to decode N frames at a given timestamp.</li>329<li><b>Encoding time</b> — wall-clock to encode the whole clip, end-to-end.</li>330<li><b>Encoding fps</b> — encode throughput in frames per second, derived from the encoding time and the episode length. Higher means faster encoding; the handy complement to the raw time column.</li>331</ul>332<p><b>Fidelity</b> — every decoded frame is compared against the uncompressed source, pixel-for-pixel.</p>333<ul>334<li><b>PSNR (dB)</b> — peak signal-to-noise ratio in dB. Logarithmic, so +3 dB ≈ half the error. 40+ is excellent, 30 is acceptable, 20 is visible artefacts. Higher is closer to the source.</li>335<li><b>SSIM (0–1)</b> — structural similarity index, dimensionless. Perceptual — weights luminance, contrast, structure. 0.95+ is good, 0.80 is degraded. Higher is structure better preserved.</li>336<li><b>MSE (px²)</b> — mean-squared error in squared 8-bit pixel intensities (0–65025). Lower is less pixel error; 0 = identical.</li>337</ul>338</p>339 340<p>341<h3 id="access">Access patterns</h3>342<p>343Decode cost depends heavily on <i>how</i> frames are requested. <code>1_frame</code> pays the full seek+IDR cost per sample; <code>6_frames</code> amortizes it across contiguous frames; <code>2_frames_4_space</code> probes the worst case where the decoder must step across two distant windows.344</p>345 346<h3 id="repro">Reproduce locally</h3>347<p>The full sweep is open source. First, set <code>HF_RESULTS_REPO_ID</code> to the Hugging Face Hub dataset where you want results pushed:</p>348<pre style="background:var(--hf-gray-900);color:var(--hf-gray-100);padding:var(--space-4);border-radius:var(--radius-md);font-size:var(--fs-xs);overflow:auto">export HF_RESULTS_REPO_ID=lerobot/video-benchmark-results</pre>349<p>Then run the benchmark:</p>350<pre style="background:var(--hf-gray-900);color:var(--hf-gray-100);padding:var(--space-4);border-radius:var(--radius-md);font-size:var(--fs-xs);overflow:auto">python benchmark/video/run_video_benchmark.py \\351 --output-dir outputs/video_benchmark \\352 --repo-ids lerobot/pusht_image lerobot/kitchen \\353 --vcodec h264 hevc libsvtav1 \\354 --pix-fmt yuv420p yuv444p \\355 --g 2 10 40 \\356 --crf 10 20 30 \\357 --timestamps-modes 1_frame 2_frames 6_frames \\358 --backends pyav torchcodec \\359 --num-samples 50</pre>360 361<h3 id="contrib">Contribute</h3>362<p>363Submit your own configurations through the <b>Submit</b> tab. A background worker picks them up and pushes results to the Hub so the whole community benefits from the same measurements.364</p>365"""366 367 368RGB_KEEP_KEYS: tuple[str, ...] = (369 "repo_id", "vcodec", "encoder", "decoder", "pix_fmt", "g", "crf", "timestamps_mode", "backend",370 "video_images_size_ratio", "video_images_load_time_ratio",371 "median_load_time_video_ms", "std_load_time_video_ms",372 "median_load_time_images_ms", "std_load_time_images_ms",373 "median_psnr", "std_psnr", "median_ssim", "std_ssim",374 "median_mse", "std_mse", "encoding_fps", "encoding_time_ms", "created_at",375 "lerobot_version", "num_samples",376)377 378RGB = BenchmarkConfig(379 key="rgb",380 label="RGB",381 title="Video Encoding & Decoding Benchmark",382 subtitle=(383 "A live leaderboard for trade-offs between compression ratio, "384 "decoding speed, and image fidelity across video codecs, pixel "385 "formats, GOP sizes and CRF settings — measured on real robotics "386 "datasets."387 ),388 results_dataset=RESULTS_DATASET,389 submissions_dataset=SUBMISSIONS_DATASET,390 columns=COLUMNS,391 repos=REPOS,392 filter_options={393 "vcodec": VCODECS,394 "pix_fmt": PIX_FMTS,395 "backend": BACKENDS,396 "g": [str(v) for v in G_VALUES],397 "crf": [str(v) for v in CRF_VALUES],398 },399 filter_order=["vcodec", "pix_fmt", "backend", "g", "crf"],400 leaderboard_axes=LEADERBOARD_AXES,401 leaderboard_cats=LEADERBOARD_CATS,402 leaderboard_group_keys=("vcodec", "pix_fmt", "g", "crf", "backend"),403 submit_options=SUBMIT_OPTIONS,404 submit_defaults=SUBMIT_DEFAULTS,405 full_sweep=FULL_SWEEP,406 param_groups=PARAM_GROUPS,407 param_notes=PARAM_NOTES,408 about_html=ABOUT_HTML,409 keep_keys=RGB_KEEP_KEYS,410 scatter_x="video_images_size_ratio",411 scatter_y="median_psnr",412 codec_count=len(VCODECS),413 backend_count=len(BACKENDS),414)415 416 417# --------------------------------------------------------------------------- #418# Depth benchmark419#420# Mirrors the RGB structure for the depth Hub dataset421# ``lerobot/depth-benchmark-results``. Reuses the RGB ``TS_MODES`` /422# ``BACKENDS`` vocabularies defined above.423# --------------------------------------------------------------------------- #424DEPTH_COLUMNS: list[dict[str, Any]] = [425 # Config426 {"key": "repo_id", "label": "Dataset", "short": "Dataset", "group": "Config",427 "desc": "Hugging Face Hub dataset repo ID for the depth episode."},428 {"key": "vcodec", "label": "Codec", "short": "Codec", "group": "Config",429 "desc": "Video codec. Generic codecs (hevc, av1) are supported, or you can pin a specific encoding library (e.g. libaom-av1)."},430 {"key": "encoder", "label": "Encoder", "short": "Encoder", "group": "Config", "default_hidden": True,431 "desc": "Encoding library used to produce the video."},432 {"key": "decoder", "label": "Decoder", "short": "Decoder", "group": "Config", "default_hidden": True,433 "desc": "Decoding library used to read the video."},434 {"key": "pix_fmt", "label": "Pixel format", "short": "Pixel format", "group": "Config",435 "desc": "Pixel format. Depth uses high-bit-depth gray formats (e.g. gray12le, gray16le)."},436 {"key": "g", "label": "GOP", "short": "GOP", "group": "Config",437 "desc": "Group Of Pictures — the keyframe interval. A small value keeps seeks cheap but makes files bigger, while a large value compresses better at the cost of expensive random access."},438 {"key": "crf", "label": "CRF", "short": "CRF", "group": "Config",439 "desc": "Constant Rate Factor — the quality knob: lower gives bigger files and higher fidelity, higher gives smaller files and more loss (ignored when lossless)."},440 {"key": "lossless", "label": "Lossless", "short": "Lossless", "group": "Config",441 "desc": "Whether the codec runs in a mathematically lossless mode."},442 {"key": "use_log", "label": "Log encode", "short": "Log encode", "group": "Config",443 "desc": "Whether depth is log-transformed before quantization, allocating more precision to near depths."},444 {"key": "depth_min", "label": "Depth min (m)", "short": "Depth min", "group": "Config",445 "fmt_spec": "%.3f",446 "desc": "Lower bound of the depth range mapped into the encoded value range, in meters."},447 {"key": "depth_max", "label": "Depth max (m)", "short": "Depth max", "group": "Config",448 "fmt_spec": "%.3f",449 "desc": "Upper bound of the depth range mapped into the encoded value range, in meters."},450 {"key": "shift", "label": "Shift (m)", "short": "Shift", "group": "Config",451 "fmt_spec": "%.3f",452 "desc": "Offset applied to depth before encoding, in meters."},453 {"key": "timestamps_mode", "label": "Access pattern", "short": "Access", "group": "Config",454 "desc": "How frames are requested. 1_frame: seek, decode one frame, done. 2_frames: two adjacent frames. 6_frames: contiguous window. 2_frames_4_space: two samples four frames apart — worst-case for GOP-heavy settings."},455 {"key": "backend", "label": "Backend", "short": "Backend", "group": "Config",456 "desc": "Video decoding library (pyav)."},457 458 # Compression459 {"key": "video_images_size_ratio", "label": "Video/Image\nsize ratio ↓", "short": "Video/Image size ratio",460 "group": "Compression", "metric": True, "lower": True, "fmt_spec": "%.4f",461 "desc": "Encoded video size ÷ sum of original depth image sizes. Lower means better compression — the video takes less disk than the raw frames."},462 {"key": "video_images_load_time_ratio", "label": "Video/Image\nload ratio ↓", "short": "Video/Image load ratio",463 "group": "Compression", "metric": True, "lower": True, "fmt_spec": "%.4f",464 "desc": "Video decoding time ÷ image load time for the same frames. Below 1 means the video decodes faster than reading the raw frames; above 1 means you pay for the compression at read time."},465 466 # Speed467 {"key": "median_load_time_video_ms", "label": "Decoding (ms) ↓", "short": "Decoding",468 "group": "Speed", "metric": True, "lower": True, "fmt_spec": "%.2f", "std_key": "std_load_time_video_ms",469 "desc": "Median wall-clock time to decode N frames from the compressed video."},470 {"key": "encoding_time_ms", "label": "Encoding (ms) ↓", "short": "Encoding",471 "group": "Speed", "metric": True, "lower": True, "fmt_spec": "%.2f",472 "desc": "Wall time to encode the whole episode once."},473 {"key": "encoding_fps", "label": "Encoding (fps) ↑", "short": "Encoding fps",474 "group": "Speed", "metric": True, "higher": True, "fmt_spec": "%.1f",475 "desc": "Encoding throughput — frames per second across the whole episode."},476 477 # Quality478 {"key": "median_rmse_m", "label": "RMSE (m) ↓", "short": "RMSE",479 "group": "Quality", "metric": True, "lower": True, "fmt_spec": "%.4f", "std_key": "std_rmse_m",480 "desc": "Root-mean-squared error between decoded and source depth, in meters. Lower means the decoded depth is closer to the source."},481 {"key": "median_mae_m", "label": "MAE (m) ↓", "short": "MAE",482 "group": "Quality", "metric": True, "lower": True, "fmt_spec": "%.4f", "std_key": "std_mae_m",483 "desc": "Mean absolute error between decoded and source depth, in meters. Lower means a smaller average depth error."},484 {"key": "median_absrel", "label": "AbsRel ↓", "short": "AbsRel",485 "group": "Quality", "metric": True, "lower": True, "fmt_spec": "%.4f", "std_key": "std_absrel",486 "desc": "Absolute relative error: mean(|decoded - source| / source). Lower means less error relative to the true depth."},487 {"key": "median_delta1", "label": "δ<1.25 ↑", "short": "δ1",488 "group": "Quality", "metric": True, "higher": True, "fmt_spec": "%.4f", "std_key": "std_delta1",489 "desc": "Fraction of pixels with max(d/d*, d*/d) < 1.25. Higher means more pixels land within the accuracy threshold (1.0 = all correct)."},490 {"key": "median_delta2", "label": "δ<1.25² ↑", "short": "δ2",491 "group": "Quality", "metric": True, "higher": True, "fmt_spec": "%.4f", "std_key": "std_delta2",492 "desc": "Fraction of pixels within the 1.25² accuracy threshold. Higher means more pixels within tolerance."},493 {"key": "median_delta3", "label": "δ<1.25³ ↑", "short": "δ3",494 "group": "Quality", "metric": True, "higher": True, "fmt_spec": "%.4f", "std_key": "std_delta3",495 "desc": "Fraction of pixels within the 1.25³ accuracy threshold. Higher means more pixels within tolerance."},496 497 # Quantized quality (mirror set; hidden by default)498 {"key": "quant_median_rmse_m", "label": "Quant RMSE (m) ↓", "short": "Quant RMSE",499 "group": "Quantized quality", "metric": True, "lower": True, "fmt_spec": "%.4f", "std_key": "quant_std_rmse_m",500 "desc": "RMSE from quantization alone (no codec), in meters. Lower means quantization preserves depth better."},501 {"key": "quant_median_mae_m", "label": "Quant MAE (m) ↓", "short": "Quant MAE",502 "group": "Quantized quality", "metric": True, "lower": True, "fmt_spec": "%.4f", "std_key": "quant_std_mae_m",503 "desc": "MAE from quantization alone, in meters. Lower means quantization preserves depth better."},504 {"key": "quant_median_absrel", "label": "Quant AbsRel ↓", "short": "Quant AbsRel",505 "group": "Quantized quality", "metric": True, "lower": True, "fmt_spec": "%.4f", "std_key": "quant_std_absrel",506 "desc": "Absolute relative error from quantization alone. Lower means quantization adds less relative error."},507 {"key": "quant_median_delta1", "label": "Quant δ<1.25 ↑", "short": "Quant δ1",508 "group": "Quantized quality", "metric": True, "higher": True, "fmt_spec": "%.4f", "std_key": "quant_std_delta1",509 "desc": "δ<1.25 accuracy from quantization alone. Higher means quantization keeps more pixels within tolerance."},510 {"key": "quant_median_delta2", "label": "Quant δ<1.25² ↑", "short": "Quant δ2",511 "group": "Quantized quality", "metric": True, "higher": True, "fmt_spec": "%.4f", "std_key": "quant_std_delta2",512 "desc": "δ<1.25² accuracy from quantization alone. Higher means quantization keeps more pixels within tolerance."},513 {"key": "quant_median_delta3", "label": "Quant δ<1.25³ ↑", "short": "Quant δ3",514 "group": "Quantized quality", "metric": True, "higher": True, "fmt_spec": "%.4f", "std_key": "quant_std_delta3",515 "desc": "δ<1.25³ accuracy from quantization alone. Higher means quantization keeps more pixels within tolerance."},516]517 518DEPTH_REPOS: list[str] = ["lerobot/outdoor-depth"]519DEPTH_VCODECS: list[str] = ["hevc", "libaom-av1"]520DEPTH_PIX_FMTS: list[str] = ["gray12le"]521DEPTH_BACKENDS: list[str] = ["pyav"]522DEPTH_LOSSLESS: list[str] = ["True", "False"]523DEPTH_USE_LOG: list[str] = ["True", "False"]524DEPTH_G_VALUES: list[int] = [1, 2, 10, 40]525DEPTH_CRF_VALUES: list[int] = [0, 10, 20, 30]526 527DEPTH_LEADERBOARD_AXES: list[str] = [528 "encoding_time_ms",529 "median_load_time_video_ms",530 "video_images_size_ratio",531 "median_rmse_m",532 "median_absrel",533 "median_delta1",534]535 536DEPTH_LEADERBOARD_CATS: dict[str, dict[str, Any]] = {537 "Overall": {538 "desc": "Balanced across encoding, decoding, size, and depth fidelity.",539 "weights": {540 "encoding_time_ms": 1, "median_load_time_video_ms": 1,541 "video_images_size_ratio": 1, "median_rmse_m": 1,542 "median_absrel": 1, "median_delta1": 1,543 },544 },545 "Quality": {546 "desc": "Pure depth fidelity: RMSE, AbsRel, δ<1.25 only.",547 "weights": {548 "encoding_time_ms": 0, "median_load_time_video_ms": 0,549 "video_images_size_ratio": 0, "median_rmse_m": 1,550 "median_absrel": 1, "median_delta1": 1,551 },552 },553 "Encoding": {554 "desc": "Encoding throughput and output size, fidelity ignored.",555 "weights": {556 "encoding_time_ms": 1, "median_load_time_video_ms": 0,557 "video_images_size_ratio": 1, "median_rmse_m": 0,558 "median_absrel": 0, "median_delta1": 0,559 },560 },561 "Decoding": {562 "desc": "Pure decoding latency at the selected access pattern.",563 "weights": {564 "encoding_time_ms": 0, "median_load_time_video_ms": 1,565 "video_images_size_ratio": 0, "median_rmse_m": 0,566 "median_absrel": 0, "median_delta1": 0,567 },568 },569}570 571DEPTH_SUBMIT_OPTIONS: dict[str, list[str]] = {572 "repos": DEPTH_REPOS,573 "vcodecs": DEPTH_VCODECS,574 "pix_fmts": DEPTH_PIX_FMTS,575 "lossless": DEPTH_LOSSLESS,576 "use_log": DEPTH_USE_LOG,577 "g": SUBMIT_OPTIONS["g"],578 "crf": SUBMIT_OPTIONS["crf"],579 "timestamps_modes": TS_MODES,580 "backends": DEPTH_BACKENDS,581}582 583DEPTH_SUBMIT_DEFAULTS: dict[str, Any] = {584 "repos": DEPTH_REPOS[:1],585 "vcodecs": ["hevc"],586 "pix_fmts": ["gray12le"],587 "lossless": ["True"],588 "use_log": ["True"],589 "g": ["2"],590 "crf": ["0"],591 "timestamps_modes": ["1_frame"],592 "backends": ["pyav"],593 "depth_min": None,594 "depth_max": None,595 "shift": None,596 "samples_per_config": 50,597}598 599DEPTH_FULL_SWEEP: dict[str, Any] = {600 "repos": list(DEPTH_REPOS),601 "vcodecs": list(DEPTH_VCODECS),602 "pix_fmts": list(DEPTH_PIX_FMTS),603 "lossless": list(DEPTH_LOSSLESS),604 "use_log": list(DEPTH_USE_LOG),605 "g": [str(v) for v in DEPTH_G_VALUES],606 "crf": [str(v) for v in DEPTH_CRF_VALUES],607 "timestamps_modes": list(TS_MODES),608 "backends": list(DEPTH_BACKENDS),609 "depth_min": None,610 "depth_max": None,611 "shift": None,612 "samples_per_config": 50,613}614 615DEPTH_PARAM_GROUPS: list[dict[str, Any]] = [616 {"t": "Inputs — what you benchmark",617 "desc": "The corpus. Each dataset is a LeRobot episode recording with a few minutes of depth observations.",618 "keys": ["repo_id"]},619 {"t": "Encoding — how the depth video is compressed",620 "desc": "Passed to the FFmpeg/PyAV encoder. These are the knobs operators actually tune.",621 "keys": ["vcodec", "pix_fmt", "g", "crf", "lossless"]},622 {"t": "Quantization — how depth maps to pixels",623 "desc": "How continuous depth (meters) is mapped into the encoder's integer pixel range before compression.",624 "keys": ["use_log", "depth_min", "depth_max", "shift"]},625 {"t": "Decoding — how you read it back",626 "desc": "The other half of the equation. The same MP4 can decode very differently depending on library and access pattern.",627 "keys": ["backend", "timestamps_mode"]},628 {"t": "Fidelity metrics — decoded depth vs. source",629 "desc": "We decode the compressed video, re-read the source depth frames, and compare pixel-for-pixel in meters.",630 "keys": ["median_rmse_m", "median_mae_m", "median_absrel", "median_delta1", "median_delta2", "median_delta3"]},631 {"t": "Performance metrics — how fast, how small",632 "desc": "The cost side. Run on a single CPU thread unless noted.",633 "keys": ["encoding_time_ms", "encoding_fps", "video_images_size_ratio", "median_load_time_video_ms", "video_images_load_time_ratio"]},634]635 636DEPTH_ABOUT_HTML: str = """\637<h3 id="what">What this is</h3>638<p>A benchmark for encoding and decoding <i>depth</i> frames from robotics639datasets as video. Depth maps are single-channel, high-bit-depth images;640this page quantifies the size/speed wins of video encoding against the depth641error introduced by quantization and lossy codecs.</p>642 643<p>644<h3 id="metrics">Metrics</h3>645<p>The size and speed metrics are shared with the RGB benchmark; the fidelity metrics are depth-specific because reconstruction error is measured in meters rather than on pixel intensities.</p>646<p><b>Size & speed</b></p>647<ul>648<li><b>Video/image size ratio</b> — encoded video ÷ sum of source depth frames. Lower means better compression — the video takes less disk than the raw frames.</li>649<li><b>Video/image load ratio</b> — video decoding time ÷ depth image load time for the same frames. <1 means the video is faster to read than the raw frames; >1 means you pay for the compression at read time.</li>650<li><b>Decoding time</b> — median wall-clock to decode N frames at a given timestamp.</li>651<li><b>Encoding time</b> — wall-clock to encode the whole clip, end-to-end.</li>652<li><b>Encoding fps</b> — encode throughput in frames per second, derived from the encoding time and the episode length. Higher means faster encoding; the handy complement to the raw time column.</li>653</ul>654<p><b>Depth fidelity</b> — every decoded frame is de-quantized back to metric depth and compared against the source depth map, pixel-for-pixel; errors are reported in meters. Invalid pixels (zero / no return) are excluded so they don't skew the error.</p>655<ul>656<li><b>RMSE (m)</b> — root-mean-squared error between decoded and source depth, in meters. Squares the residuals, so it punishes large per-pixel misses harder than MAE. Lower is decoded depth closer to the source.</li>657<li><b>MAE (m)</b> — mean absolute error in meters. The plain average miss per pixel. Lower is smaller average miss.</li>658<li><b>AbsRel</b> — mean of <code>|decoded − source| / source</code>. A relative error, so a 5 cm miss at 1 m counts far more than the same miss at 10 m; the standard depth-estimation metric. Lower is less error relative to the true depth.</li>659<li><b>δ<1.25 / 1.25² / 1.25³</b> — accuracy thresholds: the fraction of pixels whose ratio <code>max(d/d*, d*/d)</code> stays under 1.25, 1.25² and 1.25³. Higher is more pixels within tolerance (1.0 = every pixel within threshold); the cubed threshold is the most forgiving.</li>660<li><b>Quantized variants</b> (<code>Quant …</code> columns, hidden by default) — the very same metrics computed after quantization but <i>before</i> the codec. They isolate the error you lose just by squeezing continuous depth into an integer pixel range, so the gap between a metric and its <code>Quant</code> twin is the codec's own contribution.</li>661</ul>662</p>663 664<p>665<h3 id="access">Access patterns</h3>666<p>667Decode cost depends heavily on <i>how</i> frames are requested. <code>1_frame</code> pays the full seek+IDR cost per sample; <code>6_frames</code> amortizes it across contiguous frames; <code>2_frames_4_space</code> probes the worst case where the decoder must step across two distant windows.668</p>669</p>670 671<h3 id="repro">Reproduce locally</h3>672<p>The full sweep is open source. First, set <code>HF_RESULTS_REPO_ID</code> to the Hugging Face Hub dataset where you want results pushed:</p>673<pre style="background:var(--hf-gray-900);color:var(--hf-gray-100);padding:var(--space-4);border-radius:var(--radius-md);font-size:var(--fs-xs);overflow:auto">export HF_RESULTS_REPO_ID=lerobot/depth-benchmark-results</pre>674<p>Then run the benchmark:</p>675<pre style="background:var(--hf-gray-900);color:var(--hf-gray-100);padding:var(--space-4);border-radius:var(--radius-md);font-size:var(--fs-xs);overflow:auto">python benchmark/video/run_video_benchmark.py \\676 --output-dir outputs/depth_benchmark \\677 --repo-ids lerobot/outdoor-depth \\678 --vcodec hevc libaom-av1 \\679 --pix-fmt gray12le \\680 --lossless true false \\681 --use-log true false \\682 --g 2 10 40 \\683 --crf 0 10 20 30 \\684 --timestamps-modes 1_frame 2_frames 6_frames \\685 --backends pyav \\686 --num-samples 50</pre>687 688<h3 id="contrib">Contribute</h3>689<p>690Submit your own configurations through the <b>Submit</b> tab. A background worker picks them up and pushes results to the Hub so the whole community benefits from the same measurements.691</p>692"""693 694DEPTH_KEEP_KEYS: tuple[str, ...] = (695 "repo_id", "vcodec", "encoder", "decoder", "pix_fmt", "g", "crf", "lossless", "use_log",696 "depth_min", "depth_max", "shift", "timestamps_mode", "backend",697 "resolution", "num_pixels", "video_size_bytes", "images_size_bytes",698 "video_images_size_ratio", "video_images_load_time_ratio",699 "median_load_time_video_ms", "std_load_time_video_ms",700 "median_load_time_images_ms", "std_load_time_images_ms",701 "encoding_fps", "encoding_time_ms",702 "median_rmse_m", "std_rmse_m", "median_mae_m", "std_mae_m",703 "median_absrel", "std_absrel",704 "median_delta1", "std_delta1", "median_delta2", "std_delta2",705 "median_delta3", "std_delta3",706 "quant_median_rmse_m", "quant_std_rmse_m", "quant_median_mae_m", "quant_std_mae_m",707 "quant_median_absrel", "quant_std_absrel",708 "quant_median_delta1", "quant_std_delta1", "quant_median_delta2", "quant_std_delta2",709 "quant_median_delta3", "quant_std_delta3",710 "created_at", "lerobot_version", "num_samples",711)712 713DEPTH = BenchmarkConfig(714 key="depth",715 label="Depth",716 title="Depth Encoding & Decoding Benchmark",717 subtitle=(718 "Trade-offs between compression, decoding speed, and depth fidelity "719 "(RMSE, AbsRel, δ accuracy) across codecs, high-bit-depth pixel "720 "formats, and depth range mappings."721 ),722 results_dataset=RESULTS_DATASET_DEPTH,723 submissions_dataset=SUBMISSIONS_DATASET,724 columns=DEPTH_COLUMNS,725 repos=DEPTH_REPOS,726 filter_options={727 "vcodec": DEPTH_VCODECS,728 "pix_fmt": DEPTH_PIX_FMTS,729 "lossless": DEPTH_LOSSLESS,730 "use_log": DEPTH_USE_LOG,731 "backend": DEPTH_BACKENDS,732 "g": [str(v) for v in DEPTH_G_VALUES],733 "crf": [str(v) for v in DEPTH_CRF_VALUES],734 },735 filter_order=["vcodec", "pix_fmt", "lossless", "use_log", "backend", "g", "crf"],736 leaderboard_axes=DEPTH_LEADERBOARD_AXES,737 leaderboard_cats=DEPTH_LEADERBOARD_CATS,738 leaderboard_group_keys=("vcodec", "pix_fmt", "g", "crf", "lossless", "use_log", "backend"),739 submit_options=DEPTH_SUBMIT_OPTIONS,740 submit_defaults=DEPTH_SUBMIT_DEFAULTS,741 full_sweep=DEPTH_FULL_SWEEP,742 param_groups=DEPTH_PARAM_GROUPS,743 param_notes={},744 about_html=DEPTH_ABOUT_HTML,745 keep_keys=DEPTH_KEEP_KEYS,746 scatter_x="video_images_size_ratio",747 scatter_y="median_delta1",748 codec_count=len(DEPTH_VCODECS),749 backend_count=len(DEPTH_BACKENDS),750)751 752BENCHMARKS: list[BenchmarkConfig] = [RGB, DEPTH]753