kalanosdev/lerobot-grader
0
1"""2grading.py — canonical Kalanos RDQ metadata-only grading core.3 4SINGLE SOURCE OF TRUTH for metadata-only ("0.1-meta") scoring. Imported by:5 - batch_grade.py (bulk grading of the launch list)6 - app.py (the Hugging Face Space on-demand grader)7 8so that a dataset graded on the Space and the same dataset graded in the batch9produce byte-identical *scores* — and, when HF returns the same metadata,10byte-identical *reports*. All scoring logic lives here and nowhere else; do not11re-implement any part of it in app.py or batch_grade.py. Changing a weight or a12check here changes both surfaces at once.13 14Metadata-only grade: `format` and `coverage` are MEASURED from repository15metadata; `sync` and `outliers` are NOT measured and are scored null. The16disclosure in `notes` must travel with the report — these grades are not a17substitute for a measured (full) grade and must not be published publicly18without that disclosure intact.19"""20import io21import json22import os23import time24from datetime import datetime, timezone25 26import requests27 28HF = "https://huggingface.co"29GRADER_VERSION = "0.1-meta"30REPORT_KIND = "metadata"31 32# Default hard cap for the repo-size tree walk. batch_grade.py keeps the full33# 45s (throughput over latency); the Space passes a smaller value so an34# on-demand grade of a huge repo can't hang the UI (size never affects scores).35SIZE_WALK_DEADLINE_S = 4536 37 38def UTC():39 return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")40 41 42def _auth_headers():43 """Attach an HF token only if one is present in the environment.44 Public datasets resolve fine without it; this keeps the free public Space45 working with no secret set, while private/gated repos work when HF_TOKEN is."""46 tok = os.environ.get("HF_TOKEN")47 return {"Authorization": f"Bearer {tok}"} if tok else {}48 49 50def hf_api(path):51 r = requests.get(f"{HF}/api/{path}", timeout=30, headers=_auth_headers())52 if r.status_code == 404:53 return None54 r.raise_for_status()55 return r.json()56 57 58def hf_file(slug, path):59 r = requests.get(f"{HF}/datasets/{slug}/resolve/main/{path}", timeout=60, headers=_auth_headers())60 if r.status_code in (401, 403, 404):61 return None62 r.raise_for_status()63 return r64 65 66def fetch_repo_size_bytes(slug, deadline_s=SIZE_WALK_DEADLINE_S):67 """Repo size in bytes. Returns (size_bytes, complete):68 - complete=True -> size_bytes is the real total (from usedStorage, or a tree69 walk that finished before the cap).70 - complete=False, size_bytes not None -> LOWER BOUND: the tree walk hit the71 page/time cap before finishing (huge repos, e.g. droid_lerobot's ~1.7TB /72 tens of thousands of files).73 - complete=False, size_bytes is None -> couldn't determine size at all.74 75 Tries the dataset info endpoint's usedStorage first: one request regardless of76 repo size. Falls back to summing the paginated tree API only if that's absent,77 with a hard page/time cap so one huge repo can't stall the whole batch (or the78 interactive Space, which passes a shorter deadline_s)."""79 headers = _auth_headers()80 81 try:82 r = requests.get(f"{HF}/api/datasets/{slug}?expand[]=usedStorage", headers=headers, timeout=30)83 if r.status_code == 200:84 used = r.json().get("usedStorage")85 if isinstance(used, int) and used > 0:86 return used, True87 except Exception:88 pass89 90 url = f"{HF}/api/datasets/{slug}/tree/main?recursive=true"91 total = 092 pages = 093 deadline = time.time() + deadline_s # hard cap per dataset — large repos become a lower bound, not a hang94 try:95 while url and pages < 200 and time.time() < deadline:96 r = requests.get(url, headers=headers, timeout=20)97 if r.status_code == 404:98 return None, False99 r.raise_for_status()100 for item in r.json():101 if item.get("type") == "file" and isinstance(item.get("size"), int):102 total += item["size"]103 url = r.links.get("next", {}).get("url")104 pages += 1105 return total, (url is None) # url is None => walked the whole tree; else capped out106 except Exception:107 return (total or None), False108 109 110def fetch_metadata(slug, size_deadline_s=SIZE_WALK_DEADLINE_S):111 """Real facts from HF: license, gated, and LeRobot meta if present.112 Returns None if the dataset does not exist (or the API 404s)."""113 info = hf_api(f"datasets/{slug}")114 if info is None:115 return None116 card = info.get("cardData") or {}117 meta = {118 "license": (card.get("license") if isinstance(card.get("license"), str) else (card.get("license") or [""])[0]) or "unknown",119 "gated": bool(info.get("gated")),120 "downloads": info.get("downloads", 0),121 "last_modified": info.get("lastModified", ""),122 "sha": (info.get("sha") or "")[:7],123 "lerobot": None, "episodes_meta": None, "size_bytes": None, "size_complete": False,124 }125 meta["size_bytes"], meta["size_complete"] = fetch_repo_size_bytes(slug, deadline_s=size_deadline_s)126 for path in ("meta/info.json",):127 r = hf_file(slug, path)128 if r is not None:129 try:130 meta["lerobot"] = r.json()131 except Exception:132 pass133 # episode stats (v2 jsonl or v3 parquet listing is heavy; jsonl is cheap when present)134 r = hf_file(slug, "meta/episodes.jsonl")135 if r is not None:136 lengths = []137 for line in io.StringIO(r.text):138 try:139 lengths.append(json.loads(line).get("length", 0))140 except Exception:141 pass142 if lengths:143 meta["episodes_meta"] = {"count": len(lengths), "lengths": lengths}144 return meta145 146 147def hist(values, bins):148 counts = [0] * (len(bins) - 1)149 for v in values:150 for i in range(len(bins) - 1):151 if bins[i] <= v < bins[i + 1]:152 counts[i] += 1153 break154 else:155 counts[-1] += 1156 return {"bins": bins, "counts": counts}157 158 159def gini(xs):160 xs = sorted(x for x in xs if x >= 0)161 n = len(xs)162 s = sum(xs)163 if n == 0 or s == 0:164 return 0.0165 g = 0.0166 for i, x in enumerate(xs, 1):167 g += (2 * i - n - 1) * x168 return round(g / (n * s), 3)169 170 171def meta_grade(slug, m):172 """Metadata-only v0 grade. Format & coverage are measured; sync & outliers are173 provisional placeholders (null) disclosed in the notes. Do NOT publish these174 publicly without the disclosure intact.175 176 `m` is the dict returned by fetch_metadata(). This is the ONE function whose177 output defines a metadata grade; both the batch and the Space call it."""178 lr = m.get("lerobot") or {}179 fps = lr.get("fps", 0)180 total_eps = lr.get("total_episodes") or (m.get("episodes_meta") or {}).get("count") or 0181 total_frames = lr.get("total_frames", 0)182 tasks = lr.get("total_tasks", 0)183 features = lr.get("features", {}) or {}184 cams = [k for k in features if "image" in k or features[k].get("dtype") in ("video", "image")]185 fmt_checks = {186 "has_lerobot_meta": lr != {},187 "fps_declared": bool(fps),188 "features_declared": bool(features),189 "chunked_layout": bool(lr.get("data_path")),190 }191 fmt_score = 100.0 * sum(fmt_checks.values()) / len(fmt_checks) if lr else 40.0192 lens = (m.get("episodes_meta") or {}).get("lengths") or []193 secs = [l / fps for l in lens] if fps and lens else []194 ep_hist = hist(secs, [0, 10, 20, 30, 45, 60, 90, 120]) if secs else {"bins": [0, 1], "counts": [total_eps]}195 task_balance = 0.0 # per-task counts not in cheap metadata; neutral196 cov_score = min(100.0, 30 + 8 * tasks + 6 * len(cams)) if lr else 50.0197 hours = round(total_frames / fps / 3600, 1) if fps and total_frames else 0198 # meta/info.json's codebase_version already includes a leading "v" (e.g. "v2.0");199 # strip it before prepending our own so we never emit "LeRobot vv2.0".200 raw_version = str(lr.get("codebase_version", "")).strip()201 version = raw_version.lstrip("vV")202 fmt_label = f"LeRobot v{version}" if version else ("LeRobot" if lr else "unknown")203 204 size_bytes = m.get("size_bytes")205 size_complete = m.get("size_complete")206 size_gb = round(size_bytes / 1e9, 2) if isinstance(size_bytes, int) else 0207 if not isinstance(size_bytes, int):208 size_note = ["Repo size could not be determined; size_gb=0 is a placeholder, not a measurement."]209 elif not size_complete:210 size_note = [f"Repo size is a LOWER BOUND ({size_gb} GB): the file listing was capped before finishing "211 f"(large repo). Re-run with a longer cap or rely on usedStorage for an exact total."]212 else:213 size_note = []214 215 return {216 "report_id": "",217 "report_kind": REPORT_KIND,218 "grader_version": GRADER_VERSION,219 "graded_at": UTC(),220 "source": {"kind": "hf", "ref": slug, "url": f"{HF}/datasets/{slug}",221 "commit": m["sha"], "license": m["license"].lower(), "gated": m["gated"]},222 "dataset": {"format": fmt_label,223 "episodes": total_eps, "hours": hours,224 "embodiment": lr.get("robot_type", "unknown"),225 "sensors": cams or ["unknown"], "size_gb": size_gb},226 "scores": {"total": round((fmt_score + cov_score) / 2, 1),227 "sync": None, "coverage": round(cov_score, 1),228 "outliers": None, "format": round(fmt_score, 1)},229 "metrics": {230 "sync": {"median_drift_ms": 0, "p95_drift_ms": 0, "max_drift_ms": 0, "episodes_over_10ms": 0, "dropped_frames_pct": 0},231 "coverage": {"task_families": tasks, "environments": 0, "episodes_per_task_min": 0,232 "episodes_per_task_max": 0, "gini_task_balance": task_balance, "lighting_variation_score": 0},233 "outliers": {"flagged_episodes": 0, "flagged_pct": 0, "categories": {}},234 "format": {"schema_valid_pct": 100 if lr else 0, "calibration_present_pct": 0,235 "units_consistent": bool(lr), "timestamps_monotonic_pct": 0},236 },237 "histograms": {"drift_ms": {"bins": [0, 1], "counts": [0]}, "episode_length_s": ep_hist},238 "flagged_sample": [],239 "notes": [240 f"Metadata-level report: file structure, coverage and episode statistics are measured from repository metadata. "241 f"Timestamp synchronization and trajectory outliers are not yet measured and are not scored.",242 f"{total_eps:,} episodes, {tasks} task(s), {len(cams)} camera stream(s), fps={fps}.",243 f"HF downloads: {m['downloads']:,}; last modified {m['last_modified'][:10]}.",244 *size_note,245 ],246 }247