Mushari440/benchmark
0
1"""Arabic LLM Leaderboard — Docker Space backend.2 3Serves the static single-page leaderboard (index.html) AND provides a gated4model-submission endpoint:5 6 GET / -> index.html7 GET /api/me -> {"user": <hf-username|null>}8 GET /login -> redirect to Hugging Face OAuth9 GET /login/callback -> OAuth callback, sets the session10 GET /logout -> clears the session11 POST /api/submit -> validate a model on the Hub + enqueue it (PENDING)12 13Submissions require the visitor to sign in with their own Hugging Face account14(OAuth). Their username is recorded on the request for provenance. The actual15write to the `requests` dataset is done with the Space's OWN token (a Space16secret, never exposed to the browser) — the submitter's token can't write to a17dataset they don't own.18"""19import datetime20import html21import json22import os23import re24import secrets25import threading26import time27 28import requests29from fastapi import FastAPI, HTTPException, Request30from fastapi.responses import FileResponse, HTMLResponse, JSONResponse, RedirectResponse31from typing import List, Optional32 33from pydantic import BaseModel34from starlette.middleware.sessions import SessionMiddleware35 36from huggingface_hub import HfApi37from huggingface_hub.utils import (38 GatedRepoError,39 HfHubHTTPError,40 RepositoryNotFoundError,41)42 43# ----------------------------------------------------------------------------- config44HF_CO = "https://huggingface.co"45HERE = os.path.dirname(os.path.abspath(__file__))46 47HF_TOKEN = os.environ.get("HF_TOKEN") # Space secret with write access to REQUESTS_REPO48REQUESTS_REPO = os.environ.get("REQUESTS_REPO", "Mushari440/requests")49RESULTS_REPO = os.environ.get("RESULTS_REPO", "Mushari440/results")50# Private scores live in a separate PRIVATE dataset, never in the public one --51# hiding a row in the UI would still leave it readable on the Hub.52RESULTS_PRIVATE_REPO = os.environ.get("RESULTS_PRIVATE_REPO", "Mushari440/results-private")53VISIBILITIES = {"public", "private"}54 55# Injected by HF when `hf_oauth: true` is set in the README frontmatter.56OAUTH_CLIENT_ID = os.environ.get("OAUTH_CLIENT_ID")57OAUTH_CLIENT_SECRET = os.environ.get("OAUTH_CLIENT_SECRET")58OAUTH_SCOPES = os.environ.get("OAUTH_SCOPES", "openid profile")59OPENID_PROVIDER_URL = os.environ.get("OPENID_PROVIDER_URL", HF_CO).rstrip("/")60SPACE_HOST = os.environ.get("SPACE_HOST", "") # e.g. mushari440-benchmark.hf.space61 62# Closed-source models are served through a paid API billed to OUR key, so only63# the leaderboard owner may queue them. Open-weight models run on our own GPUs and64# cost nothing, so anyone signed in may submit those.65LEADERBOARD_OWNER = os.environ.get("LEADERBOARD_OWNER", "Mushari440")66SOURCES = {"open", "closed"}67# Guard rails for the owner-only partial-run controls.68MAX_EXAMPLES_CAP = int(os.environ.get("MAX_EXAMPLES_CAP", "1000"))69SUBTASK_RE = re.compile(r"^[a-z0-9_]{2,64}$")70 71MODEL_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]*/[A-Za-z0-9][A-Za-z0-9._-]*$")72REVISION_RE = re.compile(r"^[A-Za-z0-9._/-]{1,120}$")73PRECISIONS = {"bfloat16", "float16"}74WEIGHT_TYPES = {"Original", "Adapter", "Delta"}75TYPE_MAP = {76 "fine-tuned": "🔶 fine-tuned",77 "instruction-tuned": "⭕ instruction-tuned",78 "pretrained": "🟢 pretrained",79 "RL-tuned": "🟦 RL-tuned",80}81BLOCK_RESUBMIT = {"PENDING", "RUNNING", "FINISHED"} # FAILED is allowed to be re-submitted82 83api = HfApi(endpoint=HF_CO, token=HF_TOKEN) # writes to REQUESTS_REPO + reads the queue84# Validate submitted models as an ANONYMOUS client sees them, so the Space's write token is85# never used as a "confused deputy" to read repos the public can't (private/gated-for-owner).86api_public = HfApi(endpoint=HF_CO, token=None)87 88_write_lock = threading.Lock() # serialize dedupe-check + write within this process89# Short-TTL in-process guards to close the eventually-consistent-read gap (the queue read via90# the CDN lags a few seconds behind a just-written commit).91_RECENT_TTL = 120 # seconds92_recent_keys = {} # (model, precision) -> monotonic ts (just-submitted, not yet visible)93 94 95def _prune_recent(now):96 for k in [k for k, t in _recent_keys.items() if now - t > _RECENT_TTL]:97 del _recent_keys[k]98 99# ----------------------------------------------------------------------------- app100app = FastAPI(title="Arabic LLM Leaderboard")101# SameSite=None + Secure so the session cookie also works when the Space is embedded102# in an iframe on huggingface.co. Signed with the OAuth client secret (stable, private).103# Sign the session with the OAuth client secret (stable + private). If it is somehow absent,104# fall back to a RANDOM per-process key — forged cookies become impossible (the only cost is105# that sessions don't survive a restart). Never a hardcoded constant (would be forgeable).106app.add_middleware(107 SessionMiddleware,108 secret_key=OAUTH_CLIENT_SECRET or secrets.token_hex(32),109 same_site="none",110 https_only=True,111 max_age=8 * 60 * 60,112)113 114_oauth = None115if OAUTH_CLIENT_ID and OAUTH_CLIENT_SECRET:116 from authlib.integrations.starlette_client import OAuth117 118 _oauth = OAuth()119 _oauth.register(120 name="hf",121 client_id=OAUTH_CLIENT_ID,122 client_secret=OAUTH_CLIENT_SECRET,123 server_metadata_url=f"{OPENID_PROVIDER_URL}/.well-known/openid-configuration",124 client_kwargs={"scope": OAUTH_SCOPES},125 )126 127 128def _redirect_uri() -> str:129 return f"https://{SPACE_HOST}/login/callback"130 131 132def _origin_ok(request: Request) -> bool:133 """CSRF guard: the write must originate from our own page. Both the direct host and the134 huggingface.co iframe embed serve the page from https://{SPACE_HOST}, so that single origin135 is sufficient (and huggingface.co itself never posts here)."""136 origin = request.headers.get("origin") or ""137 return bool(SPACE_HOST) and origin == f"https://{SPACE_HOST}"138 139 140def _list_requests():141 """All request dicts currently in the queue (live read via the raw file API)."""142 headers = {"Authorization": f"Bearer {HF_TOKEN}"} if HF_TOKEN else {}143 out = []144 for path in api.list_repo_files(REQUESTS_REPO, repo_type="dataset"):145 if not path.endswith(".json"):146 continue147 try:148 r = requests.get(149 f"{HF_CO}/datasets/{REQUESTS_REPO}/raw/main/{path}",150 headers=headers,151 timeout=20,152 )153 r.raise_for_status()154 data = r.json()155 except (requests.RequestException, ValueError):156 continue157 if isinstance(data, dict) and "model" in data:158 # Carry the repo path: the manage endpoints need it to delete or159 # rewrite the exact file, and it cannot be reconstructed reliably160 # (the filename encodes precision/weight_type as submitted).161 data["_path"] = path162 out.append(data)163 return out164 165 166# ----------------------------------------------------------------------------- routes167@app.get("/", include_in_schema=False)168def index():169 return FileResponse(os.path.join(HERE, "index.html"))170 171@app.get("/healthz", include_in_schema=False)172def healthz():173 return {"ok": True, "oauth": bool(_oauth)}174 175@app.get("/api/me")176def me(request: Request):177 user = request.session.get("user")178 # `is_owner` only decides whether the UI OFFERS the closed-source option.179 # /api/submit re-checks server-side, so faking this client-side gains nothing.180 return {"user": user, "is_owner": bool(user) and user == LEADERBOARD_OWNER}181 182@app.get("/login", include_in_schema=False)183async def login(request: Request):184 if _oauth is None:185 raise HTTPException(503, "Sign-in is not configured on this Space yet.")186 return await _oauth.hf.authorize_redirect(request, _redirect_uri())187 188@app.get("/login/callback", include_in_schema=False)189async def login_callback(request: Request):190 if _oauth is None:191 raise HTTPException(503, "Sign-in is not configured on this Space yet.")192 try:193 token = await _oauth.hf.authorize_access_token(request)194 except Exception as e: # OAuthError (incl. attacker-supplied ?error=), state/nonce, network195 # NEVER reflect the exception into the page: authorize_access_token raises from the196 # attacker-controlled `error_description` query param BEFORE any state check, so echoing197 # it would be a reflected-XSS sink. Log server-side only; show a static message.198 print(f"[oauth] sign-in failed: {type(e).__name__}: {e}")199 return HTMLResponse(200 "<!doctype html><meta charset=utf-8>"201 "<p style='font-family:system-ui'>Sign-in failed or was cancelled. "202 "<a href='/'>Return to the leaderboard</a>.</p>",203 status_code=400,204 )205 info = token.get("userinfo") or {}206 if not info:207 try:208 info = await _oauth.hf.userinfo(token=token)209 except Exception:210 info = {}211 username = info.get("preferred_username") or info.get("name") or info.get("sub")212 if not username:213 return HTMLResponse("<p>Could not read your Hugging Face username.</p>", status_code=400)214 request.session["user"] = str(username)215 safe_user = html.escape(str(username)) # display-name fallback can contain markup216 # Small page so the flow also works when opened in a new tab from an iframe.217 return HTMLResponse(218 "<!doctype html><meta charset=utf-8>"219 "<body style='font-family:system-ui;background:#071411;color:#eaf3ef;"220 "display:flex;align-items:center;justify-content:center;height:100vh;margin:0'>"221 f"<div style='text-align:center'><p>Signed in as <b>{safe_user}</b>.</p>"222 "<p><a style='color:#34d399' href='/'>Return to the leaderboard →</a></p>"223 "<script>try{if(window.opener){window.opener.focus();window.close();}"224 "else{location.href='/?signedin=1';}}catch(e){location.href='/?signedin=1';}</script>"225 "</div></body>"226 )227 228@app.get("/logout", include_in_schema=False)229def logout(request: Request):230 request.session.pop("user", None)231 return RedirectResponse("/", status_code=302)232 233 234def _require_owner(request: Request, check_origin: bool = True) -> str:235 """Every management route funnels through here. The identity comes from the236 signed session, so a crafted request cannot impersonate the owner.237 238 check_origin is the CSRF guard and applies to STATE-CHANGING requests only.239 Browsers omit the Origin header on same-origin GETs, so enforcing it on a read240 endpoint rejects our own page. Reads are still session-gated, and a cross-origin241 read cannot see the response anyway (no CORS headers are served)."""242 user = request.session.get("user")243 if not user:244 raise HTTPException(401, "Please sign in with Hugging Face.")245 if user != LEADERBOARD_OWNER:246 raise HTTPException(403, "Only the leaderboard maintainer can manage models.")247 if check_origin and not _origin_ok(request):248 raise HTTPException(403, "Request blocked (bad origin).")249 if not HF_TOKEN:250 raise HTTPException(503, "The Space has no write token configured.")251 return user252 253 254# ── Benchmark versions ───────────────────────────────────────────────────────255# Results are namespaced per version: <version>/<org>/results_<model>.json.256# LEGACY_VERSION also owns the un-prefixed files at the repo root, which is where257# every score lived before versioning. Keep this list in step with VERSIONS in258# index.html.259VERSIONS = [260 v.strip()261 for v in os.environ.get("VERSIONS", "v7,v8,v9,v10,v11").split(",")262 if v.strip()263]264LEGACY_VERSION = os.environ.get("LEGACY_VERSION", "v7")265 266 267def _clean_version(version: str | None) -> str:268 v = (version or LEGACY_VERSION).strip()269 if v not in VERSIONS:270 raise HTTPException(400, f"Unknown benchmark version {v!r}.")271 return v272 273 274def _results_path(model: str, version: str | None = None) -> str:275 org, _, name = model.partition("/")276 return f"{_clean_version(version)}/{org}/results_{name or org}.json"277 278 279def _results_paths(model: str, version: str | None = None) -> list[str]:280 """Every path a score for this model may sit at, newest layout first. The bare281 path is only in play for the legacy version, and only until migration finishes."""282 v = _clean_version(version)283 org, _, name = model.partition("/")284 paths = [f"{v}/{org}/results_{name or org}.json"]285 if v == LEGACY_VERSION:286 paths.append(f"{org}/results_{name or org}.json")287 return paths288 289 290class ModelRef(BaseModel):291 model: str292 version: str | None = None293 # "version" removes only this version's score and leaves the queue entry, so the294 # model can be re-evaluated. "all" removes the queue entry and every version's295 # score. Deleting a v8 score used to destroy the shared request too, which took296 # the v7 row's metadata with it and left no way to re-run without resubmitting.297 scope: str = "version"298 299 300@app.get("/api/manage")301def manage(request: Request, version: str | None = None):302 """Every request + where its scores live, per benchmark version. Owner-only."""303 _require_owner(request, check_origin=False) # read-only GET304 api = HfApi(token=HF_TOKEN)305 have = {}306 for repo, key in ((RESULTS_REPO, "public"), (RESULTS_PRIVATE_REPO, "private")):307 try:308 for f in api.list_repo_files(repo, repo_type="dataset"):309 if f.endswith(".json"):310 have.setdefault(f, set()).add(key)311 except Exception:312 pass313 rows = []314 for r in _list_requests():315 m = r.get("model", "")316 # Presence per version, so the panel can show at a glance which boards a317 # model is actually on. `result_in` stays for the selected version because318 # the older UI read it.319 by_version = {}320 for v in VERSIONS:321 where = set()322 for path in _results_paths(m, v):323 where |= have.get(path, set())324 if where:325 by_version[v] = sorted(where)326 rows.append({327 "model": m,328 "status": r.get("status"),329 "submitted_time": r.get("submitted_time"),330 "submitted_by": r.get("submitted_by"),331 "source": r.get("source", "open"),332 "visibility": r.get("visibility", "public"),333 "result_in": by_version.get(_clean_version(version), []),334 "results_by_version": by_version,335 })336 rows.sort(key=lambda x: x.get("submitted_time") or "", reverse=True)337 return {"models": rows, "owner": LEADERBOARD_OWNER,338 "versions": VERSIONS, "version": _clean_version(version)}339 340 341@app.post("/api/model/delete")342def model_delete(request: Request, body: ModelRef):343 """Remove a model's request AND its score from both results repos."""344 _require_owner(request)345 model = (body.model or "").strip()346 if not MODEL_RE.match(model):347 raise HTTPException(400, "Bad model id.")348 api = HfApi(token=HF_TOKEN)349 scope = (body.scope or "version").strip().lower()350 if scope not in ("version", "all"):351 raise HTTPException(400, "scope must be 'version' or 'all'.")352 removed = []353 if scope == "all":354 for r in _list_requests():355 if r.get("model") == model and r.get("_path"):356 try:357 api.delete_file(path_in_repo=r["_path"], repo_id=REQUESTS_REPO,358 repo_type="dataset",359 commit_message=f"Remove request {model}")360 removed.append(f"request:{r['_path']}")361 except Exception as e:362 print(f"[delete] request {model}: {type(e).__name__}: {e}")363 version = _clean_version(body.version)364 targets = ([p for v in VERSIONS for p in _results_paths(model, v)]365 if scope == "all" else _results_paths(model, version))366 for repo in (RESULTS_REPO, RESULTS_PRIVATE_REPO):367 for path in dict.fromkeys(targets):368 try:369 api.delete_file(path_in_repo=path, repo_id=repo,370 repo_type="dataset",371 commit_message=f"Remove {version} results for {model}")372 removed.append(f"results:{repo}:{path}")373 except Exception:374 pass # simply not present there375 if not removed:376 raise HTTPException(377 404,378 f"{model} has no score on {version}." if scope == "version"379 else f"Nothing found for {model}.")380 what = f"every version of {model}" if scope == "all" else f"{model} from {version}"381 return {"ok": True, "removed": removed,382 "message": f"Deleted {what} ({len(removed)} file(s))."}383 384 385@app.post("/api/model/rerun")386def model_rerun(request: Request, body: ModelRef):387 """Queue a model for re-evaluation. RERUN also clears the worker's done-log,388 which a plain PENDING does not -- that is why a re-submit alone never ran."""389 _require_owner(request)390 model = (body.model or "").strip()391 api = HfApi(token=HF_TOKEN)392 hit = None393 for r in _list_requests():394 if r.get("model") == model:395 hit = r396 break397 if not hit:398 raise HTTPException(404, f"{model} is not in the queue.")399 if not hit.get("_path"):400 raise HTTPException(500, "Could not resolve the request file path.")401 payload = {k: v for k, v in hit.items() if k != "_path"}402 payload["status"] = "RERUN"403 api.upload_file(404 path_or_fileobj=json.dumps(payload, ensure_ascii=False, indent=1).encode("utf-8"),405 path_in_repo=hit["_path"], repo_id=REQUESTS_REPO, repo_type="dataset",406 commit_message=f"{model} -> RERUN")407 return {"ok": True,408 "message": f"{model} queued for re-evaluation (RERUN). It lands on "409 f"whichever version the worker is running (its RUN_ID)."}410 411 412# Statuses a requeue may revive. A model that ran, or tried and failed, can run413# again. Everything else was set deliberately and a sweep must not quietly undo it:414# HELD is parked on purpose, and CANCELLED / SUPERSEDED / REJECTED were excluded on415# purpose -- nvidia/Qwen3.6-35B-A3B-NVFP4 is CANCELLED because the A100 cannot run416# NVFP4 at all, so reviving it would fail on every sweep from here on.417REQUEUABLE = {"FINISHED", "FAILED"}418ALREADY_QUEUED = {"PENDING", "RUNNING", "RERUN"}419 420 421class RequeueRef(BaseModel):422 version: str | None = None423 # Default to only what is missing, so re-running a half-finished sweep does not424 # discard the models that already landed.425 only_missing: bool = True426 427 428@app.post("/api/requeue-all")429def requeue_all(request: Request, body: RequeueRef):430 """Set every queued model to RERUN so a whole version can be evaluated.431 432 The worker only ever picks up PENDING / RUNNING / RERUN, so after a sweep every433 model sits at FINISHED and a new version would evaluate nothing at all. This is434 the switch that starts the next version's run."""435 _require_owner(request)436 version = _clean_version(body.version)437 api = HfApi(token=HF_TOKEN)438 439 have = set()440 for repo in (RESULTS_REPO, RESULTS_PRIVATE_REPO):441 try:442 have |= {f for f in api.list_repo_files(repo, repo_type="dataset")443 if f.endswith(".json")}444 except Exception:445 pass446 447 queued, skipped = [], {}448 for r in _list_requests():449 model = r.get("model")450 if not model or not r.get("_path"):451 continue452 status = str(r.get("status") or "").upper()453 if status in ALREADY_QUEUED:454 skipped[model] = "already queued"455 continue456 if status not in REQUEUABLE:457 skipped[model] = f"{status.lower()} on purpose"458 continue459 if body.only_missing and any(p in have for p in _results_paths(model, version)):460 skipped[model] = f"already scored on {version}"461 continue462 payload = {k: v for k, v in r.items() if k != "_path"}463 payload["status"] = "RERUN"464 try:465 api.upload_file(466 path_or_fileobj=json.dumps(payload, ensure_ascii=False, indent=1).encode("utf-8"),467 path_in_repo=r["_path"], repo_id=REQUESTS_REPO, repo_type="dataset",468 commit_message=f"{model} -> RERUN (requeue for {version})")469 queued.append(model)470 except Exception as e:471 skipped[model] = f"upload failed ({type(e).__name__})"472 print(f"[requeue-all] {model}: {type(e).__name__}: {e}")473 reasons: dict[str, int] = {}474 for why in skipped.values():475 reasons[why] = reasons.get(why, 0) + 1476 tail = (" Left alone: "477 + ", ".join(f"{n} {why}" for why, n in sorted(reasons.items()))) if reasons else ""478 msg = (f"Queued {len(queued)} model(s) for {version}.{tail}" if queued479 else f"Nothing to queue for {version}.{tail}")480 return {"ok": True, "queued": queued, "skipped": skipped,481 "skipped_reasons": reasons, "message": msg}482 483 484@app.get("/api/private-results")485def private_results(request: Request, version: str | None = None):486 """The private scores, served only to the owner. The browser cannot read that487 dataset directly, so this endpoint is the only way they reach the board."""488 _require_owner(request, check_origin=False) # read-only GET489 api = HfApi(token=HF_TOKEN)490 out = []491 v = _clean_version(version)492 try:493 files = [f for f in api.list_repo_files(RESULTS_PRIVATE_REPO, repo_type="dataset")494 # `_`-prefixed files are side-car metrics, not model rows495 if f.endswith(".json") and not f.split("/")[-1].startswith("_")]496 except Exception:497 files = []498 499 def _in_version(path: str) -> bool:500 if path.startswith(f"{v}/"):501 return True502 # Un-prefixed root files predate versioning and belong to the legacy board.503 return (v == LEGACY_VERSION and path.count("/") == 1504 and not any(path.startswith(f"{o}/") for o in VERSIONS))505 506 seen, keep = set(), []507 for f in sorted(files, key=lambda f: not f.startswith(f"{v}/")):508 if not _in_version(f):509 continue510 stem = f.split("/")[-1]511 if stem in seen: # prefixed copy wins over a not-yet-removed root one512 continue513 seen.add(stem)514 keep.append(f)515 files = keep516 for f in files:517 try:518 url = f"{HF_CO}/datasets/{RESULTS_PRIVATE_REPO}/raw/main/{f}"519 r = requests.get(url, headers={"Authorization": f"Bearer {HF_TOKEN}"}, timeout=20)520 r.raise_for_status()521 d = r.json()522 d["_private"] = True523 out.append(d)524 except Exception as e:525 print(f"[private-results] {f}: {type(e).__name__}: {e}")526 return {"results": out}527 528 529@app.get("/api/private-metrics")530def private_metrics(request: Request, version: str | None = None):531 """Owner-only extra columns: policy A/B scores and the judge timestamp.532 533 Kept in the PRIVATE dataset, not the public results repo, so the numbers are534 not merely hidden in the UI -- an anonymous reader cannot fetch them at all.535 Returns {} rather than raising when the file is absent, so a version that has536 no side-car simply renders without the extra columns.537 """538 _require_owner(request, check_origin=False) # read-only GET539 v = _clean_version(version)540 url = f"{HF_CO}/datasets/{RESULTS_PRIVATE_REPO}/raw/main/{v}/_metrics_extra.json"541 try:542 r = requests.get(url, headers={"Authorization": f"Bearer {HF_TOKEN}"}, timeout=20)543 if r.status_code == 404:544 return {"models": {}}545 r.raise_for_status()546 return r.json()547 except Exception as e:548 print(f"[private-metrics] {v}: {type(e).__name__}: {e}")549 return {"models": {}}550 551 552class SubmitBody(BaseModel):553 model: str554 revision: str = "main"555 precision: str = "bfloat16"556 model_type: str = "fine-tuned"557 weight_type: str = "Original"558 base_model: str = ""559 # "open" -> open-weight repo on the Hub, run locally on our GPUs (anyone)560 # "closed" -> served via the OpenRouter API, billed to us (owner only)561 source: str = "open"562 # Partial-run controls, OWNER ONLY (see the gate in submit()). A capped or563 # subtask-filtered score is not comparable with the full-corpus board, so these564 # must never be settable by an outside submitter.565 max_examples: Optional[int] = None566 subtasks: List[str] = []567 # "public" -> score published to the public board (default, anyone)568 # "private" -> score goes to the private dataset, visible only to the owner569 visibility: str = "public"570 571 572@app.post("/api/submit")573def submit(request: Request, body: SubmitBody):574 user = request.session.get("user")575 if not user:576 raise HTTPException(401, "Please sign in with Hugging Face before submitting.")577 if not _origin_ok(request):578 raise HTTPException(403, "Request blocked (bad origin).")579 if not HF_TOKEN:580 raise HTTPException(503, "The Space has no write token configured; submissions are disabled.")581 582 model = (body.model or "").strip()583 if not MODEL_RE.match(model):584 raise HTTPException(400, "Model id must look like 'org/name' (letters, digits, . _ - only).")585 if body.precision not in PRECISIONS:586 raise HTTPException(400, "Precision must be 'bfloat16' or 'float16'.")587 precision = body.precision588 if body.weight_type not in WEIGHT_TYPES:589 raise HTTPException(400, "Weight type must be Original, Adapter, or Delta.")590 weight_type = body.weight_type591 if body.model_type not in TYPE_MAP:592 raise HTTPException(400, "Unknown model type.")593 revision = (body.revision or "main").strip() or "main"594 if not REVISION_RE.match(revision):595 raise HTTPException(400, "Invalid revision.")596 597 # ── THE GATE ────────────────────────────────────────────────────────────598 # Authoritative because `user` comes from the signed OAuth session, not the599 # request body. A hand-crafted POST cannot get past this.600 source = (body.source or "open").strip().lower()601 if source not in SOURCES:602 raise HTTPException(400, "Source must be 'open' or 'closed'.")603 # Partial-run controls: owner-only, and rejected loudly rather than silently604 # dropped, so a non-owner is never misled into thinking they got a quick run.605 max_examples = body.max_examples606 subtasks = [str(s).strip() for s in (body.subtasks or []) if str(s).strip()]607 visibility = (body.visibility or "public").strip().lower()608 if visibility not in VISIBILITIES:609 raise HTTPException(400, "Visibility must be 'public' or 'private'.")610 if visibility == "private" and user != LEADERBOARD_OWNER:611 raise HTTPException(612 403, "Private evaluations are restricted to the leaderboard maintainer.")613 614 if (max_examples is not None or subtasks) and user != LEADERBOARD_OWNER:615 raise HTTPException(616 403,617 "Limiting the item count or the subtask list is restricted to the "618 "leaderboard maintainer — a partial run does not produce a score "619 "comparable with the models already on the board.",620 )621 if max_examples is not None:622 if not (1 <= max_examples <= MAX_EXAMPLES_CAP):623 raise HTTPException(400, f"max_examples must be between 1 and {MAX_EXAMPLES_CAP}.")624 if subtasks:625 bad = [s for s in subtasks if not SUBTASK_RE.match(s)]626 if bad:627 raise HTTPException(400, f"Invalid subtask id(s): {bad[:5]}")628 subtasks = sorted(set(subtasks))629 630 if source == "closed" and user != LEADERBOARD_OWNER:631 raise HTTPException(632 403,633 "Closed-source (API) models can only be submitted by the leaderboard "634 "maintainer, because they are billed per token. Please submit an "635 "open-weight model from the Hub instead.",636 )637 638 # Closed-source models are API slugs (openai/gpt-5.2), not Hub repos, so every639 # check below would 404 on them. The owner gate above already ran; the worker640 # validates the slug against OpenRouter at run time.641 if source == "closed":642 params, likes, lic = None, 0, "proprietary"643 else:644 # 1) The model must exist and be a public, non-gated, loadable Transformers repo. Validate645 # with the ANONYMOUS client so we only ever see what the public sees.646 try:647 info = api_public.model_info(model, revision=revision, files_metadata=False)648 except GatedRepoError:649 raise HTTPException(400, "That model is gated — the evaluator can't download it. Submit a public model.")650 except RepositoryNotFoundError:651 raise HTTPException(404, f"'{model}' was not found on the Hub (it must be a public model repo).")652 except HfHubHTTPError:653 raise HTTPException(400, f"Could not read '{model}' at revision '{revision}' as a public model.")654 655 if getattr(info, "private", False):656 raise HTTPException(400, "That model is private. Submit a public model.")657 if getattr(info, "gated", False):658 raise HTTPException(400, "That model is gated. Submit a non-gated public model.")659 siblings = [getattr(s, "rfilename", "") for s in (getattr(info, "siblings", None) or [])]660 if "config.json" not in siblings:661 raise HTTPException(400, "That repo has no config.json — it doesn't look like a Transformers model the evaluator can run.")662 663 # parameter count (billions), best-effort664 params = None665 st = getattr(info, "safetensors", None)666 total = getattr(st, "total", None) if st else None667 if isinstance(total, (int, float)) and total > 0:668 params = round(total / 1e9, 3)669 if params is None:670 m = re.search(r"(\d+(?:\.\d+)?)\s*[bB]\b", model)671 if m:672 params = float(m.group(1))673 674 likes = int(getattr(info, "likes", 0) or 0)675 card = getattr(info, "card_data", None)676 lic = None677 if card is not None:678 try:679 lic = card.get("license")680 except Exception:681 lic = getattr(card, "license", None)682 lic = lic or "?"683 684 org, _, name = model.partition("/")685 path_in_repo = f"{org}/{name}_eval_request_False_{precision}_{weight_type}.json"686 687 key = (model, precision)688 with _write_lock:689 now_m = time.monotonic()690 _prune_recent(now_m)691 692 # 2) De-duplicate: don't re-queue something already pending/running/finished. Check both693 # the (eventually-consistent) remote queue AND just-written keys held in-process.694 existing = _list_requests()695 for r in existing:696 if r.get("model") == model and r.get("precision") == precision \697 and (r.get("status", "") or "").upper() in BLOCK_RESUBMIT:698 raise HTTPException(699 409,700 f"{model} ({precision}) is already {str(r.get('status')).lower()} in the queue.",701 )702 if key in _recent_keys:703 raise HTTPException(409, f"{model} ({precision}) was just submitted — it is already queued.")704 # No cap on how many distinct models a user may queue — only the same model+precision705 # is blocked from being queued twice (the dedupe check above).706 707 now = datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")708 payload = {709 "model": model,710 "base_model": (body.base_model or "").strip(),711 "revision": revision,712 "precision": precision,713 "weight_type": weight_type,714 "status": "PENDING",715 "submitted_time": now,716 "model_type": TYPE_MAP[body.model_type],717 "likes": likes,718 "params": params, # float billions, or null when genuinely unknown719 "license": lic,720 "private": False,721 "submitted_by": user,722 # Read by the worker: "closed" routes to the OpenRouter API path (no723 # download, no GPU). Absent or "open" keeps the existing GPU route, so724 # every request queued before this change behaves exactly as before.725 "source": source,726 # Owner-only partial-run controls; absent on a normal full submission.727 # The worker re-checks the submitter before honouring them.728 **({"max_examples": max_examples} if max_examples is not None else {}),729 **({"subtasks": subtasks} if subtasks else {}),730 # Read by the worker: routes the score to the private dataset.731 **({"visibility": visibility} if visibility != "public" else {}),732 }733 try:734 api.upload_file(735 path_or_fileobj=json.dumps(payload, ensure_ascii=False, indent=1).encode("utf-8"),736 path_in_repo=path_in_repo,737 repo_id=REQUESTS_REPO,738 repo_type="dataset",739 commit_message=f"Submit {model} ({precision}) — by {user}",740 )741 except Exception as e:742 print(f"[submit] upload failed for {model}: {type(e).__name__}: {e}")743 raise HTTPException(502, "Failed to enqueue the submission. Please try again shortly.")744 745 # Record in-process so an immediate duplicate can't slip past the remote read lag.746 _recent_keys[key] = now_m747 748 return JSONResponse(749 {750 "ok": True,751 "message": f"Submitted {model} — it is now PENDING. The worker will evaluate it and it will appear on the board.",752 "params": params,753 "path": path_in_repo,754 }755 )756