Mushari440/benchmark

1"""Exercise the real manage/delete/requeue routes with the Hub stubbed out."""2import sys, types, json, importlib.util3 4import os5os.environ["HF_TOKEN"] = "stub-token"6os.environ["LEADERBOARD_OWNER"] = "Mushari440"7 8# ---- fake Hub state -------------------------------------------------------9FILES = {10    "Mushari440/results": {11        "v7/org/results_alpha.json", "v7/org/results_beta.json",12        "v8/org/results_alpha.json",13    },14    "Mushari440/results-private": set(),15    "Mushari440/requests": {"org/alpha.json", "org/beta.json", "org/gamma.json"},16}17REQ = {18    "org/alpha.json": {"model": "org/alpha", "status": "FINISHED"},19    "org/beta.json":  {"model": "org/beta",  "status": "FINISHED"},20    "org/gamma.json": {"model": "org/gamma", "status": "PENDING"},21}22COMMITS = []23 24class FakeApi:25    def __init__(self, *a, **k): pass26    def list_repo_files(self, repo_id=None, repo_type=None, *a, **k):27        return sorted(FILES.get(repo_id or a[0] if a else repo_id, set()))28    def delete_file(self, path_in_repo=None, repo_id=None, **k):29        if path_in_repo not in FILES.get(repo_id, set()):30            raise RuntimeError("404")31        FILES[repo_id].discard(path_in_repo)32        if repo_id == "Mushari440/requests":33            REQ.pop(path_in_repo, None)      # the queue entry lives in both fakes34        COMMITS.append(("delete", repo_id, path_in_repo))35    def upload_file(self, path_or_fileobj=None, path_in_repo=None, repo_id=None, **k):36        body = json.loads(path_or_fileobj.decode() if isinstance(path_or_fileobj, bytes) else path_or_fileobj)37        REQ[path_in_repo] = body38        COMMITS.append(("upload", repo_id, path_in_repo, body.get("status")))39 40spec = importlib.util.spec_from_file_location("spaceapp", os.path.join(os.path.dirname(__file__), "app.py"))41app = importlib.util.module_from_spec(spec)42sys.modules["spaceapp"] = app43spec.loader.exec_module(app)44 45app.HfApi = FakeApi46app.api = FakeApi()47app._require_owner = lambda *a, **k: "Mushari440"48app._list_requests = lambda: [dict(v, _path=k) for k, v in REQ.items()]49 50fails = []51def check(name, got, want):52    ok = got == want53    if not ok:54        fails.append(name)55        print(f"  FAIL  {name}\n        got  {got}\n        want {want}")56    else:57        print(f"  PASS  {name}")58 59R = types.SimpleNamespace(headers={}, cookies={}, session={})60 61check("v10 is registered by the API", "v10" in app.VERSIONS, True)62check("v11 is registered by the API", "v11" in app.VERSIONS, True)63 64print("manage reports presence per version")65rows = {m["model"]: m for m in app.manage(R, version="v8")["models"]}66check("alpha scored on v7 and v8", sorted(rows["alpha" if "alpha" in rows else "org/alpha"]["results_by_version"]), ["v7", "v8"])67check("beta scored on v7 only",    sorted(rows["org/beta"]["results_by_version"]), ["v7"])68check("gamma scored nowhere",      rows["org/gamma"]["results_by_version"], {})69check("result_in follows the asked version", rows["org/beta"]["result_in"], [])70 71print("\ndelete scope='version' touches only that version")72app.model_delete(R, app.ModelRef(model="org/alpha", version="v8", scope="version"))73check("v8 score gone",        "v8/org/results_alpha.json" in FILES["Mushari440/results"], False)74check("v7 score untouched",   "v7/org/results_alpha.json" in FILES["Mushari440/results"], True)75check("request kept",         "org/alpha.json" in REQ, True)76 77print("\ndelete scope='all' removes the model everywhere")78app.model_delete(R, app.ModelRef(model="org/beta", version="v7", scope="all"))79check("v7 score gone",  "v7/org/results_beta.json" in FILES["Mushari440/results"], False)80check("request gone",   "org/beta.json" in REQ, False)81 82print("\nrequeue never revives a deliberately-set status")83REQ.clear()84REQ.update({85    "org/fin.json":   {"model": "org/fin",   "status": "FINISHED"},86    "org/fail.json":  {"model": "org/fail",  "status": "FAILED"},87    "org/held.json":  {"model": "org/held",  "status": "HELD"},88    "org/canc.json":  {"model": "org/canc",  "status": "CANCELLED"},89    "org/sup.json":   {"model": "org/sup",   "status": "SUPERSEDED"},90    "org/rej.json":   {"model": "org/rej",   "status": "REJECTED"},91    "org/pend.json":  {"model": "org/pend",  "status": "PENDING"},92})93FILES["Mushari440/results"] = set()94out = app.requeue_all(R, app.RequeueRef(version="v8", only_missing=True))95check("FINISHED is revived", "org/fin" in out["queued"], True)96check("FAILED is revived",   "org/fail" in out["queued"], True)97check("HELD is left parked",       out["skipped"].get("org/held"),  "held on purpose")98check("CANCELLED stays cancelled", out["skipped"].get("org/canc"),  "cancelled on purpose")99check("SUPERSEDED stays out",      out["skipped"].get("org/sup"),   "superseded on purpose")100check("REJECTED stays out",        out["skipped"].get("org/rej"),   "rejected on purpose")101check("PENDING not double-queued", out["skipped"].get("org/pend"),  "already queued")102check("only the two revivable ones", sorted(out["queued"]), ["org/fail", "org/fin"])103 104print("\nrequeue only_missing skips models already scored on the target version")105FILES["Mushari440/results"] = {"v7/org/results_alpha.json", "v8/org/results_alpha.json"}106REQ.clear()107REQ["org/alpha.json"] = {"model": "org/alpha", "status": "FINISHED"}108REQ["org/delta.json"] = {"model": "org/delta", "status": "FINISHED"}109REQ["org/gamma.json"] = {"model": "org/gamma", "status": "PENDING"}110out = app.requeue_all(R, app.RequeueRef(version="v8", only_missing=True))111check("alpha skipped (already on v8)", out["skipped"].get("org/alpha"), "already scored on v8")112check("delta queued",                  "org/delta" in out["queued"], True)113check("gamma skipped (already PENDING)", out["skipped"].get("org/gamma"), "already queued")114check("delta status is RERUN",         REQ["org/delta.json"]["status"], "RERUN")115 116print("\nrequeue only_missing=False re-queues scored models too")117REQ["org/alpha.json"] = {"model": "org/alpha", "status": "FINISHED"}118out = app.requeue_all(R, app.RequeueRef(version="v8", only_missing=False))119check("alpha queued this time", "org/alpha" in out["queued"], True)120 121print("\nFAILURES:", fails or "none")122sys.exit(1 if fails else 0)123