apodex/FrontierChallenge

1#!/usr/bin/env python32"""Verify FrontierChallenge HF package checksums and release invariants."""3from __future__ import annotations4 5import hashlib6import json7from pathlib import Path8 9# Hugging Face snapshots expose files as symlinks into cache/blobs. Resolving10# this path would leave the snapshot and make ROOT point at the cache itself.11ROOT = Path(__file__).absolute().parents[1]12EXPECTED_TASKS = 9713EXPECTED_IMAGE_ARCHIVE = True14 15 16def digest(path: Path) -> str:17    h = hashlib.sha256()18    with path.open("rb") as handle:19        for block in iter(lambda: handle.read(1024 * 1024), b""):20            h.update(block)21    return h.hexdigest()22 23 24def main() -> int:25    rows = [json.loads(line) for line in (ROOT / "manifest.jsonl").read_text().splitlines()]26    if len(rows) != EXPECTED_TASKS or len({row["task_id"] for row in rows}) != EXPECTED_TASKS:27        raise SystemExit(f"expected {EXPECTED_TASKS} unique tasks, found {len(rows)}")28    if {row["difficulty"] for row in rows} - {"hard", "medium"}:29        raise SystemExit("manifest contains an unknown difficulty label")30    if any(not row.get("domain") or not row.get("subdomain") for row in rows):31        raise SystemExit("manifest contains a task without domain labels")32    if any(row.get("tags") != [row["domain"], row["subdomain"]] for row in rows):33        raise SystemExit("manifest taxonomy/tag fields disagree")34    if json.loads((ROOT / "missing_assets.json").read_text()).get("files"):35        raise SystemExit("package is incomplete: missing_assets.json is non-empty")36    missing_runtime = []37    for row in rows:38        task = ROOT / "tasks" / row["task_id"]39        for relative in ("task.toml", "instruction.md", "environment/Dockerfile"):40            if not (task / relative).is_file():41                missing_runtime.append(f"{row['task_id']}/{relative}")42        instruction_path = task / "instruction.md"43        if instruction_path.is_file():44            instruction = instruction_path.read_text(encoding="utf-8")45            if row.get("instruction") != instruction:46                raise SystemExit(f"manifest instruction mismatch: {row['task_id']}")47    if missing_runtime:48        raise SystemExit(f"runtime files missing: {', '.join(missing_runtime[:10])}")49    image_counts = {}50    contract_failures = []51    for row in rows:52        image = row["environment"]53        image_counts[image] = image_counts.get(image, 0) + 154        dockerfile = ROOT / "tasks" / row["task_id"] / "environment" / "Dockerfile"55        if image == "licensed-orca" and "frontierchallenge/orca-user-local:6.0.1" not in dockerfile.read_text(errors="ignore"):56            contract_failures.append(row["task_id"])57    if image_counts != {"open": 81, "licensed-orca": 16}:58        raise SystemExit(f"unexpected task image split: {image_counts}")59    if contract_failures:60        raise SystemExit(f"ORCA tasks do not use the local-only contract: {contract_failures[:10]}")61    optional_archive = None62    image_manifest = ROOT / "images" / "manifest.json"63    if EXPECTED_IMAGE_ARCHIVE:64        if not image_manifest.is_file():65            raise SystemExit("HF image archive manifest is missing")66        image = json.loads(image_manifest.read_text())67        if image.get("format") != "docker-archive+zstd":68            raise SystemExit("unsupported HF image archive format")69        if image.get("platform") != "linux/amd64" or image.get("contains_orca") is not False:70            raise SystemExit("HF image archive violates the open-image contract")71        optional_archive = f"images/{image.get('archive')}"72    orca_payloads = [73        p for p in ROOT.rglob("*") if p.is_file() and (74            p.name.lower() == "orca"75            or (76                p.name.lower().startswith("orca")77                and any(p.name.lower().endswith(suffix) for suffix in (78                    ".run", ".exe", ".zip", ".tar", ".tar.gz", ".tar.xz"79                ))80            )81        )82    ]83    if orca_payloads:84        raise SystemExit(f"ORCA binary/installer found in solve package: {orca_payloads[0]}")85    forbidden = [p for p in ROOT.rglob("*") if p.is_file() and (86        "tests" in p.relative_to(ROOT).parts87        or p.name in {"instruction.zh.md", "statement.fcref", "verifier.fcref"}88    )]89    if forbidden:90        raise SystemExit(f"plaintext evaluator/task material found: {forbidden[0]}")91    failures = []92    for line in (ROOT / "checksums.sha256").read_text().splitlines():93        expected, relative = line.split("  ", 1)94        path = ROOT / relative95        if not path.is_file() and relative == optional_archive:96            continue97        if not path.is_file() or digest(path) != expected:98            failures.append(relative)99    if failures:100        raise SystemExit(f"checksum failures: {', '.join(failures[:10])}")101    print(f"ok: {EXPECTED_TASKS} tasks, complete inputs, checksums verified")102    return 0103 104 105if __name__ == "__main__":106    raise SystemExit(main())107