FluidInference/verdict-coreml

1"""Measure Verdict LUT8 against FP16 on fixed Decision Index requests without using gold labels."""2 3from __future__ import annotations4 5import argparse6import gzip7import json8import platform9import statistics10import subprocess11import time12from collections import Counter13from pathlib import Path14 15import coremltools as ct16import numpy as np17from transformers import AutoTokenizer18 19from assets import ROOT, sha256, verify_assets20from decision_index_engine import adapt_question, as_text21from native_reference import build_request, decode, load_calibrator22from preprocessing import prepare23 24 25def selected_requests(rows_path: Path, tokenizer, limit: int, length: int):26    """First ten eligible rows per family, before inspecting either model's output."""27    family_counts = Counter()28    skipped = Counter()29    selected = []30    with gzip.open(rows_path, "rt") as stream:31        for line in stream:32            row = json.loads(line)33            family = row["family"]34            if family_counts[family] >= 10:35                continue36            if len(row["questions"]) != 1:37                skipped["multiple_questions"] += 138                continue39            question = next(iter(row["questions"].values()))40            if question["type"] not in ("choice", "noul"):41                skipped["unsupported_question_type"] += 142                continue43            try:44                request = build_request(as_text(row["state"]), adapt_question(question))45            except ValueError:46                skipped["invalid_or_overcapacity"] += 147                continue48            tokens = len(tokenizer(request.text, truncation=False)["input_ids"])49            if tokens > length:50                skipped["overlength"] += 151                continue52            selected.append((row["id"], family, request, tokens))53            family_counts[family] += 154            if len(selected) >= limit:55                break56    return selected, family_counts, skipped57 58 59def main() -> None:60    parser = argparse.ArgumentParser(description=__doc__)61    parser.add_argument("--rows", type=Path, required=True, help="pinned public Decision Index selected-rows.jsonl.gz")62    parser.add_argument("--limit", type=int, default=100)63    parser.add_argument("--length", type=int, default=128)64    parser.add_argument("--repeats", type=int, default=5)65    args = parser.parse_args()66    if args.length != 128:67        raise ValueError("the predeclared LUT8 validation protocol is L128 only")68    if args.limit != 100:69        raise ValueError("the predeclared validation manifest uses exactly 100 requests")70    source = verify_assets(required=("config.json", "tokenizer.json", "tokenizer_config.json", "calibrator.json"))71    tokenizer = AutoTokenizer.from_pretrained(source)72    config = json.loads((source / "config.json").read_text())73    calibrator = load_calibrator(source)74    selected, families, skipped = selected_requests(args.rows, tokenizer, args.limit, args.length)75    if len(selected) != args.limit:76        raise ValueError(f"only {len(selected)} eligible fixed requests; expected {args.limit}")77 78    packages = {79        "fp16": ROOT / "build" / "verdict_fp16_L128_candidates25.mlpackage",80        "lut8": ROOT / "build" / "verdict_lut8_kmeans_per_tensor_L128_candidates25.mlpackage",81    }82    models = {name: ct.models.MLModel(str(path), compute_units=ct.ComputeUnit.ALL) for name, path in packages.items()}83    for _, _, request, _ in selected[:2]:84        arrays = prepare(tokenizer, config["class_token_index"], request.text, args.length, 25)85        for model in models.values():86            model.predict(arrays)87 88    rows = []89    timings = {name: [] for name in models}90    for index, (row_id, family, request, tokens) in enumerate(selected):91        arrays = prepare(tokenizer, config["class_token_index"], request.text, args.length, 25)92        results = {}93        for name, model in models.items():94            output = model.predict(arrays)95            results[name] = decode(output["logits"], request, calibrator)96            if index < 20:97                for _ in range(args.repeats):98                    start = time.perf_counter()99                    model.predict(arrays)100                    timings[name].append((time.perf_counter() - start) * 1000)101        reference = results["fp16"]102        compressed = results["lut8"]103        differences = [abs(reference["probabilities"][key] - compressed["probabilities"][key]) for key in request.ids]104        rows.append(105            {106                "id": row_id,107                "family": family,108                "tokens": tokens,109                "candidates": len(request.ids),110                "fp16_selected_id": reference["selected_id"],111                "lut8_selected_id": compressed["selected_id"],112                "selection_agrees": reference["selected_id"] == compressed["selected_id"],113                "abstention_agrees": reference["is_abstention"] == compressed["is_abstention"],114                "max_probability_error": max(differences),115            }116        )117    errors = np.array([row["max_probability_error"] for row in rows])118    selection_agreement = sum(row["selection_agrees"] for row in rows) / len(rows)119    abstention_agreement = sum(row["abstention_agrees"] for row in rows) / len(rows)120    gates = {121        "min_selection_agreement": 0.99,122        "min_abstention_agreement": 0.99,123        "max_p95_probability_error": 0.02,124        "max_worst_probability_error": 0.10,125    }126    report = {127        "suite_file": args.rows.name,128        "suite_sha256": sha256(args.rows),129        "selection_protocol": "First 10 eligible rows per family in suite order, 100 total; no gold labels used",130        "selected_row_ids": [row["id"] for row in rows],131        "families": dict(families),132        "skipped_before_limit": dict(skipped),133        "packages": {name: path.name for name, path in packages.items()},134        "hardware": {135            "chip": subprocess.run(136                ["sysctl", "-n", "machdep.cpu.brand_string"], capture_output=True, text=True137            ).stdout.strip(),138            "macos": platform.mac_ver()[0],139        },140        "questions": len(rows),141        "selection_agreement": selection_agreement,142        "abstention_agreement": abstention_agreement,143        "p95_probability_error": float(np.percentile(errors, 95)),144        "worst_probability_error": float(errors.max()),145        "median_model_call_ms": {name: statistics.median(values) for name, values in timings.items()},146        "gates": gates,147        "rows": rows,148    }149    report["passed"] = (150        selection_agreement >= gates["min_selection_agreement"]151        and abstention_agreement >= gates["min_abstention_agreement"]152        and report["p95_probability_error"] <= gates["max_p95_probability_error"]153        and report["worst_probability_error"] <= gates["max_worst_probability_error"]154    )155    target = ROOT / "reports" / "lut8-L128-suite-parity.json"156    target.write_text(json.dumps(report, indent=2) + "\n")157    print(158        json.dumps({key: value for key, value in report.items() if key not in ("rows", "selected_row_ids")}, indent=2)159    )160    if not report["passed"]:161        raise SystemExit(1)162 163 164if __name__ == "__main__":165    main()166