rafmacalaba/data-use-annotate

1#!/usr/bin/env python32"""Build the gold-controls queue from local human-adjudicated spans.3 4Joins outputs/gliner_datause_v3_probe_human473.jsonl (key, set, surface)5to analysis/v24_sample_review/gliner2_review.jsonl6(spans[].key -> input passage, start/end, origin). Scores are the7singlepass infer-head H100 predictions8(outputs/gliner-datause-catchall-infer-probe/human473_predictions.jsonl);9truth labels are deliberately excluded (blind controls).10Splits: annotator190 (set=annotator) + jdc283 (set=jdc).11 12    uv run python human_labeling/build_gold_queue.py [--out human_labeling/queue_human473.json]13"""14 15import argparse16import json17import sys18from pathlib import Path19 20sys.path.insert(0, str(Path(__file__).resolve().parent))21 22from probe_labels import decide  # noqa: E40223 24REPO = Path(__file__).resolve().parent.parent25HUMAN = REPO / "outputs" / "gliner_datause_v3_probe_human473.jsonl"26REVIEW = REPO / "analysis" / "v24_sample_review" / "gliner2_review.jsonl"27 28 29def main() -> None:30    ap = argparse.ArgumentParser()31    ap.add_argument("--out", default=str(REPO / "human_labeling" / "queue_human473.json"))32    a = ap.parse_args()33 34    # singlepass (infer-head) rescore, published H100 predictions; 23 of 47335    # out-of-grid spans absent here -> head_score None, band "unscored"36    SP = REPO / "outputs" / "gliner-datause-catchall-infer-probe" / "human473_predictions.jsonl"37    sp = {}38    if SP.exists():39        for line in SP.read_text().splitlines():40            if line.strip():41                o = json.loads(line)42                sp[o["key"]] = float(o.get("head_score"))43    # review spans by key -> (passage, start, end, origin)44    ctx: dict[str, tuple] = {}45    for line in REVIEW.read_text().splitlines():46        if not line.strip():47            continue48        row = json.loads(line)49        for s in row.get("spans", []):50            if s.get("key"):51                ctx[s["key"]] = (row.get("input", ""), s.get("start"),52                                 s.get("end"), row.get("origin"))53    items: list[dict] = []54    missing: list[str] = []55    from collections import Counter56    for line in HUMAN.read_text().splitlines():57        if not line.strip():58            continue59        r = json.loads(line)60        key = r["key"]61        subset = "annotator190" if r.get("set") == "annotator" else "jdc283"62        score = sp.get(key)  # singlepass infer-head (H100); None = out-of-grid63        if key in ctx:64            passage, start, end, origin = ctx[key]65            items.append({66                "key": key, "surface": r.get("surface"), "ctx": passage,67                "ctx_missing": False, "head_score": score,68                "start": start, "end": end,69                "band": decide(score, origin), "origin": origin,70                "specificity": "", "split": "holdout",71                "queue": "human473", "subset": subset,72                "scored_by": "rafmacalaba/gliner-datause-catchall-infer-probe (H100)",73            })74        else:75            missing.append(key)76    Path(a.out).write_text("\n".join(json.dumps(i) for i in items) + "\n")77    print(f"queued={len(items)} missing_ctx={len(missing)} "78          f"scored={sum(1 for i in items if i['head_score'] is not None)} "79          f"{Counter(i['subset'] for i in items)} -> {a.out}")80    if missing:81        print("missing:", missing[:5])82 83 84if __name__ == "__main__":85    main()86