J94/bit-vector-tensor-control-policy

1#!/usr/bin/env python32from __future__ import annotations3 4import argparse5import json6import sys7from collections import Counter, defaultdict8from datetime import datetime, timezone9from pathlib import Path10from typing import Any11 12ROOT = Path(__file__).resolve().parents[1]13HIGH_CONF_PATH = ROOT / "runs" / "benchmark" / "golden-xmlc-20260421T120703Z" / "xmlc_seed_high_confidence.jsonl"14WEAK_PATH = ROOT / "runs" / "benchmark" / "golden-xmlc-20260421T120703Z" / "xmlc_seed.jsonl"15TAG_SPEC_PATH = ROOT / "configs" / "golden_xmlc_tags_v0.json"16SCHEMA_PATH = ROOT / "schemas" / "corpus_reasoning_packet_v0.json"17RUNS_DIR = ROOT / "runs" / "analysis"18LATEST_POINTER_PATH = RUNS_DIR / "corpus_reasoning_pilot_latest.json"19 20 21def utc_stamp() -> str:22    return datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")23 24 25def load_json(path: Path) -> dict[str, Any]:26    return json.loads(path.read_text(encoding="utf-8"))27 28 29def load_jsonl(path: Path) -> list[dict[str, Any]]:30    rows: list[dict[str, Any]] = []31    with path.open("r", encoding="utf-8") as handle:32        for line in handle:33            if line.strip():34                rows.append(json.loads(line))35    return rows36 37 38def algebra_for_count(count: int, *, high: int = 120, low: int = 1) -> str:39    if count <= 0:40        return "0"41    if count >= high:42        return "inf"43    if count >= low:44        return "f"45    return "u"46 47 48def algebra_for_ratio(value: float | None, *, strong: float, weak: float) -> str:49    if value is None:50        return "u"51    if value <= weak:52        return "0"53    if value >= strong:54        return "inf" if value >= 0.9 else "f"55    return "f"56 57 58def bit_for_truth(value: bool | None) -> str:59    if value is None:60        return "u"61    return "1" if value else "0"62 63 64def compact_lineage_ids(doc_ids: list[str], limit: int = 12) -> list[str]:65    return sorted(doc_ids)[:limit]66 67 68def source_scope_from_counts(counts: Counter[str]) -> list[str]:69    ordered = [source for source, count in counts.most_common() if count > 0]70    return ordered or ["unknown"]71 72 73def build_packet(74    *,75    label_id: str,76    weak_rows: list[dict[str, Any]],77    high_rows: list[dict[str, Any]],78) -> dict[str, Any]:79    weak_count = len(weak_rows)80    high_count = len(high_rows)81    source_counts = Counter(row.get("corpus_type", "unknown") for row in weak_rows)82    doc_ids = [row.get("doc_id", "") for row in high_rows or weak_rows]83    timestamps = [row.get("timestamp", "") for row in weak_rows if row.get("timestamp")]84    co_labels = Counter()85    for row in weak_rows:86        for other in row.get("weak_labels", []):87            if other != label_id:88                co_labels[other] += 189    co_count = sum(co_labels.values())90    prompt_count = source_counts.get("prompt_packet", 0)91    history_count = source_counts.get("history_event", 0)92    high_ratio = (high_count / weak_count) if weak_count else None93    cross_source_ratio = None94    if weak_count:95        cross_source_ratio = min(prompt_count, history_count) / weak_count if min(prompt_count, history_count) else 0.096 97    coherent = None98    if weak_count:99        coherent = high_ratio is not None and high_ratio >= 0.25100    novel = high_count > 0101    conflicted = True if co_count > weak_count else False if weak_count else None102    stale = None103    promotable = bool(coherent and not conflicted and high_count >= 5)104 105    if timestamps:106        time_trend = [107            "0",108            algebra_for_count(max(1, weak_count // 4), high=80),109            algebra_for_count(max(1, weak_count // 2), high=80),110            algebra_for_count(high_count, high=80),111        ]112    else:113        time_trend = ["u", "u", "u", "u"]114 115    packet = {116        "abstraction_id": label_id,117        "observer_basis": {118          "unit": "golden_tag",119          "time_window": "full_seed_window",120          "source_scope": source_scope_from_counts(source_counts),121        },122        "bits": {123            "coherent": bit_for_truth(coherent),124            "novel": bit_for_truth(novel),125            "stale": bit_for_truth(stale),126            "conflicted": bit_for_truth(conflicted),127            "promotable": bit_for_truth(promotable),128        },129        "vectors": {130            "motif_weights": [131                algebra_for_count(high_count, high=120),132                algebra_for_count(co_count, high=90),133                algebra_for_ratio(high_ratio, strong=0.6, weak=0.0),134                "f" if promotable else "0",135            ],136            "source_weights": [137                algebra_for_count(prompt_count, high=160),138                algebra_for_count(history_count, high=80),139                algebra_for_ratio(cross_source_ratio, strong=0.25, weak=0.0),140            ],141            "time_trend": time_trend,142            "uncertainty_decomposition": [143                "u" if not timestamps else "0",144                "f" if conflicted else "0",145                "f" if high_ratio is not None and high_ratio < 0.5 else "0",146            ],147        },148        "tensor_slice": {149            "coverage": algebra_for_count(weak_count, high=180),150            "coherence": "f" if coherent else ("0" if coherent is False else "u"),151            "evidence": algebra_for_count(high_count, high=120),152            "compression_loss": "0" if high_ratio is not None and high_ratio >= 0.4 else ("f" if high_ratio is not None else "u"),153            "drift": "u",154            "promotion_readiness": "f" if promotable else "0",155        },156        "lineage": {157            "source_packet_ids": compact_lineage_ids(doc_ids),158            "parent_abstraction_ids": sorted(co_labels.keys())[:6],159        },160    }161    return packet162 163 164def build_packets(output_dir: Path) -> dict[str, Any]:165    tag_spec = load_json(TAG_SPEC_PATH)166    _schema = load_json(SCHEMA_PATH)167    weak_rows = load_jsonl(WEAK_PATH)168    high_rows = load_jsonl(HIGH_CONF_PATH)169 170    weak_by_label: dict[str, list[dict[str, Any]]] = defaultdict(list)171    high_by_label: dict[str, list[dict[str, Any]]] = defaultdict(list)172    for row in weak_rows:173        for label_id in row.get("weak_labels", []):174            weak_by_label[label_id].append(row)175    for row in high_rows:176        for label_id in row.get("high_confidence_labels", []):177            high_by_label[label_id].append(row)178 179    packets_dir = output_dir / "packets"180    packets_dir.mkdir(parents=True, exist_ok=True)181    packet_summaries: list[dict[str, Any]] = []182    packet_paths: list[str] = []183    for label in tag_spec["labels"]:184        label_id = label["id"]185        packet = build_packet(186            label_id=label_id,187            weak_rows=weak_by_label.get(label_id, []),188            high_rows=high_by_label.get(label_id, []),189        )190        packet_path = packets_dir / f"{label_id}.json"191        packet_path.write_text(json.dumps(packet, indent=2, sort_keys=True) + "\n", encoding="utf-8")192        packet_paths.append(str(packet_path))193        packet_summaries.append(194            {195                "abstraction_id": label_id,196                "path": str(packet_path),197                "bits": packet["bits"],198                "tensor_slice": packet["tensor_slice"],199                "observer_basis": packet["observer_basis"],200            }201        )202 203    registry = {204        "version": "corpus_reasoning_registry_v0",205        "generated_at": datetime.now(timezone.utc).isoformat(timespec="seconds"),206        "run_dir": str(output_dir),207        "packet_count": len(packet_summaries),208        "abstraction_ids": [item["abstraction_id"] for item in packet_summaries],209        "packets": packet_summaries,210    }211    registry_path = output_dir / "registry.json"212    registry_path.write_text(json.dumps(registry, indent=2, sort_keys=True) + "\n", encoding="utf-8")213 214    summary_lines = [215        "# Corpus Reasoning Pilot",216        "",217        "Clean product one-liner: this run materializes one BVT reasoning packet per golden abstraction so Codex can reason over packet state before reopening raw corpus.",218        "",219        f"- Generated at: `{registry['generated_at']}`",220        f"- Packet count: `{registry['packet_count']}`",221        "",222        "| Abstraction | Coherent | Conflicted | Promotable | Evidence | Compression loss |",223        "| --- | --- | --- | --- | --- | --- |",224    ]225    for item in packet_summaries:226        bits = item["bits"]227        tensor = item["tensor_slice"]228        summary_lines.append(229            f"| `{item['abstraction_id']}` | `{bits['coherent']}` | `{bits['conflicted']}` | `{bits['promotable']}` | `{tensor['evidence']}` | `{tensor['compression_loss']}` |"230        )231    (output_dir / "summary.md").write_text("\n".join(summary_lines) + "\n", encoding="utf-8")232 233    latest_pointer = {234        "generated_at": registry["generated_at"],235        "run_dir": str(output_dir),236        "registry_path": str(registry_path),237        "packet_paths": packet_paths,238    }239    LATEST_POINTER_PATH.write_text(json.dumps(latest_pointer, indent=2, sort_keys=True) + "\n", encoding="utf-8")240    return {241        "registry_path": str(registry_path),242        "summary_path": str(output_dir / "summary.md"),243        "latest_pointer_path": str(LATEST_POINTER_PATH),244    }245 246 247def main() -> int:248    parser = argparse.ArgumentParser(description="Build corpus reasoning packets for the golden abstractions.")249    parser.add_argument("--output-dir", default="")250    args = parser.parse_args()251    output_dir = Path(args.output_dir) if args.output_dir else (RUNS_DIR / f"corpus_reasoning_pilot-{utc_stamp()}")252    result = build_packets(output_dir)253    json.dump(result, sys.stdout, indent=2)254    sys.stdout.write("\n")255    return 0256 257 258if __name__ == "__main__":259    raise SystemExit(main())260