JetLaggedByData/scifi-forge

1"""2v3_agentic/evaluate/benchmark.py3Full V1 vs V2 vs V3 benchmark report.4 5Reads:6  v1_baseline/eval_results_v1.json        (from v1_baseline/evaluate.py)7  v2_finetuned/eval_results_v2.json       (from v2_finetuned/evaluate.py)8  data/stories/*/story.json               (from pipeline/runner.py)9 10Writes:11  mlflow_runs/benchmark_report.json       (full metrics for Streamlit + README)12  mlflow_runs/v2_loss_curve.json          (if MLflow DB is available)13 14Logs everything to a single MLflow run: "v1_vs_v2_vs_v3_benchmark"15 16Run:17  python v3_agentic/evaluate/benchmark.py18  python v3_agentic/evaluate/benchmark.py --rerun-v1  # re-evaluate V1 first19  python v3_agentic/evaluate/benchmark.py --rerun-v2  # re-evaluate V2 first20"""21 22import sys23import json24import argparse25import subprocess26from pathlib import Path27 28import mlflow29 30sys.path.insert(0, str(Path(__file__).resolve().parents[2]))31 32from v3_agentic.evaluate.consistency_scorer import (33    get_avg_consistency_score,34    get_avg_revision_cycles,35    get_score_distribution,36    get_per_story_summary,37)38 39 40# ── Paths ─────────────────────────────────────────────────────────────────41ROOT          = Path(__file__).resolve().parents[2]42V1_RESULTS    = ROOT / "v1_baseline"  / "eval_results_v1.json"43V2_RESULTS    = ROOT / "v2_finetuned" / "eval_results_v2.json"44REPORT_PATH   = ROOT / "mlflow_runs"  / "benchmark_report.json"45LOSS_PATH     = ROOT / "mlflow_runs"  / "v2_loss_curve.json"46 47 48# ── Loaders ───────────────────────────────────────────────────────────────49 50def _load_json(path: Path) -> dict:51    if path.exists():52        return json.loads(path.read_text(encoding="utf-8"))53    return {}54 55 56def _run_subprocess(cmd: list[str]) -> bool:57    """Run a subprocess and return True on success."""58    print(f"  Running: {' '.join(cmd)}")59    result = subprocess.run(cmd, capture_output=False)60    return result.returncode == 061 62 63# ── V1 metrics ────────────────────────────────────────────────────────────64 65def get_v1_metrics(rerun: bool = False) -> dict:66    """67    Load V1 metrics. Re-runs evaluate.py if rerun=True or results missing.68    """69    if rerun or not V1_RESULTS.exists():70        print("Running V1 evaluation...")71        ok = _run_subprocess([72            sys.executable, str(ROOT / "v1_baseline" / "evaluate.py")73        ])74        if not ok:75            print("  ⚠️  V1 evaluation failed — using zeros.")76            return {}77 78    data = _load_json(V1_RESULTS)79    return {80        "v1_char_perplexity":         data.get("char_perplexity", 0.0),81        "v1_bleu2":                   data.get("bleu2", 0.0),82        "v1_inference_chars_per_sec": data.get("inference_chars_per_sec", 0.0),83        "v1_avg_sentence_length":     data.get("avg_sentence_length_chars", 0.0),84    }85 86 87# ── V2 metrics ────────────────────────────────────────────────────────────88 89def get_v2_metrics(rerun: bool = False) -> dict:90    """91    Load V2 metrics. Re-runs evaluate.py if rerun=True or results missing.92    """93    if rerun or not V2_RESULTS.exists():94        print("Running V2 evaluation...")95        ok = _run_subprocess([96            sys.executable, str(ROOT / "v2_finetuned" / "evaluate.py"),97            "--samples", "50",98        ])99        if not ok:100            print("  ⚠️  V2 evaluation failed — using zeros.")101            return {}102 103    data = _load_json(V2_RESULTS)104    return {105        "v2_word_perplexity":           data.get("word_perplexity", 0.0),106        "v2_bleu2":                     data.get("bleu2", 0.0),107        "v2_bleu4":                     data.get("bleu4", 0.0),108        "v2_inference_tokens_per_sec":  data.get("inference_tokens_per_sec", 0.0),109        "v2_genre_consistency_score":   data.get("genre_consistency_score", 0.0),110    }111 112 113# ── V3 metrics ────────────────────────────────────────────────────────────114 115def get_v3_metrics() -> dict:116    """Compute V3 metrics from stored story JSONs — no model inference."""117    stories_dir = ROOT / "data" / "stories"118    dist = get_score_distribution(stories_dir)119 120    return {121        "v3_avg_consistency_score": get_avg_consistency_score(stories_dir),122        "v3_avg_revision_cycles":   get_avg_revision_cycles(stories_dir),123        "v3_score_mean":            dist.get("mean", 0.0),124        "v3_score_median":          dist.get("median", 0.0),125        "v3_score_stdev":           dist.get("stdev", 0.0),126        "v3_chapters_excellent":    dist.get("excellent", 0),127        "v3_chapters_good":         dist.get("good", 0),128        "v3_chapters_poor":         dist.get("poor", 0),129    }130 131 132# ── Improvement deltas ────────────────────────────────────────────────────133 134def compute_deltas(v1: dict, v2: dict) -> dict:135    """136    Compute V1→V2 improvement percentages for the benchmark table.137    These are the headline numbers for the README and LinkedIn post.138    """139    deltas = {}140 141    v1_ppl = v1.get("v1_char_perplexity", 0)142    v2_ppl = v2.get("v2_word_perplexity", 0)143    if v1_ppl > 0 and v2_ppl > 0:144        deltas["perplexity_pct_change"] = round((v2_ppl - v1_ppl) / v1_ppl * 100, 1)145 146    v1_bleu = v1.get("v1_bleu2", 0)147    v2_bleu = v2.get("v2_bleu2", 0)148    if v1_bleu > 0:149        deltas["bleu2_pct_change"] = round((v2_bleu - v1_bleu) / max(v1_bleu, 1e-9) * 100, 1)150 151    return deltas152 153 154# ── MLflow loss curve export ──────────────────────────────────────────────155 156def export_loss_curve() -> None:157    """158    Pull V2 training loss from MLflow SQLite DB and save as JSON159    for the Model Arena training loss chart.160    Non-fatal if MLflow DB is unavailable.161    """162    try:163        client = mlflow.tracking.MlflowClient()164        runs   = client.search_runs(165            experiment_ids=["0"],166            filter_string="tags.mlflow.runName = 'v2_qlora_finetune'",167            order_by=["start_time DESC"],168            max_results=1,169        )170        if not runs:171            return172 173        run_id  = runs[0].info.run_id174        history = client.get_metric_history(run_id, "train_loss")175        curve   = [{"step": m.step, "loss": m.value} for m in history]176 177        LOSS_PATH.parent.mkdir(parents=True, exist_ok=True)178        LOSS_PATH.write_text(json.dumps(curve, indent=2))179        print(f"  Loss curve exported → {LOSS_PATH} ({len(curve)} steps)")180 181    except Exception as exc:182        print(f"  ⚠️  Could not export loss curve: {exc}")183 184 185# ── Report writer ─────────────────────────────────────────────────────────186 187def write_report(v1: dict, v2: dict, v3: dict, deltas: dict) -> None:188    """Write full benchmark report JSON and a human-readable summary."""189    report = {190        "v1": v1,191        "v2": v2,192        "v3": v3,193        "deltas": deltas,194        "per_story": get_per_story_summary(ROOT / "data" / "stories"),195    }196    REPORT_PATH.parent.mkdir(parents=True, exist_ok=True)197    REPORT_PATH.write_text(json.dumps(report, indent=2))198 199    print("\n── Benchmark Report ─────────────────────────────────────────────")200    print("\n  V1 LSTM:")201    for k, v in v1.items():202        print(f"    {k}: {v}")203    print("\n  V2 QLoRA:")204    for k, v in v2.items():205        print(f"    {k}: {v}")206    print("\n  V3 Agentic:")207    for k, v in v3.items():208        print(f"    {k}: {v}")209    print("\n  Deltas (V1 → V2):")210    for k, v in deltas.items():211        arrow = "▼" if "perplexity" in k and v < 0 else "▲"212        print(f"    {k}: {arrow} {v:+.1f}%")213    print(f"\n  Saved → {REPORT_PATH}")214 215 216# ── Main ──────────────────────────────────────────────────────────────────217 218def run_full_benchmark(rerun_v1: bool = False, rerun_v2: bool = False) -> dict:219    """220    Full V1 vs V2 vs V3 benchmark. Logs to MLflow, writes JSON report.221    Returns the full report dict.222    """223    print("SciFi Forge — Full Benchmark\n")224 225    print("Loading V1 metrics...")226    v1 = get_v1_metrics(rerun=rerun_v1)227 228    print("Loading V2 metrics...")229    v2 = get_v2_metrics(rerun=rerun_v2)230 231    print("Computing V3 metrics from stored stories...")232    v3 = get_v3_metrics()233 234    deltas = compute_deltas(v1, v2)235 236    with mlflow.start_run(run_name="v1_vs_v2_vs_v3_benchmark"):237        all_metrics = {**v1, **v2, **v3, **deltas}238        mlflow.log_metrics({239            k: float(v) for k, v in all_metrics.items()240            if isinstance(v, (int, float))241        })242        mlflow.log_artifact(str(REPORT_PATH)) if REPORT_PATH.exists() else None243        print("\n  Metrics logged to MLflow.")244 245    print("\nExporting V2 loss curve from MLflow...")246    export_loss_curve()247 248    write_report(v1, v2, v3, deltas)249    return {"v1": v1, "v2": v2, "v3": v3, "deltas": deltas}250 251 252if __name__ == "__main__":253    parser = argparse.ArgumentParser(description="SciFi Forge — full benchmark")254    parser.add_argument("--rerun-v1", action="store_true",255                        help="Re-run V1 evaluate.py before loading results")256    parser.add_argument("--rerun-v2", action="store_true",257                        help="Re-run V2 evaluate.py before loading results")258    args = parser.parse_args()259    run_full_benchmark(rerun_v1=args.rerun_v1, rerun_v2=args.rerun_v2)260