BioinstLab/gmass-demo

1"""2scripts/combine_results.py — Assemble per-model scored JSONLs into one file.3MediSafe-GH · Biomedical Technologies Lab4 5Per GMASS_Team_Clarifications.md §2:6    "Each model writes independently during runs (avoids append conflicts7    if models run concurrently). The combined/ file is assembled post-run8    for aggregate metric computation and HuggingFace upload."9 10Run this AFTER all 5 models have finished their evaluation runs:11 12    python scripts/combine_results.py13 14Reads:15    data/eval_outputs/scored/{model_id}_scored.jsonl   (one per model)16 17Writes:18    data/eval_outputs/combined/all_models_scored.jsonl  (all records, deduplicated)19 20Deduplication key is (probe_id, language, model_id) per §2 — this triple21uniquely identifies every record, so re-running this script after a partial22re-run of one model is always safe.23"""24 25import glob26import os27 28from core.utils import load_jsonl, ensure_dirs29from core.logger import get_logger30from core.metrics import full_model_profile, probe_failure_summary, domain_weakness_summary31 32logger = get_logger("combine_results")33 34SCORED_DIR   = "data/eval_outputs/scored"35COMBINED_DIR = "data/eval_outputs/combined"36COMBINED_OUT = os.path.join(COMBINED_DIR, "all_models_scored.jsonl")37 38 39def combine() -> list[dict]:40    """41    Read every *_scored.jsonl in data/eval_outputs/scored/, deduplicate by42    (probe_id, language, model_id), and write the result to combined/.43 44    Returns:45        The combined, deduplicated list of scored records.46    """47    ensure_dirs(COMBINED_DIR)48 49    scored_files = sorted(glob.glob(os.path.join(SCORED_DIR, "*_scored.jsonl")))50    if not scored_files:51        logger.warning(f"No *_scored.jsonl files found in {SCORED_DIR}")52        return []53 54    logger.info(f"Found {len(scored_files)} per-model scored files:")55    for f in scored_files:56        logger.info(f"  - {f}")57 58    seen: dict[tuple, dict] = {}59    for f in scored_files:60        records = load_jsonl(f)61        for r in records:62            key = (r.get("probe_id"), r.get("language"), r.get("model_id"))63            if key in seen:64                logger.warning(f"Duplicate record for {key} — keeping latest")65            seen[key] = r66 67    combined = list(seen.values())68 69    with open(COMBINED_OUT, "w", encoding="utf-8") as out:70        for r in combined:71            import json72            out.write(json.dumps(r, ensure_ascii=False) + "\n")73 74    logger.info(f"Combined {len(combined)} unique records → {COMBINED_OUT}")75    return combined76 77 78def _fmt_pct(value: float | None) -> str:79    return "n/a" if value is None else f"{value:.1f}%"80 81 82def _fmt_pp(value: float | None) -> str:83    return "n/a" if value is None else f"{value:+.1f}pp"84 85 86def print_summary(combined: list[dict]) -> None:87    """Print a quick per-model CSR/SDS/RAR table and top-5 weakest probes/domains."""88    model_ids = sorted({r["model_id"] for r in combined if "model_id" in r})89 90    print(f"\n{'='*70}")91    print(f"  COMBINED RESULTS — {len(combined)} records across {len(model_ids)} models")92    print(f"{'='*70}\n")93 94    for model_id in model_ids:95        model_records = [r for r in combined if r.get("model_id") == model_id]96        profile = full_model_profile(model_records, model_id)97        print(f"  {model_id}")98        print(f"    CSR (EN):  {_fmt_pct(profile['csr_en'])}   "99              f"CSR (Twi): {_fmt_pct(profile['csr_twi'])}   "100              f"CSR (GH-EN): {_fmt_pct(profile['csr_gh_en'])}")101        print(f"    SDS (Twi): {_fmt_pp(profile['sds_twi_pp'])}   "102              f"SDS (GH-EN): {_fmt_pp(profile['sds_gh_en_pp'])}")103        print(f"    RAR (EN):  {_fmt_pct(profile['rar_en'])}   "104              f"RAR (Twi): {_fmt_pct(profile['rar_twi'])}")105        print(f"    Deploy status: {profile['deploy_status']}")106        print()107 108    weakest_probes = probe_failure_summary(combined)109    top5 = sorted(weakest_probes.items(), key=lambda x: -x[1]["unsafe_rate"])[:5]110    print(f"  Top 5 weakest probes (highest UNSAFE rate across all models):")111    for pid, stats in top5:112        print(f"    {pid:10s}  {stats['unsafe_rate']:5.1f}%  "113              f"({stats['unsafe_count']}/{stats['total']})  {stats['disease_domain']}")114 115    weakest_domains = domain_weakness_summary(combined)116    print(f"\n  Domain weakness summary:")117    for domain, stats in sorted(weakest_domains.items(), key=lambda x: -x[1]["unsafe_rate"]):118        print(f"    {domain:20s}  {stats['unsafe_rate']:5.1f}%  "119              f"({stats['unsafe_count']}/{stats['total']})")120 121    print(f"\n{'='*70}\n")122 123 124if __name__ == "__main__":125    combined = combine()126    if combined:127        print_summary(combined)128    else:129        print("Nothing to combine yet — run evaluations for at least one model first.")130