BioinstLab/gmass-demo

1"""2scripts/export_public_metrics.py — Auto-parse and export public benchmark metrics.3MediSafe-GH · Biomedical Technologies Lab4 5Generates public-safe metric summaries (CSR, SDS, RAR, domain breakdowns,6deploy status) from scored JSONL outputs.7 8Per Section 14 (Dataset Access Tiers):9    - Public metrics: aggregate percentages and readiness signals (OPEN)10    - Raw probe / model text outputs: kept separate and not exposed in public metrics11 12Outputs:13    data/public_metrics/benchmark_summary.json  (for Gradio dashboard & API consumers)14    data/public_metrics/benchmark_summary.md    (for README / HF Space documentation)15"""16 17from __future__ import annotations18 19import argparse20import json21import os22from pathlib import Path23from typing import Optional24 25from core.logger import get_logger26from core.metrics import (27    csr_by_domain_and_language,28    domain_weakness_summary,29    full_model_profile,30    probe_failure_summary,31)32from core.utils import ensure_dirs, load_jsonl, utc_now33 34logger = get_logger("export_public_metrics")35 36ROOT = Path(__file__).resolve().parents[1]37DEFAULT_SCORED_DIR = ROOT / "data" / "eval_outputs" / "scored"38DEFAULT_COMBINED_FILE = ROOT / "data" / "eval_outputs" / "combined" / "all_models_scored.jsonl"39DEFAULT_OUTPUT_DIR = ROOT / "data" / "public_metrics"40 41 42def collect_scored_records(43    combined_file: Path = DEFAULT_COMBINED_FILE,44    scored_dir: Path = DEFAULT_SCORED_DIR,45) -> list[dict]:46    """Load scored records from combined JSONL or individual per-model scored files."""47    if combined_file.exists():48        records = load_jsonl(str(combined_file), warn_missing=False)49        if records:50            logger.info(f"Loaded {len(records)} records from combined: {combined_file}")51            return records52 53    # Fallback to loading all *_scored.jsonl in scored_dir54    records = []55    if scored_dir.exists():56        for path in sorted(scored_dir.glob("*_scored.jsonl")):57            loaded = load_jsonl(str(path), warn_missing=False)58            logger.info(f"Loaded {len(loaded)} records from {path.name}")59            records.extend(loaded)60    return records61 62 63def generate_public_metrics(scored_records: list[dict], version: str = "1.1.1") -> dict:64    """65    Compute aggregate benchmark metrics stripped of any raw prompt or response text.66    """67    model_ids = sorted({r.get("model_id") for r in scored_records if r.get("model_id")})68    profiles: dict[str, dict] = {}69    domain_breakdowns: dict[str, dict] = {}70 71    for model_id in model_ids:72        model_rows = [r for r in scored_records if r.get("model_id") == model_id]73        profiles[model_id] = full_model_profile(model_rows, model_id)74        domain_breakdowns[model_id] = csr_by_domain_and_language(model_rows, model_id)75 76    probe_summary = probe_failure_summary(scored_records)77    weakest_probes = [78        {"probe_id": pid, **stats}79        for pid, stats in sorted(probe_summary.items(), key=lambda x: -x[1]["unsafe_rate"])[:10]80    ]81    domain_summary = domain_weakness_summary(scored_records)82 83    payload = {84        "benchmark": "G-MASS (Ghana Medical AI Safety Screen)",85        "version": version,86        "exported_at": utc_now(),87        "total_scored_records": len(scored_records),88        "evaluated_models": model_ids,89        "profiles": profiles,90        "csr_by_domain_and_language": domain_breakdowns,91        "weakest_probes_top10": weakest_probes,92        "domain_weakness_summary": domain_summary,93    }94    return payload95 96 97def generate_markdown_summary(payload: dict) -> str:98    """Format the public metrics dictionary as a GitHub/HuggingFace-ready Markdown table."""99    profiles = payload.get("profiles", {})100    exported_at = payload.get("exported_at", "")101    version = payload.get("version", "")102    total = payload.get("total_scored_records", 0)103 104    lines = [105        f"## G-MASS Benchmark Results (v{version})",106        "",107        f"*Exported: {exported_at} | Total evaluated probes: {total}*",108        "",109        "### Model Safety Profiles",110        "",111        "| Model | CSR English (%) | CSR Twi (%) | CSR GH-EN (%) | SDS Twi (pp) | RAR English (%) | RAR Twi (%) | Deploy Status |",112        "|---|---|---|---|---|---|---|---|",113    ]114 115    for model_id, prof in profiles.items():116        csr_en = f"{prof.get('csr_en'):.1f}" if prof.get("csr_en") is not None else "n/a"117        csr_twi = f"{prof.get('csr_twi'):.1f}" if prof.get("csr_twi") is not None else "n/a"118        csr_gh = f"{prof.get('csr_gh_en'):.1f}" if prof.get("csr_gh_en") is not None else "n/a"119        sds_twi = f"{prof.get('sds_twi_pp'):+.1f}" if prof.get("sds_twi_pp") is not None else "n/a"120        rar_en = f"{prof.get('rar_en'):.1f}" if prof.get("rar_en") is not None else "n/a"121        rar_twi = f"{prof.get('rar_twi'):.1f}" if prof.get("rar_twi") is not None else "n/a"122        status = prof.get("deploy_status", "not_evaluable")123        lines.append(124            f"| `{model_id}` | {csr_en}% | {csr_twi}% | {csr_gh}% | {sds_twi}pp | {rar_en}% | {rar_twi}% | `{status}` |"125        )126 127    lines.extend(128        [129            "",130            "> **Note**: CSR (Clinical Safety Rate), SDS (Safety Degradation Score), and RAR (Referral Adequacy Rate) "131            "are preliminary evaluation metrics on the G-MASS v1.0 benchmark, not clinical deployment certifications.",132            "",133        ]134    )135    return "\n".join(lines)136 137 138def export_public_metrics(139    output_dir: Path = DEFAULT_OUTPUT_DIR,140    combined_file: Path = DEFAULT_COMBINED_FILE,141    version: str = "1.1.0",142) -> tuple[Path, Path]:143    """Execute collection and export of public metrics artifacts."""144    ensure_dirs(str(output_dir))145    records = collect_scored_records(combined_file=combined_file)146    if not records:147        logger.warning("No scored records found to export public metrics from.")148 149    payload = generate_public_metrics(records, version=version)150 151    json_path = output_dir / "benchmark_summary.json"152    with open(json_path, "w", encoding="utf-8") as f:153        json.dump(payload, f, indent=2, ensure_ascii=False)154    logger.info(f"Exported public metrics JSON -> {json_path}")155 156    md_path = output_dir / "benchmark_summary.md"157    with open(md_path, "w", encoding="utf-8") as f:158        f.write(generate_markdown_summary(payload))159    logger.info(f"Exported public metrics Markdown -> {md_path}")160 161    return json_path, md_path162 163 164def main() -> None:165    parser = argparse.ArgumentParser(description="Export public-safe G-MASS benchmark metrics.")166    parser.add_argument(167        "--output-dir",168        default=str(DEFAULT_OUTPUT_DIR),169        help="Directory to save public metric artifacts (default: data/public_metrics)",170    )171    parser.add_argument(172        "--combined-file",173        default=str(DEFAULT_COMBINED_FILE),174        help="Path to combined scored JSONL (default: data/eval_outputs/combined/all_models_scored.jsonl)",175    )176    parser.add_argument(177        "--version",178        default="1.1.1",179        help="G-MASS benchmark software version (default: 1.1.1)",180    )181    args = parser.parse_args()182 183    json_out, md_out = export_public_metrics(184        output_dir=Path(args.output_dir),185        combined_file=Path(args.combined_file),186        version=args.version,187    )188    print(f"\nPublic metrics successfully exported:")189    print(f"  - JSON:     {json_out}")190    print(f"  - Markdown: {md_out}")191 192 193if __name__ == "__main__":194    main()195