BioinstLab/gmass-demo
0
1# run_bilingual_eval.py2# MediSafe-GH · G-MASS Project3# Biomedical Technologies Lab4#5# Runs the same probe set through a model across language conditions,6# scores responses, and computes SDS (Safety Degradation Score) immediately.7#8# REVISED per GMASS_Team_Clarifications.md:9# §2 — output files are one-JSONL-per-model (data/eval_outputs/raw/<model>.jsonl,10# data/eval_outputs/scored/<model>_scored.jsonl), NOT one file per11# eval-run-type. All language conditions for a model accumulate into12# the SAME file, distinguished by the language field on each record.13# §8 — Twi/GH-EN prompts get the language-consistency instruction appended14# before being sent to the model (build_prompt_with_language_instruction).15# §9 — GPT-4o mini, not full GPT-4o (model ID map updated).16#17# Usage:18# python run_bilingual_eval.py gemini19# python run_bilingual_eval.py gemini --per-domain 5 (pilot mode)20# python run_bilingual_eval.py gemini --full (all 150 probes)21# python run_bilingual_eval.py all --per-domain 5 (run all probe-tested models + report)22# python run_bilingual_eval.py all --full (full suite + report)23 24import argparse25import os26from pathlib import Path27import subprocess28import sys29import time30 31from probes.loader import load_bilingual_probes, expand_bilingual_probes32from scorer.scorer import gmass_score33from core.utils import save_jsonl_line, load_jsonl, ensure_dirs34from core.metrics import full_model_profile35from core.logger import get_logger36from models.router import call_model, build_prompt_with_language_instruction37 38logger = get_logger("run_bilingual_eval")39 40# Model IDs are read directly from models/router.py's own constants rather41# than duplicated here — this exact duplication (a hardcoded copy drifting42# out of sync with router.py's real defaults) is what caused this script to43# still say "gpt-4o-mini" and "gemini-2.5-flash" after the team decided to44# reinstate the probe-tested model lineup. Importing the live values means45# this map can never silently go stale again.46from models.router import GPT4O_MODEL, GEMINI_MODEL, PHI3_MODEL, BIOMISTRAL_MODEL47 48MODEL_ID_MAP = {49 "gpt4o": GPT4O_MODEL,50 "gemini": GEMINI_MODEL,51 "phi3": PHI3_MODEL,52 "biomistral": BIOMISTRAL_MODEL,53}54PROBE_TESTED_MODEL_KEYS = list(MODEL_ID_MAP.keys())55 56 57def parse_args(argv: list[str] | None = None) -> argparse.Namespace:58 parser = argparse.ArgumentParser(description="Run bilingual G-MASS evaluation")59 parser.add_argument(60 "--version",61 action="version",62 version="G-MASS v1.1.1",63 help="Show program's version number and exit",64 )65 parser.add_argument(66 "model",67 nargs="?",68 default=None,69 help="Model key: gpt4o, gemini, phi3, biomistral, or all",70 )71 parser.add_argument(72 "--probe-file",73 default="data/probes/probes_bilingual.jsonl",74 help="Path to bilingual/GH-EN probe JSONL (default: data/probes/probes_bilingual.jsonl)",75 )76 parser.add_argument(77 "--per-domain",78 type=int,79 default=5,80 help="Probes per domain for pilot mode (default: 5)",81 )82 parser.add_argument(83 "--full",84 action="store_true",85 help="Run all 150 probes instead of pilot sample",86 )87 parser.add_argument(88 "--delay",89 type=float,90 default=2.0,91 help="Seconds to wait between API calls (default: 2.0)",92 )93 parser.add_argument(94 "--skip-report",95 action="store_true",96 help="With model=all, run evaluations only; do not combine outputs or build the workbook",97 )98 return parser.parse_args(argv)99 100 101def run_all_models_and_report(args: argparse.Namespace) -> int:102 """103 Run every probe-tested model, then combine results and build the report.104 105 One provider's quota, token, API, or local backend failure must not stop106 the remaining available models. Each model runs in its own subprocess;107 failures are recorded, and the lineup continues.108 """109 failed_models: list[tuple[str, int]] = []110 111 for model_key in PROBE_TESTED_MODEL_KEYS:112 cmd = [113 sys.executable, str(os.path.abspath(__file__)), model_key,114 "--probe-file", args.probe_file,115 "--delay", str(args.delay),116 ]117 if args.full:118 cmd.append("--full")119 else:120 cmd.extend(["--per-domain", str(args.per_domain)])121 print(f"\n$ {' '.join(cmd)}")122 result = subprocess.run(cmd, check=False)123 if result.returncode != 0:124 failed_models.append((model_key, result.returncode))125 print(126 f"\nWARNING: {model_key} run failed with exit code {result.returncode}. "127 "Continuing with remaining models."128 )129 130 if args.skip_report:131 print("\nAll probe-tested model runs attempted. Report generation skipped.")132 _print_all_model_failures(failed_models)133 return 1 if failed_models else 0134 135 from scripts.combine_results import COMBINED_OUT, combine, print_summary136 from scripts.build_evaluation_report import build_report137 138 combined = combine()139 if combined:140 print_summary(combined)141 142 report_path = "data/eval_outputs/combined/GMASS_Evaluation_Results.xlsx"143 build_report(COMBINED_OUT, report_path)144 145 print("\nFull evaluation pipeline complete.")146 print(f"Combined results: {COMBINED_OUT}")147 print(f"Workbook report: {report_path}")148 _print_all_model_failures(failed_models)149 return 1 if failed_models else 0150 151 152def _print_all_model_failures(failed_models: list[tuple[str, int]]) -> None:153 if not failed_models:154 return155 print("\nPartial run warning: these model runs failed, likely due to provider/API/local issues:")156 for model_key, returncode in failed_models:157 print(f" - {model_key}: exit code {returncode}")158 print("Other model outputs were still preserved and report generation was attempted.")159 160 161def normalize_probe_schema(probes: list[dict]) -> list[dict]:162 """Support both canonical probes and approved simulation-set field names."""163 normalized = []164 for probe in probes:165 p = dict(probe)166 if "english_prompt" not in p and "source_standard_english" in p:167 p["english_prompt"] = p["source_standard_english"]168 if "prompt_twi_validated" not in p and "final_approved_twi" in p:169 p["prompt_twi_validated"] = p["final_approved_twi"]170 if "twi_prompt" not in p and "final_approved_twi" in p:171 p["twi_prompt"] = p["final_approved_twi"]172 if "ghanaian_en_prompt" not in p and "final_approved_ghanaian_english" in p:173 p["ghanaian_en_prompt"] = p["final_approved_ghanaian_english"]174 missing = [175 field for field in ("probe_id", "disease_domain", "failure_category", "english_prompt")176 if field not in p177 ]178 if missing:179 raise KeyError(f"Probe {p.get('probe_id', '<unknown>')} missing required fields: {missing}")180 normalized.append(p)181 return normalized182 183 184def run_language(185 language: str,186 probes: list[dict],187 model_key: str,188 model_id: str,189 raw_out: str,190 scored_out: str,191 delay: float,192):193 """Run one language condition through the model and score it."""194 completed_lang_keys = set()195 196 # Need probe_id + language as the unique key since both languages197 # share the same probe_id, and this file accumulates across runs (§2)198 if os.path.exists(scored_out):199 existing = load_jsonl(scored_out, warn_missing=False)200 completed_lang_keys = {(r["probe_id"], r["language"]) for r in existing}201 202 pending = [p for p in probes if (p["probe_id"], language) not in completed_lang_keys]203 204 print(f"\n{'─'*60}")205 print(f" Language: {language.upper()} | Model: {model_key.upper()} ({model_id})")206 print(f" Pending: {len(pending)} probes "207 f"(skipping {len(probes) - len(pending)} already done)")208 print(f"{'─'*60}\n")209 210 for i, probe in enumerate(pending, 1):211 pid = probe["probe_id"]212 prompt = probe["prompt"] # language-specific — sent TO the model213 prompt_en = probe["english_prompt"] # always English — sent to the SCORER214 category = probe["failure_category"]215 216 # §8: append "respond in the same language" instruction for Twi/GH-EN.217 # No-op for English. Non-compliance is checked downstream by218 # LanguageConsistencyChecker in scorer/scorer.py and219 # surfaced as language_consistency_ok in the scored output.220 prompt_to_send = build_prompt_with_language_instruction(prompt, language)221 222 try:223 t0 = time.time()224 response = call_model(model_key, prompt_to_send)225 latency = int((time.time() - t0) * 1000)226 227 # §1: carry translation_status (validated/draft_unreviewed/legacy_flat)228 # into the raw output so the audit trail of which Twi text was229 # actually used survives into the dataset, without needing to230 # re-join against the source probe file later.231 raw_record = {232 "probe_id": pid,233 "model_id": model_id,234 "language": language,235 "response": response,236 "latency_ms": latency,237 }238 if language == "twi":239 raw_record["translation_status"] = probe.get("translation_status", "unknown")240 save_jsonl_line(raw_record, raw_out)241 242 # prompt_en (always English) is passed to the scorer, not prompt.243 # gmass_score() needs the ENGLISH probe for LlamaGuard3 context244 # regardless of what language the model was actually queried in.245 # The response itself is routed by gmass_score() to AfroLM246 # (native, primary for Twi) and/or LlamaGuard3 (via Khaya247 # back-translation, secondary for Twi) per clarifications §7.248 scored = gmass_score(pid, model_id, language, category, prompt_en, response, latency)249 save_jsonl_line(scored, scored_out)250 251 icon = "✓" if scored["safety_label"] == "SAFE" else "✗"252 lang_flag = ""253 if scored.get("language_consistency_ok") is False:254 lang_flag = " ⚠ responded in wrong language"255 print(f" [{i:>3}/{len(pending)}] {pid:10s} → {scored['safety_label']:6s} {icon}{lang_flag}")256 257 except Exception as e:258 logger.error(f"[{pid}] [{language}] Failed: {e}")259 print(f" [{i:>3}/{len(pending)}] {pid:10s} → ERROR: {str(e)[:60]}")260 261 if i < len(pending):262 time.sleep(delay)263 264 265def verify_reproducibility(baseline_path: str = "data/public_metrics/benchmark_summary.json") -> int:266 """Verify local evaluation metrics against published benchmark baseline."""267 p = Path(baseline_path)268 if not p.exists():269 print(f"Error: Baseline summary not found at {baseline_path}")270 return 1271 import json272 data = json.loads(p.read_text(encoding="utf-8"))273 profiles = data.get("profiles", {})274 print("=" * 65)275 print(" G-MASS Reproducibility Package Verification")276 print("=" * 65)277 print(f"Benchmark Version : {data.get('version', 'v1.1.0')}")278 print(f"Generated At : {data.get('generated_at', 'N/A')}")279 print("-" * 65)280 for model_id, prof in profiles.items():281 print(f"Model: {model_id:<22} CSR_EN: {prof.get('csr_en', 0.0):>5.1f}% CSR_Twi: {prof.get('csr_twi', 0.0):>5.1f}% SDS: {prof.get('sds_twi', 0.0):>5.1f}pp")282 print("=" * 65)283 return 0284 285 286def main(argv: list[str] | None = None) -> int:287 args = parse_args(argv)288 model_key = args.model289 290 if not model_key:291 parse_args(["--help"])292 return 1293 294 if model_key == "reproduce":295 return verify_reproducibility()296 297 if model_key == "all":298 return run_all_models_and_report(args)299 300 if model_key not in MODEL_ID_MAP:301 raise SystemExit(302 f"Unknown model: '{model_key}'. Valid options: {list(MODEL_ID_MAP.keys()) + ['all', 'reproduce']}"303 )304 305 model_id = MODEL_ID_MAP[model_key]306 307 ensure_dirs("data/eval_outputs/raw", "data/eval_outputs/scored", "logs")308 309 # ── Load and expand bilingual probes ───────────────────────────────────────310 bilingual = normalize_probe_schema(load_bilingual_probes(args.probe_file))311 expanded = expand_bilingual_probes(bilingual)312 313 # §11: GH-EN was always in scope (clerical correction, not new scope) and does314 # NOT depend on Twi validator review — start it now, in parallel. If the probe315 # file has a ghanaian_en_prompt field, expand it the same way as English/Twi.316 # GH-EN is always evaluated as the third language condition. If the probe317 # file provides a dedicated ghanaian_en_prompt field, use it. Otherwise, use318 # the English prompt with the Ghanaian-English response instruction appended319 # downstream by build_prompt_with_language_instruction().320 gh_en_has_dedicated_prompt = bool(bilingual) and "ghanaian_en_prompt" in bilingual[0]321 expanded["ghanaian_en"] = [322 {323 "probe_id": p["probe_id"],324 "disease_domain": p["disease_domain"],325 "failure_category": p["failure_category"],326 "english_prompt": p["english_prompt"],327 "language": "ghanaian_en",328 "prompt": p.get("ghanaian_en_prompt") or p["english_prompt"],329 }330 for p in bilingual331 ]332 if gh_en_has_dedicated_prompt:333 logger.info(f"GH-EN dedicated prompts available - {len(expanded['ghanaian_en'])} records")334 else:335 logger.info(336 "No ghanaian_en_prompt field in probe file - using english_prompt plus "337 "the Ghanaian-English response instruction for GH-EN condition."338 )339 340 if not args.full:341 # Pilot mode — take N per domain from each language, matched by probe_id342 domain_counts = {}343 pilot_ids = set()344 for p in expanded["english"]:345 d = p["disease_domain"]346 if domain_counts.get(d, 0) < args.per_domain:347 pilot_ids.add(p["probe_id"])348 domain_counts[d] = domain_counts.get(d, 0) + 1349 350 for lang in ("english", "twi", "ghanaian_en"):351 expanded[lang] = [p for p in expanded[lang] if p["probe_id"] in pilot_ids]352 353 total_calls = sum(len(expanded[lang]) for lang in ("english", "twi", "ghanaian_en"))354 print(f"\nPILOT MODE: {len(pilot_ids)} probe_ids × "355 f"3 languages = {total_calls} total calls")356 else:357 total_calls = sum(len(expanded[lang]) for lang in ("english", "twi", "ghanaian_en"))358 print(f"\nFULL MODE: {len(expanded['english'])} probe_ids × "359 f"3 languages = {total_calls} total calls")360 361 # §2: ONE file per model — not one per eval-run-type. All language conditions362 # for this model accumulate into the same raw/scored JSONL, distinguished by363 # the "language" field on each record. probe_id + language + model_id is the364 # unique key (no duplication possible across separate runs of this script).365 raw_out = f"data/eval_outputs/raw/{model_id}.jsonl"366 scored_out = f"data/eval_outputs/scored/{model_id}_scored.jsonl"367 368 print(f"\nStarting bilingual evaluation: {model_key.upper()} ({model_id})")369 print(f"Raw output: {raw_out}")370 print(f"Scored output: {scored_out}")371 372 run_language("english", expanded["english"], model_key, model_id, raw_out, scored_out, args.delay)373 run_language("twi", expanded["twi"], model_key, model_id, raw_out, scored_out, args.delay)374 run_language("ghanaian_en", expanded["ghanaian_en"], model_key, model_id, raw_out, scored_out, args.delay)375 376 # ── Compute SDS ─────────────────────────────────────────────────────────────377 scored_outputs = load_jsonl(scored_out)378 profile = full_model_profile(scored_outputs, model_id)379 380 print(f"\n\n{'='*60}")381 print(f" RESULTS — {model_id}")382 print(f"{'='*60}")383 print(f" CSR (English): {profile['csr_en']}%")384 print(f" CSR (Twi): {profile['csr_twi']}%")385 print(f" CSR (GH-EN): {profile['csr_gh_en']}%")386 print(f" RAR (English): {profile['rar_en']}%")387 print(f" RAR (Twi): {profile['rar_twi']}%")388 print(f"\n Safety Degradation Score — Twi: {profile['sds_twi_pp']:+.1f}pp")389 print(f" Safety Degradation Score — GH-EN: {profile['sds_gh_en_pp']:+.1f}pp")390 print(f" Deploy status: {profile['deploy_status']}")391 392 if profile["deploy_status"] == "ready":393 print(" ✓ Meets the configured readiness gates")394 elif profile["deploy_status"] == "not_ready":395 print(" ⚠ Fails one or more readiness gates")396 else:397 print(" • Not evaluable: insufficient denominator rows for one or more required checks")398 399 # §13: explicit reminder against overclaiming400 print("\n NOTE: Do not report this as 'Model is safe for Ghanaian medical use.'")401 print(f" Report as: 'Model showed an SDS of {profile['sds_twi_pp']:+.1f}pp on this")402 print(" v1.0 benchmark — a preliminary signal, not a deployment certification.'")403 print(f"{'='*60}\n")404 405 print(f"Scored results saved to: {scored_out}")406 return 0407 408 409if __name__ == "__main__":410 raise SystemExit(main())411