PerturbReason/PerturbReason_dataset_code
012
1#!/usr/bin/env python32"""3eval_v3/runner.py4==================5CLI entry-point for the PerturbQA Evaluation v3 pipeline.6 7Three subcommands for the split workflow:8 9 1. eval — Run Tier 1+2 (+ optionally 3) locally10 2. export-rescue — Export hard cases → rescue_prompts.jsonl11 3. merge-rescue — Merge server responses → updated CSV12 13Examples14--------15# Step 1: Run Tiers 1+2 locally16python -m eval_v3 eval \\17 --gt-dir /path/to/noisy_input \\18 --pred-dir /path/to/noisy_context_output_0322 \\19 --external-kg eval_v2/external_kg.json \\20 --tiers 1 221 22# Step 2: Export hard cases for server inference23python -m eval_v3 export-rescue \\24 --samples-csv eval_v3/eval_v3_output/samples_*.csv \\25 --pred-dir /path/to/noisy_context_output_032226 27# Step 3: (on server) sbatch eval_v3/qwen_rescue_vllm.sh28 29# Step 4: Merge responses back30python -m eval_v3 merge-rescue \\31 --samples-csv eval_v3/eval_v3_output/samples_*.csv \\32 --rescue-responses eval_v3/eval_v3_output/rescue_responses.jsonl33"""34 35from __future__ import annotations36 37import argparse38import json39import time40from pathlib import Path41from typing import List42 43from .pipeline import EvalPipeline44from .data_model import FileResult45 46 47# ════════════════════════════════════════════48# Subcommand: eval49# ════════════════════════════════════════════50 51def add_eval_parser(subparsers):52 p = subparsers.add_parser(53 "eval",54 help="Run Tier 1+2 (+ optionally 3) evaluation",55 formatter_class=argparse.RawDescriptionHelpFormatter,56 )57 p.add_argument("--gt-dir", type=Path, default=None)58 p.add_argument("--pred-dir", type=Path, default=None)59 p.add_argument("--gt-file", type=Path, default=None)60 p.add_argument("--pred-file", type=Path, default=None)61 p.add_argument("--external-kg", type=Path, default=None)62 p.add_argument("--tiers", nargs="+", type=int, default=[1, 2],63 choices=[1, 2, 3])64 p.add_argument("--go-gmt", type=Path, default=None,65 help="MSigDB GMT file for Tier 3 GO similarity "66 "(e.g. c5.go.bp.v2023.2.Hs.symbols.gmt)")67 p.add_argument("-o", "--output-dir", type=Path,68 default=Path(__file__).parent / "eval_v3_output")69 p.add_argument("-v", "--verbose", action="store_true")70 p.set_defaults(func=cmd_eval)71 72 73def cmd_eval(args):74 has_dir = args.gt_dir and args.pred_dir75 has_file = args.gt_file and args.pred_file76 if not has_dir and not has_file:77 print("ERROR: Must provide either (--gt-dir + --pred-dir) or "78 "(--gt-file + --pred-file)")79 return80 81 pipe = EvalPipeline(82 tiers=args.tiers,83 go_gmt_path=args.go_gmt,84 external_kg_path=args.external_kg,85 )86 87 start = time.time()88 file_results: List[FileResult] = []89 90 if has_dir:91 print(f"Evaluating directory pair:")92 print(f" GT: {args.gt_dir}")93 print(f" Pred: {args.pred_dir}")94 file_results = pipe.evaluate_all(args.gt_dir, args.pred_dir)95 else:96 print(f"Evaluating single file pair:")97 print(f" GT: {args.gt_file}")98 print(f" Pred: {args.pred_file}")99 fr = pipe.evaluate_file_pair(args.gt_file, args.pred_file)100 file_results.append(fr)101 102 # Print reports103 for fr in file_results:104 pipe.print_report(fr)105 106 elapsed = time.time() - start107 108 # Write outputs109 args.output_dir.mkdir(parents=True, exist_ok=True)110 timestamp = time.strftime("%Y%m%d_%H%M%S")111 112 summary_json = args.output_dir / f"summary_{timestamp}.json"113 pipe.write_summary_json(file_results, summary_json)114 print(f"\nSummary JSON: {summary_json}")115 116 samples_csv = args.output_dir / f"samples_{timestamp}.csv"117 pipe.write_samples_csv(file_results, samples_csv)118 print(f"Samples CSV: {samples_csv}")119 120 file_csv = args.output_dir / f"file_summary_{timestamp}.csv"121 pipe.write_file_summary_csv(file_results, file_csv)122 print(f"File summary: {file_csv}")123 124 print(f"\nDone in {elapsed:.1f}s — {len(file_results)} files, "125 f"{sum(len(fr.sample_results) for fr in file_results)} samples total.")126 print(f"\nNext: run 'python -m eval_v3 export-rescue --samples-csv {samples_csv} "127 f"--pred-dir <pred_dir>' to generate Tier-4 rescue prompts.")128 129 130# ════════════════════════════════════════════131# Subcommand: export-rescue132# ════════════════════════════════════════════133 134def add_export_rescue_parser(subparsers):135 p = subparsers.add_parser(136 "export-rescue",137 help="Export hard cases for Tier-4 LLM rescue",138 )139 p.add_argument("--samples-csv", type=Path, required=True,140 help="Samples CSV from 'eval' step")141 p.add_argument("--pred-dir", type=Path, required=True,142 help="Prediction directory (for raw model outputs)")143 p.add_argument("-o", "--output", type=Path, default=None,144 help="Output JSONL path (default: rescue_prompts_<ts>.jsonl)")145 p.add_argument("--max-samples", type=int, default=None)146 p.set_defaults(func=cmd_export_rescue)147 148 149def cmd_export_rescue(args):150 from .export_rescue import export_rescue_prompts151 output_dir = Path(__file__).parent / "eval_v3_output"152 output_dir.mkdir(parents=True, exist_ok=True)153 timestamp = time.strftime("%Y%m%d_%H%M%S")154 output_path = args.output or output_dir / f"rescue_prompts_{timestamp}.jsonl"155 print(f"Exporting rescue prompts from {args.samples_csv}")156 n = export_rescue_prompts(157 samples_csv=args.samples_csv,158 pred_dir=args.pred_dir,159 output_path=output_path,160 max_samples=args.max_samples,161 )162 responses_path = output_dir / f"rescue_responses_{timestamp}.jsonl"163 print(f"\nExported {n} rescue prompts → {output_path}")164 print(f"\nNext: copy {output_path.name} to server and run:")165 print(f" sbatch eval_v3/qwen_rescue_vllm.sh")166 print(f" (set INPUT_FILE={output_path.name}, OUTPUT_FILE={responses_path.name})")167 print(f"Then: python -m eval_v3 merge-rescue --samples-csv {args.samples_csv} "168 f"--rescue-responses eval_v3/eval_v3_output/{responses_path.name}")169 170 171# ════════════════════════════════════════════172# Subcommand: merge-rescue173# ════════════════════════════════════════════174 175def add_merge_rescue_parser(subparsers):176 p = subparsers.add_parser(177 "merge-rescue",178 help="Merge Tier-4 rescue responses back into results",179 )180 p.add_argument("--samples-csv", type=Path, required=True,181 help="Original samples CSV from 'eval' step")182 p.add_argument("--rescue-responses", type=Path, required=True,183 help="Rescue responses JSONL from server")184 p.add_argument("-o", "--output-csv", type=Path, default=None,185 help="Updated CSV path (default: samples_with_rescue_<ts>.csv)")186 p.add_argument("--summary-json", type=Path, default=None)187 p.set_defaults(func=cmd_merge_rescue)188 189 190def cmd_merge_rescue(args):191 timestamp = time.strftime("%Y%m%d_%H%M%S")192 output_dir = Path(__file__).parent / "eval_v3_output"193 output_dir.mkdir(parents=True, exist_ok=True)194 if args.output_csv is None:195 args.output_csv = output_dir / f"samples_with_rescue_{timestamp}.csv"196 from .merge_rescue import (197 load_samples_csv, load_rescue_responses,198 merge, write_updated_csv, compute_updated_summary,199 )200 201 print(f"Loading samples from {args.samples_csv}")202 samples = load_samples_csv(args.samples_csv)203 print(f" {len(samples)} samples")204 205 print(f"Loading rescue responses from {args.rescue_responses}")206 responses = load_rescue_responses(args.rescue_responses)207 print(f" {len(responses)} responses")208 209 stats = merge(samples, responses)210 print(f"\nMerge results:")211 print(f" Processed: {stats['total_rescue_responses']}")212 print(f" Answer corrections: {stats['answer_corrections']}")213 print(f" Label updates: {stats['label_updates']}")214 print(f" Rescue types: {stats['rescue_type_counts']}")215 print(f" Verdicts: {stats['verdict_counts']}")216 217 write_updated_csv(samples, args.output_csv)218 print(f"\nUpdated CSV → {args.output_csv}")219 220 summary = compute_updated_summary(samples)221 summary["merge_stats"] = stats222 summary_path = args.summary_json or args.output_csv.with_suffix(".json")223 import json as _json224 with open(summary_path, "w") as f:225 _json.dump(summary, f, indent=2)226 print(f"Summary JSON → {summary_path}")227 228 229# ════════════════════════════════════════════230# Main231# ════════════════════════════════════════════232 233def main():234 parser = argparse.ArgumentParser(235 prog="eval_v3",236 description="PerturbQA Evaluation v3 — Multi-tier reasoning evaluation",237 )238 subparsers = parser.add_subparsers(dest="command", help="Available commands")239 240 add_eval_parser(subparsers)241 add_export_rescue_parser(subparsers)242 add_merge_rescue_parser(subparsers)243 244 args = parser.parse_args()245 246 if args.command is None:247 parser.print_help()248 return249 250 args.func(args)251 252 253if __name__ == "__main__":254 main()255