PerturbReason/PerturbReason_dataset_code

1"""2eval_v3/metric_llm.py3======================4Tier-4 — LLM Rescue prompt builders and response parsers.5 6This module is a **library only** — it does NOT run inference.7The actual workflow is:8 9  1. Run Tier 1+2 → samples CSV10  2. export_rescue.py → rescue_prompts.jsonl  (uses this module)11  3. qwen_rescue_vllm.py on server → rescue_responses.jsonl12  4. merge_rescue.py → updated samples CSV   (uses this module)13 14Rescue types (matched to M5 error taxonomy):15  - parse_fail           — Tier-1 could not extract an answer16  - wrong_unclassified   — wrong answer, no symbolic diagnosis17  - wrong_correct_chain  — chain looks valid but answer is wrong18  - wrong_sign_flip      — isolated sign-flip error19  - correct_hallucinated — correct answer but hallucinated edges20  - correct_wrong_reason — correct answer but poor reasoning coverage21  - bleurt_high_wrong    — high BLEUrT but wrong answer22"""23 24from __future__ import annotations25 26import re27import textwrap28from typing import Any, Dict, Optional29 30 31# ────────────────────────────────────────────32# Hard-case label sets33# ────────────────────────────────────────────34 35HARD_LABELS = {36    "CORRECT_HALLUCINATED_EDGE",37    "CORRECT_RIGHT_FOR_WRONG_REASON",38    "WRONG_CORRECT_CHAIN",39}40 41BLEURT_HIGH_THRESH = 0.342 43 44# ────────────────────────────────────────────45# Rescue type selection (from flat CSV fields)46# ────────────────────────────────────────────47 48def select_rescue_type(49    error_label: str,50    answer_parse_fail: bool = False,51    bleurt_score: Optional[float] = None,52    answer_correct: bool = False,53) -> str:54    """55    Return the rescue type key, or "" if no rescue needed.56 57    Called by export_rescue.py on CSV row fields (no SampleResult objects).58    """59    if answer_parse_fail:60        return "parse_fail"61 62    if error_label == "WRONG_UNCLASSIFIED":63        return "wrong_unclassified"64    if error_label == "WRONG_CORRECT_CHAIN":65        return "wrong_correct_chain"66    if error_label == "WRONG_SIGN_FLIP":67        return "wrong_sign_flip"68    if error_label == "CORRECT_HALLUCINATED_EDGE":69        return "correct_hallucinated"70    if error_label == "CORRECT_RIGHT_FOR_WRONG_REASON":71        return "correct_wrong_reason"72 73    if (bleurt_score is not None74            and bleurt_score >= BLEURT_HIGH_THRESH75            and not answer_correct):76        return "bleurt_high_wrong"77 78    return ""79 80 81# ────────────────────────────────────────────82# Prompt builders (take flat strings, not objects)83# ────────────────────────────────────────────84 85def _prompt_parse_fail(86    model_output: str, perturbation: str, cell_type: str,87    effect_gene: str, gt_answer: str, **_kw,88) -> str:89    return textwrap.dedent(f"""\90        You are a biology assistant evaluating a model's prediction.91 92        Task: After applying **{perturbation}** in cell type **{cell_type}**, \93does the expression of **{effect_gene}** go UP, DOWN, or remain UNCHANGED?94 95        Ground-truth answer: **{gt_answer}**96 97        The model produced the following output but no final answer could be extracted \98automatically. Read it and determine what direction the model intended to predict.99 100        --- MODEL OUTPUT ---101        {model_output[:1200]}102        --- END ---103 104        Reply with exactly ONE word: up / down / unchanged, or "unclear" if truly undeterminable.105    """).strip()106 107 108def _prompt_wrong_unclassified(109    model_output: str, perturbation: str, cell_type: str,110    effect_gene: str, gt_answer: str, model_answer: str = "", **_kw,111) -> str:112    return textwrap.dedent(f"""\113        You are evaluating the reasoning quality of a biology model.114 115        Setting: **{perturbation}** applied to **{cell_type}**.116        Question: Does **{effect_gene}** expression go **{gt_answer.upper()}**?117 118        The model answered **{(model_answer or '').upper()}** — which is WRONG.119        Deterministic checks found no clear single root cause for the error.120 121        --- MODEL REASONING ---122        {model_output[:1500]}123        --- END ---124 125        Is the model's reasoning chain internally consistent and biologically plausible, \126even though the final answer is wrong?127 128        Reply: YES or NO, then ONE sentence explaining the root cause of the error.129    """).strip()130 131 132def _prompt_wrong_correct_chain(133    model_output: str, perturbation: str, cell_type: str,134    effect_gene: str, gt_answer: str, model_answer: str = "", **_kw,135) -> str:136    return textwrap.dedent(f"""\137        You are auditing a biology model's chain-of-thought.138 139        Setting: **{perturbation}** applied to **{cell_type}**.140        Effect gene: **{effect_gene}**141        Ground truth: **{gt_answer.upper()}**142        Model answer: **{(model_answer or '').upper()}** (WRONG)143 144        The model's extracted reasoning path appears structurally valid (connected, \145uses real edges, consistent signs), yet the final answer disagrees with the ground truth.146 147        --- MODEL REASONING ---148        {model_output[:1500]}149        --- END ---150 151        Possible explanations:152          ANSWER_BUG — the reasoning is sound but the model wrote the wrong final label.153          CHAIN_BUG  — the chain has a subtle biological error not captured by simple metrics.154          ALTERNATE  — the chain follows a valid alternative pathway.155 156        Reply: ANSWER_BUG, CHAIN_BUG, or ALTERNATE, then ONE sentence justifying.157    """).strip()158 159 160def _prompt_wrong_sign_flip(161    model_output: str, perturbation: str, cell_type: str,162    effect_gene: str, gt_answer: str, model_answer: str = "", **_kw,163) -> str:164    return textwrap.dedent(f"""\165        You are evaluating a biology model's reasoning.166 167        Setting: **{perturbation}** applied to **{cell_type}**.168        Effect gene: **{effect_gene}**169        Ground truth: **{gt_answer.upper()}**170        Model answer: **{(model_answer or '').upper()}** (WRONG)171 172        Sign-consistency checks detected that some triplets have their regulatory \173direction flipped relative to the external knowledge graph.174 175        --- MODEL REASONING ---176        {model_output[:1500]}177        --- END ---178 179        Is the sign-flip error the SOLE cause of the wrong answer, or are there \180additional reasoning problems?181 182        Reply: SOLE or ADDITIONAL, then ONE sentence.183    """).strip()184 185 186def _prompt_correct_hallucinated(187    model_output: str, perturbation: str, cell_type: str,188    effect_gene: str, gt_answer: str, **_kw,189) -> str:190    return textwrap.dedent(f"""\191        You are evaluating a biology model's reasoning quality.192 193        Setting: **{perturbation}** applied to **{cell_type}**.194        Effect gene: **{effect_gene}**195        Ground truth: **{gt_answer.upper()}**196        Model answer: **{gt_answer.upper()}** (CORRECT)197 198        The model got the right answer, but some edges in its reasoning chain \199are NOT found in the reference knowledge graph (hallucinated edges).200 201        --- MODEL REASONING ---202        {model_output[:1500]}203        --- END ---204 205        Is the model's reasoning chain biologically plausible despite the hallucinated edges?206 207        Reply: YES or NO, then ONE sentence explaining.208    """).strip()209 210 211def _prompt_correct_wrong_reason(212    model_output: str, perturbation: str, cell_type: str,213    effect_gene: str, gt_answer: str, **_kw,214) -> str:215    return textwrap.dedent(f"""\216        You are evaluating a biology model's reasoning quality.217 218        Setting: **{perturbation}** applied to **{cell_type}**.219        Effect gene: **{effect_gene}**220        Ground truth: **{gt_answer.upper()}**221        Model answer: **{gt_answer.upper()}** (CORRECT)222 223        The model got the right answer, but its reasoning path has poor coverage \224of the ground-truth causal path (low recall or disconnected path).225 226        --- MODEL REASONING ---227        {model_output[:1500]}228        --- END ---229 230        Is the model's reasoning internally consistent and biologically plausible, \231even though it doesn't match the reference path well?232 233        Reply: YES or NO, then ONE sentence explaining.234    """).strip()235 236 237def _prompt_bleurt_high_wrong(238    model_output: str, perturbation: str, cell_type: str,239    effect_gene: str, gt_answer: str, model_answer: str = "",240    bleurt_score: Optional[float] = None, **_kw,241) -> str:242    thresh = bleurt_score if bleurt_score is not None else BLEURT_HIGH_THRESH243    return textwrap.dedent(f"""\244        You are evaluating a biology model's prediction.245 246        Setting: **{perturbation}** applied to **{cell_type}**.247        Effect gene: **{effect_gene}**248        Ground truth: **{gt_answer.upper()}**249        Extracted model answer: **{model_answer or '(none)'}** (parsed as WRONG)250 251        The model's text is highly similar to the reference answer \252(BLEUrT = {thresh:.2f}), suggesting the reasoning may be correct.253 254        --- MODEL OUTPUT ---255        {model_output[:1200]}256        --- END ---257 258        Based on the text, what direction does the model predict?259        Reply with ONE word: up / down / unchanged / unclear.260    """).strip()261 262 263_PROMPT_BUILDERS = {264    "parse_fail":           _prompt_parse_fail,265    "wrong_unclassified":   _prompt_wrong_unclassified,266    "wrong_correct_chain":  _prompt_wrong_correct_chain,267    "wrong_sign_flip":      _prompt_wrong_sign_flip,268    "correct_hallucinated": _prompt_correct_hallucinated,269    "correct_wrong_reason": _prompt_correct_wrong_reason,270    "bleurt_high_wrong":    _prompt_bleurt_high_wrong,271}272 273 274def build_rescue_prompt(275    rescue_type: str,276    model_output: str,277    perturbation: str = "",278    cell_type: str = "",279    effect_gene: str = "",280    gt_answer: str = "",281    model_answer: str = "",282    bleurt_score: Optional[float] = None,283) -> str:284    """285    Build a rescue prompt for the given rescue type.286 287    Public API used by export_rescue.py.288    """289    builder = _PROMPT_BUILDERS.get(rescue_type)290    if builder is None:291        raise ValueError(f"Unknown rescue type: {rescue_type}")292    return builder(293        model_output=model_output,294        perturbation=perturbation,295        cell_type=cell_type,296        effect_gene=effect_gene,297        gt_answer=gt_answer,298        model_answer=model_answer,299        bleurt_score=bleurt_score,300    )301 302 303# ────────────────────────────────────────────304# Response parser (used by merge_rescue.py)305# ────────────────────────────────────────────306 307def _find_keyword(text: str, keywords: list) -> tuple:308    """Return (matched_keyword_upper, text_after_match) for the first keyword found."""309    pattern = r"\b(" + "|".join(re.escape(k) for k in keywords) + r")\b"310    m = re.search(pattern, text, re.IGNORECASE)311    if m:312        return m.group(1).upper(), text[m.end():].strip()313    return "", text314 315 316def parse_rescue_response(317    rescue_type: str,318    response: str,319    gt_answer: str = "",320) -> Dict[str, Any]:321    """322    Parse the LLM's raw text response for a given rescue type.323 324    Searches for verdict keywords anywhere in the response text rather than325    relying on the first word, which handles cases where the model echoes326    back prompt instructions before giving its verdict.327 328    Returns dict with: verdict, answer, explanation, corrects_answer.329    Public API used by merge_rescue.py.330    """331    text = response.strip()332    verdict: str = ""333    answer: Optional[str] = None334    corrects: Optional[bool] = None335    explanation: str = ""336 337    if rescue_type == "parse_fail":338        kw, expl = _find_keyword(text, ["UP", "DOWN", "UNCHANGED"])339        answer = kw.lower() if kw else "unclear"340        corrects = (answer == gt_answer.lower()) if kw else False341        explanation = expl342        verdict = kw or "UNCLEAR"343 344    elif rescue_type == "wrong_unclassified":345        kw, expl = _find_keyword(text, ["YES", "NO"])346        verdict = kw if kw else "NO"347        explanation = expl348 349    elif rescue_type == "wrong_correct_chain":350        kw, expl = _find_keyword(text, ["ANSWER_BUG", "ALTERNATE", "CHAIN_BUG"])351        if kw == "ANSWER_BUG":352            verdict = "ANSWER_BUG"353        elif kw == "ALTERNATE":354            verdict = "ALTERNATE"355        else:356            verdict = "CHAIN_BUG"357        explanation = expl358 359    elif rescue_type == "wrong_sign_flip":360        kw, expl = _find_keyword(text, ["SOLE", "ADDITIONAL"])361        verdict = kw if kw else "ADDITIONAL"362        explanation = expl363 364    elif rescue_type in ("correct_hallucinated", "correct_wrong_reason"):365        kw, expl = _find_keyword(text, ["YES", "NO"])366        verdict = kw if kw else "NO"367        explanation = expl368 369    elif rescue_type == "bleurt_high_wrong":370        kw, expl = _find_keyword(text, ["UP", "DOWN", "UNCHANGED"])371        answer = kw.lower() if kw else "unclear"372        corrects = (answer == gt_answer.lower()) if kw else False373        explanation = expl374        verdict = kw or "UNCLEAR"375 376    else:377        # Unknown rescue type — fall back to first-word extraction378        parts = text.split(None, 1)379        verdict = parts[0].upper() if parts else ""380        explanation = parts[1].strip() if len(parts) > 1 else ""381 382    return {383        "verdict": verdict,384        "answer": answer,385        "explanation": explanation,386        "corrects_answer": corrects,387    }388