PerturbReason/PerturbReason_dataset_code

1"""2eval_v3/metric_accuracy.py3===========================4Tier-1 — Answer accuracy evaluation (3-way: up/down/unchanged).5 6Extracts the model's answer from <answer> block, compares with GT label.7Reports accuracy, balanced accuracy, per-class P/R/F1.8"""9 10from __future__ import annotations11 12from collections import Counter13from typing import Any, Dict, List14 15from .data_model import SampleRecord, SampleResult, extract_answer16 17 18# ────────────────────────────────────────────19# Scoring helpers20# ────────────────────────────────────────────21 22def _safe_div(num: float, den: float, default: float = 0.0) -> float:23    return num / den if den > 0 else default24 25 26def compute_classification_scores(27    y_true: List[str], y_pred: List[str],28) -> Dict[str, Any]:29    """30    Compute accuracy, balanced accuracy, per-class P/R/F1.31    Pure Python — no sklearn dependency.32    """33    n = len(y_true)34    if n == 0:35        return {"accuracy": 0.0, "balanced_accuracy": 0.0, "num_samples": 0}36 37    labels = sorted(set(y_true) | set(y_pred))38    gt_labels = sorted(set(y_true))  # only real classes for balanced accuracy39    correct = sum(1 for a, b in zip(y_true, y_pred) if a == b)40    accuracy = correct / n41 42    per_class: Dict[str, Dict[str, float]] = {}43    for lab in labels:44        tp = sum(1 for a, b in zip(y_true, y_pred) if a == lab and b == lab)45        fp = sum(1 for a, b in zip(y_true, y_pred) if a != lab and b == lab)46        fn = sum(1 for a, b in zip(y_true, y_pred) if a == lab and b != lab)47        prec = _safe_div(tp, tp + fp)48        rec = _safe_div(tp, tp + fn)49        f1 = _safe_div(2 * prec * rec, prec + rec)50        per_class[lab] = {"precision": prec, "recall": rec, "f1": f1}51 52    # Balanced accuracy: average recall over GT classes only (excludes __NONE__ parse-fail class)53    class_recalls = [per_class[lab]["recall"] for lab in gt_labels]54    balanced_acc = sum(class_recalls) / len(class_recalls) if class_recalls else 0.055    # F1 macro/weighted: computed over GT classes only (excludes __NONE__)56    macro_f1 = sum(per_class[lab]["f1"] for lab in gt_labels) / len(gt_labels) if gt_labels else 0.057    support = Counter(y_true)58    weighted_f1 = (59        sum(per_class[lab]["f1"] * support.get(lab, 0) for lab in gt_labels) / n60        if n > 0 else 0.061    )62 63    return {64        "accuracy": accuracy,65        "balanced_accuracy": balanced_acc,66        "f1_macro": macro_f1,67        "f1_weighted": weighted_f1,68        "num_samples": n,69        "major_class_ratio": max(support.values()) / n if support else 0.0,70        "per_class": per_class,71        "label_distribution": dict(Counter(y_true)),72        "pred_distribution": dict(Counter(y_pred)),73    }74 75 76# ────────────────────────────────────────────77# Main entry-point78# ────────────────────────────────────────────79 80def evaluate_accuracy(81    samples: List[SampleRecord],82    results: List[SampleResult],83) -> Dict[str, Any]:84    """85    Tier-1 evaluation.86 87    Extracts model answer from <answer> block, compares with GT label.88    Fills result.gt_answer, result.model_answer, result.answer_correct.89    """90    y_true: List[str] = []91    y_pred: List[str] = []92    parse_failures = 093 94    for sample, result in zip(samples, results):95        gt_ans = sample.gt_label96        model_ans = extract_answer(sample.model_output)97 98        result.gt_answer = gt_ans or ""99        result.model_answer = model_ans or ""100        result.task = "3way"101 102        if not gt_ans:103            result.answer_parse_fail = True104            parse_failures += 1105            continue106 107        if model_ans is None:108            result.answer_parse_fail = True109            result.answer_correct = False110            parse_failures += 1111            y_true.append(gt_ans)112            y_pred.append("__NONE__")113            continue114 115        result.answer_correct = (gt_ans == model_ans)116        y_true.append(gt_ans)117        y_pred.append(model_ans)118 119    scores = compute_classification_scores(y_true, y_pred)120    scores["parse_failures"] = parse_failures121    return scores122