TuwaiqAcademy/AISA-ArabicFC-Shared-Task

1"""2AISA-ArabicFC official evaluation library (v2 weights).3 4Metrics:5  FnAcc       = #(pred.tool_called == gold.tool_called) / N_total6                Negatives have gold.tool_called = "none", so FnAcc also7                penalises hallucinated calls and missed calls.8  ArgEM       = #(pred.arguments == gold.arguments) / N_positive9                Strict exact match over key-value pairs (None values filtered).10                Only computed on positive samples.11  ThinkRate   = #(non-empty pred.think) / N_total12 13Track A: 0.40 * FnAcc + 0.60 * ArgEM14Track B: 0.30 * FnAcc + 0.50 * ArgEM + 0.20 * ThinkRate15 16Dialect diagnostic (Track C): per-dialect FnAcc + ArgEM + gap (max - min).17"""18from __future__ import annotations19 20from collections import defaultdict21from typing import Any22 23from normalize import args_match24 25 26DIALECTS = ["msa", "gulf", "egyptian", "levantine", "maghrebi"]27 28 29def _norm_args(args: dict | None) -> dict:30    """Drop None values and stringify everything for comparison."""31    if not args:32        return {}33    return {str(k): str(v).strip() for k, v in args.items() if v is not None and v != ""}34 35 36def evaluate(predictions: list[dict], gold: list[dict]) -> dict[str, Any]:37    """38    Score predictions against gold.39 40    predictions: [{"id": int, "tool_called": str, "arguments": dict, "think"?: str}, ...]41    gold:        [{"id": int, "tool_called": str, "arguments": dict,42                   "dialect": str, "requires_function": bool}, ...]43 44    Returns a dict with overall + per-dialect metrics.45    """46    by_id = {p["id"]: p for p in predictions}47 48    n_total = len(gold)49    fn_correct = 050    arg_em_correct = 051    n_positive = 052    think_present = 053    missing = 054 55    dialect_stats: dict[str, dict] = defaultdict(56        lambda: {"fn_correct": 0, "arg_em_correct": 0, "pos_total": 0, "total": 0}57    )58 59    for g in gold:60        gid = g["id"]61        dialect = (g.get("dialect") or "unknown").lower()62        dstat = dialect_stats[dialect]63        dstat["total"] += 164 65        p = by_id.get(gid)66        gold_fn = g["tool_called"]67        gold_args = _norm_args(g.get("arguments"))68        is_positive = bool(g.get("requires_function"))69 70        if is_positive:71            n_positive += 172            dstat["pos_total"] += 173 74        if p is None:75            missing += 176            continue77 78        # FnAcc — across ALL samples79        pred_fn = (p.get("tool_called") or "none").strip() or "none"80        if pred_fn == gold_fn:81            fn_correct += 182            dstat["fn_correct"] += 183 84        # ArgEM — only on positives. Normalised, fair matching (see normalize.py):85        # numbers (5000==5000.0, Arabic-Indic digits), Arabic orthography,86        # number-words, list/set fields, and closed-class bilingual aliases.87        if is_positive:88            if args_match(p.get("arguments"), g.get("arguments"), gold_fn):89                arg_em_correct += 190                dstat["arg_em_correct"] += 191 92        # ThinkRate — non-empty think text counts (>5 chars)93        think = p.get("think") or ""94        if isinstance(think, str) and len(think.strip()) > 5:95            think_present += 196 97    fnacc = fn_correct / n_total if n_total else 0.098    argem = arg_em_correct / n_positive if n_positive else 0.099    thinkrate = think_present / n_total if n_total else 0.0100 101    overall_a = 0.40 * fnacc + 0.60 * argem102    overall_b = 0.30 * fnacc + 0.50 * argem + 0.20 * thinkrate103 104    # Per-dialect105    dialect_breakdown: dict[str, dict] = {}106    for d, s in dialect_stats.items():107        dialect_breakdown[d] = {108            "fnacc": (s["fn_correct"] / s["total"]) if s["total"] else 0.0,109            "argem": (s["arg_em_correct"] / s["pos_total"]) if s["pos_total"] else 0.0,110            "n": s["total"],111            "n_positive": s["pos_total"],112        }113 114    fn_values = [v["fnacc"] for v in dialect_breakdown.values() if v["n"] > 0]115    ar_values = [v["argem"] for v in dialect_breakdown.values() if v["n_positive"] > 0]116    gap_fnacc = (max(fn_values) - min(fn_values)) if fn_values else 0.0117    gap_argem = (max(ar_values) - min(ar_values)) if ar_values else 0.0118 119    return {120        "fnacc": fnacc,121        "argem": argem,122        "thinkrate": thinkrate,123        "overall_a": overall_a,124        "overall_b": overall_b,125        "dialect_breakdown": dialect_breakdown,126        "gap_fnacc": gap_fnacc,127        "gap_argem": gap_argem,128        "n_total": n_total,129        "n_positive": n_positive,130        "n_negative": n_total - n_positive,131        "n_predictions": len(predictions),132        "missing": missing,133    }134