admesh/agentic-intent-classifier

1from __future__ import annotations2 3import argparse4import json5import sys6from pathlib import Path7 8BASE_DIR = Path(__file__).resolve().parent.parent9if str(BASE_DIR) not in sys.path:10    sys.path.insert(0, str(BASE_DIR))11 12from combined_inference import classify_query13from config import BASE_DIR, INTENT_HEAD_CONFIG, ensure_artifact_dirs14from model_runtime import get_head15from schemas import validate_classify_response16 17DEFAULT_THRESHOLDS = [0.0, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, 0.45]18SAFE_INTENT_TYPES = {"ambiguous", "personal_reflection", "support"}19OBVIOUS_INTENT_TYPES = {"informational", "commercial", "transactional"}20SWEEP_SUITE_PATH = BASE_DIR / "examples" / "intent_threshold_sweep_suite.json"21OUTPUT_PATH = BASE_DIR / "artifacts" / "evaluation" / "intent_threshold_sweep.json"22 23 24def load_jsonl(path: Path) -> list[dict]:25    with path.open("r", encoding="utf-8") as handle:26        return [json.loads(line) for line in handle]27 28 29def load_json(path: Path) -> list[dict]:30    return json.loads(path.read_text(encoding="utf-8"))31 32 33def round_score(value: float) -> float:34    return round(float(value), 4)35 36 37def write_json(path: Path, payload: dict) -> None:38    path.parent.mkdir(parents=True, exist_ok=True)39    path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8")40 41 42def evaluate_intent_head_threshold(threshold: float) -> dict:43    head = get_head("intent_type")44    dataset_specs = [45        ("val", INTENT_HEAD_CONFIG.split_paths["val"]),46        ("test", INTENT_HEAD_CONFIG.split_paths["test"]),47        ("hard_cases", BASE_DIR / "data" / "hard_cases.jsonl"),48        ("third_wave_cases", BASE_DIR / "data" / "third_wave_cases.jsonl"),49    ]50    rows = []51    for suite_name, path in dataset_specs:52        for item in load_jsonl(path):53            rows.append({"suite": suite_name, **item})54 55    predictions = head.predict_batch([row["text"] for row in rows], confidence_threshold=threshold)56    obvious_total = 057    obvious_fallback = 058    ambiguous_total = 059    ambiguous_bad_allow = 060    intent_only_safe_pred = 061 62    for row, prediction in zip(rows, predictions):63        predicted_label = prediction["label"]64        head_would_fallback = (not prediction["meets_confidence_threshold"]) or (predicted_label in SAFE_INTENT_TYPES)65 66        if row[INTENT_HEAD_CONFIG.label_field] in OBVIOUS_INTENT_TYPES:67            obvious_total += 168            if head_would_fallback:69                obvious_fallback += 170 71        if row[INTENT_HEAD_CONFIG.label_field] == "ambiguous":72            ambiguous_total += 173            if not head_would_fallback:74                ambiguous_bad_allow += 175 76        if head_would_fallback and predicted_label in SAFE_INTENT_TYPES:77            intent_only_safe_pred += 178 79    return {80        "obvious_prompt_count": obvious_total,81        "obvious_false_fallback_rate": round_score(obvious_fallback / obvious_total) if obvious_total else 0.0,82        "ambiguous_prompt_count": ambiguous_total,83        "ambiguous_bad_allow_rate": round_score(ambiguous_bad_allow / ambiguous_total) if ambiguous_total else 0.0,84        "safe_predicate_rate": round_score(intent_only_safe_pred / len(rows)) if rows else 0.0,85    }86 87 88def evaluate_combined_threshold(threshold: float) -> dict:89    suite = load_json(SWEEP_SUITE_PATH)90    benchmark = load_json(BASE_DIR / "examples" / "demo_prompt_suite.json")91 92    obvious_total = 093    obvious_false_fallback = 094    safe_total = 095    safe_bad_allow = 096    intent_only = 097    phase_only = 098    both = 099    policy_safe = 0100 101    suite_outputs = []102    for item in suite:103        payload = validate_classify_response(classify_query(item["input"], threshold_overrides={"intent_type": threshold}))104        fallback = payload["model_output"].get("fallback")105        fallback_applied = fallback is not None106        failed_components = set((fallback or {}).get("failed_components", []))107 108        if item["expected_outcome"] == "pass":109            obvious_total += 1110            if fallback_applied:111                obvious_false_fallback += 1112        else:113            safe_total += 1114            if not fallback_applied:115                safe_bad_allow += 1116 117        if fallback_applied:118            if failed_components == {"intent_type"}:119                intent_only += 1120            elif failed_components == {"decision_phase"}:121                phase_only += 1122            elif failed_components == {"intent_type", "decision_phase"}:123                both += 1124            else:125                policy_safe += 1126 127        suite_outputs.append(128            {129                "input": item["input"],130                "expected_outcome": item["expected_outcome"],131                "fallback_applied": fallback_applied,132                "failed_components": sorted(failed_components),133                "intent_type": payload["model_output"]["classification"]["intent"]["type"],134                "decision_phase": payload["model_output"]["classification"]["intent"]["decision_phase"],135                "intent_confidence": payload["model_output"]["classification"]["intent"]["component_confidence"]["intent_type"]["confidence"],136                "phase_confidence": payload["model_output"]["classification"]["intent"]["component_confidence"]["decision_phase"]["confidence"],137            }138        )139 140    benchmark_fallbacks = []141    for item in benchmark:142        payload = validate_classify_response(classify_query(item["input"], threshold_overrides={"intent_type": threshold}))143        fallback = payload["model_output"].get("fallback")144        fallback_applied = fallback is not None145        failed_components = set((fallback or {}).get("failed_components", []))146        benchmark_fallbacks.append(147            {148                "fallback_applied": fallback_applied,149                "intent_only": failed_components == {"intent_type"},150                "phase_only": failed_components == {"decision_phase"},151                "both": failed_components == {"intent_type", "decision_phase"},152            }153        )154 155    total_suite_fallbacks = intent_only + phase_only + both + policy_safe156    benchmark_total_fallbacks = sum(1 for item in benchmark_fallbacks if item["fallback_applied"])157    return {158        "suite_path": str(SWEEP_SUITE_PATH),159        "obvious_prompt_count": obvious_total,160        "false_fallback_rate_on_obvious_prompts": round_score(obvious_false_fallback / obvious_total) if obvious_total else 0.0,161        "safe_prompt_count": safe_total,162        "bad_allow_rate_on_safe_prompts": round_score(safe_bad_allow / safe_total) if safe_total else 0.0,163        "fallback_responsibility": {164            "intent_only": intent_only,165            "phase_only": phase_only,166            "both": both,167            "policy_safe": policy_safe,168            "intent_share_of_threshold_fallbacks": round_score(169                (intent_only + both) / (intent_only + phase_only + both)170            )171            if (intent_only + phase_only + both)172            else 0.0,173            "phase_share_of_threshold_fallbacks": round_score(174                (phase_only + both) / (intent_only + phase_only + both)175            )176            if (intent_only + phase_only + both)177            else 0.0,178            "fallback_rate": round_score(total_suite_fallbacks / len(suite)) if suite else 0.0,179        },180        "benchmark_fallback_rate": round_score(benchmark_total_fallbacks / len(benchmark)) if benchmark else 0.0,181        "benchmark_intent_only_fallback_rate": round_score(182            sum(1 for item in benchmark_fallbacks if item["intent_only"]) / len(benchmark)183        )184        if benchmark185        else 0.0,186        "benchmark_phase_only_fallback_rate": round_score(187            sum(1 for item in benchmark_fallbacks if item["phase_only"]) / len(benchmark)188        )189        if benchmark190        else 0.0,191        "suite_outputs": suite_outputs,192    }193 194 195def pick_recommended_threshold(results: list[dict]) -> dict:196    return min(197        results,198        key=lambda item: (199            item["combined"]["bad_allow_rate_on_safe_prompts"],200            item["head"]["ambiguous_bad_allow_rate"],201            item["combined"]["false_fallback_rate_on_obvious_prompts"],202            abs(item["combined"]["fallback_responsibility"]["intent_share_of_threshold_fallbacks"] - 0.5),203            item["combined"]["benchmark_fallback_rate"],204            item["threshold"],205        ),206    )207 208 209def apply_threshold(threshold: float) -> None:210    calibration_path = INTENT_HEAD_CONFIG.calibration_path211    payload = json.loads(calibration_path.read_text(encoding="utf-8"))212    payload["confidence_threshold"] = round_score(threshold)213    payload["threshold_selection_mode"] = "manual_sweep"214    write_json(calibration_path, payload)215 216 217def main() -> None:218    parser = argparse.ArgumentParser(description="Sweep candidate intent_type thresholds and compare end-to-end behavior.")219    parser.add_argument(220        "--thresholds",221        nargs="*",222        type=float,223        default=DEFAULT_THRESHOLDS,224        help="Candidate thresholds to evaluate.",225    )226    parser.add_argument(227        "--apply-threshold",228        type=float,229        default=None,230        help="Optional threshold to write into the intent_type calibration artifact after evaluation.",231    )232    args = parser.parse_args()233 234    ensure_artifact_dirs()235    thresholds = [round_score(value) for value in args.thresholds]236    results = []237    for threshold in thresholds:238        results.append(239            {240                "threshold": threshold,241                "head": evaluate_intent_head_threshold(threshold),242                "combined": evaluate_combined_threshold(threshold),243            }244        )245 246    recommended = pick_recommended_threshold(results)247    output = {248        "thresholds": thresholds,249        "results": results,250        "recommended_threshold": recommended["threshold"],251    }252    write_json(OUTPUT_PATH, output)253 254    if args.apply_threshold is not None:255        apply_threshold(args.apply_threshold)256        output["applied_threshold"] = round_score(args.apply_threshold)257        write_json(OUTPUT_PATH, output)258 259    print(json.dumps(output, indent=2))260 261 262if __name__ == "__main__":263    main()264