cjoyy/ops-agent

1import json2from pathlib import Path3 4from graph_supervisor import supervisor5 6 7DATASET_PATH = Path("eval/golden_dataset.json")8 9 10def load_dataset():11    with DATASET_PATH.open("r", encoding="utf-8") as file:12        return json.load(file)13 14 15def classify_ticket(ticket: str) -> str:16    result = supervisor(17        {18            "messages": [ticket],19            "resolved_categories": [],20            "needs_followup": False,21            "followup_category": None,22            "runbook_context": "",23            "debug": False,24        }25    )26    return result["category"]27 28 29def main():30    dataset = load_dataset()31    mistakes = []32 33    for item in dataset:34        ticket = item["ticket"]35        expected = item["expected_category"]36        actual = classify_ticket(ticket)37 38        if actual != expected:39            mistakes.append(40                {41                    "ticket": ticket,42                    "expected": expected,43                    "actual": actual,44                }45            )46 47    total = len(dataset)48    correct = total - len(mistakes)49    accuracy = correct / total if total else 050 51    print(f"Accuracy: {accuracy:.0%} ({correct}/{total})")52 53    if not mistakes:54        print("No misclassified tickets.")55        return56 57    print("\nMisclassified tickets:")58    for mistake in mistakes:59        print("- Ticket:", mistake["ticket"])60        print("  Expected:", mistake["expected"])61        print("  Actual:", mistake["actual"])62 63 64if __name__ == "__main__":65    main()66