HaofanWen/causal_debugging_agent

1import json2import re3import sys4from pathlib import Path5from sacrebleu import corpus_bleu6import javalang7 8# Paths9GOLD_PATH = Path("bug_data/debug_dataset.jsonl")10PRED_PATH = Path("debug_output.json")11OUT_PATH  = Path("evaluate_debug.json")12 13def extract_code(text: str) -> str:14    m = re.search(r"```(?:[^\n]*)\n([\s\S]*?)```", text)15    if m:16        return m.group(1).strip()17    return text.strip()18 19def normalize_whitespace(s: str) -> str:20    return re.sub(r"\s+", "", s)21 22def load_gold(path: Path) -> dict:23    gold = {}24    with open(path, encoding="utf-8") as f:25        for line in f:26            rec = json.loads(line)27            tid  = rec.get("task_id")28            code = rec.get("Final answer", "").strip()29            if tid:30                gold[tid] = code31    return gold32 33def load_preds(path: Path) -> dict:34    preds = {}35    with open(path, encoding="utf-8") as f:36        data = json.load(f)  # single JSON list37        for rec in data:38            tid = rec.get("task_id")39            ans = rec.get("submitted_answer", "").strip()40            if tid:41                preds[tid] = ans42    return preds43 44def extract_subtrees(code: str) -> set:45    """46    Wrap the snippet in a dummy class, parse into AST, and collect (node_type, child_types) tuples.47    """48    # 1) Wrap in a minimal class49    wrapped = f"public class Dummy {{\n{code}\n}}"50    try:51        tree = javalang.parse.parse(wrapped)52    except Exception:53        return set()  # parse error → empty54 55    subs = set()56    def visit(node):57        from javalang.tree import Node58        if not isinstance(node, Node):59            return60        # collect this node type and its direct child node types61        child_types = tuple(62            type(child).__name__63            for _, child in node.filter(lambda x: isinstance(x, Node))64        )65        subs.add((type(node).__name__, child_types))66        # recurse into children67        for child in node.children:68            if isinstance(child, Node):69                visit(child)70            elif isinstance(child, list):71                for c in child:72                    if isinstance(c, Node):73                        visit(c)74 75    visit(tree)76    return subs77 78def ast_score(pred_code: str, ref_code: str) -> float:79    """80    AST score = |subtrees(pred) ∩ subtrees(ref)| / |subtrees(ref)|81    """82    pred_subs = extract_subtrees(pred_code)83    ref_subs  = extract_subtrees(ref_code)84    if not ref_subs:85        return 0.086    overlap = pred_subs & ref_subs87    return len(overlap) / len(ref_subs)88 89def main():90    if not GOLD_PATH.exists():91        print(f"Error: gold file not found at {GOLD_PATH}", file=sys.stderr)92        sys.exit(1)93    if not PRED_PATH.exists():94        print(f"Error: predictions file not found at {PRED_PATH}", file=sys.stderr)95        sys.exit(1)96 97    gold_map = load_gold(GOLD_PATH)98    pred_map = load_preds(PRED_PATH)99 100    total = len(gold_map)101    if total == 0:102        print("No gold records to evaluate.", file=sys.stderr)103        sys.exit(1)104 105    per_task_em  = {}106    per_task_ast = {}107    hyps = []108    refs = []109    em_count = 0110    ast_sum = 0.0111 112    for tid, gold_code in gold_map.items():113        pred_text = pred_map.get(tid, "")114        pred_code = extract_code(pred_text)115 116        # Exact Match117        em = int(normalize_whitespace(gold_code) == normalize_whitespace(pred_code))118        per_task_em[tid] = em119        em_count += em120 121        # AST Score122        a_score = ast_score(pred_code, gold_code)123        per_task_ast[tid] = round(a_score, 4)124        ast_sum += a_score125 126        # for BLEU127        hyps.append(pred_code)128        refs.append(gold_code)129 130    em_score  = em_count / total131    bleu_score = corpus_bleu(hyps, [refs]).score132    avg_ast   = ast_sum / total133 134    result = {135        "exact_match": round(em_score, 4),136        "bleu":        round(bleu_score, 2),137        "ast":         round(avg_ast, 4),138        "per_task_em":  per_task_em,139        "per_task_ast": per_task_ast140    }141 142    with open(OUT_PATH, "w", encoding="utf-8") as outf:143        json.dump(result, outf, indent=2, ensure_ascii=False)144 145    print(f"Evaluated {total} examples")146    print(f"Exact Match = {em_score:.4f}")147    print(f"BLEU        = {bleu_score:.2f}")148    print(f"AST Score   = {avg_ast:.4f}")149    print(f"Results saved to {OUT_PATH}")150 151if __name__ == "__main__":152    main()153