DCAgent2/bfcl-parity

1"""2BFCL evaluation script for task: simple_python_3253 4This script evaluates the agent's function calling output against ground truth.5"""6import json7import sys8from pathlib import Path9 10 11def load_result():12    """Load the result.json file generated by the agent."""13    result_path = Path("/app/result.json")14    if not result_path.exists():15        raise FileNotFoundError("result.json not found. Agent must write output to /app/result.json")16    17    with open(result_path, 'r') as f:18        return json.load(f)19 20 21def normalize_function_name(name: str) -> str:22    """Normalize function name by replacing dots with underscores."""23    return name.replace('.', '_')24 25 26def compare_function_calls(predicted, ground_truth):27    """28    Compare predicted function calls against ground truth.29 30    Args:31        predicted: List of function call dicts from agent32        ground_truth: List of acceptable function call dicts33    34    Returns:35        bool: True if prediction matches any acceptable ground truth36    """37    if not isinstance(predicted, list):38        return False39    40    if len(predicted) == 0 and len(ground_truth) == 0:41        return True42    43    if len(predicted) != len(ground_truth):44        return False45    46    # Precompute ground truth entries47    gt_entries = []48    for gt_call in ground_truth:49        if not isinstance(gt_call, dict) or len(gt_call) != 1:50            return False51        gt_func_name = list(gt_call.keys())[0]52        gt_entries.append((normalize_function_name(gt_func_name), gt_call[gt_func_name]))53 54    # Order-independent matching: each predicted call must match55    # exactly one ground truth call, but in any order.56    # Matches upstream BFCL ast_checker.py parallel_function_checker_no_order.57    matched_gt = set()58    for pred_call in predicted:59        if not isinstance(pred_call, dict) or len(pred_call) != 1:60            return False61 62        pred_func_name = list(pred_call.keys())[0]63        pred_params = pred_call[pred_func_name]64        pred_func_name_norm = normalize_function_name(pred_func_name)65 66        found = False67        for j, (gt_func_name_norm, gt_params) in enumerate(gt_entries):68            if j in matched_gt:69                continue70            if pred_func_name_norm == gt_func_name_norm and compare_parameters(pred_params, gt_params):71                matched_gt.add(j)72                found = True73                break74 75        if not found:76            return False77 78    return True79 80 81def compare_parameters(pred_params, gt_params):82    """83    Compare predicted parameters against ground truth parameters.84    85    Ground truth may contain multiple acceptable values for each parameter.86    Rejects unexpected parameters not in ground truth (matches upstream BFCL behavior).87    """88    if not isinstance(pred_params, dict) or not isinstance(gt_params, dict):89        return False90    91    # Reject unexpected parameters not in ground truth92    for param_name in pred_params:93        if param_name not in gt_params:94            return False95    96    # Check all ground truth parameters97    for param_name, acceptable_values in gt_params.items():98        if param_name not in pred_params:99            # Parameter missing - check if it's in acceptable values (empty string means optional)100            if "" not in acceptable_values and None not in acceptable_values:101                return False102            continue103        104        pred_value = pred_params[param_name]105        106        # Check if predicted value matches any acceptable value107        if not isinstance(acceptable_values, list):108            acceptable_values = [acceptable_values]109        110        # Special case: if acceptable_values is empty list, check if pred_value is also empty list111        if len(acceptable_values) == 0:112            if pred_value != []:113                return False114            continue115        116        matched = False117        for acceptable_value in acceptable_values:118            if values_equal(pred_value, acceptable_value):119                matched = True120                break121        122        if not matched:123            return False124    125    return True126 127 128def standardize_string(s):129    """Standardize string by removing punctuation/whitespace and lowercasing.130    Matches upstream BFCL ast_checker.py standardize_string behavior."""131    import re132    return re.sub(r"[ ,./\-_*^]", "", s).lower().replace("'", '"')133 134 135def values_equal(v1, v2):136    """Check if two values are equal, handling type conversions."""137    # Handle empty string as "not provided" or default138    if v2 == "" or v2 is None:139        return True140 141    # Direct equality142    if v1 == v2:143        return True144 145    # Try numeric comparison146    try:147        if float(v1) == float(v2):148            return True149    except (ValueError, TypeError):150        pass151 152    # Try string comparison with standardization153    # Matches upstream BFCL ast_checker.py string_checker behavior:154    # strips spaces, commas, periods, slashes, hyphens, underscores, asterisks, carets155    if isinstance(v1, str) and isinstance(v2, str):156        if standardize_string(v1) == standardize_string(v2):157            return True158    elif str(v1).lower() == str(v2).lower():159        return True160    161    # Handle list/array comparison162    if isinstance(v1, list) and isinstance(v2, list):163        if len(v1) != len(v2):164            return False165        return all(values_equal(a, b) for a, b in zip(v1, v2))166    167    return False168 169 170def main():171    """Main evaluation function."""172    # Load ground truth173    ground_truth = [{'sports.match_results': {'team1': ['Chicago Bulls'], 'team2': ['Los Angeles Lakers'], 'season': ['']}}]174    175    try:176        # Load agent's result177        result = load_result()178        179        # Compare against ground truth180        if compare_function_calls(result, ground_truth):181            print("✓ Test passed: Function call matches ground truth")182            return 0183        else:184            print("✗ Test failed: Function call does not match ground truth")185            print(f"Predicted: {result}")186            print(f"Expected one of: {ground_truth}")187            return 1188    189    except Exception as e:190        print(f"✗ Test failed with error: {e}")191        import traceback192        traceback.print_exc()193        return 1194 195 196if __name__ == "__main__":197    exit_code = main()198    sys.exit(exit_code)199