YashR05/pullrequest-arena

1try:2    from ..models import ReviewAction3except ImportError:4    from models import ReviewAction5 6def _clip(score: float) -> float:7    """Strictly bind score to the open interval (0, 1).8    The OpenEnv Phase-2 validator rejects 0.0 and 1.0 exactly,9    so we clamp to [0.01, 0.99]."""10    try:11        s = float(score)12    except (TypeError, ValueError):13        s = 0.514    return max(0.01, min(0.99, s))15 16def _get_action_match(action: ReviewAction, expected: str) -> float:17    if action.type == "approve":18        return 0.019    if action.type == expected:20        return 1.021    if action.type == "submit_patch" and expected in ["request_changes", "suggest_fix", "submit_patch"]:22        return 1.023    if action.type == "comment":24        return 0.525    # If they use suggest_fix instead of request_changes or vice versa, still give partial/full credit26    return 0.527 28def _get_keyword_quality(action: ReviewAction, keywords: list) -> float:29    comment = (action.comment or "").lower()30    hits = sum(1 for kw in keywords if kw.lower() in comment)31    if hits >= 2:32        return 1.033    if hits == 1:34        return 0.535    return 0.036 37def _base_grade(action: ReviewAction, task: dict) -> float:38    action_match = _get_action_match(action, task.get("expected_action", "request_changes"))39    keyword_quality = _get_keyword_quality(action, task.get("keywords", []))40    41    # If the agent attempts a patch, award bonus points if the patch is decent42    patch_score = 0.043    if action.type == "submit_patch" and action.patch:44        patch = action.patch.strip()45        expected_patch = task.get("expected_patch", "")46        if expected_patch and expected_patch in patch:47            patch_score = 1.048        elif len(patch) > 5:49            patch_score = 0.550            51    raw = (0.5 * action_match) + (0.3 * keyword_quality) + (0.2 * patch_score)52    return _clip(raw)53 54def grade_task_1(action: ReviewAction, task: dict) -> float:55    return _clip(_base_grade(action, task))56 57def grade_task_2(action: ReviewAction, task: dict) -> float:58    return _clip(_base_grade(action, task))59 60def grade_task_3(action: ReviewAction, task: dict) -> float:61    return _clip(_base_grade(action, task))62 63def grade_task_4(action: ReviewAction, task: dict) -> float:64    return _clip(_base_grade(action, task))65 66def grade_task_5(action: ReviewAction, task: dict) -> float:67    return _clip(_base_grade(action, task))68 69def grade_task_6(action: ReviewAction, task: dict) -> float:70    return _clip(_base_grade(action, task))71 72def grade_task_7(action: ReviewAction, task: dict) -> float:73    if action.type == "approve":74        return _clip(0.01)75        76    comment = (action.comment or "").lower()77    action_match = _get_action_match(action, task.get("expected_action", "request_changes"))78    79    if "sql injection" in comment and ("description" in comment or "misleading" in comment):80        keyword_quality = 1.081    else:82        keyword_quality = _get_keyword_quality(action, task.get("keywords", []))83        84    raw = (0.6 * action_match) + (0.4 * keyword_quality)85    return _clip(raw)86 87def grade_task_8(action: ReviewAction, task: dict) -> float:88    comment = (action.comment or "").lower()89    action_match = _get_action_match(action, task.get("expected_action", "request_changes"))90    91    if ("logging" in comment or "log" in comment) and "card" in comment:92        keyword_quality = 1.093    else:94        keyword_quality = 0.095        96    raw = (0.6 * action_match) + (0.4 * keyword_quality)97    return _clip(raw)98 99def grade_task_9(action: ReviewAction, task: dict) -> float:100    comment = (action.comment or "").lower()101    action_match = _get_action_match(action, task.get("expected_action", "request_changes"))102    103    keyword_quality = _get_keyword_quality(action, task.get("keywords", []))104    105    if "config.py" in comment:106        action_match = 1.0107    elif "middleware.py" in comment:108        keyword_quality = 0.0109        110    raw = (0.6 * action_match) + (0.4 * keyword_quality)111    return _clip(raw)112 113# Individual functions for openenv.yaml registration114def grade_task_10(a, t): return _clip(_base_grade(a, t))115def grade_task_11(a, t): return _clip(_base_grade(a, t))116def grade_task_12(a, t): return _clip(_base_grade(a, t))117def grade_task_13(a, t): return _clip(_base_grade(a, t))118def grade_task_14(a, t): return _clip(_base_grade(a, t))119def grade_task_15(a, t): return _clip(_base_grade(a, t))120def grade_task_16(a, t): return _clip(_base_grade(a, t))121def grade_task_17(a, t): return _clip(_base_grade(a, t))122def grade_task_18(a, t): return _clip(_base_grade(a, t))123def grade_task_19(a, t): return _clip(_base_grade(a, t))124 125def route_grader(action: ReviewAction, task: dict) -> float:126    task_id = str(task.get("id", ""))127    graders = {128        "1": grade_task_1,129        "2": grade_task_2,130        "3": grade_task_3,131        "4": grade_task_4,132        "5": grade_task_5,133        "6": grade_task_6,134        "7": grade_task_7,135        "8": grade_task_8,136        "9": grade_task_9,137        "10": grade_task_10,138        "11": grade_task_11,139        "12": grade_task_12,140        "13": grade_task_13,141        "14": grade_task_14,142        "15": grade_task_15,143        "16": grade_task_16,144        "17": grade_task_17,145        "18": grade_task_18,146        "19": grade_task_19,147    }148    grader_func = graders.get(task_id)149    if grader_func:150        return _clip(grader_func(action, task))151    return _clip(_base_grade(action, task))152