albert-einstein-09/codedark

1"""2CodeDark Scoring System3 4Reward computation with multi-metric scoring:5- 80% correctness (binary: exact match within tolerance)6- 10% efficiency (fewer turns = better)7- 10% token cost (lower usage = better)8"""9 10from typing import Any, Optional, Tuple11import ast12 13 14def normalize_value(val: Any) -> Any:15    """Normalize a value for comparison.16 17    Handles:18    - String to float/int conversion19    - String to list/dict parsing20    - Float rounding for precision21    - Percentage stripping22    """23    if val is None:24        return None25 26    # Already a proper type - just normalize floats27    if isinstance(val, float):28        return round(val, 4)29    if isinstance(val, int):30        return float(val)31    if isinstance(val, (list, dict)):32        return val33 34    # String handling35    if isinstance(val, str):36        val = val.strip()37 38        # Try to parse as list/dict first39        if val.startswith("[") or val.startswith("{"):40            try:41                return ast.literal_eval(val)42            except (ValueError, SyntaxError):43                pass44 45        # Try float (strip % if present)46        try:47            return round(float(val.rstrip("%")), 4)48        except ValueError:49            pass50 51        # Return as lowercase string52        return val.lower()53 54    return val55 56 57def parse_markdown_table(text: str) -> Optional[list]:58    """Parse markdown table to list of dicts.59 60    Handles tables like:61    | job | mean | std |62    |-----|------|-----|63    | retired | 40.97 | 9.74 |64    """65    if not isinstance(text, str):66        return None67 68    lines = text.strip().split("\n")69 70    # Find table lines (contain |)71    table_lines = [line for line in lines if "|" in line]72    if len(table_lines) < 3:  # Need header + separator + at least 1 row73        return None74 75    # Parse header76    header_line = table_lines[0]77    headers = [h.strip().lower() for h in header_line.split("|") if h.strip()]78    if not headers:79        return None80 81    # Skip separator line (contains ---)82    data_start = 183    if "---" in table_lines[1] or "--|" in table_lines[1]:84        data_start = 285 86    # Parse data rows87    rows = []88    for line in table_lines[data_start:]:89        cells = [c.strip() for c in line.split("|") if c.strip()]90        if len(cells) != len(headers):91            continue92 93        row = {}94        for h, c in zip(headers, cells):95            # Clean number formatting (commas, currency symbols)96            c_clean = c.replace(",", "").replace("€", "").replace("$", "").strip()97            try:98                if "." in c_clean:99                    row[h] = round(float(c_clean), 4)100                else:101                    row[h] = int(c_clean)102            except ValueError:103                row[h] = c.lower()104        rows.append(row)105 106    return rows if rows else None107 108 109def compare_answers(submitted: Any, expected: Any, tolerance: float = 0.01) -> bool:110    """Compare answers with support for structured data and numeric tolerance.111 112    Handles:113    - Type mismatches (string "33.53" vs float 33.53)114    - Floating point precision (rounds to 4 decimals)115    - Nested structures (lists, dicts)116    - String parsing for lists/dicts117    """118    # Normalize both values119    submitted_n = normalize_value(submitted)120    expected_n = normalize_value(expected)121 122    # Null checks123    if submitted_n is None and expected_n is None:124        return True125    if submitted_n is None or expected_n is None:126        return False127 128    # Same type comparison after normalization129    if type(submitted_n) == type(expected_n):130        if isinstance(expected_n, list):131            if len(submitted_n) != len(expected_n):132                return False133            # Check if list contains dicts (structured data) - use order-sensitive134            if expected_n and isinstance(expected_n[0], dict):135                return all(136                    compare_answers(s, e, tolerance)137                    for s, e in zip(submitted_n, expected_n)138                )139            # Simple values list - order-insensitive (we don't tell models to sort)140            submitted_sorted = sorted([str(x).lower().strip() for x in submitted_n])141            expected_sorted = sorted([str(x).lower().strip() for x in expected_n])142            return submitted_sorted == expected_sorted143 144        if isinstance(expected_n, dict):145            if set(submitted_n.keys()) != set(expected_n.keys()):146                return False147            return all(148                compare_answers(submitted_n[k], expected_n[k], tolerance)149                for k in expected_n150            )151 152        if isinstance(expected_n, float):153            return abs(submitted_n - expected_n) <= tolerance154 155        # String comparison156        return str(submitted_n) == str(expected_n)157 158    # Type mismatch after normalization - try numeric comparison159    try:160        sub_f = (161            float(submitted_n) if not isinstance(submitted_n, (list, dict)) else None162        )163        exp_f = float(expected_n) if not isinstance(expected_n, (list, dict)) else None164        if sub_f is not None and exp_f is not None:165            return abs(sub_f - exp_f) <= tolerance166    except (ValueError, TypeError):167        pass168 169    # Try markdown table parsing if expected is list and submitted is string170    if isinstance(expected_n, list) and isinstance(submitted_n, str):171        parsed = parse_markdown_table(submitted)  # Use original, not normalized172        if parsed is not None:173            return compare_answers(parsed, expected, tolerance)174 175    # Fallback: string comparison176    return str(submitted_n).lower() == str(expected_n).lower()177 178 179def score_correctness(submitted: Any, expected: Any, tolerance: float = 0.01) -> float:180    """Score the submitted answer correctness. Weight: 0.80181 182    Scoring:183    - 0.80: Exact match184    - 0.20: Almost there (rounding or 100x scale error)185    - 0.00: Wrong186 187    Args:188        submitted: Submitted answer189        expected: Expected answer190        tolerance: Numeric tolerance for comparison191 192    Returns:193        Correctness score (0.0, 0.20, or 0.80)194    """195    if submitted is None:196        return 0.0197 198    try:199        # Try numeric comparison first200        submitted_f = float(submitted)201        expected_f = float(expected)202 203        # Exact match (within tolerance)204        if abs(submitted_f - expected_f) < tolerance:205            return 0.80206 207        if expected_f != 0:208            ratio = submitted_f / expected_f209 210            # 100x scale error (decimal vs percentage)211            # e.g., 0.0959 vs 9.59 or 9.59 vs 0.0959212            if 0.009 < ratio < 0.011 or 99 < ratio < 101:213                return 0.20214 215            # Rounding error (within 1% of expected)216            if 0.99 < ratio < 1.01:217                return 0.20218 219    except (ValueError, TypeError):220        # Structured data comparison (lists, dicts)221        if compare_answers(submitted, expected, tolerance=tolerance):222            return 0.80223 224    return 0.0225 226 227def score_efficiency(turns: int, max_turns: int, is_correct: bool) -> float:228    """Score based on turns used (fewer = better). Weight: 0.10229 230    Only applies if answer is correct.231 232    Args:233        turns: Number of turns used234        max_turns: Maximum turns allowed235        is_correct: Whether the answer was correct236 237    Returns:238        Efficiency score (0.0 to 0.10)239    """240    if not is_correct:241        return 0.0242 243    # Scale: 1 turn = 0.10, max_turns = 0.01244    efficiency = max(0.0, 1.0 - (turns / max_turns))245    return 0.10 * efficiency246 247 248def score_token_cost(249    input_tokens: int,250    output_tokens: int,251    is_correct: bool,252    input_price: float = 1.0,253    output_price: float = 5.0,254    target_cost: float = 0.01,255    max_cost: float = 0.10,256) -> Tuple[float, float]:257    """Score based on token cost (lower = better). Weight: 0.10258 259    Only applies if answer is correct.260 261    Args:262        input_tokens: Number of input tokens used263        output_tokens: Number of output tokens used264        is_correct: Whether the answer was correct265        input_price: Price per 1M input tokens (default $1)266        output_price: Price per 1M output tokens (default $5)267        target_cost: Cost for full score (default $0.01)268        max_cost: Cost for zero score (default $0.10)269 270    Returns:271        Tuple of (token_score, cost_usd)272    """273    if not is_correct:274        return 0.0, 0.0275 276    # Calculate cost in dollars277    cost = (input_tokens * input_price / 1_000_000) + (278        output_tokens * output_price / 1_000_000279    )280 281    # Scale: <$0.01 = full score, >$0.10 = 0282    if cost <= target_cost:283        efficiency = 1.0284    elif cost >= max_cost:285        efficiency = 0.0286    else:287        efficiency = 1.0 - ((cost - target_cost) / (max_cost - target_cost))288 289    return 0.10 * efficiency, cost290 291 292def compute_reward(293    submitted: Any,294    expected: Any,295    tolerance: float,296    turns: int,297    max_turns: int,298    input_tokens: int = 0,299    output_tokens: int = 0,300) -> Tuple[float, float, float, float]:301    """Compute total reward from all components.302 303    Args:304        submitted: Submitted answer305        expected: Expected answer306        tolerance: Numeric tolerance307        turns: Number of turns used308        max_turns: Maximum turns allowed309        input_tokens: Number of input tokens (optional)310        output_tokens: Number of output tokens (optional)311 312    Returns:313        Tuple of (total_reward, correctness, efficiency, token_cost_usd)314    """315    # Correctness (0.80 weight)316    correctness = score_correctness(submitted, expected, tolerance)317    is_correct = correctness > 0318 319    # Efficiency (0.10 weight)320    efficiency = score_efficiency(turns, max_turns, is_correct)321 322    # Token cost (0.10 weight)323    # If tokens not tracked, estimate from turns324    if input_tokens == 0 and output_tokens == 0:325        input_tokens = turns * 1000326        output_tokens = turns * 500327 328    token_score, cost_usd = score_token_cost(input_tokens, output_tokens, is_correct)329 330    total_reward = correctness + efficiency + token_score331 332    return total_reward, correctness, efficiency, cost_usd333