Ginnipahwa05/Meta-Pytorch

1#!/usr/bin/env python32"""3Simulate the complete scoring flow to trace where 0.0 or 1.0 values leak through.4"""5 6from graders import EasyTaskGrader, MediumTaskGrader, HardTaskGrader, BaseGrader7from environment import make_env8 9def simulate_grade_flow():10    """Simulate a complete episode and trace all score values."""11    12    print("=" * 80)13    print("SCORING FLOW ANALYSIS")14    print("=" * 80)15    16    # Test case 1: Easy task, perfect resolution17    print("\n[TEST 1] Easy task - Perfect resolution (0 damage, few steps)")18    env = make_env(task_id="easy_0", seed=0)19    env.reset()20    21    # Simulate perfect episode: quickly resolve with no damage22    env.current_step = 323    env.damage_score = 0.024    env.resolved_incidents = ["test_incident"]25    env.actions_log = [26        {"action_type": "query_logs"},27        {"action_type": "query_logs"},28        {"action_type": "resolve_incident"},29    ]30    31    grade = env.get_grade()32    print(f"  Raw grade score: {grade['score']}")33    print(f"  Raw correctness: {grade['correctness']}")34    print(f"  Raw efficiency: {grade['efficiency']}")35    print(f"  Raw damage: {grade['damage']}")36    print(f"  Is score exactly 0.0? {grade['score'] == 0.0}")37    print(f"  Is score exactly 1.0? {grade['score'] == 1.0}")38    print(f"  Is correctness exactly 0.0? {grade['correctness'] == 0.0}")39    print(f"  Is correctness exactly 1.0? {grade['correctness'] == 1.0}")40    print(f"  Is efficiency exactly 0.0? {grade['efficiency'] == 0.0}")41    print(f"  Is efficiency exactly 1.0? {grade['efficiency'] == 1.0}")42    print(f"  Is damage exactly 0.0? {grade['damage'] == 0.0}")43    print(f"  Is damage exactly 1.0? {grade['damage'] == 1.0}")44    45    # Test case 2: Easy task, failed episode46    print("\n[TEST 2] Easy task - Failed resolution (high damage, many steps)")47    env2 = make_env(task_id="easy_0", seed=0)48    env2.reset()49    50    env2.current_step = 1551    env2.damage_score = 0.9552    env2.resolved_incidents = []53    env2.actions_log = []54    55    grade2 = env2.get_grade()56    print(f"  Raw grade score: {grade2['score']}")57    print(f"  Raw correctness: {grade2['correctness']}")58    print(f"  Raw efficiency: {grade2['efficiency']}")59    print(f"  Raw damage: {grade2['damage']}")60    print(f"  Is score exactly 0.0? {grade2['score'] == 0.0}")61    print(f"  Is score exactly 1.0? {grade2['score'] == 1.0}")62    print(f"  Is correctness exactly 0.0? {grade2['correctness'] == 0.0}")63    print(f"  Is correctness exactly 1.0? {grade2['correctness'] == 1.0}")64    print(f"  Is efficiency exactly 0.0? {grade2['efficiency'] == 0.0}")65    print(f"  Is efficiency exactly 1.0? {grade2['efficiency'] == 1.0}")66    print(f"  Is damage exactly 0.0? {grade2['damage'] == 0.0}")67    print(f"  Is damage exactly 1.0? {grade2['damage'] == 1.0}")68 69    # Test case 3: Check component clamping directly70    print("\n[TEST 3] Direct component clamping")71    print(f"  safe_score(0.0) = {BaseGrader.safe_score(0.0)}")72    print(f"  safe_score(1.0) = {BaseGrader.safe_score(1.0)}")73    print(f"  safe_score(0.5) = {BaseGrader.safe_score(0.5)}")74    print(f"  MIN_SCORE = {BaseGrader.MIN_SCORE}")75    print(f"  MAX_SCORE = {BaseGrader.MAX_SCORE}")76 77    # Test case 4: Check compute_final_score directly78    print("\n[TEST 4] Direct final score computation")79    min_val = BaseGrader.MIN_SCORE80    max_val = BaseGrader.MAX_SCORE81    mid_val = (min_val + max_val) / 282    83    print(f"  compute_final_score({min_val}, {min_val}, {min_val}) = {BaseGrader.compute_final_score(min_val, min_val, min_val)}")84    print(f"  compute_final_score({max_val}, {max_val}, {max_val}) = {BaseGrader.compute_final_score(max_val, max_val, max_val)}")85    print(f"  compute_final_score({mid_val}, {mid_val}, {mid_val}) = {BaseGrader.compute_final_score(mid_val, mid_val, mid_val)}")86    print(f"  compute_final_score(0.0, 0.0, 0.0) = {BaseGrader.compute_final_score(0.0, 0.0, 0.0)}")87    print(f"  compute_final_score(1.0, 1.0, 1.0) = {BaseGrader.compute_final_score(1.0, 1.0, 1.0)}")88 89    # Test case 5: Simulation with API wrapper90    print("\n[TEST 5] API endpoint wrapping")91    def safe_openenv_score(value: float) -> float:92        min_score = 0.1 + 1e-493        max_score = 0.9 - 1e-494        return min(max_score, max(min_score, float(value)))95    96    test_values = [0.0, 0.1, 0.5, 0.9, 1.0, BaseGrader.MIN_SCORE, BaseGrader.MAX_SCORE]97    for val in test_values:98        wrapped = safe_openenv_score(val)99        print(f"  safe_openenv_score({val}) = {wrapped}, is 0.0? {wrapped == 0.0}, is 1.0? {wrapped == 1.0}")100 101    print("\n" + "=" * 80)102 103if __name__ == "__main__":104    simulate_grade_flow()105