Ginnipahwa05/Meta-Pytorch
0
1"""2DevOps War Room - Task Definitions3Three variants (easy, medium, hard) with deterministic incident generation4"""5 6import random7from typing import Dict, List, Tuple8from models import (9 ServiceName, Alert, LogEntry, TaskDefinition10)11 12 13class IncidentScenario:14 """Pre-defined incident scenarios with root causes and manifestations"""15 16 # Easy scenarios: single clear incident17 EASY_SCENARIOS = [18 {19 "name": "Database Connection Pool Exhausted",20 "root_cause": "db_connection_leak",21 "affected_service": ServiceName.DB,22 "cascade_to": [ServiceName.PAYMENTS, ServiceName.API_GATEWAY],23 "alerts": [24 "Database latency spike (>5000ms)",25 "Failed connections from payments service",26 ],27 "logs": {28 ServiceName.DB: [29 "ERROR: Connection pool exhausted (1000/1000 active)",30 "ERROR: New connection requests timing out",31 ],32 ServiceName.PAYMENTS: [33 "ERROR: Failed to connect to database",34 "ERROR: Checkout transaction failed",35 ]36 },37 "misleading_logs": [38 ("cache", "WARN: Cache hit rate dropped to 60%"), # Not the issue39 ("auth", "INFO: User login spike (+200%)"), # Normal seasonal traffic40 ]41 },42 {43 "name": "Payment Service Deployment Failed",44 "root_cause": "bad_payment_deployment",45 "affected_service": ServiceName.PAYMENTS,46 "cascade_to": [ServiceName.API_GATEWAY],47 "alerts": [48 "Payment service error rate 45%",49 "Checkout endpoint returning 500s",50 ],51 "logs": {52 ServiceName.PAYMENTS: [53 "ERROR: NullPointerException in PaymentProcessor.process()",54 "ERROR: Service startup failed: Missing configuration value",55 ],56 },57 "misleading_logs": [58 ("db", "INFO: Query response time 200ms (normal)"),59 ("cache", "INFO: Eviction policy triggered"),60 ]61 },62 ]63 64 # Medium scenarios: multiple services, misleading signals65 MEDIUM_SCENARIOS = [66 {67 "name": "Cache Failure + API Gateway Overload",68 "root_cause": "cache_backend_failure",69 "affected_services": [ServiceName.CACHE, ServiceName.API_GATEWAY],70 "cascade_to": [ServiceName.AUTH, ServiceName.PAYMENTS],71 "alerts": [72 "Cache service down",73 "API Gateway latency spike (>2000ms)",74 "Auth service latency increased",75 ],76 "logs": {77 ServiceName.CACHE: [78 "ERROR: Redis connection lost",79 "ERROR: Unable to reconnect to cache backend",80 ],81 ServiceName.API_GATEWAY: [82 "WARN: Cache misses increasing (70%)",83 "WARN: Queuing requests due to high latency",84 "INFO: Rate limiting triggered",85 ],86 ServiceName.AUTH: [87 "WARN: Session lookups timing out",88 "WARN: Database hit rate increased",89 ]90 },91 "misleading_logs": [92 ("payments", "INFO: Transaction volume 15% above baseline"),93 ("db", "WARN: CPU at 75% (normal for this query volume)"),94 ]95 },96 {97 "name": "Traffic Spike + Inadequate Scaling",98 "root_cause": "insufficient_api_gateway_replicas",99 "affected_services": [ServiceName.API_GATEWAY],100 "cascade_to": [ServiceName.PAYMENTS, ServiceName.DB],101 "alerts": [102 "API Gateway CPU at 90%+",103 "API Gateway request queue depth > 10k",104 "Downstream services experiencing timeouts",105 ],106 "logs": {107 ServiceName.API_GATEWAY: [108 "WARN: Request queue depth: 12000",109 "ERROR: Requests timing out due to queue saturation",110 "INFO: Incoming request rate: 5000 req/s",111 ],112 ServiceName.PAYMENTS: [113 "ERROR: Timeout calling API Gateway",114 ]115 },116 "misleading_logs": [117 ("auth", "INFO: Auth service responding normally (<100ms)"),118 ("cache", "INFO: Cache operational, hit rate 85%"),119 ]120 },121 ]122 123 # Hard scenarios: cascading failures with tradeoffs124 HARD_SCENARIOS = [125 {126 "name": "Cascading Auth + Payment Failures",127 "root_cause": "auth_token_validation_bug",128 "primary_affected": ServiceName.AUTH,129 "cascade_sequence": [130 {131 "step_range": (0, 3),132 "service": ServiceName.AUTH,133 "error": "Auth token validation returning 50% false negatives",134 "impact": "Users cannot authenticate"135 },136 {137 "step_range": (4, 7),138 "service": ServiceName.PAYMENTS,139 "error": "Payment failures due to invalid auth",140 "impact": "Cascades from auth to payments"141 },142 {143 "step_range": (8, 12),144 "service": ServiceName.API_GATEWAY,145 "error": "Queue saturates from retries",146 "impact": "System-wide degradation"147 }148 ],149 "quick_fix_trap": {150 "description": "Restarting auth might help, but doesn't fix root issue",151 "penalty_delay_steps": 5,152 },153 "correct_fix": "Rollback auth to previous stable version",154 "misleading_logs": [155 ("db", "WARN: Connection pool at 85%"),156 ("cache", "INFO: Recent deployment completed"),157 ]158 },159 {160 "name": "Multi-Service Outage with Tradeoffs",161 "root_cause": "database_replication_lag",162 "affected": [ServiceName.DB, ServiceName.PAYMENTS, ServiceName.AUTH],163 "tradeoff_scenario": {164 "option_a": {165 "action": "Restart DB primary (fixes consistency)",166 "benefit": "Replication catches up",167 "cost": "5 minute downtime",168 "time_penalty": 50, # Reduces reward169 },170 "option_b": {171 "action": "Wait for replication to catch up",172 "benefit": "Zero downtime",173 "cost": "Services degraded for 10 steps",174 "flexibility": True, # Can switch to A later175 }176 },177 "misleading_logs": [178 ("api_gateway", "INFO: CPU usage declining"),179 ("cache", "WARN: Eviction rate increased"),180 ]181 }182 ]183 184 185class TaskGenerator:186 """Generate task-specific scenarios deterministically"""187 188 SCORE_MIN_EXCLUSIVE = 0.1189 SCORE_MAX_EXCLUSIVE = 0.9190 191 @staticmethod192 def grader_metadata(difficulty: str) -> Dict:193 return {194 "type": "deterministic",195 "difficulty": difficulty,196 "score_range": {197 "min_exclusive": TaskGenerator.SCORE_MIN_EXCLUSIVE,198 "max_exclusive": TaskGenerator.SCORE_MAX_EXCLUSIVE,199 },200 }201 202 @staticmethod203 def generate_easy_task(seed: int = 0) -> Dict:204 """Easy task: Single incident, clear logs"""205 random.seed(seed)206 scenario = IncidentScenario.EASY_SCENARIOS[seed % len(IncidentScenario.EASY_SCENARIOS)]207 208 return {209 "task_id": f"easy_{seed}",210 "difficulty": "easy",211 "name": scenario["name"],212 "description": f"Identify root cause: {scenario['name']}",213 "max_steps": 15,214 "num_incidents": 1,215 "has_cascading_failures": False,216 "num_misleading_logs": len(scenario.get("misleading_logs", [])),217 "grader": TaskGenerator.grader_metadata("easy"),218 "scenario": scenario,219 }220 221 @staticmethod222 def generate_medium_task(seed: int = 0) -> Dict:223 """Medium task: Multiple services, misleading signals"""224 random.seed(seed)225 scenario = IncidentScenario.MEDIUM_SCENARIOS[seed % len(IncidentScenario.MEDIUM_SCENARIOS)]226 227 return {228 "task_id": f"medium_{seed}",229 "difficulty": "medium",230 "name": scenario["name"],231 "description": f"Resolve multiple incidents: {scenario['name']}",232 "max_steps": 25,233 "num_incidents": 2,234 "has_cascading_failures": True,235 "num_misleading_logs": len(scenario.get("misleading_logs", [])),236 "grader": TaskGenerator.grader_metadata("medium"),237 "scenario": scenario,238 }239 240 @staticmethod241 def generate_hard_task(seed: int = 0) -> Dict:242 """Hard task: Cascading failures, tradeoffs"""243 random.seed(seed)244 scenario = IncidentScenario.HARD_SCENARIOS[seed % len(IncidentScenario.HARD_SCENARIOS)]245 246 return {247 "task_id": f"hard_{seed}",248 "difficulty": "hard",249 "name": scenario["name"],250 "description": f"Resolve complex incident: {scenario['name']}",251 "max_steps": 30,252 "num_incidents": 3,253 "has_cascading_failures": True,254 "num_misleading_logs": 4,255 "grader": TaskGenerator.grader_metadata("hard"),256 "scenario": scenario,257 }258 259 260TASK_DEFINITIONS = {261 "easy_0": TaskGenerator.generate_easy_task(seed=0),262 "easy_1": TaskGenerator.generate_easy_task(seed=1),263 "medium_0": TaskGenerator.generate_medium_task(seed=0),264 "medium_1": TaskGenerator.generate_medium_task(seed=1),265 "hard_0": TaskGenerator.generate_hard_task(seed=0),266 "hard_1": TaskGenerator.generate_hard_task(seed=1),267}268