dkAmulet/sql-query-optimizer
0
1#!/usr/bin/env python32"""3validate_local.py — Pre-submission validator for SQL Query Optimizer.4 5Mimics the official hackathon validation checklist:6 1. openenv.yaml — exists and has required fields7 2. Required files present8 3. Dockerfile exists and is non-trivial9 4. inference.py named correctly and is at root10 5. Environment instantiation — reset()/step()/state() all work11 6. All 3 tasks enumerable12 7. All graders return scores in [0.0, 1.0]13 8. Baseline results reproducible14 15Run:16 python validate_local.py17 18Exits with code 0 (all pass) or 1 (any failure).19"""20from __future__ import annotations21 22import importlib23import json24import os25import sys26import time27from pathlib import Path28 29ROOT = Path(__file__).parent30 31PASS = "✓"32FAIL = "✗"33WARN = "⚠"34_results: list[tuple[str, bool, str]] = []35 36 37def check(name: str, condition: bool, detail: str = "") -> bool:38 _results.append((name, condition, detail))39 mark = PASS if condition else FAIL40 line = f" {mark} {name}"41 if detail:42 line += f" — {detail}"43 print(line)44 return condition45 46 47def section(title: str) -> None:48 print(f"\n{'─' * 62}")49 print(f" {title}")50 print("─" * 62)51 52 53# ─────────────────────────────────────────────────────────── checks ──────────54 55def check_required_files():56 section("1. Required Files")57 required = [58 "openenv.yaml",59 "Dockerfile",60 "requirements.txt",61 "inference.py",62 "README.md",63 "models.py",64 "db.py",65 "tasks.py",66 "env.py",67 "app.py",68 ]69 all_present = True70 for fname in required:71 present = (ROOT / fname).exists()72 check(f"{fname} present", present)73 all_present = all_present and present74 return all_present75 76 77def check_openenv_yaml():78 section("2. openenv.yaml Validation")79 try:80 import yaml81 with open(ROOT / "openenv.yaml", encoding="utf-8") as f:82 cfg = yaml.safe_load(f)83 except ImportError:84 # Fallback: parse manually85 import re86 with open(ROOT / "openenv.yaml", encoding="utf-8") as f:87 raw = f.read()88 cfg = {}89 for key in ["name","version","description","tasks","endpoints","reward_range"]:90 cfg[key] = key in raw91 92 check("openenv.yaml readable (yaml not installed, minimal check)", True,93 "install pyyaml for full validation")94 return True95 96 required_keys = ["name","version","description","tasks","endpoints","reward_range"]97 for k in required_keys:98 check(f"openenv.yaml has '{k}'", k in cfg)99 100 tasks_cfg = cfg.get("tasks", [])101 check("openenv.yaml has >= 3 tasks", len(tasks_cfg) >= 3, f"found {len(tasks_cfg)}")102 103 for t in tasks_cfg:104 has_id = "id" in t105 has_diff = "difficulty" in t106 has_max = "max_steps" in t107 check(f" task '{t.get('id','?')}' has id/difficulty/max_steps", has_id and has_diff and has_max)108 109 rr = cfg.get("reward_range", [])110 check("reward_range is [0.0, 1.0]", list(rr) == [0.0, 1.0], f"got {rr}")111 112 return True113 114 115def check_dockerfile():116 section("3. Dockerfile")117 df = ROOT / "Dockerfile"118 if not df.exists():119 check("Dockerfile exists", False)120 return False121 122 content = df.read_text(encoding="utf-8")123 check("Dockerfile has FROM instruction", "FROM" in content)124 check("Dockerfile exposes port 7860", "7860" in content)125 check("Dockerfile has CMD or ENTRYPOINT", "CMD" in content or "ENTRYPOINT" in content)126 check("Dockerfile has COPY instruction", "COPY" in content)127 check("Dockerfile is non-trivial (> 5 lines)", content.count("\n") > 5,128 f"{content.count(chr(10))} lines")129 return True130 131 132def check_inference_script():133 section("4. inference.py")134 inf = ROOT / "inference.py"135 if not inf.exists():136 check("inference.py at project root", False)137 return False138 139 content = inf.read_text(encoding="utf-8")140 check("inference.py at project root", True)141 check("inference.py reads API_BASE_URL", "API_BASE_URL" in content)142 check("inference.py reads MODEL_NAME", "MODEL_NAME" in content)143 check("inference.py reads HF_TOKEN", "HF_TOKEN" in content)144 check("inference.py uses OpenAI client", "OpenAI" in content)145 check("inference.py has main() function", "def main" in content)146 return True147 148 149def check_environment_api():150 section("5. Environment API (reset/step/state)")151 sys.path.insert(0, str(ROOT))152 153 try:154 from env import SQLQueryOptimizerEnv155 from models import SQLAction, SQLObservation, StepResult, EnvironmentState156 except ImportError as e:157 check("Environment imports successfully", False, str(e))158 print(f"\n {WARN} Install dependencies: pip install pydantic fastapi openai")159 return False160 161 check("Environment imports successfully", True)162 163 env = SQLQueryOptimizerEnv()164 165 # reset()166 try:167 obs = env.reset()168 is_obs = isinstance(obs, SQLObservation)169 check("reset() returns SQLObservation", is_obs)170 check("reset() observation has task_id", hasattr(obs, "task_id") and obs.task_id != "")171 check("reset() observation has schema_ddl", hasattr(obs, "schema_ddl") and len(obs.schema_ddl) > 50)172 check("reset() observation has slow_query", hasattr(obs, "slow_query") and len(obs.slow_query) > 10)173 except Exception as e:174 check("reset() works without error", False, str(e))175 env.close()176 return False177 178 # step()179 try:180 result = env.step(SQLAction(181 optimized_query="SELECT user_id, username, email FROM users WHERE is_active = 1"182 ))183 is_step = isinstance(result, StepResult)184 check("step() returns StepResult", is_step)185 check("step() reward.value in [0,1]",186 0.0 <= result.reward.value <= 1.0, f"got {result.reward.value:.3f}")187 check("step() has done flag", isinstance(result.done, bool))188 check("step() has info dict", isinstance(result.info, dict))189 except Exception as e:190 check("step() works without error", False, str(e))191 env.close()192 return False193 194 # state()195 try:196 st = env.state()197 is_state = isinstance(st, EnvironmentState)198 check("state() returns EnvironmentState", is_state)199 check("state() has task_id", hasattr(st, "task_id") and st.task_id != "")200 except Exception as e:201 check("state() works without error", False, str(e))202 203 env.close()204 return True205 206 207def check_tasks():208 section("6. Tasks Enumerable (>= 3)")209 sys.path.insert(0, str(ROOT))210 211 try:212 from env import SQLQueryOptimizerEnv213 except ImportError:214 check("Tasks check skipped (pydantic not installed)", True, "install dependencies")215 return True216 217 env = SQLQueryOptimizerEnv()218 tasks = env.list_tasks()219 check("list_tasks() returns >= 3 tasks", len(tasks) >= 3, f"got {len(tasks)}")220 221 difficulties = [t.get("difficulty") for t in tasks]222 check("Has easy task", "easy" in difficulties)223 check("Has medium task", "medium" in difficulties)224 check("Has hard task", "hard" in difficulties)225 226 for t in tasks:227 check(f" task '{t['task_id']}' has all required fields",228 all(k in t for k in ["task_id","name","difficulty","max_steps","description"]))229 230 env.close()231 return True232 233 234def check_graders():235 section("7. Graders Return [0.0, 1.0]")236 sys.path.insert(0, str(ROOT))237 238 try:239 from env import SQLQueryOptimizerEnv240 from models import SQLAction241 from tasks import TASK_ORDER242 except ImportError:243 check("Grader check skipped (pydantic not installed)", True)244 return True245 246 env = SQLQueryOptimizerEnv()247 queries = [248 "SELECT * FROM users WHERE is_active = 1",249 "SELECT user_id, username, email FROM users WHERE is_active = 1",250 "COMPLETELY BROKEN SQL",251 "SELECT order_id, user_id, total_amount FROM orders WHERE user_id IN (SELECT user_id FROM users WHERE country='USA' AND is_active=1) AND status='delivered'",252 ]253 254 all_ok = True255 for task_id in TASK_ORDER:256 env.reset(task_id)257 for q in queries:258 try:259 r = env.step(SQLAction(optimized_query=q))260 v = r.reward.value261 in_range = 0.0 <= v <= 1.0262 if not in_range:263 all_ok = False264 print(f" {FAIL} task={task_id} score={v:.3f} OUT OF RANGE")265 if r.done:266 env.reset(task_id)267 except Exception as exc:268 # step() after done raises RuntimeError — that's expected269 if "Episode is over" in str(exc):270 env.reset(task_id)271 else:272 all_ok = False273 print(f" {FAIL} Unexpected exception: {exc}")274 275 check("All grader scores in [0.0, 1.0]", all_ok)276 env.close()277 return all_ok278 279 280def check_baseline_reproducibility():281 section("8. Baseline Reproducibility (determinism check)")282 sys.path.insert(0, str(ROOT))283 284 try:285 from env import SQLQueryOptimizerEnv286 from models import SQLAction287 except ImportError:288 check("Reproducibility check skipped (pydantic not installed)", True)289 return True290 291 q = "SELECT user_id, username, email FROM users WHERE is_active = 1"292 scores = []293 for _ in range(3):294 env = SQLQueryOptimizerEnv()295 env.reset("select_star_removal")296 r = env.step(SQLAction(optimized_query=q))297 scores.append(r.reward.value)298 env.close()299 300 all_same = len(set(scores)) == 1301 check("Same query produces identical scores across runs",302 all_same, f"scores={scores}")303 return all_same304 305 306# ─────────────────────────────────────────────────────────── runner ───────────307 308def main():309 print("╔══════════════════════════════════════════════════════════╗")310 print("║ SQL Query Optimizer — Pre-Submission Validator ║")311 print("╚══════════════════════════════════════════════════════════╝")312 print(f"\n Working directory: {ROOT}\n")313 314 t0 = time.time()315 316 check_required_files()317 check_openenv_yaml()318 check_dockerfile()319 check_inference_script()320 check_environment_api()321 check_tasks()322 check_graders()323 check_baseline_reproducibility()324 325 elapsed = time.time() - t0326 passed = sum(1 for _, ok, _ in _results if ok)327 failed = sum(1 for _, ok, _ in _results if not ok)328 total = len(_results)329 330 print(f"\n{'═' * 62}")331 print(f" Results: {passed}/{total} passed ({failed} failed) [{elapsed:.2f}s]")332 333 if failed:334 print(f"\n {FAIL} FAILED CHECKS:")335 for name, ok, detail in _results:336 if not ok:337 print(f" {FAIL} {name} {detail}")338 print("═" * 62)339 print(f"\n Submission is NOT ready. Fix the {failed} failed check(s) above.")340 sys.exit(1)341 else:342 print(f" {PASS} ALL CHECKS PASSED — ready to submit!")343 print("═" * 62)344 sys.exit(0)345 346 347if __name__ == "__main__":348 main()