ritvik360/nl2sql-bench
0
1# nl2sql-bench/openenv.yaml2# OpenEnv environment manifest — validated by `openenv validate`3 4name: nl2sql-bench5version: "0.1.0"6description: >7 Natural Language to SQL query generation environment for RL training.8 An agent iteratively writes and refines SQLite queries against a synthetic9 e-commerce database to answer business questions. Multi-turn episodes with10 dense, shaped rewards. Three difficulty tasks: easy (single-table),11 medium (JOIN + GROUP BY), hard (window functions + CTEs).12 13author: "nl2sql-bench team"14license: MIT15tags:16 - openenv17 - nl2sql18 - sql19 - analytics20 - rl-training21 - deterministic22 - multi-turn23 24# ── Task definitions ────────────────────────────────────────────────────────25tasks:26 - name: simple-filter27 difficulty: easy28 description: >29 Single-table SELECT with WHERE, ORDER BY, and LIMIT.30 Tests basic SQL fluency. Expected solve rate: high.31 max_steps: 532 reward_range: [0.0, 1.0]33 34 - name: join-aggregation35 difficulty: medium36 description: >37 Multi-table JOINs with GROUP BY, HAVING, and aggregation functions38 (COUNT, SUM, AVG, ROUND). Tests relational reasoning.39 max_steps: 540 reward_range: [0.0, 1.0]41 42 - name: analytics-window43 difficulty: hard44 description: >45 Advanced analytics using CTEs, window functions (DENSE_RANK,46 ROW_NUMBER, running SUM), and nested subqueries. Tests multi-step47 planning and SQLite-specific syntax.48 max_steps: 549 reward_range: [0.0, 1.0]50 51# ── Action / Observation space ──────────────────────────────────────────────52action_space:53 type: object54 properties:55 query:56 type: string57 description: "A SQLite SELECT query string."58 59observation_space:60 type: object61 properties:62 question:63 type: string64 description: "Natural-language question the agent must answer."65 schema_context:66 type: string67 description: "Compact database schema description for the agent."68 task_name:69 type: string70 description: "Active task identifier."71 last_query:72 type: string73 description: "The SQL query submitted on the previous step."74 last_result:75 type: array76 description: "Up to 10 rows returned by the last query (list of dicts)."77 last_error:78 type: string79 nullable: true80 description: "SQLite error string if last query failed, else null."81 result_columns:82 type: array83 description: "Column names of last_result."84 step:85 type: integer86 description: "Current step number (1-indexed; 0 after reset)."87 max_steps:88 type: integer89 description: "Maximum steps per episode."90 done:91 type: boolean92 description: "True when episode ends (exact match or step limit reached)."93 reward:94 type: number95 nullable: true96 description: "Reward for the most recent step [0.0, 1.0]."97 score:98 type: number99 description: "Normalised cumulative episode score [0.0, 1.0]."100 101# ── Reward function description ─────────────────────────────────────────────102reward:103 type: shaped104 range: [0.0, 1.0]105 components:106 - name: syntax_ok107 weight: 0.10108 description: "Query executes without SQLite error."109 - name: columns_match110 weight: 0.20111 description: "Returned column names match ground truth exactly."112 - name: row_count_match113 weight: 0.20114 description: "Number of returned rows matches ground truth."115 - name: exact_match116 weight: 0.50117 description: "Full result set matches ground truth (order-aware for ORDER BY)."118 - name: step_penalty119 weight: -0.05120 description: "Deducted per step beyond the first (encourages efficiency)."121 122# ── Deployment ──────────────────────────────────────────────────────────────123server:124 port: 7860125 dockerfile: Dockerfile126 healthcheck: /health127 128# ── Baseline ────────────────────────────────────────────────────────────────129baseline:130 script: inference.py131 model: Qwen/Qwen2.5-72B-Instruct132 expected_scores:133 simple-filter: 0.70134 join-aggregation: 0.45135 analytics-window: 0.25136