training-monkey/dataoncallenv

1# openenv.yaml2# Required by the OpenEnv spec validator.3# This file describes your environment to the Hub.4 5name: dataoncallenv6version: "2.0.0"7description: >8  RL environment for data pipeline debugging agents. The agent acts as an9  on-call data analyst investigating broken reports — diagnosing root causes10  across three realistic bug scenarios. Features: partial observability11  (tables must be discovered), query cost budgets, realistic dbt/Airflow logs,12  anti-cheat constraints, and tiered evaluation with investigation quality scoring.13 14domain: data-engineering15author: RLBenchmarkingTeam16entrypoint: api.app:app17tasks:18  - id: 119    difficulty: easy20    title: "Weekly revenue report shows $0 for international sales"21    description: "Silent NULL JOIN caused by currency code case mismatch"22    optimal_steps: 523    optimal_cost: 7.024 25  - id: 226    difficulty: medium27    title: "Monthly active users dropped 8% on Feb 1st"28    description: "Timezone migration caused double-counting at month boundary"29    optimal_steps: 730    optimal_cost: 10.031 32  - id: 333    difficulty: hard34    title: "Cloud Storage revenue overstated by exactly 3.7x"35    description: "Fanout bug from non-unique JOIN key in product_promotions"36    optimal_steps: 837    optimal_cost: 13.038 39observation_space:40  task_id: integer41  result: any42  steps_taken: integer43  done: boolean44  max_steps: integer45  cost_spent: float46  budget_remaining: float47 48action_space:49  tool: string    # run_sql | inspect_schema | check_logs | check_airflow | diff_report | list_tables | submit50  query: string51  reasoning: string   # optional — rewarded by grader52 53reward_range: [0.0, 1.0]54 55reward_components:56  - name: diagnosis_correct57    max: 0.2558    description: "Tiered: exact root cause (0.25), category match (0.15), symptom only (0.08)"59  - name: fix_valid60    max: 0.2561    description: "Agent's proposed fix query returns correct output vs ground truth"62  - name: efficiency63    max: 0.1564    description: "Combined step efficiency + cost efficiency vs optimal"65  - name: reasoning_quality66    max: 0.1067    description: "Fraction of actions that include reasoning field"68  - name: investigation_quality69    max: 0.1070    description: "Logical debugging methodology: discovery → schema → logs → hypothesis → verify"71  - name: false_positive_penalty72    max: -0.1573    description: "Penalty for irrelevant table access, duplicate queries, or memorized answers"74 75features:76  - partial_observability: "Tables hidden until agent calls list_tables()"77  - query_costs: "Each tool has a fixed cost; total budget 20.0 per episode"78  - realistic_logs: "Expanded dbt logs + Airflow DAG run history with noise entries"79  - anti_cheat: "SELECT * blocked, results capped at 50 rows, min 2 steps before submit"80  - tiered_evaluation: "Diagnosis scored by depth of understanding, not binary"81 82tags:83  - openenv84  - data-engineering85  - sql86  - debugging87  - real-world88  - agentic89  - partial-observability90 91endpoints:92  health: GET /health93  reset:  POST /reset94  step:   POST /step95  state:  GET /state96  tasks:  GET /tasks