SamuelChien821/devopsbench-100
DevOpsBench-100 DevOpsBench-100 is a synthetic long-horizon software-engineering / SRE agent benchmark: 100 tasks over one executable world ("NovaCart", a mid-size e-commerce SaaS) with 72 SQLite tables, 1451 seeded rows, a 38-file monorepo with 417 commits, and 97 MCP tools spanning a first-party engineering stack (tickets, PRs, CI, deployments, canaries, migrations, feature flags, metrics, alerts, incidents, chat, knowledge base) plus deliberately disagreeing vendor-shaped… See the full description on the dataset page: https://huggingface.co/datasets/SamuelChien821/devopsbench-100.
02.4k
1CREATE TABLE alert_firings (2 firing_id INTEGER PRIMARY KEY,3 rule_id INTEGER NOT NULL,4 fingerprint TEXT NOT NULL,5 day INTEGER NOT NULL,6 silenced INTEGER NOT NULL DEFAULT 0,7 inhibited_by INTEGER,8 paged_incident INTEGER -- NULL when it never reached a human9);10CREATE TABLE alert_rules (11 rule_id INTEGER PRIMARY KEY,12 name TEXT NOT NULL,13 service_label TEXT NOT NULL, -- may name a service that no longer exists14 expr TEXT NOT NULL,15 severity TEXT NOT NULL,16 group_by TEXT NOT NULL DEFAULT '',17 routes_to TEXT NOT NULL DEFAULT ''18);19CREATE TABLE alert_silences (20 silence_id INTEGER PRIMARY KEY,21 matcher TEXT NOT NULL,22 created_by TEXT NOT NULL,23 reason TEXT NOT NULL,24 expires_day INTEGER NOT NULL25);26CREATE TABLE alerts (27 alert_id INTEGER PRIMARY KEY AUTOINCREMENT,28 service TEXT NOT NULL,29 metric TEXT NOT NULL,30 severity TEXT NOT NULL DEFAULT 'high',31 status TEXT NOT NULL DEFAULT 'firing',32 message TEXT NOT NULL DEFAULT ''33);34CREATE TABLE answers (35 answer_id INTEGER PRIMARY KEY AUTOINCREMENT,36 question_id TEXT NOT NULL,37 answer TEXT NOT NULL,38 sources TEXT NOT NULL DEFAULT '[]',39 assumptions TEXT NOT NULL DEFAULT ''40);41CREATE TABLE approval_policy (42 policy_id INTEGER PRIMARY KEY,43 action TEXT NOT NULL, -- the irreversible act requiring sign-off44 approver_role TEXT NOT NULL,45 rationale TEXT NOT NULL46);47CREATE TABLE approval_requests (48 request_id INTEGER PRIMARY KEY AUTOINCREMENT,49 action TEXT NOT NULL,50 target TEXT NOT NULL DEFAULT '',51 reason TEXT NOT NULL DEFAULT '',52 decision TEXT NOT NULL DEFAULT 'pending', -- pending|approved|denied53 responder TEXT NOT NULL DEFAULT '',54 response TEXT NOT NULL DEFAULT ''55);56CREATE TABLE audit_events (57 seq INTEGER PRIMARY KEY AUTOINCREMENT,58 tool TEXT NOT NULL,59 service TEXT NOT NULL DEFAULT '',60 detail TEXT NOT NULL DEFAULT '{}'61);62CREATE TABLE authored_docs (63 doc_id INTEGER PRIMARY KEY AUTOINCREMENT,64 title TEXT NOT NULL,65 body TEXT NOT NULL,66 author TEXT NOT NULL DEFAULT 'agent'67);68CREATE TABLE canary_assessments (69 assessment_id INTEGER PRIMARY KEY AUTOINCREMENT,70 deployment_id INTEGER NOT NULL,71 service TEXT NOT NULL,72 verdict TEXT NOT NULL,73 detail TEXT NOT NULL DEFAULT ''74);75CREATE TABLE channels (76 channel TEXT PRIMARY KEY,77 purpose TEXT NOT NULL DEFAULT ''78);79CREATE TABLE ci_runs (80 run_id INTEGER PRIMARY KEY AUTOINCREMENT,81 service TEXT NOT NULL,82 pr_number INTEGER,83 status TEXT NOT NULL,84 detail TEXT NOT NULL DEFAULT ''85);86CREATE TABLE ci_stages (87 stage_id INTEGER PRIMARY KEY AUTOINCREMENT,88 run_id INTEGER NOT NULL,89 stage TEXT NOT NULL,90 status TEXT NOT NULL,91 detail TEXT NOT NULL DEFAULT ''92);93CREATE TABLE code_exercises (94 exercise_id INTEGER PRIMARY KEY,95 service TEXT NOT NULL,96 path TEXT NOT NULL UNIQUE,97 func TEXT NOT NULL,98 spec TEXT NOT NULL,99 starter TEXT NOT NULL,100 visible_tests TEXT NOT NULL, -- json [[name, body], ...]101 hidden_tests TEXT NOT NULL -- json, never returned by any read tool102);103CREATE TABLE code_submissions (104 submission_id INTEGER PRIMARY KEY AUTOINCREMENT,105 path TEXT NOT NULL,106 content TEXT NOT NULL,107 visible_passed INTEGER NOT NULL DEFAULT 0,108 visible_total INTEGER NOT NULL DEFAULT 0,109 hidden_passed INTEGER NOT NULL DEFAULT 0,110 hidden_total INTEGER NOT NULL DEFAULT 0,111 detail TEXT NOT NULL DEFAULT ''112);113CREATE TABLE commits (114 commit_id INTEGER PRIMARY KEY AUTOINCREMENT,115 sha TEXT UNIQUE NOT NULL,116 service TEXT NOT NULL,117 author TEXT NOT NULL,118 day INTEGER NOT NULL,119 message TEXT NOT NULL,120 files TEXT NOT NULL DEFAULT '',121 additions INTEGER NOT NULL DEFAULT 0,122 deletions INTEGER NOT NULL DEFAULT 0123);124CREATE TABLE confluence_pages (125 page_id INTEGER PRIMARY KEY,126 space TEXT NOT NULL,127 title TEXT NOT NULL,128 body TEXT NOT NULL,129 last_updated_day INTEGER NOT NULL,130 stale INTEGER NOT NULL DEFAULT 0131);132CREATE TABLE contract_rules (133 rule_id INTEGER PRIMARY KEY,134 producer_service TEXT NOT NULL,135 endpoint TEXT NOT NULL,136 consumer_service TEXT NOT NULL,137 consumer_key TEXT NOT NULL,138 consumer_required_value TEXT NOT NULL,139 message TEXT NOT NULL140);141CREATE TABLE db_grants (142 grant_id INTEGER PRIMARY KEY,143 service TEXT NOT NULL,144 component TEXT NOT NULL, -- an infra_components name145 role TEXT NOT NULL,146 state TEXT NOT NULL, -- active | revoked | missing147 changed_day INTEGER NOT NULL,148 note TEXT NOT NULL DEFAULT ''149);150CREATE TABLE deployments (151 deployment_id INTEGER PRIMARY KEY AUTOINCREMENT,152 service TEXT NOT NULL,153 environment TEXT NOT NULL,154 version TEXT NOT NULL,155 status TEXT NOT NULL,156 canary_percent INTEGER NOT NULL DEFAULT 100157);158CREATE TABLE diagnoses (159 diagnosis_id INTEGER PRIMARY KEY AUTOINCREMENT,160 scope TEXT NOT NULL,161 fault_detected INTEGER NOT NULL,162 service TEXT NOT NULL DEFAULT '',163 fault_type TEXT NOT NULL DEFAULT '',164 offending_key TEXT NOT NULL DEFAULT '',165 evidence TEXT NOT NULL DEFAULT ''166);167CREATE TABLE documents (168 doc_id INTEGER PRIMARY KEY,169 kind TEXT NOT NULL,170 title TEXT NOT NULL,171 service TEXT NOT NULL DEFAULT '',172 author TEXT NOT NULL DEFAULT '',173 day INTEGER NOT NULL DEFAULT 0,174 body TEXT NOT NULL175);176CREATE TABLE env_state (177 service TEXT NOT NULL,178 environment TEXT NOT NULL,179 kind TEXT NOT NULL,180 key TEXT NOT NULL,181 value TEXT NOT NULL,182 PRIMARY KEY (service, environment, kind, key)183);184CREATE TABLE error_events (185 event_id INTEGER PRIMARY KEY AUTOINCREMENT,186 fingerprint TEXT NOT NULL,187 service TEXT NOT NULL,188 title TEXT NOT NULL,189 culprit TEXT NOT NULL DEFAULT '',190 events INTEGER NOT NULL DEFAULT 1,191 status TEXT NOT NULL DEFAULT 'unresolved'192);193CREATE TABLE feature_flags (194 flag_id INTEGER PRIMARY KEY AUTOINCREMENT,195 key TEXT NOT NULL,196 service TEXT NOT NULL,197 description TEXT NOT NULL DEFAULT '',198 environment TEXT NOT NULL,199 enabled INTEGER NOT NULL DEFAULT 0,200 rollout_percent INTEGER NOT NULL DEFAULT 0,201 UNIQUE (key, environment)202);203CREATE TABLE github_issues (204 number INTEGER PRIMARY KEY,205 repo TEXT NOT NULL,206 title TEXT NOT NULL,207 state TEXT NOT NULL, -- only open|closed exists208 labels TEXT NOT NULL DEFAULT '',209 created_day INTEGER NOT NULL210);211CREATE TABLE incidents (212 incident_id INTEGER PRIMARY KEY AUTOINCREMENT,213 severity TEXT NOT NULL,214 title TEXT NOT NULL,215 service TEXT NOT NULL,216 status TEXT NOT NULL DEFAULT 'open',217 commander TEXT NOT NULL DEFAULT ''218);219CREATE TABLE infra_components (220 component_id INTEGER PRIMARY KEY,221 name TEXT UNIQUE NOT NULL,222 kind TEXT NOT NULL,223 status TEXT NOT NULL DEFAULT 'healthy',224 detail TEXT NOT NULL DEFAULT ''225);226CREATE TABLE issue_links (227 source TEXT NOT NULL,228 target TEXT NOT NULL,229 kind TEXT NOT NULL, -- duplicates | relates | implements230 PRIMARY KEY (source, target)231);232CREATE TABLE jira_issues (233 key TEXT PRIMARY KEY,234 project TEXT NOT NULL,235 summary TEXT NOT NULL,236 issue_type TEXT NOT NULL,237 status TEXT NOT NULL, -- per-project workflow, NOT open/closed238 resolution TEXT NOT NULL DEFAULT '',239 priority TEXT NOT NULL,240 component TEXT NOT NULL DEFAULT '',241 assignee TEXT NOT NULL DEFAULT '',242 created_day INTEGER NOT NULL,243 updated_day INTEGER NOT NULL244);245CREATE TABLE k8s_deployments (246 service TEXT PRIMARY KEY,247 desired_replicas INTEGER NOT NULL,248 ready_replicas INTEGER NOT NULL,249 strategy TEXT NOT NULL DEFAULT 'RollingUpdate',250 storage_class TEXT NOT NULL DEFAULT ''251);252CREATE TABLE k8s_events (253 event_id INTEGER PRIMARY KEY,254 namespace TEXT NOT NULL,255 pod TEXT NOT NULL,256 reason TEXT NOT NULL, -- OOMKilled | CrashLoopBackOff | BackOff | Killing257 message TEXT NOT NULL,258 count INTEGER NOT NULL DEFAULT 1,259 day INTEGER NOT NULL260);261CREATE TABLE k8s_nodes (262 node TEXT PRIMARY KEY,263 ready TEXT NOT NULL, -- True | False | Unknown264 condition TEXT NOT NULL, -- Ready | DiskPressure | MemoryPressure | KernelDeadlock265 message TEXT NOT NULL,266 cpu_used_pct INTEGER NOT NULL,267 disk_used_pct INTEGER NOT NULL,268 labels TEXT NOT NULL DEFAULT '',269 kernel_version TEXT NOT NULL DEFAULT '',270 taints TEXT NOT NULL DEFAULT ''271);272CREATE TABLE k8s_pods (273 pod TEXT PRIMARY KEY,274 namespace TEXT NOT NULL,275 service TEXT NOT NULL,276 image_tag TEXT NOT NULL, -- the ONLY ground truth for what is running277 phase TEXT NOT NULL,278 restarts INTEGER NOT NULL DEFAULT 0,279 memory_limit_mb INTEGER NOT NULL,280 memory_usage_mb INTEGER NOT NULL,281 node TEXT NOT NULL DEFAULT '', -- '' when the pod was never scheduled282 pending_reason TEXT NOT NULL DEFAULT ''283);284CREATE TABLE linear_issues (285 identifier TEXT PRIMARY KEY,286 team TEXT NOT NULL,287 title TEXT NOT NULL,288 state TEXT NOT NULL,289 priority INTEGER NOT NULL, -- 0=none 1=urgent 2=high 3=normal 4=low290 label TEXT NOT NULL DEFAULT '',291 created_day INTEGER NOT NULL292);293CREATE TABLE local_deploy_log (294 row_id INTEGER PRIMARY KEY AUTOINCREMENT,295 service TEXT NOT NULL,296 version TEXT NOT NULL,297 environment TEXT NOT NULL, -- CS-07: includes 'nonprod-*' spellings298 day INTEGER NOT NULL,299 was_rollback INTEGER NOT NULL DEFAULT 0 -- CS-26: do rollbacks count?300);301CREATE TABLE logs (302 log_id INTEGER PRIMARY KEY,303 service TEXT NOT NULL,304 environment TEXT NOT NULL DEFAULT 'production',305 level TEXT NOT NULL,306 message TEXT NOT NULL307);308CREATE TABLE messages (309 message_id INTEGER PRIMARY KEY AUTOINCREMENT,310 channel TEXT NOT NULL,311 author TEXT NOT NULL,312 body TEXT NOT NULL313);314CREATE TABLE metric_rules (315 rule_id INTEGER PRIMARY KEY,316 service TEXT NOT NULL,317 metric TEXT NOT NULL,318 kind TEXT NOT NULL,319 ckey TEXT NOT NULL DEFAULT '',320 cvalue TEXT NOT NULL DEFAULT '',321 value REAL NOT NULL322);323CREATE TABLE migration_requirements (324 req_id INTEGER PRIMARY KEY,325 service TEXT NOT NULL,326 module TEXT NOT NULL,327 migration_name TEXT NOT NULL328);329CREATE TABLE migrations (330 migration_id INTEGER PRIMARY KEY AUTOINCREMENT,331 service TEXT NOT NULL,332 name TEXT NOT NULL,333 environment TEXT NOT NULL DEFAULT '',334 status TEXT NOT NULL DEFAULT 'pending',335 UNIQUE (service, name, environment)336);337CREATE TABLE network_paths (338 path_id INTEGER PRIMARY KEY,339 from_service TEXT NOT NULL,340 to_target TEXT NOT NULL,341 state TEXT NOT NULL, -- open | refused | timeout342 observed_day INTEGER NOT NULL,343 note TEXT NOT NULL DEFAULT ''344);345CREATE TABLE oncall (346 team TEXT PRIMARY KEY,347 engineer TEXT NOT NULL348);349CREATE TABLE owner_spreadsheet (350 row_id INTEGER PRIMARY KEY,351 service_label TEXT NOT NULL, -- CS-01: yet another spelling352 owning_team TEXT NOT NULL,353 slack_channel TEXT NOT NULL DEFAULT '',354 last_reviewed_day INTEGER NOT NULL,355 week_start TEXT NOT NULL DEFAULT 'sunday' -- CS-24: its own week convention356);357CREATE TABLE pd_change_events (358 change_id INTEGER PRIMARY KEY AUTOINCREMENT,359 pd_service_id TEXT NOT NULL,360 summary TEXT NOT NULL,361 day INTEGER NOT NULL362);363CREATE TABLE pd_incidents (364 incident_number INTEGER PRIMARY KEY,365 title TEXT NOT NULL,366 pd_service_id TEXT NOT NULL,367 urgency TEXT NOT NULL, -- high|low (NOT the same as priority)368 priority TEXT NOT NULL DEFAULT '',369 status TEXT NOT NULL,370 created_day INTEGER NOT NULL,371 resolved_day INTEGER372);373CREATE TABLE pd_oncall (374 schedule_id TEXT NOT NULL,375 schedule_name TEXT NOT NULL,376 escalation_policy TEXT NOT NULL,377 user_name TEXT NOT NULL,378 day INTEGER NOT NULL,379 PRIMARY KEY (schedule_id, day)380);381CREATE TABLE pd_services (382 pd_service_id TEXT PRIMARY KEY,383 name TEXT NOT NULL, -- CS-01: a THIRD spelling of the service384 escalation_policy TEXT NOT NULL,385 status TEXT NOT NULL DEFAULT 'active'386);387CREATE TABLE pr_changes (388 change_id INTEGER PRIMARY KEY AUTOINCREMENT,389 pr_number INTEGER NOT NULL,390 change_type TEXT NOT NULL,391 payload TEXT NOT NULL392);393CREATE TABLE prom_series (394 series_id INTEGER PRIMARY KEY,395 metric TEXT NOT NULL,396 label_service TEXT NOT NULL, -- Prometheus label spelling397 label_env TEXT NOT NULL,398 day INTEGER NOT NULL,399 value REAL NOT NULL,400 counter_reset INTEGER NOT NULL DEFAULT 0401);402CREATE TABLE pull_requests (403 number INTEGER PRIMARY KEY,404 service TEXT NOT NULL,405 title TEXT NOT NULL,406 body TEXT NOT NULL DEFAULT '',407 author TEXT NOT NULL DEFAULT '',408 ticket_key TEXT NOT NULL DEFAULT '',409 status TEXT NOT NULL DEFAULT 'open',410 merged_version TEXT NOT NULL DEFAULT ''411);412CREATE TABLE remediation_proposals (413 proposal_id INTEGER PRIMARY KEY,414 incident_ref TEXT NOT NULL,415 author TEXT NOT NULL,416 summary TEXT NOT NULL,417 detail TEXT NOT NULL418);419CREATE TABLE repo_files (420 file_id INTEGER PRIMARY KEY AUTOINCREMENT,421 service TEXT NOT NULL,422 path TEXT UNIQUE NOT NULL,423 language TEXT NOT NULL,424 owner TEXT NOT NULL DEFAULT '',425 loc INTEGER NOT NULL DEFAULT 0,426 content TEXT NOT NULL427);428CREATE TABLE repo_state (429 service TEXT NOT NULL,430 kind TEXT NOT NULL,431 key TEXT NOT NULL,432 value TEXT NOT NULL,433 PRIMARY KEY (service, kind, key)434);435CREATE TABLE runtime_stats (436 service TEXT PRIMARY KEY,437 heap_used_pct INTEGER NOT NULL,438 gc_pause_p99_ms INTEGER NOT NULL,439 gc_collections_per_min INTEGER NOT NULL,440 threads INTEGER NOT NULL441);442CREATE TABLE sentry_issues (443 issue_id TEXT PRIMARY KEY,444 project_slug TEXT NOT NULL, -- Sentry's own naming445 title TEXT NOT NULL,446 level TEXT NOT NULL,447 events INTEGER NOT NULL, -- sampled448 users_affected INTEGER NOT NULL DEFAULT 0,449 first_seen_day INTEGER NOT NULL,450 last_seen_day INTEGER NOT NULL,451 status TEXT NOT NULL DEFAULT 'unresolved'452);453CREATE TABLE sentry_projects (454 slug TEXT PRIMARY KEY,455 platform TEXT NOT NULL,456 sample_rate REAL NOT NULL -- why Sentry counts < Prometheus counts457);458CREATE TABLE service_aliases (459 canonical TEXT NOT NULL,460 alias TEXT NOT NULL,461 system TEXT NOT NULL,462 PRIMARY KEY (alias, system)463);464CREATE TABLE service_dependencies (465 service TEXT NOT NULL,466 depends_on TEXT NOT NULL,467 kind TEXT NOT NULL,468 PRIMARY KEY (service, depends_on)469);470CREATE TABLE service_metrics (471 service TEXT NOT NULL,472 environment TEXT NOT NULL,473 metric TEXT NOT NULL,474 value REAL NOT NULL,475 PRIMARY KEY (service, environment, metric)476);477CREATE TABLE services (478 service_id INTEGER PRIMARY KEY,479 name TEXT UNIQUE NOT NULL,480 kind TEXT NOT NULL,481 team TEXT NOT NULL,482 tier INTEGER NOT NULL,483 language TEXT NOT NULL,484 description TEXT NOT NULL DEFAULT '',485 repo_version TEXT NOT NULL486);487CREATE TABLE slos (488 slo_id INTEGER PRIMARY KEY,489 service TEXT NOT NULL,490 metric TEXT NOT NULL,491 threshold REAL NOT NULL,492 description TEXT NOT NULL DEFAULT ''493);494CREATE TABLE status_page (495 post_id INTEGER PRIMARY KEY AUTOINCREMENT,496 state TEXT NOT NULL,497 title TEXT NOT NULL,498 body TEXT NOT NULL DEFAULT ''499);500CREATE TABLE status_page_posts (501 post_id INTEGER PRIMARY KEY,502 title TEXT NOT NULL,503 impact TEXT NOT NULL, -- none|minor|major|critical504 state TEXT NOT NULL,505 published_day INTEGER NOT NULL,506 linked_incident INTEGER507);508CREATE TABLE tests_catalog (509 test_id INTEGER PRIMARY KEY,510 service TEXT NOT NULL,511 suite TEXT NOT NULL,512 name TEXT NOT NULL,513 status TEXT NOT NULL DEFAULT 'passing',514 quarantined INTEGER NOT NULL DEFAULT 0515);516CREATE TABLE tickets (517 ticket_id INTEGER PRIMARY KEY AUTOINCREMENT,518 key TEXT UNIQUE NOT NULL,519 type TEXT NOT NULL,520 title TEXT NOT NULL,521 description TEXT NOT NULL DEFAULT '',522 status TEXT NOT NULL DEFAULT 'open',523 priority TEXT NOT NULL DEFAULT 'medium',524 assignee TEXT NOT NULL DEFAULT '',525 service TEXT NOT NULL DEFAULT ''526);527CREATE TABLE tool_calls (528 call_seq INTEGER PRIMARY KEY AUTOINCREMENT,529 tool TEXT NOT NULL530);531CREATE TABLE traffic_profile (532 route_id INTEGER PRIMARY KEY,533 service TEXT NOT NULL,534 route TEXT NOT NULL,535 rps INTEGER NOT NULL,536 share_pct INTEGER NOT NULL DEFAULT 100537);538CREATE TABLE versions (539 version_id INTEGER PRIMARY KEY AUTOINCREMENT,540 service TEXT NOT NULL,541 version TEXT NOT NULL,542 state_json TEXT NOT NULL,543 requires_migration TEXT NOT NULL DEFAULT '',544 UNIQUE (service, version)545);546CREATE TABLE vulnerabilities (547 vuln_id INTEGER PRIMARY KEY,548 cve TEXT NOT NULL,549 package TEXT NOT NULL,550 service TEXT NOT NULL,551 severity TEXT NOT NULL,552 fixed_version TEXT NOT NULL,553 status TEXT NOT NULL DEFAULT 'open'554);555CREATE TABLE workspace_files (556 path TEXT PRIMARY KEY,557 content TEXT NOT NULL,558 seeded INTEGER NOT NULL DEFAULT 1 -- 0 once the agent has written it559);560CREATE TABLE workspace_runs (561 run_id INTEGER PRIMARY KEY AUTOINCREMENT,562 path TEXT NOT NULL,563 exit_code INTEGER NOT NULL,564 stdout TEXT NOT NULL DEFAULT '',565 stderr TEXT NOT NULL DEFAULT '',566 timed_out INTEGER NOT NULL DEFAULT 0567);568DELETE FROM "sqlite_sequence";569 