CoolFace
Datasetpublic

SamuelChien821/devopsbench-100

DevOpsBench-100 DevOpsBench-100 is a synthetic long-horizon software-engineering / SRE agent benchmark: 100 tasks over one executable world ("NovaCart", a mid-size e-commerce SaaS) with 72 SQLite tables, 1451 seeded rows, a 38-file monorepo with 417 commits, and 97 MCP tools spanning a first-party engineering stack (tickets, PRs, CI, deployments, canaries, migrations, feature flags, metrics, alerts, incidents, chat, knowledge base) plus deliberately disagreeing vendor-shaped… See the full description on the dataset page: https://huggingface.co/datasets/SamuelChien821/devopsbench-100.

sourceHugging Facecc-by-4.0updated 23d agoView on Hugging Face
0likes2.4kdownloads
schema.sql569 linesDownload Raw Back to world
1CREATE TABLE alert_firings (2    firing_id INTEGER PRIMARY KEY,3    rule_id INTEGER NOT NULL,4    fingerprint TEXT NOT NULL,5    day INTEGER NOT NULL,6    silenced INTEGER NOT NULL DEFAULT 0,7    inhibited_by INTEGER,8    paged_incident INTEGER          -- NULL when it never reached a human9);10CREATE TABLE alert_rules (11    rule_id INTEGER PRIMARY KEY,12    name TEXT NOT NULL,13    service_label TEXT NOT NULL,   -- may name a service that no longer exists14    expr TEXT NOT NULL,15    severity TEXT NOT NULL,16    group_by TEXT NOT NULL DEFAULT '',17    routes_to TEXT NOT NULL DEFAULT ''18);19CREATE TABLE alert_silences (20    silence_id INTEGER PRIMARY KEY,21    matcher TEXT NOT NULL,22    created_by TEXT NOT NULL,23    reason TEXT NOT NULL,24    expires_day INTEGER NOT NULL25);26CREATE TABLE alerts (27    alert_id INTEGER PRIMARY KEY AUTOINCREMENT,28    service TEXT NOT NULL,29    metric TEXT NOT NULL,30    severity TEXT NOT NULL DEFAULT 'high',31    status TEXT NOT NULL DEFAULT 'firing',32    message TEXT NOT NULL DEFAULT ''33);34CREATE TABLE answers (35    answer_id INTEGER PRIMARY KEY AUTOINCREMENT,36    question_id TEXT NOT NULL,37    answer TEXT NOT NULL,38    sources TEXT NOT NULL DEFAULT '[]',39    assumptions TEXT NOT NULL DEFAULT ''40);41CREATE TABLE approval_policy (42    policy_id INTEGER PRIMARY KEY,43    action TEXT NOT NULL,          -- the irreversible act requiring sign-off44    approver_role TEXT NOT NULL,45    rationale TEXT NOT NULL46);47CREATE TABLE approval_requests (48    request_id INTEGER PRIMARY KEY AUTOINCREMENT,49    action TEXT NOT NULL,50    target TEXT NOT NULL DEFAULT '',51    reason TEXT NOT NULL DEFAULT '',52    decision TEXT NOT NULL DEFAULT 'pending',   -- pending|approved|denied53    responder TEXT NOT NULL DEFAULT '',54    response TEXT NOT NULL DEFAULT ''55);56CREATE TABLE audit_events (57    seq INTEGER PRIMARY KEY AUTOINCREMENT,58    tool TEXT NOT NULL,59    service TEXT NOT NULL DEFAULT '',60    detail TEXT NOT NULL DEFAULT '{}'61);62CREATE TABLE authored_docs (63    doc_id INTEGER PRIMARY KEY AUTOINCREMENT,64    title TEXT NOT NULL,65    body TEXT NOT NULL,66    author TEXT NOT NULL DEFAULT 'agent'67);68CREATE TABLE canary_assessments (69    assessment_id INTEGER PRIMARY KEY AUTOINCREMENT,70    deployment_id INTEGER NOT NULL,71    service TEXT NOT NULL,72    verdict TEXT NOT NULL,73    detail TEXT NOT NULL DEFAULT ''74);75CREATE TABLE channels (76    channel TEXT PRIMARY KEY,77    purpose TEXT NOT NULL DEFAULT ''78);79CREATE TABLE ci_runs (80    run_id INTEGER PRIMARY KEY AUTOINCREMENT,81    service TEXT NOT NULL,82    pr_number INTEGER,83    status TEXT NOT NULL,84    detail TEXT NOT NULL DEFAULT ''85);86CREATE TABLE ci_stages (87    stage_id INTEGER PRIMARY KEY AUTOINCREMENT,88    run_id INTEGER NOT NULL,89    stage TEXT NOT NULL,90    status TEXT NOT NULL,91    detail TEXT NOT NULL DEFAULT ''92);93CREATE TABLE code_exercises (94    exercise_id INTEGER PRIMARY KEY,95    service TEXT NOT NULL,96    path TEXT NOT NULL UNIQUE,97    func TEXT NOT NULL,98    spec TEXT NOT NULL,99    starter TEXT NOT NULL,100    visible_tests TEXT NOT NULL,   -- json [[name, body], ...]101    hidden_tests TEXT NOT NULL     -- json, never returned by any read tool102);103CREATE TABLE code_submissions (104    submission_id INTEGER PRIMARY KEY AUTOINCREMENT,105    path TEXT NOT NULL,106    content TEXT NOT NULL,107    visible_passed INTEGER NOT NULL DEFAULT 0,108    visible_total INTEGER NOT NULL DEFAULT 0,109    hidden_passed INTEGER NOT NULL DEFAULT 0,110    hidden_total INTEGER NOT NULL DEFAULT 0,111    detail TEXT NOT NULL DEFAULT ''112);113CREATE TABLE commits (114    commit_id INTEGER PRIMARY KEY AUTOINCREMENT,115    sha TEXT UNIQUE NOT NULL,116    service TEXT NOT NULL,117    author TEXT NOT NULL,118    day INTEGER NOT NULL,119    message TEXT NOT NULL,120    files TEXT NOT NULL DEFAULT '',121    additions INTEGER NOT NULL DEFAULT 0,122    deletions INTEGER NOT NULL DEFAULT 0123);124CREATE TABLE confluence_pages (125    page_id INTEGER PRIMARY KEY,126    space TEXT NOT NULL,127    title TEXT NOT NULL,128    body TEXT NOT NULL,129    last_updated_day INTEGER NOT NULL,130    stale INTEGER NOT NULL DEFAULT 0131);132CREATE TABLE contract_rules (133    rule_id INTEGER PRIMARY KEY,134    producer_service TEXT NOT NULL,135    endpoint TEXT NOT NULL,136    consumer_service TEXT NOT NULL,137    consumer_key TEXT NOT NULL,138    consumer_required_value TEXT NOT NULL,139    message TEXT NOT NULL140);141CREATE TABLE db_grants (142    grant_id INTEGER PRIMARY KEY,143    service TEXT NOT NULL,144    component TEXT NOT NULL,       -- an infra_components name145    role TEXT NOT NULL,146    state TEXT NOT NULL,           -- active | revoked | missing147    changed_day INTEGER NOT NULL,148    note TEXT NOT NULL DEFAULT ''149);150CREATE TABLE deployments (151    deployment_id INTEGER PRIMARY KEY AUTOINCREMENT,152    service TEXT NOT NULL,153    environment TEXT NOT NULL,154    version TEXT NOT NULL,155    status TEXT NOT NULL,156    canary_percent INTEGER NOT NULL DEFAULT 100157);158CREATE TABLE diagnoses (159    diagnosis_id INTEGER PRIMARY KEY AUTOINCREMENT,160    scope TEXT NOT NULL,161    fault_detected INTEGER NOT NULL,162    service TEXT NOT NULL DEFAULT '',163    fault_type TEXT NOT NULL DEFAULT '',164    offending_key TEXT NOT NULL DEFAULT '',165    evidence TEXT NOT NULL DEFAULT ''166);167CREATE TABLE documents (168    doc_id INTEGER PRIMARY KEY,169    kind TEXT NOT NULL,170    title TEXT NOT NULL,171    service TEXT NOT NULL DEFAULT '',172    author TEXT NOT NULL DEFAULT '',173    day INTEGER NOT NULL DEFAULT 0,174    body TEXT NOT NULL175);176CREATE TABLE env_state (177    service TEXT NOT NULL,178    environment TEXT NOT NULL,179    kind TEXT NOT NULL,180    key TEXT NOT NULL,181    value TEXT NOT NULL,182    PRIMARY KEY (service, environment, kind, key)183);184CREATE TABLE error_events (185    event_id INTEGER PRIMARY KEY AUTOINCREMENT,186    fingerprint TEXT NOT NULL,187    service TEXT NOT NULL,188    title TEXT NOT NULL,189    culprit TEXT NOT NULL DEFAULT '',190    events INTEGER NOT NULL DEFAULT 1,191    status TEXT NOT NULL DEFAULT 'unresolved'192);193CREATE TABLE feature_flags (194    flag_id INTEGER PRIMARY KEY AUTOINCREMENT,195    key TEXT NOT NULL,196    service TEXT NOT NULL,197    description TEXT NOT NULL DEFAULT '',198    environment TEXT NOT NULL,199    enabled INTEGER NOT NULL DEFAULT 0,200    rollout_percent INTEGER NOT NULL DEFAULT 0,201    UNIQUE (key, environment)202);203CREATE TABLE github_issues (204    number INTEGER PRIMARY KEY,205    repo TEXT NOT NULL,206    title TEXT NOT NULL,207    state TEXT NOT NULL,           -- only open|closed exists208    labels TEXT NOT NULL DEFAULT '',209    created_day INTEGER NOT NULL210);211CREATE TABLE incidents (212    incident_id INTEGER PRIMARY KEY AUTOINCREMENT,213    severity TEXT NOT NULL,214    title TEXT NOT NULL,215    service TEXT NOT NULL,216    status TEXT NOT NULL DEFAULT 'open',217    commander TEXT NOT NULL DEFAULT ''218);219CREATE TABLE infra_components (220    component_id INTEGER PRIMARY KEY,221    name TEXT UNIQUE NOT NULL,222    kind TEXT NOT NULL,223    status TEXT NOT NULL DEFAULT 'healthy',224    detail TEXT NOT NULL DEFAULT ''225);226CREATE TABLE issue_links (227    source TEXT NOT NULL,228    target TEXT NOT NULL,229    kind TEXT NOT NULL,            -- duplicates | relates | implements230    PRIMARY KEY (source, target)231);232CREATE TABLE jira_issues (233    key TEXT PRIMARY KEY,234    project TEXT NOT NULL,235    summary TEXT NOT NULL,236    issue_type TEXT NOT NULL,237    status TEXT NOT NULL,          -- per-project workflow, NOT open/closed238    resolution TEXT NOT NULL DEFAULT '',239    priority TEXT NOT NULL,240    component TEXT NOT NULL DEFAULT '',241    assignee TEXT NOT NULL DEFAULT '',242    created_day INTEGER NOT NULL,243    updated_day INTEGER NOT NULL244);245CREATE TABLE k8s_deployments (246    service TEXT PRIMARY KEY,247    desired_replicas INTEGER NOT NULL,248    ready_replicas INTEGER NOT NULL,249    strategy TEXT NOT NULL DEFAULT 'RollingUpdate',250    storage_class TEXT NOT NULL DEFAULT ''251);252CREATE TABLE k8s_events (253    event_id INTEGER PRIMARY KEY,254    namespace TEXT NOT NULL,255    pod TEXT NOT NULL,256    reason TEXT NOT NULL,          -- OOMKilled | CrashLoopBackOff | BackOff | Killing257    message TEXT NOT NULL,258    count INTEGER NOT NULL DEFAULT 1,259    day INTEGER NOT NULL260);261CREATE TABLE k8s_nodes (262    node TEXT PRIMARY KEY,263    ready TEXT NOT NULL,           -- True | False | Unknown264    condition TEXT NOT NULL,       -- Ready | DiskPressure | MemoryPressure | KernelDeadlock265    message TEXT NOT NULL,266    cpu_used_pct INTEGER NOT NULL,267    disk_used_pct INTEGER NOT NULL,268    labels TEXT NOT NULL DEFAULT '',269    kernel_version TEXT NOT NULL DEFAULT '',270    taints TEXT NOT NULL DEFAULT ''271);272CREATE TABLE k8s_pods (273    pod TEXT PRIMARY KEY,274    namespace TEXT NOT NULL,275    service TEXT NOT NULL,276    image_tag TEXT NOT NULL,       -- the ONLY ground truth for what is running277    phase TEXT NOT NULL,278    restarts INTEGER NOT NULL DEFAULT 0,279    memory_limit_mb INTEGER NOT NULL,280    memory_usage_mb INTEGER NOT NULL,281    node TEXT NOT NULL DEFAULT '',       -- '' when the pod was never scheduled282    pending_reason TEXT NOT NULL DEFAULT ''283);284CREATE TABLE linear_issues (285    identifier TEXT PRIMARY KEY,286    team TEXT NOT NULL,287    title TEXT NOT NULL,288    state TEXT NOT NULL,289    priority INTEGER NOT NULL,     -- 0=none 1=urgent 2=high 3=normal 4=low290    label TEXT NOT NULL DEFAULT '',291    created_day INTEGER NOT NULL292);293CREATE TABLE local_deploy_log (294    row_id INTEGER PRIMARY KEY AUTOINCREMENT,295    service TEXT NOT NULL,296    version TEXT NOT NULL,297    environment TEXT NOT NULL,     -- CS-07: includes 'nonprod-*' spellings298    day INTEGER NOT NULL,299    was_rollback INTEGER NOT NULL DEFAULT 0   -- CS-26: do rollbacks count?300);301CREATE TABLE logs (302    log_id INTEGER PRIMARY KEY,303    service TEXT NOT NULL,304    environment TEXT NOT NULL DEFAULT 'production',305    level TEXT NOT NULL,306    message TEXT NOT NULL307);308CREATE TABLE messages (309    message_id INTEGER PRIMARY KEY AUTOINCREMENT,310    channel TEXT NOT NULL,311    author TEXT NOT NULL,312    body TEXT NOT NULL313);314CREATE TABLE metric_rules (315    rule_id INTEGER PRIMARY KEY,316    service TEXT NOT NULL,317    metric TEXT NOT NULL,318    kind TEXT NOT NULL,319    ckey TEXT NOT NULL DEFAULT '',320    cvalue TEXT NOT NULL DEFAULT '',321    value REAL NOT NULL322);323CREATE TABLE migration_requirements (324    req_id INTEGER PRIMARY KEY,325    service TEXT NOT NULL,326    module TEXT NOT NULL,327    migration_name TEXT NOT NULL328);329CREATE TABLE migrations (330    migration_id INTEGER PRIMARY KEY AUTOINCREMENT,331    service TEXT NOT NULL,332    name TEXT NOT NULL,333    environment TEXT NOT NULL DEFAULT '',334    status TEXT NOT NULL DEFAULT 'pending',335    UNIQUE (service, name, environment)336);337CREATE TABLE network_paths (338    path_id INTEGER PRIMARY KEY,339    from_service TEXT NOT NULL,340    to_target TEXT NOT NULL,341    state TEXT NOT NULL,           -- open | refused | timeout342    observed_day INTEGER NOT NULL,343    note TEXT NOT NULL DEFAULT ''344);345CREATE TABLE oncall (346    team TEXT PRIMARY KEY,347    engineer TEXT NOT NULL348);349CREATE TABLE owner_spreadsheet (350    row_id INTEGER PRIMARY KEY,351    service_label TEXT NOT NULL,   -- CS-01: yet another spelling352    owning_team TEXT NOT NULL,353    slack_channel TEXT NOT NULL DEFAULT '',354    last_reviewed_day INTEGER NOT NULL,355    week_start TEXT NOT NULL DEFAULT 'sunday'  -- CS-24: its own week convention356);357CREATE TABLE pd_change_events (358    change_id INTEGER PRIMARY KEY AUTOINCREMENT,359    pd_service_id TEXT NOT NULL,360    summary TEXT NOT NULL,361    day INTEGER NOT NULL362);363CREATE TABLE pd_incidents (364    incident_number INTEGER PRIMARY KEY,365    title TEXT NOT NULL,366    pd_service_id TEXT NOT NULL,367    urgency TEXT NOT NULL,         -- high|low  (NOT the same as priority)368    priority TEXT NOT NULL DEFAULT '',369    status TEXT NOT NULL,370    created_day INTEGER NOT NULL,371    resolved_day INTEGER372);373CREATE TABLE pd_oncall (374    schedule_id TEXT NOT NULL,375    schedule_name TEXT NOT NULL,376    escalation_policy TEXT NOT NULL,377    user_name TEXT NOT NULL,378    day INTEGER NOT NULL,379    PRIMARY KEY (schedule_id, day)380);381CREATE TABLE pd_services (382    pd_service_id TEXT PRIMARY KEY,383    name TEXT NOT NULL,            -- CS-01: a THIRD spelling of the service384    escalation_policy TEXT NOT NULL,385    status TEXT NOT NULL DEFAULT 'active'386);387CREATE TABLE pr_changes (388    change_id INTEGER PRIMARY KEY AUTOINCREMENT,389    pr_number INTEGER NOT NULL,390    change_type TEXT NOT NULL,391    payload TEXT NOT NULL392);393CREATE TABLE prom_series (394    series_id INTEGER PRIMARY KEY,395    metric TEXT NOT NULL,396    label_service TEXT NOT NULL,   -- Prometheus label spelling397    label_env TEXT NOT NULL,398    day INTEGER NOT NULL,399    value REAL NOT NULL,400    counter_reset INTEGER NOT NULL DEFAULT 0401);402CREATE TABLE pull_requests (403    number INTEGER PRIMARY KEY,404    service TEXT NOT NULL,405    title TEXT NOT NULL,406    body TEXT NOT NULL DEFAULT '',407    author TEXT NOT NULL DEFAULT '',408    ticket_key TEXT NOT NULL DEFAULT '',409    status TEXT NOT NULL DEFAULT 'open',410    merged_version TEXT NOT NULL DEFAULT ''411);412CREATE TABLE remediation_proposals (413    proposal_id INTEGER PRIMARY KEY,414    incident_ref TEXT NOT NULL,415    author TEXT NOT NULL,416    summary TEXT NOT NULL,417    detail TEXT NOT NULL418);419CREATE TABLE repo_files (420    file_id INTEGER PRIMARY KEY AUTOINCREMENT,421    service TEXT NOT NULL,422    path TEXT UNIQUE NOT NULL,423    language TEXT NOT NULL,424    owner TEXT NOT NULL DEFAULT '',425    loc INTEGER NOT NULL DEFAULT 0,426    content TEXT NOT NULL427);428CREATE TABLE repo_state (429    service TEXT NOT NULL,430    kind TEXT NOT NULL,431    key TEXT NOT NULL,432    value TEXT NOT NULL,433    PRIMARY KEY (service, kind, key)434);435CREATE TABLE runtime_stats (436    service TEXT PRIMARY KEY,437    heap_used_pct INTEGER NOT NULL,438    gc_pause_p99_ms INTEGER NOT NULL,439    gc_collections_per_min INTEGER NOT NULL,440    threads INTEGER NOT NULL441);442CREATE TABLE sentry_issues (443    issue_id TEXT PRIMARY KEY,444    project_slug TEXT NOT NULL,    -- Sentry's own naming445    title TEXT NOT NULL,446    level TEXT NOT NULL,447    events INTEGER NOT NULL,       -- sampled448    users_affected INTEGER NOT NULL DEFAULT 0,449    first_seen_day INTEGER NOT NULL,450    last_seen_day INTEGER NOT NULL,451    status TEXT NOT NULL DEFAULT 'unresolved'452);453CREATE TABLE sentry_projects (454    slug TEXT PRIMARY KEY,455    platform TEXT NOT NULL,456    sample_rate REAL NOT NULL      -- why Sentry counts < Prometheus counts457);458CREATE TABLE service_aliases (459    canonical TEXT NOT NULL,460    alias TEXT NOT NULL,461    system TEXT NOT NULL,462    PRIMARY KEY (alias, system)463);464CREATE TABLE service_dependencies (465    service TEXT NOT NULL,466    depends_on TEXT NOT NULL,467    kind TEXT NOT NULL,468    PRIMARY KEY (service, depends_on)469);470CREATE TABLE service_metrics (471    service TEXT NOT NULL,472    environment TEXT NOT NULL,473    metric TEXT NOT NULL,474    value REAL NOT NULL,475    PRIMARY KEY (service, environment, metric)476);477CREATE TABLE services (478    service_id INTEGER PRIMARY KEY,479    name TEXT UNIQUE NOT NULL,480    kind TEXT NOT NULL,481    team TEXT NOT NULL,482    tier INTEGER NOT NULL,483    language TEXT NOT NULL,484    description TEXT NOT NULL DEFAULT '',485    repo_version TEXT NOT NULL486);487CREATE TABLE slos (488    slo_id INTEGER PRIMARY KEY,489    service TEXT NOT NULL,490    metric TEXT NOT NULL,491    threshold REAL NOT NULL,492    description TEXT NOT NULL DEFAULT ''493);494CREATE TABLE status_page (495    post_id INTEGER PRIMARY KEY AUTOINCREMENT,496    state TEXT NOT NULL,497    title TEXT NOT NULL,498    body TEXT NOT NULL DEFAULT ''499);500CREATE TABLE status_page_posts (501    post_id INTEGER PRIMARY KEY,502    title TEXT NOT NULL,503    impact TEXT NOT NULL,          -- none|minor|major|critical504    state TEXT NOT NULL,505    published_day INTEGER NOT NULL,506    linked_incident INTEGER507);508CREATE TABLE tests_catalog (509    test_id INTEGER PRIMARY KEY,510    service TEXT NOT NULL,511    suite TEXT NOT NULL,512    name TEXT NOT NULL,513    status TEXT NOT NULL DEFAULT 'passing',514    quarantined INTEGER NOT NULL DEFAULT 0515);516CREATE TABLE tickets (517    ticket_id INTEGER PRIMARY KEY AUTOINCREMENT,518    key TEXT UNIQUE NOT NULL,519    type TEXT NOT NULL,520    title TEXT NOT NULL,521    description TEXT NOT NULL DEFAULT '',522    status TEXT NOT NULL DEFAULT 'open',523    priority TEXT NOT NULL DEFAULT 'medium',524    assignee TEXT NOT NULL DEFAULT '',525    service TEXT NOT NULL DEFAULT ''526);527CREATE TABLE tool_calls (528    call_seq INTEGER PRIMARY KEY AUTOINCREMENT,529    tool TEXT NOT NULL530);531CREATE TABLE traffic_profile (532    route_id INTEGER PRIMARY KEY,533    service TEXT NOT NULL,534    route TEXT NOT NULL,535    rps INTEGER NOT NULL,536    share_pct INTEGER NOT NULL DEFAULT 100537);538CREATE TABLE versions (539    version_id INTEGER PRIMARY KEY AUTOINCREMENT,540    service TEXT NOT NULL,541    version TEXT NOT NULL,542    state_json TEXT NOT NULL,543    requires_migration TEXT NOT NULL DEFAULT '',544    UNIQUE (service, version)545);546CREATE TABLE vulnerabilities (547    vuln_id INTEGER PRIMARY KEY,548    cve TEXT NOT NULL,549    package TEXT NOT NULL,550    service TEXT NOT NULL,551    severity TEXT NOT NULL,552    fixed_version TEXT NOT NULL,553    status TEXT NOT NULL DEFAULT 'open'554);555CREATE TABLE workspace_files (556    path TEXT PRIMARY KEY,557    content TEXT NOT NULL,558    seeded INTEGER NOT NULL DEFAULT 1   -- 0 once the agent has written it559);560CREATE TABLE workspace_runs (561    run_id INTEGER PRIMARY KEY AUTOINCREMENT,562    path TEXT NOT NULL,563    exit_code INTEGER NOT NULL,564    stdout TEXT NOT NULL DEFAULT '',565    stderr TEXT NOT NULL DEFAULT '',566    timed_out INTEGER NOT NULL DEFAULT 0567);568DELETE FROM "sqlite_sequence";569