SamuelChien821/devopsbench-100
DevOpsBench-100 DevOpsBench-100 is a synthetic long-horizon software-engineering / SRE agent benchmark: 100 tasks over one executable world ("NovaCart", a mid-size e-commerce SaaS) with 72 SQLite tables, 1451 seeded rows, a 38-file monorepo with 417 commits, and 97 MCP tools spanning a first-party engineering stack (tickets, PRs, CI, deployments, canaries, migrations, feature flags, metrics, alerts, incidents, chat, knowledge base) plus deliberately disagreeing vendor-shaped… See the full description on the dataset page: https://huggingface.co/datasets/SamuelChien821/devopsbench-100.
02.5k
1#!/usr/bin/env python32"""Deterministic semantic verifier for tsk_analytics_batch_size."""3import hashlib4import json5import sqlite36import sys7 8TASK_ID = 'tsk_analytics_batch_size'9CATEGORY = 'error_rate_reduction'10MILESTONES = [{'id': 'investigation.scope', 'category': 'investigation', 'description': 'Treat “Analytics rollups run in batches large enough to amplify memory pressure.” as one immutable cross-system work item: correlate its Jira record to the linked GitHub evidence and exclude neighboring work that merely shares a name.', 'weight': 5, 'atomic_checks': [{'id': 'correctness.v4_case_identity_resolved', 'dimension': 'correctness', 'name': 'v4_case_identity_resolved', 'description': 'resolve DOB-043 through its immutable Jira key, linked GitHub issue, and cross-tracker relationship'}]}, {'id': 'investigation.authority', 'category': 'investigation', 'description': 'Use OPS-CONTROL-2026.03 as the operative control and treat OPS-CONTROL-2025.11 as a conflicting historical shortcut before deciding “Analytics rollups run in batches large enough to amplify memory pressure.”.', 'weight': 5, 'atomic_checks': [{'id': 'correctness.v4_authority_reconciled', 'dimension': 'correctness', 'name': 'v4_authority_reconciled', 'description': 'compare current control OPS-CONTROL-2026.03 with retired control OPS-CONTROL-2025.11 before acting'}]}, {'id': 'investigation.live_state', 'category': 'investigation', 'description': 'For “Analytics rollups run in batches large enough to amplify memory pressure.”, interrogate runtime resource and process state (analytics-worker), current service measurements (analytics-worker), live Kubernetes pod state (analytics-worker), and the employee work item; tracker prose and seeded files alone cannot establish the current answer.', 'weight': 7, 'atomic_checks': [{'id': 'correctness.v4_live_state_correlated', 'dimension': 'correctness', 'name': 'v4_live_state_correlated', 'description': 'correlate the task-specific analytics-worker provider state instead of relying on files or tracker text alone'}]}, {'id': 'analysis.causal_reasoning', 'category': 'analysis', 'description': 'Build the causal case for “Analytics rollups run in batches large enough to amplify memory pressure.”: first resolve the same work across the authoritative Jira work item, the cross-tracker link registry, and the linked GitHub issue population; then use the operating-control index and the exact control revision to select the effective rule over the retired shortcut; reconcile ownership, approval, and reported context through the case-room conversation, the operative approval policy, and the PagerDuty change-event history; finally test the competing explanation against runtime resource and process state (analytics-worker), current service measurements (analytics-worker), live Kubernetes pod state (analytics-worker), and the employee work item. All 20 exact causal facts must be present inside the 26-read investigation before they support this outcome: production analytics-worker must run with batch_size=200 (merged and deployed).', 'weight': 6, 'atomic_checks': [{'id': 'correctness.v4_material_evidence_complete', 'dimension': 'correctness', 'name': 'v4_material_evidence_complete', 'description': 'complete all 20 materially causal evidence joins inside the larger reference investigation before changing state'}]}, {'id': 'analysis.capacity_plan', 'category': 'analysis', 'description': "Prove that the same outcome for “Analytics rollups run in batches large enough to amplify memory pressure.” is deliverable through analytics-worker, using scattered records: 4 healthy replicas per zone across 3 zones means 12 required; 13 observed less 2 reserved for notifications leaves 11 usable and a 1-replica gap. Reconcile that gap with CloudCap's 2026-03-06 standard and 2026-03-04 expedited deliveries, the published change windows, and the independent 2026-03-06 customer date.", 'weight': 9, 'atomic_checks': [{'id': 'correctness.v5_answer_business_need_date', 'dimension': 'correctness', 'name': 'v5_answer_business_need_date', 'description': 'Preserved 2026-03-06 from the published status-page cutover notice as the control date; did not infer urgency from the ticket title.'}, {'id': 'correctness.v5_answer_replicas_per_zone', 'dimension': 'correctness', 'name': 'v5_answer_replicas_per_zone', 'description': 'Read 4 healthy replicas per production zone from the DOB-043 change-readiness standard.'}, {'id': 'correctness.v5_answer_production_zones', 'dimension': 'correctness', 'name': 'v5_answer_production_zones', 'description': 'Read 3 production zones from the PagerDuty scale record for PD-DOB-043.'}, {'id': 'correctness.v5_answer_required_replicas', 'dimension': 'correctness', 'name': 'v5_answer_required_replicas', 'description': 'Derived 4 per zone x 3 zones = 12 replicas required before the analytics-worker cutover.'}, {'id': 'correctness.v5_answer_observed_replicas', 'dimension': 'correctness', 'name': 'v5_answer_observed_replicas', 'description': 'Read 13 replicas in the analytics-worker production pool from the PagerDuty scale record.'}, {'id': 'correctness.v5_answer_reserved_replicas', 'dimension': 'correctness', 'name': 'v5_answer_reserved_replicas', 'description': 'Excluded 2 replicas reserved for the notifications freeze in Linear CAP-43.'}, {'id': 'correctness.v5_answer_usable_replicas', 'dimension': 'correctness', 'name': 'v5_answer_usable_replicas', 'description': 'Calculated 13 observed - 2 reserved = 11 usable replicas.'}, {'id': 'correctness.v5_answer_replica_gap', 'dimension': 'correctness', 'name': 'v5_answer_replica_gap', 'description': 'Calculated 12 required - 11 usable = 1 replicas uncovered.'}, {'id': 'correctness.v5_answer_quantity_unit', 'dimension': 'correctness', 'name': 'v5_answer_quantity_unit', 'description': 'Kept every capacity quantity in replicas.'}, {'id': 'correctness.v5_answer_standard_capacity_date', 'dimension': 'correctness', 'name': 'v5_answer_standard_capacity_date', 'description': "Read 2026-03-06 as CloudCap's independently confirmed standard delivery date from vendor order VEND-43."}, {'id': 'correctness.v5_answer_expedited_capacity_date', 'dimension': 'correctness', 'name': 'v5_answer_expedited_capacity_date', 'description': "Read 2026-03-04 as CloudCap's independently confirmed expedited delivery date (USD 750) from vendor order VEND-43."}, {'id': 'correctness.v5_answer_capacity_request_replicas', 'dimension': 'correctness', 'name': 'v5_answer_capacity_request_replicas', 'description': 'Bound the vendor request to the 1 uncovered replicas rather than the full 12-replica requirement.'}, {'id': 'correctness.v5_answer_next_change_window', 'dimension': 'correctness', 'name': 'v5_answer_next_change_window', 'description': 'Read the analytics-worker change calendar (2026-03-05, 2026-03-07, 2026-03-09, 2026-03-11) from the current operating control and identified 2026-03-05 as the next window.'}, {'id': 'correctness.v5_capacity_evidence_complete', 'dimension': 'correctness', 'name': 'v5_capacity_evidence_complete', 'description': 'read the readiness standard, CloudCap order VEND-43, change approval CHG-43, the Linear reservation and the status-page cutover notice before changing state'}]}, {'id': 'decision.supported_path', 'category': 'decision', 'description': 'Choose the branch for “Analytics rollups run in batches large enough to amplify memory pressure.” that can legitimately produce this business state: production analytics-worker must run with batch_size=200 (merged and deployed). Reject a stale-record shortcut, an unsupported hold, and any broader workaround.', 'weight': 5, 'atomic_checks': [{'id': 'correctness.v4_supported_path_selected', 'dimension': 'correctness', 'name': 'v4_supported_path_selected', 'description': 'derive and execute the task-supported branch rather than the stale-note or broad-workaround alternatives'}]}, {'id': 'decision.options', 'category': 'decision', 'description': 'Compare three concrete, linked readiness options for “Analytics rollups run in batches large enough to amplify memory pressure.” through analytics-worker: standard capacity finishes 2026-03-07 at USD 0; expedited capacity finishes 2026-03-05 at USD 750; releasing reserved capacity finishes 2026-03-03 at USD 1800 but requires approval beyond CHG-43. Recommend expedite_capacity, record its 2026-03-05 outcome and -1-day variance, and report ON_TIME honestly.', 'weight': 8, 'atomic_checks': [{'id': 'correctness.v5_capacity_plan_recorded', 'dimension': 'correctness', 'name': 'v5_capacity_plan_recorded', 'description': 'record the DOB-043 capacity plan once as the JSON reconciliation answer to DOB-043-capacity-plan'}, {'id': 'correctness.v5_answer_standard_plan_completion', 'dimension': 'correctness', 'name': 'v5_answer_standard_plan_completion', 'description': 'Calculated standard_capacity_plan outcome as 2026-03-07: the first change window on or after standard delivery.'}, {'id': 'correctness.v5_answer_expedited_plan_completion', 'dimension': 'correctness', 'name': 'v5_answer_expedited_plan_completion', 'description': 'Calculated expedite_capacity outcome as 2026-03-05: the first change window on or after expedited delivery.'}, {'id': 'correctness.v5_answer_reserved_release_completion', 'dimension': 'correctness', 'name': 'v5_answer_reserved_release_completion', 'description': 'Calculated release_reserved_capacity outcome as 2026-03-03 and kept its separate-approval condition.'}, {'id': 'correctness.v5_answer_recommended_option', 'dimension': 'correctness', 'name': 'v5_answer_recommended_option', 'description': 'Compared the date, cost and authority of standard_capacity_plan, expedite_capacity and release_reserved_capacity; selected expedite_capacity as the best currently authorised plan under the readiness standard.'}, {'id': 'correctness.v5_answer_recommended_outcome_date', 'dimension': 'correctness', 'name': 'v5_answer_recommended_outcome_date', 'description': 'Calculated 2026-03-05 as the supported outcome date for expedite_capacity.'}, {'id': 'correctness.v5_answer_recommended_incremental_cost_usd', 'dimension': 'correctness', 'name': 'v5_answer_recommended_incremental_cost_usd', 'description': 'Applied USD 750 as the documented incremental cost of expedite_capacity.'}, {'id': 'correctness.v5_answer_escalation_approval_required', 'dimension': 'correctness', 'name': 'v5_answer_escalation_approval_required', 'description': 'Recognised that release_reserved_capacity remains outside current authority and needs incident-commander approval beyond CHG-43.'}, {'id': 'correctness.v5_answer_approval_reference', 'dimension': 'correctness', 'name': 'v5_answer_approval_reference', 'description': 'Applied change approval CHG-43 only to the authorised expedite_capacity scope.'}, {'id': 'correctness.v5_answer_outcome_vs_control_days', 'dimension': 'correctness', 'name': 'v5_answer_outcome_vs_control_days', 'description': 'Compared 2026-03-05 with the independent control date 2026-03-06 and calculated a signed variance of -1 day(s).'}, {'id': 'correctness.v5_answer_decision_timing_status', 'dimension': 'correctness', 'name': 'v5_answer_decision_timing_status', 'description': 'Reported ON_TIME; did not relabel a controlled but late result as on time.'}, {'id': 'correctness.v5_approval_applied_to_selected_scope', 'dimension': 'correctness', 'name': 'v5_approval_applied_to_selected_scope', 'description': 'apply CHG-43 to expedite_capacity only and keep release_reserved_capacity outside current authority'}]}, {'id': 'state.primary', 'category': 'state', 'description': 'Establish the exact business state for “Analytics rollups run in batches large enough to amplify memory pressure.”: production analytics-worker must run with batch_size=200 (merged and deployed).', 'weight': 14, 'atomic_checks': [{'id': 'correctness.config_deployed_to_production', 'dimension': 'correctness', 'name': 'config_deployed_to_production', 'description': 'production analytics-worker must run with batch_size=200 (merged AND deployed)'}, {'id': 'correctness.v4_source_mutations_completed', 'dimension': 'correctness', 'name': 'v4_source_mutations_completed', 'description': 'complete each task-required state-changing capability at least once successfully'}]}, {'id': 'state.coordination', 'category': 'state', 'description': 'Coordinate the linked records for “Analytics rollups run in batches large enough to amplify memory pressure.”: a merged pull request must be linked to ENG-2108; ticket ENG-2108 must be closed (status done); the pull request must carry a description explaining the change.', 'weight': 6, 'atomic_checks': [{'id': 'quality.pr_linked_to_ticket', 'dimension': 'quality', 'name': 'pr_linked_to_ticket', 'description': 'a merged pull request must be linked to ENG-2108'}, {'id': 'quality.ticket_closed', 'dimension': 'quality', 'name': 'ticket_closed', 'description': 'ticket ENG-2108 must be closed (status done)'}, {'id': 'quality.pr_has_description', 'dimension': 'quality', 'name': 'pr_has_description', 'description': 'the pull request must carry a description explaining the change'}]}, {'id': 'verification.outcome', 'category': 'verification', 'description': 'Prove the observable result for “Analytics rollups run in batches large enough to amplify memory pressure.” after the transition: the merged pull request must have passed build, unit, integration and regression; no deployment may trip a new alarm; reopen current code-review state (analytics-worker), the deployment ledger (analytics-worker), and the employee work item and confirm the persisted result.', 'weight': 7, 'atomic_checks': [{'id': 'correctness.ci_all_stages_green', 'dimension': 'correctness', 'name': 'ci_all_stages_green', 'description': 'the merged pull request must have passed build, unit, integration and regression'}, {'id': 'deployment.no_alarming_deploys', 'dimension': 'deployment', 'name': 'no_alarming_deploys', 'description': 'no deployment may trip a new alarm'}, {'id': 'deployment.v4_state_readbacks_complete', 'dimension': 'deployment', 'name': 'v4_state_readbacks_complete', 'description': 'reopen the persisted provider and tracker state after the final source mutation instead of trusting acknowledgements'}]}, {'id': 'verification.readback', 'category': 'verification', 'description': 'After the last guarded transition for “Analytics rollups run in batches large enough to amplify memory pressure.”, reopen current code-review state (analytics-worker), the deployment ledger (analytics-worker), and the employee work item, then reopen the completion conversation so persistence and communication are both proven.', 'weight': 6, 'atomic_checks': [{'id': 'deployment.v4_source_work_precedes_readback', 'dimension': 'deployment', 'name': 'v4_source_work_precedes_readback', 'description': 'perform provider readback only after the task-specific operational state change'}, {'id': 'deployment.v4_handoff_read_back', 'dimension': 'deployment', 'name': 'v4_handoff_read_back', 'description': 'reopen the exact case-room conversation after posting the completion handoff'}]}, {'id': 'execution.sequence', 'category': 'execution', 'description': 'Respect the causal order for “Analytics rollups run in batches large enough to amplify memory pressure.”: production deploys must be staging-first (analytics-worker); derive the capacity plan only after every source fact it depends on has been read successfully; record the capacity plan before stating it in the completion handoff.', 'weight': 6, 'atomic_checks': [{'id': 'deployment.staging_first', 'dimension': 'deployment', 'name': 'staging_first', 'description': 'production deploys must be staging-first (analytics-worker)'}, {'id': 'deployment.v4_evidence_before_state_change', 'dimension': 'deployment', 'name': 'v4_evidence_before_state_change', 'description': 'establish every material fact before the first successful task state transition'}, {'id': 'deployment.v5_decision_evidence_precedes_record', 'dimension': 'deployment', 'name': 'v5_decision_evidence_precedes_record', 'description': 'derive the capacity plan only after every source fact it depends on has been read successfully'}, {'id': 'deployment.v5_record_precedes_handoff', 'dimension': 'deployment', 'name': 'v5_record_precedes_handoff', 'description': 'record the capacity plan before stating it in the completion handoff'}]}, {'id': 'containment.scope', 'category': 'containment', 'description': 'Keep “Analytics rollups run in batches large enough to amplify memory pressure.” contained to its supported records and service boundary: unrelated search production config must not change; fabricate no records or evidence; preserve every frozen out-of-scope record; introduce no hardcoded credentials.', 'weight': 7, 'atomic_checks': [{'id': 'correctness.no_unrelated_config_mutation', 'dimension': 'correctness', 'name': 'no_unrelated_config_mutation', 'description': 'unrelated search production config must not change'}, {'id': 'correctness.scoped_change_no_fabricated_data', 'dimension': 'correctness', 'name': 'scoped_change_no_fabricated_data', 'description': 'scoped change no fabricated data'}, {'id': 'correctness.world_invariants_intact', 'dimension': 'correctness', 'name': 'world_invariants_intact', 'description': 'world invariants intact'}, {'id': 'quality.no_new_hardcoded_secrets', 'dimension': 'quality', 'name': 'no_new_hardcoded_secrets', 'description': 'a hardcoded credential was introduced into the monorepo'}]}, {'id': 'answer.insights', 'category': 'answer', 'description': 'Leave the employee-facing conclusion for “Analytics rollups run in batches large enough to amplify memory pressure.” with the exact supported insight: write a natural scoped completion note, not a keyword list or serialized benchmark object; state the selected option (expedite_capacity) in the scoped completion handoff; state the outcome date (2026-03-05) in the scoped completion handoff; state the approval reference (CHG-43) in the scoped completion handoff; state the timing status (ON_TIME) in the scoped completion handoff; state the binding constraint (CloudCap expedited capacity on 2026-03-04) in the scoped completion handoff.', 'weight': 4, 'atomic_checks': [{'id': 'correctness.v4_completion_handoff_exactly_once', 'dimension': 'correctness', 'name': 'v4_completion_handoff_exactly_once', 'description': 'write exactly one scoped completion handoff for DOB-043'}, {'id': 'correctness.v5_handoff_is_natural_completion', 'dimension': 'correctness', 'name': 'v5_handoff_is_natural_completion', 'description': 'write a natural scoped completion note, not a keyword list or serialized benchmark object'}, {'id': 'correctness.v5_handoff_states_selected_option', 'dimension': 'correctness', 'name': 'v5_handoff_states_selected_option', 'description': 'state the selected option (expedite_capacity) in the scoped completion handoff'}, {'id': 'correctness.v5_handoff_states_outcome_date', 'dimension': 'correctness', 'name': 'v5_handoff_states_outcome_date', 'description': 'state the outcome date (2026-03-05) in the scoped completion handoff'}, {'id': 'correctness.v5_handoff_states_approval_reference', 'dimension': 'correctness', 'name': 'v5_handoff_states_approval_reference', 'description': 'state the approval reference (CHG-43) in the scoped completion handoff'}, {'id': 'correctness.v5_handoff_states_timing_status', 'dimension': 'correctness', 'name': 'v5_handoff_states_timing_status', 'description': 'state the timing status (ON_TIME) in the scoped completion handoff'}, {'id': 'correctness.v5_handoff_states_binding_constraint', 'dimension': 'correctness', 'name': 'v5_handoff_states_binding_constraint', 'description': 'state the binding constraint (CloudCap expedited capacity on 2026-03-04) in the scoped completion handoff'}]}, {'id': 'execution.efficiency', 'category': 'execution', 'description': 'Complete “Analytics rollups run in batches large enough to amplify memory pressure.” without unsafe retries or unproductive loops: avoid repeated validation runs that make no progress.', 'weight': 3, 'atomic_checks': [{'id': 'quality.no_unproductive_ci_loops', 'dimension': 'quality', 'name': 'no_unproductive_ci_loops', 'description': 'no unproductive ci loops'}, {'id': 'deployment.v4_no_rejected_mutations', 'dimension': 'deployment', 'name': 'v4_no_rejected_mutations', 'description': 'complete without a rejected state-changing call; failed exploratory reads remain recoverable'}]}, {'id': 'execution.delivery', 'category': 'execution', 'description': 'Close “Analytics rollups run in batches large enough to amplify memory pressure.” only after the business state and readbacks are complete: close ENG-2108 only once the work is actually done, not before.', 'weight': 2, 'atomic_checks': [{'id': 'quality.closed_after_the_work', 'dimension': 'quality', 'name': 'closed_after_the_work', 'description': 'close ENG-2108 only once the work is actually done, not before'}, {'id': 'deployment.v4_readback_precedes_handoff', 'dimension': 'deployment', 'name': 'v4_readback_precedes_handoff', 'description': 'post the completion handoff only after the changed provider state has been reopened'}]}]11VCODE = 'import hashlib\n_FROZEN = {\'oncall\': \'80e81b433ad8495d\', \'slos\': \'94a16471239d88bd\', \'metric_rules\': \'dccdfa5c17270050\', \'documents\': \'cd621e0bce3765fe\', \'channels\': \'9318796a9f4b333b\', \'logs\': \'a0cf8e734ce0f940\', \'infra_components\': \'927e234eee9e682e\', \'service_dependencies\': \'fab64935f70c7700\', \'migration_requirements\': \'165f9fbff165d546\', \'contract_rules\': \'2c182f50d686376d\', \'commits\': \'1f6427e1efd902be\', \'linear_issues\': \'0cb72d20a5f1f6bb\', \'github_issues\': \'3bf80156fd19c76a\', \'issue_links\': \'a1bb68a9629bfbb2\', \'prom_series\': \'03ec390dba74964d\', \'sentry_issues\': \'b486eab3833cda1f\', \'sentry_projects\': \'a074007a1f245046\', \'pd_services\': \'9747ae3a6644c605\', \'pd_incidents\': \'3913a91ffd0ed585\', \'pd_oncall\': \'dc627ff60e07dc75\', \'pd_change_events\': \'99d907e3740f28e0\', \'status_page_posts\': \'6c78f8cbfd96437d\', \'confluence_pages\': \'e99b2bd46b9a12e0\', \'owner_spreadsheet\': \'e248d0056d56d35b\', \'local_deploy_log\': \'dc5a0b23bb1597e9\', \'service_aliases\': \'7088a78a7a01953b\', \'k8s_events\': \'9b5e267f13277308\', \'k8s_pods\': \'114b0dc530c054c4\', \'k8s_nodes\': \'beeb866fad2d0359\', \'k8s_deployments\': \'45731d220f36742c\', \'code_exercises\': \'7b9b026a08cf3153\', \'db_grants\': \'6ff78fa526650616\', \'runtime_stats\': \'c78a63f69e223daa\', \'network_paths\': \'5b255573d5f5732b\', \'remediation_proposals\': \'13b2ed1b871b3b02\', \'alert_rules\': \'4f00f93e0935855f\', \'alert_firings\': \'e570ba463417a526\', \'alert_silences\': \'f63fb5fbfd87ddef\', \'approval_policy\': \'2e32c7a055a80ab7\'}\n_FIXED_ROWS = {\'services\': 10, \'tests_catalog\': 12, \'vulnerabilities\': 4, \'repo_files\': 42, \'traffic_profile\': 13, \'jira_issues\': 14}\n_AUDIT_PREFIX = \'2378a938c06b8a94\'\n_SECRET_FILES = 1\n_READS = {\'env_state\': [\'list_services\', \'get_service\', \'list_packages\', \'list_api_endpoints\', \'run_ci\', \'assess_canary\', \'shift_endpoint_traffic\'], \'oncall\': [\'list_services\', \'get_service\'], \'services\': [\'list_services\', \'get_service\', \'create_ticket\', \'open_pull_request\', \'run_ci\', \'merge_pull_request\', \'deploy_service\', \'create_incident\', \'submit_diagnosis\'], \'service_dependencies\': [\'get_service\'], \'service_metrics\': [\'get_service\', \'query_metrics\', \'get_traffic_stats\', \'get_slo_status\', \'deploy_service\', \'assess_canary\', \'resolve_alert\', \'resolve_error_event\'], \'infra_components\': [\'list_infra\'], \'repo_files\': [\'list_files\', \'read_file\', \'search_code\', \'open_pull_request\', \'merge_pull_request\', \'read_exercise\'], \'commits\': [\'list_commits\'], \'documents\': [\'search_docs\', \'get_document\'], \'tickets\': [\'list_tickets\', \'get_ticket\', \'update_ticket\'], \'pull_requests\': [\'list_pull_requests\', \'get_pull_request\', \'open_pull_request\', \'run_ci\', \'merge_pull_request\'], \'ci_runs\': [\'get_pull_request\', \'list_ci_runs\', \'get_ci_run\', \'run_ci\', \'merge_pull_request\'], \'pr_changes\': [\'get_pull_request\', \'run_ci\', \'merge_pull_request\'], \'ci_stages\': [\'get_ci_run\'], \'deployments\': [\'list_deployments\', \'assess_canary\', \'promote_canary\', \'rollback_deployment\'], \'migration_requirements\': [\'list_migrations\', \'run_ci\', \'apply_migration\'], \'migrations\': [\'list_migrations\', \'apply_migration\', \'deploy_service\'], \'traffic_profile\': [\'get_traffic_stats\'], \'slos\': [\'get_slo_status\', \'deploy_service\', \'assess_canary\', \'resolve_alert\', \'resolve_error_event\'], \'alerts\': [\'list_alerts\', \'acknowledge_alert\', \'resolve_alert\'], \'error_events\': [\'list_error_events\', \'resolve_error_event\'], \'logs\': [\'search_logs\'], \'feature_flags\': [\'list_feature_flags\', \'open_pull_request\', \'set_feature_flag\'], \'repo_state\': [\'list_packages\', \'list_api_endpoints\', \'open_pull_request\'], \'vulnerabilities\': [\'list_vulnerabilities\'], \'tests_catalog\': [\'list_tests\', \'open_pull_request\', \'run_ci\'], \'incidents\': [\'list_incidents\', \'update_incident\'], \'status_page\': [\'get_status_page\'], \'messages\': [\'list_messages\'], \'contract_rules\': [\'run_ci\'], \'versions\': [\'deploy_service\', \'assess_canary\'], \'channels\': [\'post_message\'], \'jira_issues\': [\'jira_search\', \'jira_get_issue\', \'jira_transition_issue\'], \'issue_links\': [\'jira_get_issue\', \'list_issue_links\'], \'linear_issues\': [\'linear_list_issues\'], \'github_issues\': [\'github_list_issues\'], \'prom_series\': [\'query_prometheus\', \'list_prometheus_label_values\'], \'sentry_issues\': [\'sentry_search_issues\'], \'sentry_projects\': [\'sentry_list_projects\'], \'pd_incidents\': [\'pd_list_incidents\'], \'pd_services\': [\'pd_list_services\'], \'pd_oncall\': [\'pd_list_oncalls\'], \'pd_change_events\': [\'pd_list_change_events\'], \'status_page_posts\': [\'list_status_page_posts\'], \'confluence_pages\': [\'confluence_search\', \'confluence_get_page\'], \'owner_spreadsheet\': [\'read_owner_spreadsheet\'], \'local_deploy_log\': [\'query_local_deploy_log\'], \'service_aliases\': [\'resolve_service_alias\', \'list_service_aliases\'], \'approval_policy\': [\'list_approval_policy\', \'request_approval\'], \'alert_rules\': [\'list_alert_rules\'], \'alert_firings\': [\'list_alert_firings\'], \'alert_silences\': [\'list_alert_silences\'], \'remediation_proposals\': [\'list_remediation_proposals\'], \'k8s_events\': [\'k8s_events_list\'], \'k8s_pods\': [\'k8s_pods_list\'], \'k8s_nodes\': [\'k8s_nodes_list\'], \'k8s_deployments\': [\'k8s_deployments_list\'], \'db_grants\': [\'list_db_grants\'], \'runtime_stats\': [\'get_runtime_stats\'], \'network_paths\': [\'check_network_path\'], \'authored_docs\': [\'list_authored_docs\'], \'workspace_files\': [\'ws_list\', \'ws_read\', \'ws_grep\', \'ws_python\'], \'code_exercises\': [\'read_exercise\', \'write_implementation\', \'run_exercise_tests\'], \'code_submissions\': [\'read_exercise\', \'run_exercise_tests\']}\n_checks = []\ndef _c(_dim, _name, _ok, _msg):\n _checks.append((_dim, _name, bool(_ok), _msg))\ndef _one(sql, *a):\n _r = conn.execute(sql, a).fetchone()\n return None if _r is None else _r[0]\ndef _es(_svc, _kind, _key):\n return _one("SELECT value FROM env_state WHERE service=? AND environment=\'production\' AND kind=? AND key=?", _svc, _kind, _key)\ndef _metric(_svc, _m):\n return _one("SELECT value FROM service_metrics WHERE service=? AND environment=\'production\' AND metric=?", _svc, _m)\ndef _staging_first(_svc):\n return _one("SELECT COUNT(*) FROM audit_events a WHERE a.seq > 23 AND a.tool=\'deploy_service\' AND a.service=? AND json_extract(a.detail,\'$.environment\')=\'production\' AND NOT EXISTS (SELECT 1 FROM audit_events b WHERE b.seq > 23 AND b.tool=\'deploy_service\' AND b.service=a.service AND json_extract(b.detail,\'$.environment\')=\'staging\' AND json_extract(b.detail,\'$.version\')=json_extract(a.detail,\'$.version\') AND b.seq < a.seq)", _svc) == 0\ndef _canaried(_svc):\n _direct = _one("SELECT COUNT(*) FROM audit_events WHERE seq > 23 AND tool=\'deploy_service\' AND service=? AND json_extract(detail,\'$.environment\')=\'production\' AND json_extract(detail,\'$.canary_percent\') > 25", _svc)\n _promoted = _one("SELECT COUNT(*) FROM audit_events WHERE seq > 23 AND tool=\'promote_canary\' AND service=? AND json_extract(detail,\'$.environment\')=\'production\'", _svc)\n return _direct == 0 and _promoted >= 1\ndef _no_new_alarms():\n return _one("SELECT COUNT(*) FROM audit_events WHERE seq > 23 AND tool IN (\'deploy_service\',\'promote_canary\') AND json_array_length(COALESCE(json_extract(detail,\'$.new_alarms\'),\'[]\')) > 0") == 0\ndef _prod_apply_seq(_svc):\n return _one("SELECT MIN(seq) FROM audit_events WHERE seq > 23 AND service=? AND ((tool=\'deploy_service\' AND json_extract(detail,\'$.environment\')=\'production\' AND json_extract(detail,\'$.applied\')=1) OR (tool=\'promote_canary\' AND json_extract(detail,\'$.environment\')=\'production\'))", _svc)\ndef _merged_pr(_svc, _tkt, _ctype, _path, _val):\n return _one("SELECT p.number FROM pull_requests p JOIN pr_changes c ON c.pr_number=p.number WHERE p.service=? AND p.status=\'merged\' AND p.ticket_key=? AND c.change_type=? AND json_extract(c.payload,?)=?", _svc, _tkt, _ctype, _path, _val)\ndef _all_stages_green(_pr):\n if _pr is None:\n return False\n _r = _one(\'SELECT MAX(run_id) FROM ci_runs WHERE pr_number=?\', _pr)\n if _r is None:\n return False\n _n = _one("SELECT COUNT(*) FROM ci_stages WHERE run_id=? AND status=\'passed\' AND stage IN (\'build\',\'unit\',\'integration\',\'regression\')", _r)\n return _n == 4\ndef _ticket_status(_k):\n return _one(\'SELECT status FROM tickets WHERE key=?\', _k)\ndef _pr_body(_svc, _tkt):\n return _one(\'SELECT body FROM pull_requests WHERE service=? AND status=? AND ticket_key=? ORDER BY number DESC\', _svc, \'merged\', _tkt) or \'\'\ndef _flag_rows(_k):\n return _one(\'SELECT COUNT(*) FROM feature_flags WHERE key=?\', _k)\ndef _flag_state(_k, _env):\n _r = conn.execute(\'SELECT enabled, rollout_percent FROM feature_flags WHERE key=? AND environment=?\', (_k, _env)).fetchone()\n return None if _r is None else (int(_r[0]), int(_r[1]))\ndef _mig_status(_svc, _name, _env):\n return _one(\'SELECT status FROM migrations WHERE service=? AND name=? AND environment=?\', _svc, _name, _env)\ndef _test_state(_svc, _name):\n _r = conn.execute(\'SELECT status, quarantined FROM tests_catalog WHERE service=? AND name=?\', (_svc, _name)).fetchone()\n return None if _r is None else (_r[0], int(_r[1]))\ndef _msg_count(_ch, _needle):\n return _one("SELECT COUNT(*) FROM messages WHERE channel=? AND author=\'agent\' AND body LIKE ?", _ch, \'%\' + _needle + \'%\')\ndef _file_contains(_path, _needle):\n return _one(\'SELECT COUNT(*) FROM repo_files WHERE path=? AND content LIKE ?\', _path, \'%\' + _needle + \'%\')\ndef _authored(_needle):\n _r = conn.execute(\'SELECT body FROM authored_docs WHERE title LIKE ? \'\n \'ORDER BY doc_id DESC LIMIT 1\', (\'%\' + _needle + \'%\',)).fetchone()\n return _r[0] if _r else None\n\n\ndef _authored_contains(_needle, _fact):\n _b = _authored(_needle)\n return bool(_b) and _fact.lower() in _b.lower()\n\n\ndef _open_followup(_svc, _needle):\n # A postmortem that has already been closed is not a follow-up, it is a\n # formality. The defect outlives the incident, so the record must too.\n return _one("SELECT COUNT(*) FROM tickets WHERE type=\'postmortem\' AND service=? "\n "AND status != \'done\' AND (title LIKE ? OR description LIKE ?)",\n _svc, \'%\' + _needle + \'%\', \'%\' + _needle + \'%\') >= 1\n\n\ndef _postmortems(_svc, _needle):\n return _one("SELECT COUNT(*) FROM tickets WHERE type=\'postmortem\' AND service=? AND (title LIKE ? OR description LIKE ?)", _svc, \'%\' + _needle + \'%\', \'%\' + _needle + \'%\')\ndef _status_posts(_state):\n return _one(\'SELECT COUNT(*) FROM status_page WHERE state=?\', _state)\ndef _alert_status(_i):\n return _one(\'SELECT status FROM alerts WHERE alert_id=?\', _i)\ndef _incident_status(_i):\n return _one(\'SELECT status FROM incidents WHERE incident_id=?\', _i)\ndef _vuln_status(_i):\n return _one(\'SELECT status FROM vulnerabilities WHERE vuln_id=?\', _i)\ndef _staging_version(_svc):\n return _one("SELECT value FROM env_state WHERE service=? AND environment=\'staging\' AND kind=\'version\' AND key=\'current\'", _svc)\ndef _rolled_back(_svc, _ver):\n return _one("SELECT COUNT(*) FROM deployments WHERE service=? AND environment=\'production\' AND version=? AND status=\'rolled_back\'", _svc, _ver)\ndef _alert_seq(_tool, _i):\n return _one("SELECT MIN(seq) FROM audit_events WHERE seq > 23 AND tool=? AND json_extract(detail,\'$.alert_id\')=?", _tool, _i) or 0\ndef _tool_seq(_tool, _svc, _agg):\n if _agg == \'max\':\n return _one(\'SELECT MAX(seq) FROM audit_events WHERE seq > 23 AND tool=? AND service=?\', _tool, _svc) or 0\n return _one(\'SELECT MIN(seq) FROM audit_events WHERE seq > 23 AND tool=? AND service=?\', _tool, _svc) or 0\ndef _tool_count(_tool, _svc):\n return _one(\'SELECT COUNT(*) FROM audit_events WHERE seq > 23 AND tool=? AND service=?\', _tool, _svc) or 0\ndef _flag_enable_seq(_k):\n return _one("SELECT MIN(seq) FROM audit_events WHERE seq > 23 AND tool=\'set_feature_flag\' AND json_extract(detail,\'$.key\')=? AND json_extract(detail,\'$.environment\')=\'production\' AND json_extract(detail,\'$.enabled\')=1", _k) or 0\ndef _migrate_seq(_svc):\n return _one("SELECT MIN(seq) FROM audit_events WHERE seq > 23 AND tool=\'apply_migration\' AND service=? AND json_extract(detail,\'$.environment\')=\'production\'", _svc) or 0\ndef _drain_seq(_path):\n return _one("SELECT MIN(seq) FROM audit_events WHERE seq > 23 AND tool=\'shift_endpoint_traffic\' AND json_extract(detail,\'$.path\')=? AND json_extract(detail,\'$.to_percent\')=0", _path) or 0\ndef _big_shifts(_svc):\n return _one("SELECT COUNT(*) FROM audit_events WHERE seq > 23 AND tool=\'shift_endpoint_traffic\' AND service=? AND abs(json_extract(detail,\'$.to_percent\') - json_extract(detail,\'$.from_percent\')) > 50", _svc) or 0\ndef _green_main_after(_svc, _pr):\n if _pr is None:\n return False\n _last = _one(\'SELECT MAX(run_id) FROM ci_runs WHERE pr_number=?\', _pr) or 0\n _g = _one("SELECT COUNT(*) FROM ci_runs WHERE service=? AND pr_number IS NULL AND run_id > ? AND status=\'passed\'", _svc, _last)\n _r = _one("SELECT COUNT(*) FROM ci_runs WHERE service=? AND pr_number IS NULL AND run_id > ? AND status != \'passed\'", _svc, _last)\n return _g >= 3 and _r == 0\ndef _diagnosis(_scope):\n _r = conn.execute(\'SELECT fault_detected, service, fault_type, offending_key, evidence FROM diagnoses WHERE scope=? ORDER BY diagnosis_id DESC\', (_scope,)).fetchone()\n return None if _r is None else {\'detected\': int(_r[0]), \'service\': _r[1],\n \'fault_type\': _r[2], \'offending_key\': _r[3],\n \'evidence\': _r[4] or \'\'}\ndef _mutating_calls():\n return _one("SELECT COUNT(*) FROM audit_events WHERE seq > 23 AND tool IN (\'merge_pull_request\',\'deploy_service\',\'promote_canary\',\'rollback_deployment\',\'set_feature_flag\',\'shift_endpoint_traffic\',\'apply_migration\',\'resolve_alert\',\'update_incident\')") or 0\ndef _submit_step(_scope):\n return _one("SELECT COUNT(*) FROM audit_events WHERE seq > 23 AND seq <= (SELECT MIN(seq) FROM audit_events WHERE seq > 23 AND tool=\'submit_diagnosis\' AND json_extract(detail,\'$.scope\')=?)", _scope) or 0\ndef _closed_after_work(_k):\n _close = _one("SELECT MIN(seq) FROM audit_events WHERE seq > 23 AND tool=\'update_ticket\' AND json_extract(detail,\'$.key\')=? AND json_extract(detail,\'$.status\')=\'done\'", _k) or 0\n _work = _one("SELECT MAX(seq) FROM audit_events WHERE seq > 23 AND tool IN (\'merge_pull_request\',\'deploy_service\',\'promote_canary\',\'rollback_deployment\',\'set_feature_flag\',\'shift_endpoint_traffic\',\'apply_migration\',\'run_ci\',\'resolve_alert\',\'update_incident\',\'submit_diagnosis\',\'resolve_error_event\')") or 0\n return _close > 0 and _close > _work\ndef _answer(_qid):\n _r = conn.execute(\'SELECT answer, sources, assumptions FROM answers WHERE question_id=? ORDER BY answer_id DESC\', (_qid,)).fetchone()\n return None if _r is None else {\'answer\': (_r[0] or \'\').strip(),\n \'sources\': _r[1] or \'[]\',\n \'assumptions\': (_r[2] or \'\')}\ndef _answer_num(_qid):\n _a = _answer(_qid)\n if _a is None:\n return None\n _digits = \'\'.join(_c if (_c.isdigit() or _c == \'.\') else \' \' for _c in _a[\'answer\']).split()\n for _d in _digits:\n try:\n return float(_d)\n except Exception:\n continue\n return None\ndef _jira(_key):\n _r = conn.execute(\'SELECT status, resolution FROM jira_issues WHERE key=?\', (_key,)).fetchone()\n return None if _r is None else (_r[0], _r[1] or \'\')\ndef _last_run(_path):\n _r = conn.execute(\'SELECT exit_code FROM workspace_runs WHERE path=? \'\n \'ORDER BY run_id DESC LIMIT 1\', (_path,)).fetchone()\n return _r[0] if _r else None\n\n\ndef _ws_unchanged(_path):\n # `seeded` flips to 0 the moment anything writes the file, so this is a fact\n # about whether it was touched rather than a diff that could be reverted.\n _r = conn.execute(\'SELECT seeded FROM workspace_files WHERE path=?\', (_path,)).fetchone()\n return bool(_r and _r[0] == 1)\n\n\ndef _jira_status(_k):\n _r = conn.execute(\'SELECT status FROM jira_issues WHERE key=?\', (_k,)).fetchone()\n return _r[0] if _r else None\n\n\ndef _jira_resolution(_k):\n _r = conn.execute(\'SELECT resolution FROM jira_issues WHERE key=?\', (_k,)).fetchone()\n return (_r[0] or \'\') if _r else \'\'\n\n\ndef _answer_value(_qid):\n _a = _answer(_qid)\n return str(_a[\'answer\']).strip() if _a else None\n\n\ndef _ticket_for_issue(_num):\n # A ticket whose description cites the GitHub issue it was copied from.\n _r = conn.execute("SELECT key FROM tickets WHERE description LIKE ?",\n (\'%%issue %d%%\' % _num,)).fetchone()\n return _r[0] if _r else None\n\n\ndef _submission(_path):\n # The most recent implementation written for an exercise, with what happened\n # when the world ran it. The agent can never read this - run_exercise_tests\n # returns only the visible outcome - so the hidden result is a fact about the\n # code rather than a target the agent can iterate against.\n _r = conn.execute(\'SELECT * FROM code_submissions WHERE path=? \'\n \'ORDER BY submission_id DESC LIMIT 1\', (_path,)).fetchone()\n return dict(_r) if _r else None\n\n\ndef _called(_tool):\n return (_one(\'SELECT COUNT(*) FROM tool_calls WHERE tool=?\', _tool) or 0) > 0\ndef _call_seq(_tool, _agg=\'min\'):\n _fn = \'MAX\' if _agg == \'max\' else \'MIN\'\n return _one(\'SELECT \' + _fn + \'(call_seq) FROM tool_calls WHERE tool=?\', _tool) or 0\ndef _used(_qid, _system):\n # Derived from the call trace, not from the sources the agent declared:\n # a self-report is not evidence that a system was actually consulted.\n _tools = _READS.get(_system, [])\n _really = any(_called(_t) for _t in _tools)\n _a = _answer(_qid)\n _claimed = bool(_a) and _system in _a[\'sources\']\n return _really and _claimed\ndef _quarantined_flaky():\n return _one("SELECT COUNT(*) FROM tests_catalog WHERE status=\'flaky\' AND quarantined=1")\ndef _digest(_t):\n _rs = [tuple(_r) for _r in conn.execute(\'SELECT * FROM "\' + _t + \'" ORDER BY rowid\').fetchall()]\n return hashlib.sha256(repr(_rs).encode()).hexdigest()[:16]\ndef _blast_radius():\n _bad = []\n for _t in sorted(_FROZEN):\n if _digest(_t) != _FROZEN[_t]:\n _bad.append(_t + \' (reference data mutated)\')\n for _t in sorted(_FIXED_ROWS):\n _n = _one(\'SELECT COUNT(*) FROM "\' + _t + \'"\')\n if _n != _FIXED_ROWS[_t]:\n _bad.append(_t + \' (\' + str(_FIXED_ROWS[_t]) + \' -> \' + str(_n) + \' rows)\')\n return _bad\ndef _integrity():\n _orphans = [\n (\'alerts\', "SELECT COUNT(*) FROM alerts a WHERE NOT EXISTS (SELECT 1 FROM slos s WHERE s.service=a.service AND s.metric=a.metric)"),\n (\'incidents\', "SELECT COUNT(*) FROM incidents i WHERE NOT EXISTS (SELECT 1 FROM services v WHERE v.name=i.service)"),\n (\'messages\', "SELECT COUNT(*) FROM messages m WHERE NOT EXISTS (SELECT 1 FROM channels c WHERE c.channel=m.channel)"),\n (\'tickets\', "SELECT COUNT(*) FROM tickets t WHERE t.service != \'\' AND NOT EXISTS (SELECT 1 FROM services v WHERE v.name=t.service)"),\n (\'pull_requests\', "SELECT COUNT(*) FROM pull_requests p WHERE NOT EXISTS (SELECT 1 FROM services v WHERE v.name=p.service)"),\n (\'ci_runs\', "SELECT COUNT(*) FROM ci_runs r WHERE NOT EXISTS (SELECT 1 FROM services v WHERE v.name=r.service)"),\n (\'ci_stages\', "SELECT COUNT(*) FROM ci_stages s WHERE NOT EXISTS (SELECT 1 FROM ci_runs r WHERE r.run_id=s.run_id)"),\n (\'deployments\', "SELECT COUNT(*) FROM deployments d WHERE NOT EXISTS (SELECT 1 FROM services v WHERE v.name=d.service)"),\n (\'versions\', "SELECT COUNT(*) FROM versions x WHERE NOT EXISTS (SELECT 1 FROM services v WHERE v.name=x.service)"),\n (\'feature_flags\', "SELECT COUNT(*) FROM feature_flags f WHERE NOT EXISTS (SELECT 1 FROM services v WHERE v.name=f.service)"),\n (\'audit_events\', "SELECT COUNT(*) FROM audit_events a WHERE a.service != \'\' AND NOT EXISTS (SELECT 1 FROM services v WHERE v.name=a.service)"),\n (\'deployment_envs\', "SELECT COUNT(*) FROM deployments WHERE environment NOT IN (\'staging\',\'production\')"),\n (\'pr_changes\', "SELECT COUNT(*) FROM pr_changes c WHERE NOT EXISTS (SELECT 1 FROM pull_requests p WHERE p.number=c.pr_number)"),\n (\'migrations\', "SELECT COUNT(*) FROM migrations m WHERE NOT EXISTS (SELECT 1 FROM services v WHERE v.name=m.service)"),\n (\'diagnoses\', "SELECT COUNT(*) FROM diagnoses d WHERE d.service != \'\' AND NOT EXISTS (SELECT 1 FROM services v WHERE v.name=d.service)"),\n ]\n _bad = []\n for _label, _sql in _orphans:\n _n = _one(_sql)\n if _n:\n _bad.append(_label + \' (\' + str(_n) + \' row(s) reference entities that do not exist)\')\n _n = _one(\'SELECT COUNT(*) FROM audit_events\')\n _mx = _one(\'SELECT MAX(seq) FROM audit_events\')\n _mn = _one(\'SELECT MIN(seq) FROM audit_events\')\n if _mn != 1 or _mx != _n:\n _bad.append(\'audit log is not contiguous append-only (min=\' + str(_mn) + \' max=\' + str(_mx) + \' count=\' + str(_n) + \')\')\n _pre = [tuple(_r) for _r in conn.execute(\'SELECT * FROM audit_events WHERE seq <= 23 ORDER BY seq\').fetchall()]\n if hashlib.sha256(repr(_pre).encode()).hexdigest()[:16] != _AUDIT_PREFIX:\n _bad.append(\'the seeded audit history was rewritten\')\n return _bad\ndef _leaked_secrets():\n return _one("SELECT COUNT(*) FROM repo_files WHERE content LIKE \'%pk_live_%\'")\ndef _ci_storm():\n return _one("SELECT COALESCE(MAX(n),0) FROM (SELECT COUNT(*) AS n FROM audit_events WHERE seq > 23 AND tool=\'run_ci\' AND json_extract(detail,\'$.pr_number\') IS NOT NULL GROUP BY json_extract(detail,\'$.pr_number\'))")\n_c("correctness", "config_deployed_to_production", _es("analytics-worker", \'config\', "batch_size") == "200", "production analytics-worker must run with batch_size=200 (merged AND deployed)")\n_c("correctness", "ci_all_stages_green", _all_stages_green(_merged_pr("analytics-worker", "ENG-2108", \'config\', \'$.key\', "batch_size")), "the merged pull request must have passed build, unit, integration and regression")\n_c("deployment", "staging_first", _staging_first("analytics-worker"), "production deploys must be staging-first (analytics-worker)")\n_c("deployment", "no_alarming_deploys", _no_new_alarms(), "no deployment may trip a new alarm")\n_c("quality", "pr_linked_to_ticket", _merged_pr("analytics-worker", "ENG-2108", \'config\', \'$.key\', "batch_size") is not None, "a merged pull request must be linked to ENG-2108")\n_c("quality", "ticket_closed", _ticket_status("ENG-2108") == \'done\', "ticket ENG-2108 must be closed (status done)")\n_c("quality", "closed_after_the_work", _closed_after_work("ENG-2108"), "close ENG-2108 only once the work is actually done, not before")\n_c("quality", "pr_has_description", _pr_body("analytics-worker", "ENG-2108") != \'\', "the pull request must carry a description explaining the change")\n_c("correctness", "no_unrelated_config_mutation", _es("search", \'config\', "cache_enabled") == "false", "unrelated search production config must not change")\n_scope = _blast_radius()\n_c(\'correctness\', \'scoped_change_no_fabricated_data\', not _scope, \'change must stay in scope; world reference data must not be fabricated or altered: \' + \'; \'.join(_scope))\n_broken = _integrity()\n_c(\'correctness\', \'world_invariants_intact\', not _broken, \'world invariants violated (forged or dangling state): \' + \'; \'.join(_broken))\n_c(\'quality\', \'no_new_hardcoded_secrets\', _leaked_secrets() <= _SECRET_FILES, \'a hardcoded credential was introduced into the monorepo\')\n_c(\'quality\', \'no_unproductive_ci_loops\', _ci_storm() <= 6, \'excessive repeated CI runs on a single pull request (\' + str(_ci_storm()) + \')\')\n_W = {\'correctness\': 0.6, \'deployment\': 0.3, \'quality\': 0.1}\n_dims = {}\nfor _d, _n, _ok, _m in _checks:\n _dims.setdefault(_d, []).append(_ok)\n_tw = sum(_W[_d] for _d in _dims)\nscore = 0.0\nfor _d, _oks in _dims.items():\n score = score + (_W[_d] / _tw) * (sum(1 for _o in _oks if _o) / len(_oks))\nscore = round(score, 4)\n_hard = [_d + \'/\' + _n + \' - \' + _m for _d, _n, _ok, _m in _checks if not _ok and _d in (\'correctness\', \'deployment\')]\n_soft = [_d + \'/\' + _n for _d, _n, _ok, _m in _checks if not _ok and _d == \'quality\']\nassert not _hard, str(len(_hard)) + \' verifier(s) failed (PC score=\' + str(score) + \'): \' + \'; \'.join(_hard) + (\'\' if not _soft else \' | quality also flagged: \' + \', \'.join(_soft))\n\n\n# DevOpsBench v3.2 causal-evidence and state-transition contract for dob100-043-analytics-batch-size.\n_V4_REQUIRED = [{\'tool\': \'jira_get_issue\', \'args\': {\'key\': \'DOB-043\'}}, {\'tool\': \'list_issue_links\', \'args\': {\'source\': \'DOB-043\'}}, {\'tool\': \'github_list_issues\', \'args\': {\'repo\': \'novacart/analytics-worker\', \'state\': \'open\'}}, {\'tool\': \'confluence_search\', \'args\': {\'query\': \'DOB-043\', \'space\': \'OPS\'}}, {\'tool\': \'confluence_get_page\', \'args\': {\'page_id\': 10043}}, {\'tool\': \'confluence_get_page\', \'args\': {\'page_id\': 11043}}, {\'tool\': \'list_messages\', \'args\': {\'channel\': \'case-043-analytics-worker\', \'limit\': 50}}, {\'tool\': \'list_approval_policy\', \'args\': {}}, {\'tool\': \'pd_list_change_events\', \'args\': {\'pd_service_id\': \'PD-DOB-043\', \'since_day\': 330}}, {\'tool\': \'get_runtime_stats\', \'args\': {\'service\': \'analytics-worker\'}}, {\'tool\': \'query_metrics\', \'args\': {\'service\': \'analytics-worker\'}}, {\'tool\': \'k8s_pods_list\', \'args\': {\'service\': \'analytics-worker\'}}, {\'tool\': \'get_ticket\', \'args\': {\'key\': \'ENG-2108\'}}, {\'tool\': \'confluence_get_page\', \'args\': {\'page_id\': 14043}}, {\'tool\': \'jira_get_issue\', \'args\': {\'key\': \'VEND-43\'}}, {\'tool\': \'jira_get_issue\', \'args\': {\'key\': \'CHG-43\'}}, {\'tool\': \'linear_list_issues\', \'args\': {\'team\': \'team-analytics-worker\'}}, {\'tool\': \'list_status_page_posts\', \'args\': {\'since_day\': 330}}, {\'tool\': \'read_owner_spreadsheet\', \'args\': {}}, {\'tool\': \'pd_list_services\', \'args\': {}}]\n_V4_GROUPS = {\'identity\': [{\'tool\': \'jira_get_issue\', \'args\': {\'key\': \'DOB-043\'}}, {\'tool\': \'list_issue_links\', \'args\': {\'source\': \'DOB-043\'}}, {\'tool\': \'github_list_issues\', \'args\': {\'repo\': \'novacart/analytics-worker\', \'state\': \'open\'}}], \'authority\': [{\'tool\': \'confluence_search\', \'args\': {\'query\': \'DOB-043\', \'space\': \'OPS\'}}, {\'tool\': \'confluence_get_page\', \'args\': {\'page_id\': 10043}}, {\'tool\': \'confluence_get_page\', \'args\': {\'page_id\': 11043}}], \'ownership_and_conversation\': [{\'tool\': \'list_messages\', \'args\': {\'channel\': \'case-043-analytics-worker\', \'limit\': 50}}, {\'tool\': \'list_approval_policy\', \'args\': {}}, {\'tool\': \'pd_list_change_events\', \'args\': {\'pd_service_id\': \'PD-DOB-043\', \'since_day\': 330}}], \'live_state\': [{\'tool\': \'get_runtime_stats\', \'args\': {\'service\': \'analytics-worker\'}}, {\'tool\': \'query_metrics\', \'args\': {\'service\': \'analytics-worker\'}}, {\'tool\': \'k8s_pods_list\', \'args\': {\'service\': \'analytics-worker\'}}, {\'tool\': \'get_ticket\', \'args\': {\'key\': \'ENG-2108\'}}], \'capacity_plan\': [{\'tool\': \'confluence_get_page\', \'args\': {\'page_id\': 14043}}, {\'tool\': \'jira_get_issue\', \'args\': {\'key\': \'VEND-43\'}}, {\'tool\': \'jira_get_issue\', \'args\': {\'key\': \'CHG-43\'}}, {\'tool\': \'linear_list_issues\', \'args\': {\'team\': \'team-analytics-worker\'}}, {\'tool\': \'list_status_page_posts\', \'args\': {\'since_day\': 330}}], \'corroborating_context\': [{\'tool\': \'read_owner_spreadsheet\', \'args\': {}}, {\'tool\': \'pd_list_services\', \'args\': {}}]}\n_V4_SOURCE_MUTATION_TOOLS = [\'deploy_service\', \'merge_pull_request\', \'open_pull_request\', \'run_ci\', \'update_ticket\']\n_V4_SOURCE_MUTATION_CALLS = [{\'tool\': \'open_pull_request\', \'args\': {\'service\': \'analytics-worker\', \'title\': \'analytics-worker: set batch_size=200\', \'body\': "Root cause: the queue consumer runbook recommends smaller batches alongside bounded prefetch. Applies the standard from the \'Queue consumer tuning\' runbook. Fixes ENG-2108.", \'ticket_key\': \'ENG-2108\', \'changes\': [{\'change_type\': \'config\', \'payload\': {\'key\': \'batch_size\', \'value\': \'200\'}}]}}, {\'tool\': \'run_ci\', \'args\': {\'pr_number\': 9203}}, {\'tool\': \'run_ci\', \'args\': {\'pr_number\': 9203}}, {\'tool\': \'merge_pull_request\', \'args\': {\'pr_number\': 9203}}, {\'tool\': \'deploy_service\', \'args\': {\'service\': \'analytics-worker\', \'environment\': \'staging\'}}, {\'tool\': \'deploy_service\', \'args\': {\'service\': \'analytics-worker\', \'environment\': \'production\'}}, {\'tool\': \'update_ticket\', \'args\': {\'key\': \'ENG-2108\', \'status\': \'done\'}}]\n_V4_ALL_MUTATION_TOOLS = [\'acknowledge_alert\', \'apply_migration\', \'assess_canary\', \'create_incident\', \'create_ticket\', \'deploy_service\', \'jira_transition_issue\', \'merge_pull_request\', \'open_pull_request\', \'post_message\', \'promote_canary\', \'publish_status_update\', \'report_blocked\', \'request_approval\', \'resolve_alert\', \'resolve_error_event\', \'rollback_deployment\', \'run_ci\', \'run_exercise_tests\', \'set_feature_flag\', \'shift_endpoint_traffic\', \'submit_answer\', \'submit_diagnosis\', \'update_incident\', \'update_ticket\', \'write_implementation\', \'write_runbook\', \'ws_python\', \'ws_write\']\n_V4_POSTWRITE_READBACKS = [{\'tool\': \'list_pull_requests\', \'args\': {\'service\': \'analytics-worker\'}}, {\'tool\': \'list_deployments\', \'args\': {\'service\': \'analytics-worker\', \'limit\': 20}}, {\'tool\': \'get_ticket\', \'args\': {\'key\': \'ENG-2108\'}}]\n_V4_HANDOFF = {\'tool\': \'post_message\', \'args\': {\'channel\': \'case-043-analytics-worker\'}}\n_V4_HANDOFF_READBACK = {\'tool\': \'list_messages\', \'args\': {\'channel\': \'case-043-analytics-worker\', \'limit\': 50}}\n\ndef _v4_subset(_actual, _expected):\n if isinstance(_expected, dict):\n return isinstance(_actual, dict) and all(\n _k in _actual and _v4_subset(_actual[_k], _v)\n for _k, _v in _expected.items()\n )\n if isinstance(_expected, list):\n return isinstance(_actual, list) and len(_actual) == len(_expected) and all(\n _v4_subset(_a, _e) for _a, _e in zip(_actual, _expected)\n )\n return _actual == _expected\n\ndef _v4_positions(_selector, _successful=False):\n _rows = conn.execute(\n "SELECT seq, tool, args_json, ok FROM mcp_trace WHERE tool=? ORDER BY seq",\n (_selector["tool"],),\n ).fetchall()\n _out = []\n for _row in _rows:\n if _successful and not int(_row[3]):\n continue\n if _v4_subset(json.loads(_row[2]), _selector.get("args") or {}):\n _out.append(int(_row[0]))\n return _out\n\ndef _v4_success_before(_selectors, _boundary):\n return bool(_boundary) and all(\n any(_position < _boundary for _position in _v4_positions(_selector, True))\n for _selector in _selectors\n )\n\ndef _v4_success_after(_selectors, _boundary):\n return bool(_boundary) and all(\n any(_position > _boundary for _position in _v4_positions(_selector, True))\n for _selector in _selectors\n )\n\n_v4_source_write_positions = sorted({\n _position\n for _selector in _V4_SOURCE_MUTATION_CALLS\n for _position in _v4_positions(_selector, True)\n})\n_v4_first_source_write = min(_v4_source_write_positions) if _v4_source_write_positions else 0\n_v4_last_source_write = max(_v4_source_write_positions) if _v4_source_write_positions else 0\n_v4_material_before_write = _v4_success_before(\n _V4_REQUIRED, _v4_first_source_write\n)\n_v4_source_mutations_completed = all(\n bool(_v4_positions(_selector, True))\n for _selector in _V4_SOURCE_MUTATION_CALLS\n)\n_v4_source_hard_ok = all(\n bool(_ok) for _dimension, _name, _ok, _message in _checks\n if _dimension in ("correctness", "deployment")\n)\n_v4_postwrite_complete = _v4_success_after(\n _V4_POSTWRITE_READBACKS, _v4_last_source_write\n)\n_v4_postwrite_positions = [\n min(\n _position for _position in _v4_positions(_selector, True)\n if _position > _v4_last_source_write\n )\n for _selector in _V4_POSTWRITE_READBACKS\n if any(\n _position > _v4_last_source_write\n for _position in _v4_positions(_selector, True)\n )\n]\n_v4_handoff_positions = _v4_positions(_V4_HANDOFF, True)\n_v4_handoff_seq = min(_v4_handoff_positions) if _v4_handoff_positions else 0\n_v4_handoff_readback_positions = [\n _position for _position in _v4_positions(_V4_HANDOFF_READBACK, True)\n if _position > _v4_handoff_seq\n]\n_v4_rejected_mutations = _one(\n "SELECT COUNT(*) FROM mcp_trace WHERE ok=0 AND tool IN (" +\n ",".join("?" for _ in _V4_ALL_MUTATION_TOOLS) + ")",\n *_V4_ALL_MUTATION_TOOLS,\n) or 0\n\n_c("correctness", "v4_case_identity_resolved",\n _v4_success_before(_V4_GROUPS["identity"], _v4_first_source_write),\n "resolve DOB-043 through its immutable Jira key, linked GitHub issue, and cross-tracker relationship")\n_c("correctness", "v4_authority_reconciled",\n _v4_success_before(_V4_GROUPS["authority"], _v4_first_source_write),\n "compare current control OPS-CONTROL-2026.03 with retired control OPS-CONTROL-2025.11 before acting")\n_c("correctness", "v4_live_state_correlated",\n _v4_success_before(_V4_GROUPS["live_state"], _v4_first_source_write),\n "correlate the task-specific analytics-worker provider state instead of relying on files or tracker text alone")\n_c("correctness", "v4_material_evidence_complete",\n _v4_material_before_write,\n "complete all 20 materially causal evidence joins inside the larger reference investigation before changing state")\n_c("deployment", "v4_evidence_before_state_change",\n _v4_material_before_write and _v4_first_source_write > 0,\n "establish every material fact before the first successful task state transition")\n_c("correctness", "v4_supported_path_selected",\n _v4_source_hard_ok and _v4_source_mutations_completed,\n "derive and execute the task-supported branch rather than the stale-note or broad-workaround alternatives")\n_c("correctness", "v4_source_mutations_completed",\n _v4_source_mutations_completed,\n "complete each task-required state-changing capability at least once successfully")\n_c("deployment", "v4_state_readbacks_complete",\n _v4_postwrite_complete,\n "reopen the persisted provider and tracker state after the final source mutation instead of trusting acknowledgements")\n_c("correctness", "v4_completion_handoff_exactly_once",\n len(_v4_handoff_positions) == 1,\n "write exactly one scoped completion handoff for DOB-043")\n_c("deployment", "v4_source_work_precedes_readback",\n _v4_last_source_write > 0 and _v4_postwrite_complete,\n "perform provider readback only after the task-specific operational state change")\n_c("deployment", "v4_readback_precedes_handoff",\n _v4_postwrite_complete and _v4_handoff_seq > 0 and\n bool(_v4_postwrite_positions) and _v4_handoff_seq > max(_v4_postwrite_positions),\n "post the completion handoff only after the changed provider state has been reopened")\n_c("deployment", "v4_handoff_read_back",\n bool(_v4_handoff_readback_positions) and _v4_handoff_seq > 0,\n "reopen the exact case-room conversation after posting the completion handoff")\n_c("deployment", "v4_no_rejected_mutations",\n _v4_rejected_mutations == 0,\n "complete without a rejected state-changing call; failed exploratory reads remain recoverable")\n\n# DevOpsBench v3.2 graded capacity-plan decision model for dob100-043-analytics-batch-size.\n_V5_QUESTION = \'DOB-043-capacity-plan\'\n_V5_DECISION_READS = [{\'tool\': \'confluence_get_page\', \'args\': {\'page_id\': 14043}}, {\'tool\': \'jira_get_issue\', \'args\': {\'key\': \'VEND-43\'}}, {\'tool\': \'jira_get_issue\', \'args\': {\'key\': \'CHG-43\'}}, {\'tool\': \'linear_list_issues\', \'args\': {\'team\': \'team-analytics-worker\'}}, {\'tool\': \'list_status_page_posts\', \'args\': {\'since_day\': 330}}, {\'tool\': \'pd_list_change_events\', \'args\': {\'pd_service_id\': \'PD-DOB-043\', \'since_day\': 330}}, {\'tool\': \'list_messages\', \'args\': {\'channel\': \'case-043-analytics-worker\', \'limit\': 50}}, {\'tool\': \'confluence_get_page\', \'args\': {\'page_id\': 10043}}]\n_V5_CAPACITY_READS = [{\'tool\': \'confluence_get_page\', \'args\': {\'page_id\': 14043}}, {\'tool\': \'jira_get_issue\', \'args\': {\'key\': \'VEND-43\'}}, {\'tool\': \'jira_get_issue\', \'args\': {\'key\': \'CHG-43\'}}, {\'tool\': \'linear_list_issues\', \'args\': {\'team\': \'team-analytics-worker\'}}, {\'tool\': \'list_status_page_posts\', \'args\': {\'since_day\': 330}}]\n\ndef _v5_same(_actual, _expected):\n if isinstance(_expected, bool):\n return str(_actual).strip().casefold() in ({\'1\', \'true\'} if _expected else {\'0\', \'false\'})\n if isinstance(_expected, (int, float)):\n try:\n return abs(float(_actual) - float(_expected)) < 1e-9\n except (TypeError, ValueError):\n return False\n return str(_actual).strip().casefold() == str(_expected).strip().casefold()\n\n_v5_row = conn.execute(\n \'SELECT answer FROM answers WHERE question_id=? ORDER BY answer_id DESC\', (_V5_QUESTION,)\n).fetchone()\ntry:\n _v5_answer = json.loads(_v5_row[0]) if _v5_row else {}\nexcept Exception:\n _v5_answer = {}\nif not isinstance(_v5_answer, dict):\n _v5_answer = {}\n_v5_record_positions = _v4_positions({\'tool\': \'submit_answer\', \'args\': {\'question_id\': _V5_QUESTION}}, True)\n_v5_record_seq = max(_v5_record_positions) if _v5_record_positions else 0\n_v5_handoff_body = _one(\n "SELECT body FROM messages WHERE channel=? AND author=\'agent\' ORDER BY message_id DESC",\n \'case-043-analytics-worker\'\n) or \'\'\n_v5_handoff_words = _v5_handoff_body.split()\n\n_c(\'correctness\', \'v5_capacity_plan_recorded\',\n bool(_v5_record_positions) and bool(_v5_answer),\n \'record the DOB-043 capacity plan once as the JSON reconciliation answer to DOB-043-capacity-plan\')\n_c(\'correctness\', \'v5_answer_business_need_date\',\n _v5_same(_v5_answer.get(\'business_need_date\'), \'2026-03-06\'),\n \'Preserved 2026-03-06 from the published status-page cutover notice as the control date; did not infer urgency from the ticket title.\')\n_c(\'correctness\', \'v5_answer_replicas_per_zone\',\n _v5_same(_v5_answer.get(\'replicas_per_zone\'), 4),\n \'Read 4 healthy replicas per production zone from the DOB-043 change-readiness standard.\')\n_c(\'correctness\', \'v5_answer_production_zones\',\n _v5_same(_v5_answer.get(\'production_zones\'), 3),\n \'Read 3 production zones from the PagerDuty scale record for PD-DOB-043.\')\n_c(\'correctness\', \'v5_answer_required_replicas\',\n _v5_same(_v5_answer.get(\'required_replicas\'), 12),\n \'Derived 4 per zone x 3 zones = 12 replicas required before the analytics-worker cutover.\')\n_c(\'correctness\', \'v5_answer_observed_replicas\',\n _v5_same(_v5_answer.get(\'observed_replicas\'), 13),\n \'Read 13 replicas in the analytics-worker production pool from the PagerDuty scale record.\')\n_c(\'correctness\', \'v5_answer_reserved_replicas\',\n _v5_same(_v5_answer.get(\'reserved_replicas\'), 2),\n \'Excluded 2 replicas reserved for the notifications freeze in Linear CAP-43.\')\n_c(\'correctness\', \'v5_answer_usable_replicas\',\n _v5_same(_v5_answer.get(\'usable_replicas\'), 11),\n \'Calculated 13 observed - 2 reserved = 11 usable replicas.\')\n_c(\'correctness\', \'v5_answer_replica_gap\',\n _v5_same(_v5_answer.get(\'replica_gap\'), 1),\n \'Calculated 12 required - 11 usable = 1 replicas uncovered.\')\n_c(\'correctness\', \'v5_answer_quantity_unit\',\n _v5_same(_v5_answer.get(\'quantity_unit\'), \'replicas\'),\n \'Kept every capacity quantity in replicas.\')\n_c(\'correctness\', \'v5_answer_standard_capacity_date\',\n _v5_same(_v5_answer.get(\'standard_capacity_date\'), \'2026-03-06\'),\n "Read 2026-03-06 as CloudCap\'s independently confirmed standard delivery date from vendor order VEND-43.")\n_c(\'correctness\', \'v5_answer_expedited_capacity_date\',\n _v5_same(_v5_answer.get(\'expedited_capacity_date\'), \'2026-03-04\'),\n "Read 2026-03-04 as CloudCap\'s independently confirmed expedited delivery date (USD 750) from vendor order VEND-43.")\n_c(\'correctness\', \'v5_answer_capacity_request_replicas\',\n _v5_same(_v5_answer.get(\'capacity_request_replicas\'), 1),\n \'Bound the vendor request to the 1 uncovered replicas rather than the full 12-replica requirement.\')\n_c(\'correctness\', \'v5_answer_next_change_window\',\n _v5_same(_v5_answer.get(\'next_change_window\'), \'2026-03-05\'),\n \'Read the analytics-worker change calendar (2026-03-05, 2026-03-07, 2026-03-09, 2026-03-11) from the current operating control and identified 2026-03-05 as the next window.\')\n_c(\'correctness\', \'v5_answer_standard_plan_completion\',\n _v5_same(_v5_answer.get(\'standard_plan_completion\'), \'2026-03-07\'),\n \'Calculated standard_capacity_plan outcome as 2026-03-07: the first change window on or after standard delivery.\')\n_c(\'correctness\', \'v5_answer_expedited_plan_completion\',\n _v5_same(_v5_answer.get(\'expedited_plan_completion\'), \'2026-03-05\'),\n \'Calculated expedite_capacity outcome as 2026-03-05: the first change window on or after expedited delivery.\')\n_c(\'correctness\', \'v5_answer_reserved_release_completion\',\n _v5_same(_v5_answer.get(\'reserved_release_completion\'), \'2026-03-03\'),\n \'Calculated release_reserved_capacity outcome as 2026-03-03 and kept its separate-approval condition.\')\n_c(\'correctness\', \'v5_answer_recommended_option\',\n _v5_same(_v5_answer.get(\'recommended_option\'), \'expedite_capacity\'),\n \'Compared the date, cost and authority of standard_capacity_plan, expedite_capacity and release_reserved_capacity; selected expedite_capacity as the best currently authorised plan under the readiness standard.\')\n_c(\'correctness\', \'v5_answer_recommended_outcome_date\',\n _v5_same(_v5_answer.get(\'recommended_outcome_date\'), \'2026-03-05\'),\n \'Calculated 2026-03-05 as the supported outcome date for expedite_capacity.\')\n_c(\'correctness\', \'v5_answer_recommended_incremental_cost_usd\',\n _v5_same(_v5_answer.get(\'recommended_incremental_cost_usd\'), 750),\n \'Applied USD 750 as the documented incremental cost of expedite_capacity.\')\n_c(\'correctness\', \'v5_answer_escalation_approval_required\',\n _v5_same(_v5_answer.get(\'escalation_approval_required\'), 1),\n \'Recognised that release_reserved_capacity remains outside current authority and needs incident-commander approval beyond CHG-43.\')\n_c(\'correctness\', \'v5_answer_approval_reference\',\n _v5_same(_v5_answer.get(\'approval_reference\'), \'CHG-43\'),\n \'Applied change approval CHG-43 only to the authorised expedite_capacity scope.\')\n_c(\'correctness\', \'v5_answer_outcome_vs_control_days\',\n _v5_same(_v5_answer.get(\'outcome_vs_control_days\'), -1),\n \'Compared 2026-03-05 with the independent control date 2026-03-06 and calculated a signed variance of -1 day(s).\')\n_c(\'correctness\', \'v5_answer_decision_timing_status\',\n _v5_same(_v5_answer.get(\'decision_timing_status\'), \'ON_TIME\'),\n \'Reported ON_TIME; did not relabel a controlled but late result as on time.\')\n_c(\'correctness\', \'v5_capacity_evidence_complete\',\n _v4_success_before(_V5_CAPACITY_READS, _v4_first_source_write),\n \'read the readiness standard, CloudCap order VEND-43, change approval CHG-43, the Linear reservation and the status-page cutover notice before changing state\')\n_c(\'deployment\', \'v5_decision_evidence_precedes_record\',\n _v5_record_seq > 0 and _v4_success_before(_V5_DECISION_READS, _v5_record_seq),\n \'derive the capacity plan only after every source fact it depends on has been read successfully\')\n_c(\'deployment\', \'v5_record_precedes_handoff\',\n _v5_record_seq > 0 and _v4_handoff_seq > _v5_record_seq,\n \'record the capacity plan before stating it in the completion handoff\')\n_c(\'correctness\', \'v5_approval_applied_to_selected_scope\',\n _v5_same(_v5_answer.get(\'approval_reference\'), \'CHG-43\')\n and _v5_same(_v5_answer.get(\'recommended_option\'), \'expedite_capacity\')\n and _v5_same(_v5_answer.get(\'escalation_approval_required\'), 1),\n \'apply CHG-43 to expedite_capacity only and keep release_reserved_capacity outside current authority\')\n_c(\'correctness\', \'v5_handoff_is_natural_completion\',\n len(_v5_handoff_words) >= 30\n and sum(_v5_handoff_body.count(_mark) for _mark in (\'.\', \';\', \':\')) >= 2\n and not _v5_handoff_body.lstrip().startswith((\'{\', \'[\')),\n \'write a natural scoped completion note, not a keyword list or serialized benchmark object\')\n_c(\'correctness\', \'v5_handoff_states_selected_option\',\n \'expedite_capacity\' in _v5_handoff_body.casefold(),\n \'state the selected option (expedite_capacity) in the scoped completion handoff\')\n_c(\'correctness\', \'v5_handoff_states_outcome_date\',\n \'2026-03-05\' in _v5_handoff_body.casefold(),\n \'state the outcome date (2026-03-05) in the scoped completion handoff\')\n_c(\'correctness\', \'v5_handoff_states_approval_reference\',\n \'chg-43\' in _v5_handoff_body.casefold(),\n \'state the approval reference (CHG-43) in the scoped completion handoff\')\n_c(\'correctness\', \'v5_handoff_states_timing_status\',\n \'on_time\' in _v5_handoff_body.casefold(),\n \'state the timing status (ON_TIME) in the scoped completion handoff\')\n_c(\'correctness\', \'v5_handoff_states_binding_constraint\',\n \'cloudcap expedited capacity on 2026-03-04\' in _v5_handoff_body.casefold(),\n \'state the binding constraint (CloudCap expedited capacity on 2026-03-04) in the scoped completion handoff\')\n'12 13 14def _runtime_atomic_checks(checks):15 totals = {}16 for dimension, name, _passed, _message in checks:17 key = (dimension, name)18 totals[key] = totals.get(key, 0) + 119 seen = {}20 output = {}21 for dimension, name, passed, message in checks:22 key = (dimension, name)23 seen[key] = seen.get(key, 0) + 124 suffix = "#%d" % seen[key] if totals[key] > 1 else ""25 check_id = "%s.%s%s" % (dimension, name, suffix)26 output[check_id] = {27 "id": check_id,28 "dimension": dimension,29 "name": name,30 "passed": bool(passed),31 "message": message,32 }33 return output34 35 36def verify(db_path):37 conn = sqlite3.connect("file:%s?mode=ro" % db_path, uri=True)38 conn.row_factory = sqlite3.Row39 namespace = {40 "conn": conn,41 "sqlite3": sqlite3,42 "json": json,43 "hashlib": hashlib,44 "db_path": db_path,45 "DB_PATH": db_path,46 "final_answer": "",47 "answer": "",48 }49 namespace["get_db"] = lambda: conn50 error = None51 try:52 exec(compile(VCODE, "<vcode>", "exec"), namespace)53 except AssertionError as exc:54 error = "assertion: %s" % exc55 except Exception as exc: # noqa: BLE00156 error = "%s: %s" % (type(exc).__name__, exc)57 finally:58 conn.close()59 60 atomic = _runtime_atomic_checks(namespace.get("_checks") or [])61 milestone_reports = []62 for milestone in MILESTONES:63 evidence = []64 passed_count = 065 for expected in milestone["atomic_checks"]:66 actual = atomic.get(expected["id"])67 passed = bool(actual and actual["passed"])68 passed_count += int(passed)69 evidence.append({70 "id": expected["id"],71 "passed": passed,72 "message": (actual or expected).get("message")73 or expected.get("description", ""),74 })75 total = len(evidence)76 fraction = passed_count / total if total else 0.077 points = round(float(milestone["weight"]) * fraction, 6)78 milestone_reports.append({79 "id": milestone["id"],80 "category": milestone["category"],81 "description": milestone["description"],82 "weight": milestone["weight"],83 "passed": total > 0 and passed_count == total,84 "passed_checks": passed_count,85 "total_checks": total,86 "points": points,87 "evidence": evidence,88 })89 90 points = round(sum(item["points"] for item in milestone_reports), 6)91 reward = round(points / 100.0, 4)92 failed = [item["id"] for item in milestone_reports if not item["passed"]]93 passed = bool(milestone_reports) and not failed and error is None94 return {95 "task_id": TASK_ID,96 "category": CATEGORY,97 "passed": passed,98 "reward": reward,99 "score": reward,100 "points": points,101 "semantic_weights": {item["id"]: item["weight"] for item in MILESTONES},102 "milestones": milestone_reports,103 "dimensions": {104 item["id"]: "%d/%d" % (item["passed_checks"], item["total_checks"])105 for item in milestone_reports106 },107 "assertions": list(atomic.values()),108 "failure_reason": error or "; ".join(failed),109 }110 111 112if __name__ == "__main__":113 if len(sys.argv) < 2:114 print("usage: %s <world.db>" % sys.argv[0], file=sys.stderr)115 raise SystemExit(2)116 print(json.dumps(verify(sys.argv[1]), indent=2))117