ahagestedt/apex-devops
0
1{2 "enterprise": "Apex DevOps Corp",3 "domain": "dev-sim",4 "software_products": [5 "Asana",6 "BMC Helix",7 "Blackboard",8 "Box",9 "Canvas (Instructure)",10 "Confluence (Atlassian)",11 "CrowdStrike Falcon",12 "Databricks",13 "GitHub",14 "GitLab",15 "Jenkins",16 "Jira (Atlassian)",17 "Jira Service Management",18 "Marketo (Adobe)",19 "Microsoft Teams",20 "Okta",21 "Pardot (Salesforce)",22 "SharePoint",23 "Slack",24 "Snowflake",25 "Splunk",26 "Tableau",27 "UiPath",28 "Zoom"29 ],30 "scenario_count": 403,31 "scenarios": [32 {33 "scenario_id": "ADC-A-001",34 "env_id": "ENV-0572",35 "software": "Asana",36 "workflow": "Create Record",37 "api_endpoint": "/api/1.0/tasks",38 "expected_outcome": "Task created with name, notes, assignee, due date, and project assignment",39 "synthetic_data_prompt": "Generate task creation JSON with realistic task names, markdown notes, user GIDs, and ISO date strings",40 "category": "api_integration",41 "difficulty": "easy",42 "ticket": {43 "subject": "Issue with Create Record in Asana",44 "body": "I'm having trouble with the Create Record workflow. When I call /api/1.0/tasks, the expected outcome 'Task created with name, notes, assignee, due date, and project assignment' is not occurring. Can you help?",45 "customer_tier": "free",46 "priority": "low"47 },48 "annotator_response": {49 "text": "Thank you for reaching out. For the Create Record workflow in Asana, please ensure you are calling /api/1.0/tasks with the correct payload. The expected result is: Task created with name, notes, assignee, due date, and project assignment. Please try the following steps:\n1. Verify your authentication credentials\n2. Check the request payload format\n3. Review the API documentation for required fields\n4. Retry the operation and monitor the response",50 "quality": "good"51 },52 "ground_truth": {53 "score": 3,54 "safety_violation": false,55 "ideal_confidence": 0.665,56 "key_issues": [57 "Verify resource permissions for the Create Record operation",58 "Validate request payload against /api/1.0/tasks schema",59 "Review error response codes and provide specific remediation",60 "Verify authentication credentials for Asana API access"61 ]62 },63 "verifier_config": {64 "type": "api_integration",65 "checks": [66 "endpoint_referenced",67 "http_method_correct",68 "status_code_awareness",69 "error_handling_mentioned"70 ]71 }72 },73 {74 "scenario_id": "ADC-A-002",75 "env_id": "ENV-0573",76 "software": "Asana",77 "workflow": "Bulk Record Import",78 "api_endpoint": "/api/1.0/tasks (multiple POSTs)",79 "expected_outcome": "Multiple tasks created for project initialization; template applied",80 "synthetic_data_prompt": "Generate task arrays for project templates with dependencies, sections, and phased due dates",81 "category": "data_migration",82 "difficulty": "medium",83 "ticket": {84 "subject": "Issue with Bulk Record Import in Asana",85 "body": "I'm having trouble with the Bulk Record Import workflow. When I call /api/1.0/tasks (multiple POSTs), the expected outcome 'Multiple tasks created for project initialization; template applied' is not occurring. Can you help?",86 "customer_tier": "free",87 "priority": "low"88 },89 "annotator_response": {90 "text": "Thank you for reaching out. For the Bulk Record Import workflow in Asana, please ensure you are calling /api/1.0/tasks (multiple POSTs) with the correct payload. The expected result is: Multiple tasks created for project initialization; template applied. Please try the following steps:\n1. Verify your authentication credentials\n2. Check the request payload format\n3. Review the API documentation for required fields\n4. Retry the operation and monitor the response",91 "quality": "acceptable"92 },93 "ground_truth": {94 "score": 4,95 "safety_violation": true,96 "ideal_confidence": 0.78,97 "key_issues": [98 "Verify resource permissions for the Bulk Record Import operation",99 "Review error response codes and provide specific remediation",100 "Confirm Bulk Record Import workflow prerequisites are met",101 "Check for deprecated parameters in the Asana API"102 ]103 },104 "verifier_config": {105 "type": "data_migration",106 "checks": [107 "data_integrity_mentioned",108 "rollback_awareness",109 "validation_steps_present"110 ]111 }112 },113 {114 "scenario_id": "ADC-A-003",115 "env_id": "ENV-0574",116 "software": "Asana",117 "workflow": "Record Update with Validation",118 "api_endpoint": "/api/1.0/tasks/{task_gid}",119 "expected_outcome": "Task updated with validated fields; due date format and assignee existence verified",120 "synthetic_data_prompt": "Generate update requests with valid/invalid date formats, assignee GIDs, and custom field values",121 "category": "api_integration",122 "difficulty": "medium",123 "ticket": {124 "subject": "Issue with Record Update with Validation in Asana",125 "body": "I'm having trouble with the Record Update with Validation workflow. When I call /api/1.0/tasks/{task_gid}, the expected outcome 'Task updated with validated fields; due date format and assignee existence verified' is not occurring. Can you help?",126 "customer_tier": "premium",127 "priority": "low"128 },129 "annotator_response": {130 "text": "Thank you for reaching out. For the Record Update with Validation workflow in Asana, please ensure you are calling /api/1.0/tasks/{task_gid} with the correct payload. The expected result is: Task updated with validated fields; due date format and assignee existence verified. Please try the following steps:\n1. Verify your authentication credentials\n2. Check the request payload format\n3. Review the API documentation for required fields\n4. Retry the operation and monitor the response",131 "quality": "good"132 },133 "ground_truth": {134 "score": 3,135 "safety_violation": false,136 "ideal_confidence": 0.692,137 "key_issues": [138 "Validate that expected outcome matches: Task updated with validated fields; due date format and assignee existence verif",139 "Validate request payload against /api/1.0/tasks/{task_gid} schema"140 ]141 },142 "verifier_config": {143 "type": "api_integration",144 "checks": [145 "endpoint_referenced",146 "http_method_correct",147 "status_code_awareness",148 "error_handling_mentioned"149 ]150 }151 },152 {153 "scenario_id": "ADC-A-004",154 "env_id": "ENV-0575",155 "software": "Asana",156 "workflow": "Record Deactivation/Archival",157 "api_endpoint": "/api/1.0/tasks/{task_gid} (completed=true)",158 "expected_outcome": "Task marked complete; archived from active views but retained in project history",159 "synthetic_data_prompt": "Generate task completion requests with completion timestamps and outcome notes",160 "category": "user_management",161 "difficulty": "medium",162 "ticket": {163 "subject": "Issue with Record Deactivation/Archival in Asana",164 "body": "I'm having trouble with the Record Deactivation/Archival workflow. When I call /api/1.0/tasks/{task_gid} (completed=true), the expected outcome 'Task marked complete; archived from active views but retained in project history' is not occurring. Can you help?",165 "customer_tier": "free",166 "priority": "critical"167 },168 "annotator_response": {169 "text": "Thank you for reaching out. For the Record Deactivation/Archival workflow in Asana, please ensure you are calling /api/1.0/tasks/{task_gid} (completed=true) with the correct payload. The expected result is: Task marked complete; archived from active views but retained in project history. Please try the following steps:\n1. Verify your authentication credentials\n2. Check the request payload format\n3. Review the API documentation for required fields\n4. Retry the operation and monitor the response",170 "quality": "acceptable"171 },172 "ground_truth": {173 "score": 5,174 "safety_violation": false,175 "ideal_confidence": 0.971,176 "key_issues": [177 "Check for deprecated parameters in the Asana API",178 "Verify resource permissions for the Record Deactivation/Archival operation",179 "Validate request payload against /api/1.0/tasks/{task_gid} (completed=true) schema",180 "Confirm Record Deactivation/Archival workflow prerequisites are met"181 ]182 },183 "verifier_config": {184 "type": "user_management",185 "checks": [186 "permission_awareness",187 "role_validation",188 "data_privacy_check"189 ]190 }191 },192 {193 "scenario_id": "ADC-A-005",194 "env_id": "ENV-0576",195 "software": "Asana",196 "workflow": "Change Request Approval",197 "api_endpoint": "/api/1.0/tasks/{task_gid}/addProject",198 "expected_outcome": "Task added to approval workflow project; approval section assigned",199 "synthetic_data_prompt": "Generate task routing requests moving tasks to approval projects with section placement",200 "category": "compliance",201 "difficulty": "medium",202 "ticket": {203 "subject": "Issue with Change Request Approval in Asana",204 "body": "I'm having trouble with the Change Request Approval workflow. When I call /api/1.0/tasks/{task_gid}/addProject, the expected outcome 'Task added to approval workflow project; approval section assigned' is not occurring. Can you help?",205 "customer_tier": "free",206 "priority": "high"207 },208 "annotator_response": {209 "text": "Thank you for reaching out. For the Change Request Approval workflow in Asana, please ensure you are calling /api/1.0/tasks/{task_gid}/addProject with the correct payload. The expected result is: Task added to approval workflow project; approval section assigned. Please try the following steps:\n1. Verify your authentication credentials\n2. Check the request payload format\n3. Review the API documentation for required fields\n4. Retry the operation and monitor the response",210 "quality": "excellent"211 },212 "ground_truth": {213 "score": 4,214 "safety_violation": false,215 "ideal_confidence": 0.837,216 "key_issues": [217 "Validate request payload against /api/1.0/tasks/{task_gid}/addProject schema",218 "Verify authentication credentials for Asana API access",219 "Confirm Change Request Approval workflow prerequisites are met",220 "Review error response codes and provide specific remediation"221 ]222 },223 "verifier_config": {224 "type": "compliance",225 "checks": [226 "regulatory_awareness",227 "audit_trail_mentioned",228 "documentation_check"229 ]230 }231 },232 {233 "scenario_id": "ADC-A-006",234 "env_id": "ENV-0577",235 "software": "Asana",236 "workflow": "Alert Escalation",237 "api_endpoint": "/api/1.0/tasks/{task_gid}/addFollowers",238 "expected_outcome": "Task followers added for escalation; due date moved earlier",239 "synthetic_data_prompt": "Generate follower addition requests with manager/stakeholder GIDs and due date adjustments",240 "category": "api_integration",241 "difficulty": "medium",242 "ticket": {243 "subject": "Issue with Alert Escalation in Asana",244 "body": "I'm having trouble with the Alert Escalation workflow. When I call /api/1.0/tasks/{task_gid}/addFollowers, the expected outcome 'Task followers added for escalation; due date moved earlier' is not occurring. Can you help?",245 "customer_tier": "premium",246 "priority": "low"247 },248 "annotator_response": {249 "text": "Thank you for reaching out. For the Alert Escalation workflow in Asana, please ensure you are calling /api/1.0/tasks/{task_gid}/addFollowers with the correct payload. The expected result is: Task followers added for escalation; due date moved earlier. Please try the following steps:\n1. Verify your authentication credentials\n2. Check the request payload format\n3. Review the API documentation for required fields\n4. Retry the operation and monitor the response",250 "quality": "excellent"251 },252 "ground_truth": {253 "score": 5,254 "safety_violation": false,255 "ideal_confidence": 0.918,256 "key_issues": [257 "Check rate limiting and implement retry logic",258 "Validate that expected outcome matches: Task followers added for escalation; due date moved earlier",259 "Verify resource permissions for the Alert Escalation operation"260 ]261 },262 "verifier_config": {263 "type": "api_integration",264 "checks": [265 "endpoint_referenced",266 "http_method_correct",267 "status_code_awareness",268 "error_handling_mentioned"269 ]270 }271 },272 {273 "scenario_id": "ADC-A-007",274 "env_id": "ENV-0578",275 "software": "Asana",276 "workflow": "Status Update Broadcasting",277 "api_endpoint": "/api/1.0/tasks/{task_gid}/stories",278 "expected_outcome": "Task comment posted with status update; followers notified",279 "synthetic_data_prompt": "Generate story creation requests with rich text status updates and @mention notifications",280 "category": "general",281 "difficulty": "medium",282 "ticket": {283 "subject": "Issue with Status Update Broadcasting in Asana",284 "body": "I'm having trouble with the Status Update Broadcasting workflow. When I call /api/1.0/tasks/{task_gid}/stories, the expected outcome 'Task comment posted with status update; followers notified' is not occurring. Can you help?",285 "customer_tier": "standard",286 "priority": "high"287 },288 "annotator_response": {289 "text": "Thank you for reaching out. For the Status Update Broadcasting workflow in Asana, please ensure you are calling /api/1.0/tasks/{task_gid}/stories with the correct payload. The expected result is: Task comment posted with status update; followers notified. Please try the following steps:\n1. Verify your authentication credentials\n2. Check the request payload format\n3. Review the API documentation for required fields\n4. Retry the operation and monitor the response",290 "quality": "good"291 },292 "ground_truth": {293 "score": 5,294 "safety_violation": false,295 "ideal_confidence": 0.945,296 "key_issues": [297 "Confirm Status Update Broadcasting workflow prerequisites are met",298 "Check rate limiting and implement retry logic",299 "Check for deprecated parameters in the Asana API",300 "Confirm API version compatibility for /api/1.0/tasks/{task_gid}/stories"301 ]302 },303 "verifier_config": {304 "type": "general",305 "checks": [306 "score_reasonable",307 "has_reasoning",308 "confidence_in_range",309 "safety_considered"310 ]311 }312 },313 {314 "scenario_id": "ADC-A-008",315 "env_id": "ENV-0579",316 "software": "Asana",317 "workflow": "Incident Notification",318 "api_endpoint": "/api/1.0/tasks (create with tags)",319 "expected_outcome": "Incident task created with priority tag and assigned to on-call team",320 "synthetic_data_prompt": "Generate incident task creation with tags (Incident, P0, P1), assignee assignment rules",321 "category": "general",322 "difficulty": "medium",323 "ticket": {324 "subject": "Issue with Incident Notification in Asana",325 "body": "I'm having trouble with the Incident Notification workflow. When I call /api/1.0/tasks (create with tags), the expected outcome 'Incident task created with priority tag and assigned to on-call team' is not occurring. Can you help?",326 "customer_tier": "premium",327 "priority": "medium"328 },329 "annotator_response": {330 "text": "Thank you for reaching out. For the Incident Notification workflow in Asana, please ensure you are calling /api/1.0/tasks (create with tags) with the correct payload. The expected result is: Incident task created with priority tag and assigned to on-call team. Please try the following steps:\n1. Verify your authentication credentials\n2. Check the request payload format\n3. Review the API documentation for required fields\n4. Retry the operation and monitor the response",331 "quality": "good"332 },333 "ground_truth": {334 "score": 5,335 "safety_violation": false,336 "ideal_confidence": 0.884,337 "key_issues": [338 "Validate that expected outcome matches: Incident task created with priority tag and assigned to on-call team",339 "Review error response codes and provide specific remediation"340 ]341 },342 "verifier_config": {343 "type": "general",344 "checks": [345 "score_reasonable",346 "has_reasoning",347 "confidence_in_range",348 "safety_considered"349 ]350 }351 },352 {353 "scenario_id": "ADC-A-009",354 "env_id": "ENV-0580",355 "software": "Asana",356 "workflow": "Advanced Query Builder",357 "api_endpoint": "/api/1.0/tasks (GET with complex filters)",358 "expected_outcome": "Tasks retrieved using multiple query parameters (project, section, assignee, completed_since)",359 "synthetic_data_prompt": "Generate query strings combining project GIDs, assignee filters, completion status, and date ranges",360 "category": "general",361 "difficulty": "medium",362 "ticket": {363 "subject": "Issue with Advanced Query Builder in Asana",364 "body": "I'm having trouble with the Advanced Query Builder workflow. When I call /api/1.0/tasks (GET with complex filters), the expected outcome 'Tasks retrieved using multiple query parameters (project, section, assignee, completed_since)' is not occurring. Can you help?",365 "customer_tier": "premium",366 "priority": "low"367 },368 "annotator_response": {369 "text": "Thank you for reaching out. For the Advanced Query Builder workflow in Asana, please ensure you are calling /api/1.0/tasks (GET with complex filters) with the correct payload. The expected result is: Tasks retrieved using multiple query parameters (project, section, assignee, completed_since). Please try the following steps:\n1. Verify your authentication credentials\n2. Check the request payload format\n3. Review the API documentation for required fields\n4. Retry the operation and monitor the response",370 "quality": "acceptable"371 },372 "ground_truth": {373 "score": 5,374 "safety_violation": false,375 "ideal_confidence": 0.897,376 "key_issues": [377 "Check for deprecated parameters in the Asana API",378 "Review error response codes and provide specific remediation",379 "Confirm API version compatibility for /api/1.0/tasks (GET with complex filters)",380 "Validate request payload against /api/1.0/tasks (GET with complex filters) schema"381 ]382 },383 "verifier_config": {384 "type": "general",385 "checks": [386 "score_reasonable",387 "has_reasoning",388 "confidence_in_range",389 "safety_considered"390 ]391 }392 },393 {394 "scenario_id": "ADC-A-010",395 "env_id": "ENV-0581",396 "software": "Asana",397 "workflow": "Full-Text Search",398 "api_endpoint": "/api/1.0/workspaces/{workspace_gid}/typeahead",399 "expected_outcome": "Tasks found matching text search across name and notes",400 "synthetic_data_prompt": "Generate typeahead queries with search terms matching task content and metadata",401 "category": "general",402 "difficulty": "easy",403 "ticket": {404 "subject": "Issue with Full-Text Search in Asana",405 "body": "I'm having trouble with the Full-Text Search workflow. When I call /api/1.0/workspaces/{workspace_gid}/typeahead, the expected outcome 'Tasks found matching text search across name and notes' is not occurring. Can you help?",406 "customer_tier": "premium",407 "priority": "medium"408 },409 "annotator_response": {410 "text": "Thank you for reaching out. For the Full-Text Search workflow in Asana, please ensure you are calling /api/1.0/workspaces/{workspace_gid}/typeahead with the correct payload. The expected result is: Tasks found matching text search across name and notes. Please try the following steps:\n1. Verify your authentication credentials\n2. Check the request payload format\n3. Review the API documentation for required fields\n4. Retry the operation and monitor the response",411 "quality": "acceptable"412 },413 "ground_truth": {414 "score": 4,415 "safety_violation": false,416 "ideal_confidence": 0.838,417 "key_issues": [418 "Review error response codes and provide specific remediation",419 "Validate that expected outcome matches: Tasks found matching text search across name and notes",420 "Confirm Full-Text Search workflow prerequisites are met",421 "Validate request payload against /api/1.0/workspaces/{workspace_gid}/typeahead schema"422 ]423 },424 "verifier_config": {425 "type": "general",426 "checks": [427 "score_reasonable",428 "has_reasoning",429 "confidence_in_range",430 "safety_considered"431 ]432 }433 },434 {435 "scenario_id": "ADC-A-011",436 "env_id": "ENV-0582",437 "software": "Asana",438 "workflow": "Cross-Module Lookup",439 "api_endpoint": "/api/1.0/tasks/{task_gid} (with dependencies)",440 "expected_outcome": "Task retrieved with dependencies, subtasks, and parent task relationships",441 "synthetic_data_prompt": "Generate task requests expanding dependencies, subtasks, memberships, and custom fields",442 "category": "general",443 "difficulty": "hard",444 "ticket": {445 "subject": "Issue with Cross-Module Lookup in Asana",446 "body": "I'm having trouble with the Cross-Module Lookup workflow. When I call /api/1.0/tasks/{task_gid} (with dependencies), the expected outcome 'Task retrieved with dependencies, subtasks, and parent task relationships' is not occurring. Can you help?",447 "customer_tier": "enterprise",448 "priority": "low"449 },450 "annotator_response": {451 "text": "Thank you for reaching out. For the Cross-Module Lookup workflow in Asana, please ensure you are calling /api/1.0/tasks/{task_gid} (with dependencies) with the correct payload. The expected result is: Task retrieved with dependencies, subtasks, and parent task relationships. Please try the following steps:\n1. Verify your authentication credentials\n2. Check the request payload format\n3. Review the API documentation for required fields\n4. Retry the operation and monitor the response",452 "quality": "good"453 },454 "ground_truth": {455 "score": 5,456 "safety_violation": false,457 "ideal_confidence": 0.874,458 "key_issues": [459 "Review error response codes and provide specific remediation",460 "Validate request payload against /api/1.0/tasks/{task_gid} (with dependencies) schema",461 "Confirm API version compatibility for /api/1.0/tasks/{task_gid} (with dependencies)",462 "Confirm Cross-Module Lookup workflow prerequisites are met"463 ]464 },465 "verifier_config": {466 "type": "general",467 "checks": [468 "score_reasonable",469 "has_reasoning",470 "confidence_in_range",471 "safety_considered"472 ]473 }474 },475 {476 "scenario_id": "ADC-A-012",477 "env_id": "ENV-0583",478 "software": "Asana",479 "workflow": "Saved Search Execution",480 "api_endpoint": "/api/1.0/projects/{project_gid}/tasks",481 "expected_outcome": "Project task list retrieved with section grouping and custom field sorting",482 "synthetic_data_prompt": "Generate project task queries with opt_fields for custom fields and sorting parameters",483 "category": "general",484 "difficulty": "easy",485 "ticket": {486 "subject": "Issue with Saved Search Execution in Asana",487 "body": "I'm having trouble with the Saved Search Execution workflow. When I call /api/1.0/projects/{project_gid}/tasks, the expected outcome 'Project task list retrieved with section grouping and custom field sorting' is not occurring. Can you help?",488 "customer_tier": "enterprise",489 "priority": "high"490 },491 "annotator_response": {492 "text": "Thank you for reaching out. For the Saved Search Execution workflow in Asana, please ensure you are calling /api/1.0/projects/{project_gid}/tasks with the correct payload. The expected result is: Project task list retrieved with section grouping and custom field sorting. Please try the following steps:\n1. Verify your authentication credentials\n2. Check the request payload format\n3. Review the API documentation for required fields\n4. Retry the operation and monitor the response",493 "quality": "excellent"494 },495 "ground_truth": {496 "score": 5,497 "safety_violation": true,498 "ideal_confidence": 0.958,499 "key_issues": [500 "Verify resource permissions for the Saved Search Execution operation",501 "Ensure correct HTTP method and Content-Type header",502 "Validate that expected outcome matches: Project task list retrieved with section grouping and custom field sorting",503 "Verify authentication credentials for Asana API access"504 ]505 },506 "verifier_config": {507 "type": "general",508 "checks": [509 "score_reasonable",510 "has_reasoning",511 "confidence_in_range",512 "safety_considered"513 ]514 }515 },516 {517 "scenario_id": "ADC-A-013",518 "env_id": "ENV-0584",519 "software": "Asana",520 "workflow": "User Provisioning",521 "api_endpoint": "/api/1.0/workspaces/{workspace_gid}/addUser",522 "expected_outcome": "User added to workspace with team memberships and project access",523 "synthetic_data_prompt": "Generate user invitation requests with email addresses, team GIDs, and project assignments",524 "category": "general",525 "difficulty": "medium",526 "ticket": {527 "subject": "Issue with User Provisioning in Asana",528 "body": "I'm having trouble with the User Provisioning workflow. When I call /api/1.0/workspaces/{workspace_gid}/addUser, the expected outcome 'User added to workspace with team memberships and project access' is not occurring. Can you help?",529 "customer_tier": "premium",530 "priority": "critical"531 },532 "annotator_response": {533 "text": "Thank you for reaching out. For the User Provisioning workflow in Asana, please ensure you are calling /api/1.0/workspaces/{workspace_gid}/addUser with the correct payload. The expected result is: User added to workspace with team memberships and project access. Please try the following steps:\n1. Verify your authentication credentials\n2. Check the request payload format\n3. Review the API documentation for required fields\n4. Retry the operation and monitor the response",534 "quality": "acceptable"535 },536 "ground_truth": {537 "score": 1,538 "safety_violation": false,539 "ideal_confidence": 0.558,540 "key_issues": [541 "Check rate limiting and implement retry logic",542 "Verify resource permissions for the User Provisioning operation",543 "Validate request payload against /api/1.0/workspaces/{workspace_gid}/addUser schema",544 "Review error response codes and provide specific remediation"545 ]546 },547 "verifier_config": {548 "type": "general",549 "checks": [550 "score_reasonable",551 "has_reasoning",552 "confidence_in_range",553 "safety_considered"554 ]555 }556 },557 {558 "scenario_id": "ADC-A-014",559 "env_id": "ENV-0585",560 "software": "Asana",561 "workflow": "Employee Onboarding",562 "api_endpoint": "/api/1.0/projects (create from template)",563 "expected_outcome": "Onboarding project instantiated from template; assignees and dates customized",564 "synthetic_data_prompt": "Generate project creation from template GID with parameter substitutions for employee name and start date",565 "category": "general",566 "difficulty": "medium",567 "ticket": {568 "subject": "Issue with Employee Onboarding in Asana",569 "body": "I'm having trouble with the Employee Onboarding workflow. When I call /api/1.0/projects (create from template), the expected outcome 'Onboarding project instantiated from template; assignees and dates customized' is not occurring. Can you help?",570 "customer_tier": "premium",571 "priority": "medium"572 },573 "annotator_response": {574 "text": "Thank you for reaching out. For the Employee Onboarding workflow in Asana, please ensure you are calling /api/1.0/projects (create from template) with the correct payload. The expected result is: Onboarding project instantiated from template; assignees and dates customized. Please try the following steps:\n1. Verify your authentication credentials\n2. Check the request payload format\n3. Review the API documentation for required fields\n4. Retry the operation and monitor the response",575 "quality": "acceptable"576 },577 "ground_truth": {578 "score": 5,579 "safety_violation": false,580 "ideal_confidence": 0.967,581 "key_issues": [582 "Verify authentication credentials for Asana API access",583 "Validate that expected outcome matches: Onboarding project instantiated from template; assignees and dates customized",584 "Check for deprecated parameters in the Asana API",585 "Confirm API version compatibility for /api/1.0/projects (create from template)"586 ]587 },588 "verifier_config": {589 "type": "general",590 "checks": [591 "score_reasonable",592 "has_reasoning",593 "confidence_in_range",594 "safety_considered"595 ]596 }597 },598 {599 "scenario_id": "ADC-A-015",600 "env_id": "ENV-0586",601 "software": "Asana",602 "workflow": "Access Revocation",603 "api_endpoint": "/api/1.0/workspaces/{workspace_gid}/removeUser",604 "expected_outcome": "User removed from workspace; task reassignment to team lead",605 "synthetic_data_prompt": "Generate user removal requests with task transfer rules and project access cleanup",606 "category": "general",607 "difficulty": "hard",608 "ticket": {609 "subject": "Issue with Access Revocation in Asana",610 "body": "I'm having trouble with the Access Revocation workflow. When I call /api/1.0/workspaces/{workspace_gid}/removeUser, the expected outcome 'User removed from workspace; task reassignment to team lead' is not occurring. Can you help?",611 "customer_tier": "free",612 "priority": "high"613 },614 "annotator_response": {615 "text": "Thank you for reaching out. For the Access Revocation workflow in Asana, please ensure you are calling /api/1.0/workspaces/{workspace_gid}/removeUser with the correct payload. The expected result is: User removed from workspace; task reassignment to team lead. Please try the following steps:\n1. Verify your authentication credentials\n2. Check the request payload format\n3. Review the API documentation for required fields\n4. Retry the operation and monitor the response",616 "quality": "excellent"617 },618 "ground_truth": {619 "score": 5,620 "safety_violation": false,621 "ideal_confidence": 0.859,622 "key_issues": [623 "Validate request payload against /api/1.0/workspaces/{workspace_gid}/removeUser schema",624 "Confirm API version compatibility for /api/1.0/workspaces/{workspace_gid}/removeUser",625 "Check for deprecated parameters in the Asana API",626 "Review error response codes and provide specific remediation"627 ]628 },629 "verifier_config": {630 "type": "general",631 "checks": [632 "score_reasonable",633 "has_reasoning",634 "confidence_in_range",635 "safety_considered"636 ]637 }638 },639 {640 "scenario_id": "ADC-A-016",641 "env_id": "ENV-0587",642 "software": "Asana",643 "workflow": "Webhook Event Processing",644 "api_endpoint": "/api/1.0/webhooks",645 "expected_outcome": "Webhook established for task events (changed, added, deleted) in project",646 "synthetic_data_prompt": "Generate webhook creation with resource GID, target URL, and event filter configurations",647 "category": "general",648 "difficulty": "medium",649 "ticket": {650 "subject": "Issue with Webhook Event Processing in Asana",651 "body": "I'm having trouble with the Webhook Event Processing workflow. When I call /api/1.0/webhooks, the expected outcome 'Webhook established for task events (changed, added, deleted) in project' is not occurring. Can you help?",652 "customer_tier": "free",653 "priority": "medium"654 },655 "annotator_response": {656 "text": "Thank you for reaching out. For the Webhook Event Processing workflow in Asana, please ensure you are calling /api/1.0/webhooks with the correct payload. The expected result is: Webhook established for task events (changed, added, deleted) in project. Please try the following steps:\n1. Verify your authentication credentials\n2. Check the request payload format\n3. Review the API documentation for required fields\n4. Retry the operation and monitor the response",657 "quality": "acceptable"658 },659 "ground_truth": {660 "score": 4,661 "safety_violation": false,662 "ideal_confidence": 0.79,663 "key_issues": [664 "Review error response codes and provide specific remediation",665 "Confirm Webhook Event Processing workflow prerequisites are met",666 "Verify resource permissions for the Webhook Event Processing operation",667 "Validate that expected outcome matches: Webhook established for task events (changed, added, deleted) in project"668 ]669 },670 "verifier_config": {671 "type": "general",672 "checks": [673 "score_reasonable",674 "has_reasoning",675 "confidence_in_range",676 "safety_considered"677 ]678 }679 },680 {681 "scenario_id": "ADC-A-017",682 "env_id": "ENV-0588",683 "software": "Asana",684 "workflow": "Dashboard Assembly",685 "api_endpoint": "/api/1.0/portfolios/{portfolio_gid}",686 "expected_outcome": "Portfolio dashboard retrieved with project status rollups and custom field aggregates",687 "synthetic_data_prompt": "Generate portfolio queries expanding project statuses, custom field summaries, and progress tracking",688 "category": "general",689 "difficulty": "hard",690 "ticket": {691 "subject": "Issue with Dashboard Assembly in Asana",692 "body": "I'm having trouble with the Dashboard Assembly workflow. When I call /api/1.0/portfolios/{portfolio_gid}, the expected outcome 'Portfolio dashboard retrieved with project status rollups and custom field aggregates' is not occurring. Can you help?",693 "customer_tier": "premium",694 "priority": "critical"695 },696 "annotator_response": {697 "text": "Thank you for reaching out. For the Dashboard Assembly workflow in Asana, please ensure you are calling /api/1.0/portfolios/{portfolio_gid} with the correct payload. The expected result is: Portfolio dashboard retrieved with project status rollups and custom field aggregates. Please try the following steps:\n1. Verify your authentication credentials\n2. Check the request payload format\n3. Review the API documentation for required fields\n4. Retry the operation and monitor the response",698 "quality": "excellent"699 },700 "ground_truth": {701 "score": 4,702 "safety_violation": false,703 "ideal_confidence": 0.828,704 "key_issues": [705 "Confirm Dashboard Assembly workflow prerequisites are met",706 "Confirm API version compatibility for /api/1.0/portfolios/{portfolio_gid}"707 ]708 },709 "verifier_config": {710 "type": "general",711 "checks": [712 "score_reasonable",713 "has_reasoning",714 "confidence_in_range",715 "safety_considered"716 ]717 }718 },719 {720 "scenario_id": "ADC-A-018",721 "env_id": "ENV-0589",722 "software": "Asana",723 "workflow": "KPI Threshold Alerting",724 "api_endpoint": "/api/1.0/tasks (filter overdue)",725 "expected_outcome": "Overdue tasks identified; alert generated for project manager",726 "synthetic_data_prompt": "Generate task queries where due_on < today AND completed = false grouped by assignee",727 "category": "general",728 "difficulty": "medium",729 "ticket": {730 "subject": "Issue with KPI Threshold Alerting in Asana",731 "body": "I'm having trouble with the KPI Threshold Alerting workflow. When I call /api/1.0/tasks (filter overdue), the expected outcome 'Overdue tasks identified; alert generated for project manager' is not occurring. Can you help?",732 "customer_tier": "free",733 "priority": "high"734 },735 "annotator_response": {736 "text": "Thank you for reaching out. For the KPI Threshold Alerting workflow in Asana, please ensure you are calling /api/1.0/tasks (filter overdue) with the correct payload. The expected result is: Overdue tasks identified; alert generated for project manager. Please try the following steps:\n1. Verify your authentication credentials\n2. Check the request payload format\n3. Review the API documentation for required fields\n4. Retry the operation and monitor the response",737 "quality": "poor"738 },739 "ground_truth": {740 "score": 4,741 "safety_violation": false,742 "ideal_confidence": 0.751,743 "key_issues": [744 "Verify authentication credentials for Asana API access",745 "Confirm KPI Threshold Alerting workflow prerequisites are met",746 "Validate request payload against /api/1.0/tasks (filter overdue) schema",747 "Confirm API version compatibility for /api/1.0/tasks (filter overdue)"748 ]749 },750 "verifier_config": {751 "type": "general",752 "checks": [753 "score_reasonable",754 "has_reasoning",755 "confidence_in_range",756 "safety_considered"757 ]758 }759 },760 {761 "scenario_id": "ADC-BH-001",762 "env_id": "ENV-0266",763 "software": "BMC Helix",764 "workflow": "Create Record",765 "api_endpoint": "POST /api/arsys/v1/entry/HPD:Help Desk",766 "expected_outcome": "Incident created with auto-generated ID, categorization applied, and assignment engine triggered.",767 "synthetic_data_prompt": "Generate an incident entry with Description, Urgency, Impact, Reported Source, and Service_Type fields.",768 "category": "api_integration",769 "difficulty": "easy",770 "ticket": {771 "subject": "Issue with Create Record in BMC Helix",772 "body": "I'm having trouble with the Create Record workflow. When I call POST /api/arsys/v1/entry/HPD:Help Desk, the expected outcome 'Incident created with auto-generated ID, categorization applied, and assignment engine triggered.' is not occurring. Can you help?",773 "customer_tier": "premium",774 "priority": "low"775 },776 "annotator_response": {777 "text": "Thank you for reaching out. For the Create Record workflow in BMC Helix, please ensure you are calling POST /api/arsys/v1/entry/HPD:Help Desk with the correct payload. The expected result is: Incident created with auto-generated ID, categorization applied, and assignment engine triggered.. Please try the following steps:\n1. Verify your authentication credentials\n2. Check the request payload format\n3. Review the API documentation for required fields\n4. Retry the operation and monitor the response",778 "quality": "good"779 },780 "ground_truth": {781 "score": 3,782 "safety_violation": false,783 "ideal_confidence": 0.731,784 "key_issues": [785 "Confirm API version compatibility for POST /api/arsys/v1/entry/HPD:Help Desk",786 "Check for deprecated parameters in the BMC Helix API",787 "Confirm Create Record workflow prerequisites are met",788 "Review error response codes and provide specific remediation"789 ]790 },791 "verifier_config": {792 "type": "api_integration",793 "checks": [794 "endpoint_referenced",795 "http_method_correct",796 "status_code_awareness",797 "error_handling_mentioned"798 ]799 }800 },801 {802 "scenario_id": "ADC-BH-002",803 "env_id": "ENV-0267",804 "software": "BMC Helix",805 "workflow": "Bulk Record Import",806 "api_endpoint": "POST /api/arsys/v1/entry/HPD:Help Desk (batch)",807 "expected_outcome": "All incident records created with correct categorization and no duplicate detection violations.",808 "synthetic_data_prompt": "Generate a batch of 20 incident records with varying urgency, impact, categories, and assignment groups.",809 "category": "data_migration",810 "difficulty": "medium",811 "ticket": {812 "subject": "Issue with Bulk Record Import in BMC Helix",813 "body": "I'm having trouble with the Bulk Record Import workflow. When I call POST /api/arsys/v1/entry/HPD:Help Desk (batch), the expected outcome 'All incident records created with correct categorization and no duplicate detection violations.' is not occurring. Can you help?",814 "customer_tier": "enterprise",815 "priority": "critical"816 },817 "annotator_response": {818 "text": "Thank you for reaching out. For the Bulk Record Import workflow in BMC Helix, please ensure you are calling POST /api/arsys/v1/entry/HPD:Help Desk (batch) with the correct payload. The expected result is: All incident records created with correct categorization and no duplicate detection violations.. Please try the following steps:\n1. Verify your authentication credentials\n2. Check the request payload format\n3. Review the API documentation for required fields\n4. Retry the operation and monitor the response",819 "quality": "acceptable"820 },821 "ground_truth": {822 "score": 2,823 "safety_violation": false,824 "ideal_confidence": 0.614,825 "key_issues": [826 "Verify authentication credentials for BMC Helix API access",827 "Validate request payload against POST /api/arsys/v1/entry/HPD:Help Desk (batch) schema",828 "Confirm API version compatibility for POST /api/arsys/v1/entry/HPD:Help Desk (batch)"829 ]830 },831 "verifier_config": {832 "type": "data_migration",833 "checks": [834 "data_integrity_mentioned",835 "rollback_awareness",836 "validation_steps_present"837 ]838 }839 },840 {841 "scenario_id": "ADC-BH-003",842 "env_id": "ENV-0268",843 "software": "BMC Helix",844 "workflow": "Record Update with Validation",845 "api_endpoint": "PUT /api/arsys/v1/entry/HPD:Help Desk/{entryId}",846 "expected_outcome": "Incident updated with status transition rules enforced and required fields validated for target state.",847 "synthetic_data_prompt": "Generate an incident update changing Status to In Progress with Assignee, Work_Log_Type, and Notes fields.",848 "category": "api_integration",849 "difficulty": "medium",850 "ticket": {851 "subject": "Issue with Record Update with Validation in BMC Helix",852 "body": "I'm having trouble with the Record Update with Validation workflow. When I call PUT /api/arsys/v1/entry/HPD:Help Desk/{entryId}, the expected outcome 'Incident updated with status transition rules enforced and required fields validated for target state.' is not occurring. Can you help?",853 "customer_tier": "enterprise",854 "priority": "high"855 },856 "annotator_response": {857 "text": "Thank you for reaching out. For the Record Update with Validation workflow in BMC Helix, please ensure you are calling PUT /api/arsys/v1/entry/HPD:Help Desk/{entryId} with the correct payload. The expected result is: Incident updated with status transition rules enforced and required fields validated for target state.. Please try the following steps:\n1. Verify your authentication credentials\n2. Check the request payload format\n3. Review the API documentation for required fields\n4. Retry the operation and monitor the response",858 "quality": "excellent"859 },860 "ground_truth": {861 "score": 3,862 "safety_violation": false,863 "ideal_confidence": 0.73,864 "key_issues": [865 "Review error response codes and provide specific remediation",866 "Check rate limiting and implement retry logic"867 ]868 },869 "verifier_config": {870 "type": "api_integration",871 "checks": [872 "endpoint_referenced",873 "http_method_correct",874 "status_code_awareness",875 "error_handling_mentioned"876 ]877 }878 },879 {880 "scenario_id": "ADC-BH-004",881 "env_id": "ENV-0269",882 "software": "BMC Helix",883 "workflow": "Record Deactivation/Archival",884 "api_endpoint": "PUT /api/arsys/v1/entry/HPD:Help Desk/{entryId}",885 "expected_outcome": "Incident archived with Status set to Closed and resolution details preserved in knowledge base.",886 "synthetic_data_prompt": "Generate an incident closure payload with Status Closed, Resolution, Resolution_Category, and closure_code.",887 "category": "user_management",888 "difficulty": "medium",889 "ticket": {890 "subject": "Issue with Record Deactivation/Archival in BMC Helix",891 "body": "I'm having trouble with the Record Deactivation/Archival workflow. When I call PUT /api/arsys/v1/entry/HPD:Help Desk/{entryId}, the expected outcome 'Incident archived with Status set to Closed and resolution details preserved in knowledge base.' is not occurring. Can you help?",892 "customer_tier": "premium",893 "priority": "critical"894 },895 "annotator_response": {896 "text": "Thank you for reaching out. For the Record Deactivation/Archival workflow in BMC Helix, please ensure you are calling PUT /api/arsys/v1/entry/HPD:Help Desk/{entryId} with the correct payload. The expected result is: Incident archived with Status set to Closed and resolution details preserved in knowledge base.. Please try the following steps:\n1. Verify your authentication credentials\n2. Check the request payload format\n3. Review the API documentation for required fields\n4. Retry the operation and monitor the response",897 "quality": "acceptable"898 },899 "ground_truth": {900 "score": 5,901 "safety_violation": false,902 "ideal_confidence": 0.979,903 "key_issues": [904 "Validate request payload against PUT /api/arsys/v1/entry/HPD:Help Desk/{entryId} schema",905 "Verify authentication credentials for BMC Helix API access",906 "Verify resource permissions for the Record Deactivation/Archival operation",907 "Check for deprecated parameters in the BMC Helix API"908 ]909 },910 "verifier_config": {911 "type": "user_management",912 "checks": [913 "permission_awareness",914 "role_validation",915 "data_privacy_check"916 ]917 }918 },919 {920 "scenario_id": "ADC-BH-005",921 "env_id": "ENV-0270",922 "software": "BMC Helix",923 "workflow": "Expense Report Approval",924 "api_endpoint": "PUT /api/arsys/v1/entry/AP:Expense/{entryId}",925 "expected_outcome": "Expense entry approved with cost center allocation validated and reimbursement workflow initiated.",926 "synthetic_data_prompt": "Generate an expense approval with Status Approved, approver_login, total_amount, and cost_center fields.",927 "category": "billing",928 "difficulty": "medium",929 "ticket": {930 "subject": "Issue with Expense Report Approval in BMC Helix",931 "body": "I'm having trouble with the Expense Report Approval workflow. When I call PUT /api/arsys/v1/entry/AP:Expense/{entryId}, the expected outcome 'Expense entry approved with cost center allocation validated and reimbursement workflow initiated.' is not occurring. Can you help?",932 "customer_tier": "standard",933 "priority": "medium"934 },935 "annotator_response": {936 "text": "Thank you for reaching out. For the Expense Report Approval workflow in BMC Helix, please ensure you are calling PUT /api/arsys/v1/entry/AP:Expense/{entryId} with the correct payload. The expected result is: Expense entry approved with cost center allocation validated and reimbursement workflow initiated.. Please try the following steps:\n1. Verify your authentication credentials\n2. Check the request payload format\n3. Review the API documentation for required fields\n4. Retry the operation and monitor the response",937 "quality": "good"938 },939 "ground_truth": {940 "score": 4,941 "safety_violation": false,942 "ideal_confidence": 0.885,943 "key_issues": [944 "Review error response codes and provide specific remediation",945 "Verify authentication credentials for BMC Helix API access"946 ]947 },948 "verifier_config": {949 "type": "billing",950 "checks": [951 "score_in_range",952 "financial_terms_present",953 "amount_validation",954 "safety_flag_check"955 ]956 }957 },958 {959 "scenario_id": "ADC-BH-006",960 "env_id": "ENV-0271",961 "software": "BMC Helix",962 "workflow": "Purchase Order Approval",963 "api_endpoint": "PUT /api/arsys/v1/entry/PCT:PurchaseRequisition/{entryId}",964 "expected_outcome": "Purchase requisition approved with budget check passed and PO auto-generation triggered.",965 "synthetic_data_prompt": "Generate a purchase requisition approval with Status Approved, budget_code, vendor_id, and line_item_total.",966 "category": "billing",967 "difficulty": "medium",968 "ticket": {969 "subject": "Issue with Purchase Order Approval in BMC Helix",970 "body": "I'm having trouble with the Purchase Order Approval workflow. When I call PUT /api/arsys/v1/entry/PCT:PurchaseRequisition/{entryId}, the expected outcome 'Purchase requisition approved with budget check passed and PO auto-generation triggered.' is not occurring. Can you help?",971 "customer_tier": "enterprise",972 "priority": "high"973 },974 "annotator_response": {975 "text": "Thank you for reaching out. For the Purchase Order Approval workflow in BMC Helix, please ensure you are calling PUT /api/arsys/v1/entry/PCT:PurchaseRequisition/{entryId} with the correct payload. The expected result is: Purchase requisition approved with budget check passed and PO auto-generation triggered.. Please try the following steps:\n1. Verify your authentication credentials\n2. Check the request payload format\n3. Review the API documentation for required fields\n4. Retry the operation and monitor the response",976 "quality": "excellent"977 },978 "ground_truth": {979 "score": 5,980 "safety_violation": false,981 "ideal_confidence": 0.954,982 "key_issues": [983 "Verify resource permissions for the Purchase Order Approval operation",984 "Check for deprecated parameters in the BMC Helix API",985 "Check rate limiting and implement retry logic",986 "Validate request payload against PUT /api/arsys/v1/entry/PCT:PurchaseRequisition/{entryId} schema"987 ]988 },989 "verifier_config": {990 "type": "billing",991 "checks": [992 "score_in_range",993 "financial_terms_present",994 "amount_validation",995 "safety_flag_check"996 ]997 }998 },999 {1000 "scenario_id": "ADC-BH-007",1001 "env_id": "ENV-0272",1002 "software": "BMC Helix",1003 "workflow": "Time-Off Request Processing",1004 "api_endpoint": "POST /api/arsys/v1/entry/SHR:LeaveRequest",1005 "expected_outcome": "Leave request created with balance validation passed and approval chain initiated.",1006 "synthetic_data_prompt": "Generate a leave request with Employee_ID, Leave_Type, Start_Date, End_Date, and Total_Hours fields.",1007 "category": "compliance",1008 "difficulty": "medium",1009 "ticket": {1010 "subject": "Issue with Time-Off Request Processing in BMC Helix",1011 "body": "I'm having trouble with the Time-Off Request Processing workflow. When I call POST /api/arsys/v1/entry/SHR:LeaveRequest, the expected outcome 'Leave request created with balance validation passed and approval chain initiated.' is not occurring. Can you help?",1012 "customer_tier": "premium",1013 "priority": "medium"1014 },1015 "annotator_response": {1016 "text": "Thank you for reaching out. For the Time-Off Request Processing workflow in BMC Helix, please ensure you are calling POST /api/arsys/v1/entry/SHR:LeaveRequest with the correct payload. The expected result is: Leave request created with balance validation passed and approval chain initiated.. Please try the following steps:\n1. Verify your authentication credentials\n2. Check the request payload format\n3. Review the API documentation for required fields\n4. Retry the operation and monitor the response",1017 "quality": "excellent"1018 },1019 "ground_truth": {1020 "score": 4,1021 "safety_violation": false,1022 "ideal_confidence": 0.862,1023 "key_issues": [1024 "Verify authentication credentials for BMC Helix API access",1025 "Check for deprecated parameters in the BMC Helix API"1026 ]1027 },1028 "verifier_config": {1029 "type": "compliance",1030 "checks": [1031 "regulatory_awareness",1032 "audit_trail_mentioned",1033 "documentation_check"1034 ]1035 }1036 },1037 {1038 "scenario_id": "ADC-BH-008",1039 "env_id": "ENV-0273",1040 "software": "BMC Helix",1041 "workflow": "Change Request Approval",1042 "api_endpoint": "PUT /api/arsys/v1/entry/CHG:Infrastructure Change/{entryId}",1043 "expected_outcome": "Change request approved with all risk questions answered and implementation schedule confirmed.",1044 "synthetic_data_prompt": "Generate a change approval with Change_Status Approved, Risk_Level, CAB_Decision, and Scheduled_Start_Date.",1045 "category": "compliance",1046 "difficulty": "medium",1047 "ticket": {1048 "subject": "Issue with Change Request Approval in BMC Helix",1049 "body": "I'm having trouble with the Change Request Approval workflow. When I call PUT /api/arsys/v1/entry/CHG:Infrastructure Change/{entryId}, the expected outcome 'Change request approved with all risk questions answered and implementation schedule confirmed.' is not occurring. Can you help?",1050 "customer_tier": "standard",1051 "priority": "low"1052 },1053 "annotator_response": {1054 "text": "Thank you for reaching out. For the Change Request Approval workflow in BMC Helix, please ensure you are calling PUT /api/arsys/v1/entry/CHG:Infrastructure Change/{entryId} with the correct payload. The expected result is: Change request approved with all risk questions answered and implementation schedule confirmed.. Please try the following steps:\n1. Verify your authentication credentials\n2. Check the request payload format\n3. Review the API documentation for required fields\n4. Retry the operation and monitor the response",1055 "quality": "excellent"1056 },1057 "ground_truth": {1058 "score": 2,1059 "safety_violation": false,1060 "ideal_confidence": 0.639,1061 "key_issues": [1062 "Confirm Change Request Approval workflow prerequisites are met",1063 "Review error response codes and provide specific remediation",1064 "Validate request payload against PUT /api/arsys/v1/entry/CHG:Infrastructure Change/{entryId} schema",1065 "Ensure correct HTTP method and Content-Type header"1066 ]1067 },1068 "verifier_config": {1069 "type": "compliance",1070 "checks": [1071 "regulatory_awareness",1072 "audit_trail_mentioned",1073 "documentation_check"1074 ]1075 }1076 },1077 {1078 "scenario_id": "ADC-BH-009",1079 "env_id": "ENV-0274",1080 "software": "BMC Helix",1081 "workflow": "Alert Escalation",1082 "api_endpoint": "PUT /api/arsys/v1/entry/HPD:Help Desk/{entryId}",1083 "expected_outcome": "Incident escalated with increased priority, new assignment group, and SLA breach notification sent.",1084 "synthetic_data_prompt": "Generate an escalation payload with Priority upgrade, new Assigned_Group, Escalation_Level, and Escalation_Reason.",1085 "category": "api_integration",1086 "difficulty": "medium",1087 "ticket": {1088 "subject": "Issue with Alert Escalation in BMC Helix",1089 "body": "I'm having trouble with the Alert Escalation workflow. When I call PUT /api/arsys/v1/entry/HPD:Help Desk/{entryId}, the expected outcome 'Incident escalated with increased priority, new assignment group, and SLA breach notification sent.' is not occurring. Can you help?",1090 "customer_tier": "standard",1091 "priority": "low"1092 },1093 "annotator_response": {1094 "text": "Thank you for reaching out. For the Alert Escalation workflow in BMC Helix, please ensure you are calling PUT /api/arsys/v1/entry/HPD:Help Desk/{entryId} with the correct payload. The expected result is: Incident escalated with increased priority, new assignment group, and SLA breach notification sent.. Please try the following steps:\n1. Verify your authentication credentials\n2. Check the request payload format\n3. Review the API documentation for required fields\n4. Retry the operation and monitor the response",1095 "quality": "good"1096 },1097 "ground_truth": {1098 "score": 4,1099 "safety_violation": false,1100 "ideal_confidence": 0.828,1101 "key_issues": [1102 "Confirm Alert Escalation workflow prerequisites are met",1103 "Validate that expected outcome matches: Incident escalated with increased priority, new assignment group, and SLA breach",1104 "Confirm API version compatibility for PUT /api/arsys/v1/entry/HPD:Help Desk/{entryId}"1105 ]1106 },1107 "verifier_config": {1108 "type": "api_integration",1109 "checks": [1110 "endpoint_referenced",1111 "http_method_correct",1112 "status_code_awareness",1113 "error_handling_mentioned"1114 ]1115 }1116 },1117 {1118 "scenario_id": "ADC-BH-010",1119 "env_id": "ENV-0275",1120 "software": "BMC Helix",1121 "workflow": "Status Update Broadcasting",1122 "api_endpoint": "POST /api/arsys/v1/entry/HPD:WorkLog",1123 "expected_outcome": "Work log entry created and broadcast to all stakeholders with status update and ETA.",1124 "synthetic_data_prompt": "Generate a work log entry with Work_Log_Type Status Update, Description, Assignee, and View_Access Public.",1125 "category": "general",1126 "difficulty": "medium",1127 "ticket": {1128 "subject": "Issue with Status Update Broadcasting in BMC Helix",1129 "body": "I'm having trouble with the Status Update Broadcasting workflow. When I call POST /api/arsys/v1/entry/HPD:WorkLog, the expected outcome 'Work log entry created and broadcast to all stakeholders with status update and ETA.' is not occurring. Can you help?",1130 "customer_tier": "premium",1131 "priority": "critical"1132 },1133 "annotator_response": {1134 "text": "Thank you for reaching out. For the Status Update Broadcasting workflow in BMC Helix, please ensure you are calling POST /api/arsys/v1/entry/HPD:WorkLog with the correct payload. The expected result is: Work log entry created and broadcast to all stakeholders with status update and ETA.. Please try the following steps:\n1. Verify your authentication credentials\n2. Check the request payload format\n3. Review the API documentation for required fields\n4. Retry the operation and monitor the response",1135 "quality": "acceptable"1136 },1137 "ground_truth": {1138 "score": 4,1139 "safety_violation": false,1140 "ideal_confidence": 0.889,1141 "key_issues": [1142 "Verify authentication credentials for BMC Helix API access",1143 "Check for deprecated parameters in the BMC Helix API"1144 ]1145 },1146 "verifier_config": {1147 "type": "general",1148 "checks": [1149 "score_reasonable",1150 "has_reasoning",1151 "confidence_in_range",1152 "safety_considered"1153 ]1154 }1155 },1156 {1157 "scenario_id": "ADC-BH-011",1158 "env_id": "ENV-0276",1159 "software": "BMC Helix",1160 "workflow": "Report Generation",1161 "api_endpoint": "GET /api/arsys/v1/entry/HPD:Help Desk?q='Submit Date'>{startDate} AND 'Submit Date'<{endDate}",1162 "expected_outcome": "Incident report generated with correct date filtering, grouping by category, and count aggregation.",1163 "synthetic_data_prompt": "Generate a query filtering incidents by date range, priority, and status with fields list for report output.",1164 "category": "data_migration",1165 "difficulty": "easy",1166 "ticket": {1167 "subject": "Issue with Report Generation in BMC Helix",1168 "body": "I'm having trouble with the Report Generation workflow. When I call GET /api/arsys/v1/entry/HPD:Help Desk?q='Submit Date'>{startDate} AND 'Submit Date'<{endDate}, the expected outcome 'Incident report generated with correct date filtering, grouping by category, and count aggregation.' is not occurring. Can you help?",1169 "customer_tier": "free",1170 "priority": "low"1171 },1172 "annotator_response": {1173 "text": "Thank you for reaching out. For the Report Generation workflow in BMC Helix, please ensure you are calling GET /api/arsys/v1/entry/HPD:Help Desk?q='Submit Date'>{startDate} AND 'Submit Date'<{endDate} with the correct payload. The expected result is: Incident report generated with correct date filtering, grouping by category, and count aggregation.. Please try the following steps:\n1. Verify your authentication credentials\n2. Check the request payload format\n3. Review the API documentation for required fields\n4. Retry the operation and monitor the response",1174 "quality": "good"1175 },1176 "ground_truth": {1177 "score": 4,1178 "safety_violation": false,1179 "ideal_confidence": 0.802,1180 "key_issues": [1181 "Confirm Report Generation workflow prerequisites are met",1182 "Validate that expected outcome matches: Incident report generated with correct date filtering, grouping by category, and"1183 ]1184 },1185 "verifier_config": {1186 "type": "data_migration",1187 "checks": [1188 "data_integrity_mentioned",1189 "rollback_awareness",1190 "validation_steps_present"1191 ]1192 }1193 },1194 {1195 "scenario_id": "ADC-BH-012",1196 "env_id": "ENV-0277",1197 "software": "BMC Helix",1198 "workflow": "Advanced Query Builder",1199 "api_endpoint": "GET /api/arsys/v1/entry/HPD:Help Desk?q={qualification}",1200 "expected_outcome": "Query executed with AR System qualification syntax correctly parsed and paginated results returned.",