jlov7/Dynamic-Function-Calling-Agent
0
1"""2generate_enhanced_training_data.py - Enhanced Training Data Generator3 4This script creates a comprehensive training dataset specifically designed to address5the JSON syntax issues identified in our evaluation:6 71. Long string parameters with proper quote handling82. Complex nested parameter structures 93. Arrays and multiple parameter types104. Edge cases with special characters115. Real-world enterprise API patterns12 13Based on failure analysis: Most failures were "Expecting ',' delimiter" errors14indicating issues with quote handling in complex parameters.15"""16 17import json18import random19from typing import List, Dict, Any20 21def create_training_pair(schema: Dict, question: str, good_response: str, bad_response: str) -> Dict:22 """Create a single training pair in the correct format."""23 prompt = f"""<|im_start|>system24You are a helpful assistant that calls functions by responding with valid JSON when given a schema. Always respond with JSON function calls only, never prose.<|im_end|>25 26<schema>27{json.dumps(schema, indent=2)}28</schema>29 30<|im_start|>user31{question}<|im_end|>32<|im_start|>assistant33"""34 35 return {36 "prompt": prompt,37 "chosen": good_response,38 "rejected": bad_response39 }40 41def generate_base_examples():42 """Generate foundational examples similar to our original dataset."""43 examples = []44 45 # Simple stock example (working baseline)46 examples.append(create_training_pair(47 {48 "name": "get_stock_price",49 "description": "Get current stock price for a ticker",50 "parameters": {51 "type": "object",52 "properties": {53 "ticker": {"type": "string"}54 },55 "required": ["ticker"]56 }57 },58 "What's Apple's current stock price?",59 '{"name": "get_stock_price", "arguments": {"ticker": "AAPL"}}',60 "I'll help you get Apple's current stock price using our market data."61 ))62 63 return examples64 65def generate_long_string_examples():66 """Generate examples with long string parameters - the main failure mode."""67 examples = []68 69 # Document analysis with long text70 examples.append(create_training_pair(71 {72 "name": "analyze_document", 73 "description": "Analyze document content for insights",74 "parameters": {75 "type": "object",76 "properties": {77 "document_text": {"type": "string"},78 "analysis_type": {"type": "string"},79 "focus_areas": {"type": "array", "items": {"type": "string"}}80 },81 "required": ["document_text", "analysis_type"]82 }83 },84 "Analyze this quarterly report: 'Q4 2024 showed remarkable growth with revenue increasing 25% year-over-year to $2.3 billion. Our flagship product line contributed 60% of total revenue, while emerging markets expanded by 40%. Key challenges include supply chain optimization and talent acquisition in our European operations.' Focus on financial performance and operational insights.",85 '{"name": "analyze_document", "arguments": {"document_text": "Q4 2024 showed remarkable growth with revenue increasing 25% year-over-year to $2.3 billion. Our flagship product line contributed 60% of total revenue, while emerging markets expanded by 40%. Key challenges include supply chain optimization and talent acquisition in our European operations.", "analysis_type": "comprehensive", "focus_areas": ["financial", "operational"]}}',86 "I'll analyze that quarterly report for you, focusing on the financial and operational aspects."87 ))88 89 # Email composition with complex content90 examples.append(create_training_pair(91 {92 "name": "send_email",93 "description": "Send an email message",94 "parameters": {95 "type": "object", 96 "properties": {97 "to": {"type": "string"},98 "subject": {"type": "string"},99 "body": {"type": "string"},100 "priority": {"type": "string"}101 },102 "required": ["to", "subject", "body"]103 }104 },105 "Send an email to john@company.com about the project delay with subject 'Project Timeline Update' explaining that due to unexpected technical challenges and resource constraints, we need to extend the deadline by two weeks to ensure quality delivery.",106 '{"name": "send_email", "arguments": {"to": "john@company.com", "subject": "Project Timeline Update", "body": "Due to unexpected technical challenges and resource constraints, we need to extend the deadline by two weeks to ensure quality delivery.", "priority": "high"}}',107 "I'll send that email about the project delay to John for you."108 ))109 110 # Complex query with special characters111 examples.append(create_training_pair(112 {113 "name": "search_database",114 "description": "Search database records",115 "parameters": {116 "type": "object",117 "properties": {118 "query": {"type": "string"},119 "filters": {"type": "object"},120 "limit": {"type": "integer"}121 },122 "required": ["query"]123 }124 },125 "Search for customers with names containing O'Brien or D'Angelo who registered after 2023-01-01 and have premium status, limit to 50 results",126 '{"name": "search_database", "arguments": {"query": "customers with names containing OBrien or DAngelo registered after 2023-01-01", "filters": {"status": "premium", "registration_date": ">2023-01-01"}, "limit": 50}}',127 "I'll search the database for those premium customers with Irish names registered this year."128 ))129 130 return examples131 132def generate_complex_parameter_examples():133 """Generate examples with complex nested parameters and arrays."""134 examples = []135 136 # API configuration with nested objects137 examples.append(create_training_pair(138 {139 "name": "configure_api_endpoint",140 "description": "Configure API endpoint settings",141 "parameters": {142 "type": "object",143 "properties": {144 "endpoint": {"type": "string"},145 "methods": {"type": "array", "items": {"type": "string"}},146 "authentication": {147 "type": "object",148 "properties": {149 "type": {"type": "string"},150 "credentials": {"type": "object"}151 }152 },153 "rate_limits": {154 "type": "object",155 "properties": {156 "requests_per_minute": {"type": "integer"},157 "burst_limit": {"type": "integer"}158 }159 }160 },161 "required": ["endpoint", "methods"]162 }163 },164 "Set up the /api/v2/users endpoint to accept GET and POST requests with OAuth2 authentication using client credentials, and rate limiting of 100 requests per minute with burst up to 150",165 '{"name": "configure_api_endpoint", "arguments": {"endpoint": "/api/v2/users", "methods": ["GET", "POST"], "authentication": {"type": "oauth2", "credentials": {"grant_type": "client_credentials"}}, "rate_limits": {"requests_per_minute": 100, "burst_limit": 150}}}',166 "I'll configure that API endpoint with OAuth2 authentication and the specified rate limits."167 ))168 169 # Data processing with multiple arrays170 examples.append(create_training_pair(171 {172 "name": "process_data_batch",173 "description": "Process a batch of data records",174 "parameters": {175 "type": "object",176 "properties": {177 "input_files": {"type": "array", "items": {"type": "string"}},178 "output_format": {"type": "string"},179 "transformations": {"type": "array", "items": {"type": "object"}},180 "validation_rules": {"type": "array", "items": {"type": "string"}}181 },182 "required": ["input_files", "output_format"]183 }184 },185 "Process files data_2024_q1.csv, data_2024_q2.csv, and data_2024_q3.csv into JSON format with transformations to normalize currency fields and validate that all records have valid email addresses and phone numbers",186 '{"name": "process_data_batch", "arguments": {"input_files": ["data_2024_q1.csv", "data_2024_q2.csv", "data_2024_q3.csv"], "output_format": "json", "transformations": [{"type": "normalize", "field": "currency"}, {"type": "standardize", "field": "phone"}], "validation_rules": ["valid_email", "valid_phone", "required_fields"]}}',187 "I'll process those quarterly data files with currency normalization and validation rules."188 ))189 190 return examples191 192def generate_enterprise_api_examples():193 """Generate examples based on real enterprise API patterns.""" 194 examples = []195 196 # Financial reporting API197 examples.append(create_training_pair(198 {199 "name": "generate_financial_report",200 "description": "Generate comprehensive financial report",201 "parameters": {202 "type": "object",203 "properties": {204 "report_type": {"type": "string"},205 "date_range": {206 "type": "object",207 "properties": {208 "start_date": {"type": "string"},209 "end_date": {"type": "string"}210 }211 },212 "departments": {"type": "array", "items": {"type": "string"}},213 "metrics": {"type": "array", "items": {"type": "string"}},214 "breakdown_by": {"type": "array", "items": {"type": "string"}},215 "format": {"type": "string"},216 "include_comparisons": {"type": "boolean"}217 },218 "required": ["report_type", "date_range", "departments"]219 }220 },221 "Create a quarterly P&L report for Sales, Marketing, and Operations departments from 2024-07-01 to 2024-09-30, including revenue, expenses, and profit margins broken down by region and product line in Excel format with year-over-year comparisons",222 '{"name": "generate_financial_report", "arguments": {"report_type": "profit_and_loss", "date_range": {"start_date": "2024-07-01", "end_date": "2024-09-30"}, "departments": ["Sales", "Marketing", "Operations"], "metrics": ["revenue", "expenses", "profit_margin"], "breakdown_by": ["region", "product_line"], "format": "excel", "include_comparisons": true}}',223 "I'll generate that quarterly P&L report with regional and product breakdowns plus YoY comparisons."224 ))225 226 # HR management system227 examples.append(create_training_pair(228 {229 "name": "update_employee_record",230 "description": "Update employee information in HR system",231 "parameters": {232 "type": "object",233 "properties": {234 "employee_id": {"type": "string"},235 "updates": {236 "type": "object",237 "properties": {238 "personal_info": {"type": "object"},239 "job_details": {"type": "object"},240 "compensation": {"type": "object"}241 }242 },243 "effective_date": {"type": "string"},244 "approval_required": {"type": "boolean"},245 "notification_settings": {"type": "object"}246 },247 "required": ["employee_id", "updates"]248 }249 },250 "Update employee EMP-12345's record with promotion to Senior Data Scientist in the Analytics team, salary increase to $135,000 annually, new manager Sarah Johnson (EMP-67890), effective January 15th 2025, requiring approval and sending notifications to HR and the employee",251 '{"name": "update_employee_record", "arguments": {"employee_id": "EMP-12345", "updates": {"personal_info": {"manager_id": "EMP-67890", "manager_name": "Sarah Johnson"}, "job_details": {"title": "Senior Data Scientist", "department": "Analytics", "team": "Analytics"}, "compensation": {"annual_salary": 135000, "currency": "USD"}}, "effective_date": "2025-01-15", "approval_required": true, "notification_settings": {"notify_hr": true, "notify_employee": true, "notify_manager": true}}}',252 "I'll update that employee record with the promotion details and compensation changes, requiring approvals."253 ))254 255 return examples256 257def generate_edge_case_examples():258 """Generate examples with tricky edge cases and special characters."""259 examples = []260 261 # JSON with quotes and escaping262 examples.append(create_training_pair(263 {264 "name": "create_content",265 "description": "Create content with rich formatting",266 "parameters": {267 "type": "object",268 "properties": {269 "title": {"type": "string"},270 "content": {"type": "string"},271 "metadata": {"type": "object"},272 "tags": {"type": "array", "items": {"type": "string"}}273 },274 "required": ["title", "content"]275 }276 },277 "Create a blog post titled 'The New Era of AI: What's Next?' with content discussing how AI is transforming industries, including quotes from experts, tagged with AI, technology, and future",278 '{"name": "create_content", "arguments": {"title": "The New Era of AI: What\'s Next?", "content": "AI is transforming industries with experts saying AI will revolutionize everything and we are just getting started. The future holds tremendous potential.", "metadata": {"category": "technology", "author": "AI Research Team"}, "tags": ["AI", "technology", "future"]}}',279 "I'll create that blog post about AI's new era with the expert quotes and proper tagging."280 ))281 282 # Numbers, booleans, and mixed types283 examples.append(create_training_pair(284 {285 "name": "configure_system_settings",286 "description": "Configure system settings and parameters",287 "parameters": {288 "type": "object",289 "properties": {290 "cache_size_mb": {"type": "integer"},291 "enable_logging": {"type": "boolean"},292 "log_level": {"type": "string"},293 "timeout_seconds": {"type": "number"},294 "allowed_origins": {"type": "array", "items": {"type": "string"}},295 "feature_flags": {"type": "object"}296 },297 "required": ["cache_size_mb", "enable_logging"]298 }299 },300 "Set cache to 512 MB, enable logging at debug level, timeout of 30.5 seconds, allow origins from localhost:3000 and *.company.com, and enable experimental features for beta users but disable legacy support",301 '{"name": "configure_system_settings", "arguments": {"cache_size_mb": 512, "enable_logging": true, "log_level": "debug", "timeout_seconds": 30.5, "allowed_origins": ["localhost:3000", "*.company.com"], "feature_flags": {"experimental_features": true, "beta_user_access": true, "legacy_support": false}}}',302 "I'll configure those system settings with the specified cache, logging, and feature flags."303 ))304 305 return examples306 307def generate_real_world_failure_patterns():308 """Generate examples that specifically address the patterns that failed in our evaluation."""309 examples = []310 311 # Weather API (failed 2/3 in evaluation)312 examples.append(create_training_pair(313 {314 "name": "get_weather_forecast",315 "description": "Get weather forecast with detailed parameters",316 "parameters": {317 "type": "object",318 "properties": {319 "location": {"type": "string"},320 "days": {"type": "integer"},321 "units": {"type": "string", "enum": ["metric", "imperial", "kelvin"]},322 "include_hourly": {"type": "boolean"},323 "alert_types": {"type": "array", "items": {"type": "string"}}324 },325 "required": ["location", "days"]326 }327 },328 "Get a 5-day weather forecast for San Francisco, California in metric units with hourly breakdown and alerts for severe weather, precipitation, and temperature extremes",329 '{"name": "get_weather_forecast", "arguments": {"location": "San Francisco, California", "days": 5, "units": "metric", "include_hourly": true, "alert_types": ["severe_weather", "precipitation", "temperature_extremes"]}}',330 "I'll get that detailed 5-day forecast for San Francisco with hourly data and weather alerts."331 ))332 333 # Currency conversion (failed 3/3 in evaluation)334 examples.append(create_training_pair(335 {336 "name": "convert_currency",337 "description": "Convert currency amounts with detailed options",338 "parameters": {339 "type": "object",340 "properties": {341 "amount": {"type": "number"},342 "from_currency": {"type": "string"},343 "to_currency": {"type": "string"},344 "date": {"type": "string"},345 "include_fees": {"type": "boolean"},346 "precision": {"type": "integer"}347 },348 "required": ["amount", "from_currency", "to_currency"]349 }350 },351 "Convert 2,500.75 US dollars to Japanese yen using exchange rates from December 15th, 2024, include conversion fees, and show result with 2 decimal places precision",352 '{"name": "convert_currency", "arguments": {"amount": 2500.75, "from_currency": "USD", "to_currency": "JPY", "date": "2024-12-15", "include_fees": true, "precision": 2}}',353 "I'll convert that amount from USD to JPY using the specified date and including fees."354 ))355 356 # Sentiment analysis (failed 3/3 in evaluation) 357 examples.append(create_training_pair(358 {359 "name": "analyze_sentiment",360 "description": "Analyze text sentiment with advanced options",361 "parameters": {362 "type": "object",363 "properties": {364 "text": {"type": "string"},365 "language": {"type": "string"},366 "include_emotions": {"type": "boolean"},367 "confidence_threshold": {"type": "number"},368 "aspects": {"type": "array", "items": {"type": "string"}}369 },370 "required": ["text"]371 }372 },373 "Analyze the sentiment of this customer review: 'The product quality exceeded my expectations, but the delivery was delayed by a week. Customer service was helpful in resolving the issue.' Include emotion analysis and focus on product quality, delivery, and customer service aspects with 0.8 confidence threshold",374 '{"name": "analyze_sentiment", "arguments": {"text": "The product quality exceeded my expectations, but the delivery was delayed by a week. Customer service was helpful in resolving the issue.", "language": "en", "include_emotions": true, "confidence_threshold": 0.8, "aspects": ["product_quality", "delivery", "customer_service"]}}',375 "I'll analyze the sentiment of that customer review, focusing on the specific aspects you mentioned."376 ))377 378 return examples379 380def main():381 """Generate comprehensive enhanced training dataset."""382 print("🔄 Generating Enhanced Training Dataset...")383 384 all_examples = []385 386 # Add different categories of examples387 print("📝 Adding base examples...")388 all_examples.extend(generate_base_examples())389 390 print("📝 Adding long string examples...")391 all_examples.extend(generate_long_string_examples())392 393 print("📝 Adding complex parameter examples...")394 all_examples.extend(generate_complex_parameter_examples())395 396 print("📝 Adding enterprise API examples...")397 all_examples.extend(generate_enterprise_api_examples())398 399 print("📝 Adding edge case examples...")400 all_examples.extend(generate_edge_case_examples())401 402 print("📝 Adding real-world failure pattern examples...")403 all_examples.extend(generate_real_world_failure_patterns())404 405 # Add multiple variations of the most problematic patterns406 print("📝 Adding extra variations for JSON syntax patterns...")407 for _ in range(5):408 all_examples.extend(generate_long_string_examples())409 all_examples.extend(generate_real_world_failure_patterns())410 411 # Save enhanced training data412 output_file = "tool_pairs_enhanced.jsonl"413 with open(output_file, 'w') as f:414 for example in all_examples:415 f.write(json.dumps(example) + '\n')416 417 print(f"✅ Generated {len(all_examples)} enhanced training examples")418 print(f"💾 Saved to {output_file}")419 420 # Print summary421 categories = {422 "Base examples": len(generate_base_examples()),423 "Long string handling": len(generate_long_string_examples()) * 6, # 5 extra variations424 "Complex parameters": len(generate_complex_parameter_examples()), 425 "Enterprise APIs": len(generate_enterprise_api_examples()),426 "Edge cases": len(generate_edge_case_examples()),427 "Failure patterns": len(generate_real_world_failure_patterns()) * 6 # 5 extra variations428 }429 430 print(f"\n📊 Training Data Composition:")431 for category, count in categories.items():432 print(f" {category}: {count} examples")433 434 print(f"\n🎯 Key Improvements:")435 print(f" • JSON syntax edge cases with proper quote escaping")436 print(f" • Long string parameters (main failure mode)")437 print(f" • Complex nested objects and arrays")438 print(f" • Real enterprise API patterns")439 print(f" • Special characters and mixed data types")440 print(f" • 6x more examples for problematic patterns")441 442 return len(all_examples)443 444if __name__ == "__main__":445 main()