2008robocode-crypto/code-generation-system

1#!/usr/bin/env python32"""3Run evaluation suite and generate detailed reports.4"""5 6import sys7import json8from pathlib import Path9from datetime import datetime10 11# Add paths12sys.path.insert(0, str(Path(__file__).parent / "src"))13sys.path.insert(0, str(Path(__file__).parent / "evaluation"))14 15from evaluator import EvaluationFramework16 17 18def main():19    """Run evaluation suite."""20    21    print("\n" + "="*80)22    print("📊 EVALUATION FRAMEWORK - CODE GENERATION SYSTEM")23    print("="*80)24    print("\nRunning comprehensive evaluation on 20 test prompts...")25    print("(10 real products + 10 edge cases)\n")26    27    # Run evaluation28    evaluator = EvaluationFramework(use_llm=False)29    report = evaluator.run_evaluation(dataset_size="full")30    31    # Print formatted report32    evaluator.print_report()33    34    # Save detailed report to file35    timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")36    report_file = Path(__file__).parent / f"evaluation_report_{timestamp}.json"37    38    with open(report_file, 'w') as f:39        json.dump(report, f, indent=2, default=str)40    41    print(f"📁 Detailed report saved to: {report_file}\n")42    43    # Print key takeaways44    summary = report["summary"]45    print("\n" + "="*80)46    print("📈 KEY PERFORMANCE INDICATORS")47    print("="*80)48    49    print(f"\n✓ Success Rate: {summary.get('success_rate', 0):.1f}%")50    print(f"✓ Executable Rate: {summary.get('executable_rate', 0):.1f}%")51    print(f"✓ Average Generation Time: {summary.get('avg_latency', 0):.2f}s")52    print(f"✓ Average Retries: {summary.get('avg_retries', 0):.1f}")53    54    # Performance by category55    print(f"\n📁 Performance by Category:")56    for category, stats in summary.get("by_category", {}).items():57        success_pct = (stats["success"] / stats["total"] * 100) if stats["total"] > 0 else 058        print(f"   {category:15} {stats['success']:2}/{stats['total']} ({success_pct:5.1f}%)")59    60    # Cost analysis61    cost = report["cost_analysis"]62    print(f"\n💰 Cost vs Quality Analysis:")63    print(f"   Config Size (avg): {cost.get('avg_config_size_bytes', 0):.0f} bytes")64    print(f"   Latency (avg): {cost.get('avg_generation_latency_seconds', 0):.2f}s")65    print(f"   Quality Score: {cost.get('quality_score', 0):.1f}/100")66    print(f"   Efficiency Score: {cost.get('efficiency_score', 0):.1f}/100")67    print(f"   Recommendation: {cost.get('recommendation', 'N/A')}")68    69    print("\n" + "="*80)70 71 72if __name__ == "__main__":73    main()74