Aigenthix/Graph_RAG
0
1#!/usr/bin/env python32"""3Generate HTML report comparing RAG modes4Visualizes benchmark results with charts and tables5"""6 7import json8from pathlib import Path9from datetime import datetime10import statistics11 12 13class RAGComparisonReporter:14 """Generate HTML reports from benchmark data"""15 16 def __init__(self, results_file: str = "data/benchmark_results.json"):17 """Initialize reporter with benchmark results"""18 self.results_file = Path(results_file)19 self.data = None20 self.load_results()21 22 def load_results(self):23 """Load benchmark results from JSON"""24 if self.results_file.exists():25 with open(self.results_file, "r") as f:26 self.data = json.load(f)27 else:28 print(f"Results file not found: {self.results_file}")29 self.data = {}30 31 def generate_html(self, output_file: str = "rag_comparison_report.html"):32 """Generate complete HTML report"""33 34 html_content = f"""35<!DOCTYPE html>36<html lang="en">37<head>38 <meta charset="UTF-8">39 <meta name="viewport" content="width=device-width, initial-scale=1.0">40 <title>RAG Comparison Report</title>41 <script src="https://cdn.jsdelivr.net/npm/chart.js@4.4.0/dist/chart.umd.js"></script>42 <style>43 * {{44 margin: 0;45 padding: 0;46 box-sizing: border-box;47 }}48 49 body {{50 font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif;51 background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);52 min-height: 100vh;53 padding: 40px 20px;54 }}55 56 .container {{57 max-width: 1400px;58 margin: 0 auto;59 }}60 61 .header {{62 background: white;63 border-radius: 12px;64 padding: 40px;65 margin-bottom: 30px;66 box-shadow: 0 10px 30px rgba(0,0,0,0.2);67 text-align: center;68 }}69 70 .header h1 {{71 color: #2d3e50;72 font-size: 2.5rem;73 margin-bottom: 10px;74 }}75 76 .header p {{77 color: #7f8c8d;78 font-size: 1rem;79 }}80 81 .timestamp {{82 color: #95a5a6;83 font-size: 0.9rem;84 margin-top: 10px;85 }}86 87 .grid {{88 display: grid;89 grid-template-columns: repeat(auto-fit, minmax(350px, 1fr));90 gap: 30px;91 margin-bottom: 40px;92 }}93 94 .card {{95 background: white;96 border-radius: 12px;97 padding: 30px;98 box-shadow: 0 10px 30px rgba(0,0,0,0.2);99 }}100 101 .card h2 {{102 color: #2d3e50;103 margin-bottom: 20px;104 font-size: 1.5rem;105 border-bottom: 2px solid #667eea;106 padding-bottom: 15px;107 }}108 109 .metric {{110 display: flex;111 justify-content: space-between;112 align-items: center;113 padding: 15px 0;114 border-bottom: 1px solid #ecf0f1;115 }}116 117 .metric:last-child {{118 border-bottom: none;119 }}120 121 .metric-label {{122 color: #7f8c8d;123 font-weight: 500;124 }}125 126 .metric-value {{127 color: #2d3e50;128 font-weight: bold;129 font-size: 1.1rem;130 }}131 132 .badge {{133 display: inline-block;134 background: #667eea;135 color: white;136 padding: 8px 12px;137 border-radius: 6px;138 font-size: 0.85rem;139 font-weight: bold;140 margin: 5px 2px;141 }}142 143 .badge.simple {{ background: #3498db; }}144 .badge.agentic {{ background: #e74c3c; }}145 .badge.graph {{ background: #2ecc71; }}146 147 .chart-container {{148 position: relative;149 height: 300px;150 margin: 20px 0;151 }}152 153 table {{154 width: 100%;155 border-collapse: collapse;156 margin-top: 20px;157 }}158 159 th {{160 background: #667eea;161 color: white;162 padding: 12px;163 text-align: left;164 font-weight: 600;165 }}166 167 td {{168 padding: 12px;169 border-bottom: 1px solid #ecf0f1;170 }}171 172 tr:hover {{173 background: #f8f9fa;174 }}175 176 .comparison {{177 background: white;178 border-radius: 12px;179 padding: 30px;180 box-shadow: 0 10px 30px rgba(0,0,0,0.2);181 margin-bottom: 30px;182 }}183 184 .comparison h2 {{185 color: #2d3e50;186 margin-bottom: 20px;187 font-size: 1.5rem;188 border-bottom: 2px solid #667eea;189 padding-bottom: 15px;190 }}191 192 .winner {{193 background: #d4edda;194 border-left: 4px solid #28a745;195 padding: 15px;196 margin: 10px 0;197 border-radius: 6px;198 }}199 200 .winner strong {{201 color: #155724;202 }}203 204 .footer {{205 background: white;206 border-radius: 12px;207 padding: 20px;208 text-align: center;209 color: #7f8c8d;210 margin-top: 40px;211 box-shadow: 0 10px 30px rgba(0,0,0,0.2);212 }}213 214 @media (max-width: 768px) {{215 .grid {{216 grid-template-columns: 1fr;217 }}218 219 .header h1 {{220 font-size: 1.8rem;221 }}222 }}223 224 .section {{225 margin-bottom: 50px;226 }}227 228 .section-title {{229 color: white;230 font-size: 1.8rem;231 margin-bottom: 20px;232 font-weight: bold;233 }}234 </style>235</head>236<body>237 <div class="container">238 {self._generate_header()}239 {self._generate_executive_summary()}240 {self._generate_detailed_results()}241 {self._generate_comparisons()}242 {self._generate_recommendations()}243 {self._generate_footer()}244 </div>245 246 <script>247 {self._generate_charts_script()}248 </script>249</body>250</html>251 """252 253 output_path = Path(output_file)254 with open(output_path, "w") as f:255 f.write(html_content)256 257 print(f"✓ Report generated: {output_path}")258 return output_path259 260 def _generate_header(self) -> str:261 """Generate report header"""262 timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")263 return f"""264 <div class="header">265 <h1>🔬 RAG Comparison Report</h1>266 <p>Comprehensive benchmark of Simple, Agentic, and Graph RAG modes</p>267 <div class="timestamp">Generated: {timestamp}</div>268 </div>269 """270 271 def _generate_executive_summary(self) -> str:272 """Generate executive summary"""273 if not self.data.get("benchmarks"):274 return ""275 276 summary = "<div class='section'><div class='section-title'>📊 Executive Summary</div>"277 summary += "<div class='grid'>"278 279 for model, results in self.data["benchmarks"].items():280 modes = results.get("rag_modes", {})281 282 summary += """283 <div class="card">284 <h2>Quick Metrics</h2>285 """286 287 # Latency comparison288 if modes:289 latencies = [290 (mode, m["latency"]["mean_ms"]) for mode, m in modes.items()291 ]292 latencies.sort(key=lambda x: x[1])293 294 summary += "<div class='metric'>"295 summary += "<span class='metric-label'>⚡ Fastest</span>"296 summary += f"<span class='badge {latencies[0][0]}'>{latencies[0][0].upper()}</span>"297 summary += f"<span class='metric-value'>{latencies[0][1]:.0f}ms</span>"298 summary += "</div>"299 300 # Cost comparison301 costs = [302 (mode, m["cost_per_query_usd"]) for mode, m in modes.items()303 ]304 costs.sort(key=lambda x: x[1])305 306 summary += "<div class='metric'>"307 summary += "<span class='metric-label'>💰 Cheapest</span>"308 summary += f"<span class='badge {costs[0][0]}'>{costs[0][0].upper()}</span>"309 summary += f"<span class='metric-value'>${costs[0][1]:.4f}</span>"310 summary += "</div>"311 312 # Most comprehensive313 sources = [314 (mode, m["sources_avg"]) for mode, m in modes.items()315 ]316 sources.sort(key=lambda x: x[1], reverse=True)317 318 summary += "<div class='metric'>"319 summary += "<span class='metric-label'>📚 Most Sources</span>"320 summary += f"<span class='badge {sources[0][0]}'>{sources[0][0].upper()}</span>"321 summary += f"<span class='metric-value'>{sources[0][1]:.1f}</span>"322 summary += "</div>"323 324 summary += "</div>"325 326 summary += "</div></div>"327 return summary328 329 def _generate_detailed_results(self) -> str:330 """Generate detailed results tables"""331 if not self.data.get("benchmarks"):332 return ""333 334 html = "<div class='section'><div class='section-title'>📈 Detailed Results</div>"335 336 for model, results in self.data["benchmarks"].items():337 modes = results.get("rag_modes", {})338 339 html += """340 <div class="card">341 <h2>Model: {}</h2>342 <table>343 <tr>344 <th>RAG Mode</th>345 <th>Latency (ms)</th>346 <th>Tokens/Query</th>347 <th>Sources</th>348 <th>Cost/Query</th>349 </tr>350 """.format(model)351 352 for mode, data in modes.items():353 html += f"""354 <tr>355 <td><span class="badge {mode}">{mode.upper()}</span></td>356 <td>{data['latency']['mean_ms']:.0f}</td>357 <td>{data['tokens']['total_avg']:.0f}</td>358 <td>{data['sources_avg']:.1f}</td>359 <td>${data['cost_per_query_usd']:.4f}</td>360 </tr>361 """362 363 html += """364 </table>365 </div>366 """367 368 html += "</div>"369 return html370 371 def _generate_comparisons(self) -> str:372 """Generate comparison analysis"""373 if not self.data.get("benchmarks"):374 return ""375 376 html = "<div class='section'><div class='section-title'>🏆 Comparisons</div>"377 378 for model, results in self.data["benchmarks"].items():379 comparisons = results.get("comparisons", {})380 381 html += """382 <div class="comparison">383 <h2>Head-to-Head Comparison</h2>384 """385 386 if "fastest" in comparisons:387 fastest = comparisons["fastest"]388 html += f"""389 <div class="winner">390 <strong>⚡ Fastest Response:</strong>391 <span class="badge {fastest['mode']}">{fastest['mode'].upper()}</span>392 - {fastest['latency_ms']:.0f}ms average393 </div>394 """395 396 if "cheapest" in comparisons:397 cheapest = comparisons["cheapest"]398 html += f"""399 <div class="winner">400 <strong>💰 Most Cost-Effective:</strong>401 <span class="badge {cheapest['mode']}">{cheapest['mode'].upper()}</span>402 - ${cheapest['cost_usd']:.4f} per query403 </div>404 """405 406 if "most_comprehensive" in comparisons:407 comprehensive = comparisons["most_comprehensive"]408 html += f"""409 <div class="winner">410 <strong>📚 Most Comprehensive:</strong>411 <span class="badge {comprehensive['mode']}">{comprehensive['mode'].upper()}</span>412 - {comprehensive['sources_avg']:.1f} sources average413 </div>414 """415 416 html += """417 </div>418 """419 420 html += "</div>"421 return html422 423 def _generate_recommendations(self) -> str:424 """Generate recommendations"""425 return """426 <div class="section"><div class="section-title">💡 Recommendations</div>427 <div class="comparison">428 <h2>When to Use Each Mode</h2>429 430 <div class="winner" style="border-left-color: #3498db;">431 <strong>Simple RAG</strong> - Best for:432 <ul style="margin-left: 20px; margin-top: 10px;">433 <li>Real-time applications with <1s latency requirement</li>434 <li>Direct fact lookup and Q&A</li>435 <li>Cost-sensitive deployments</li>436 <li>High-throughput scenarios (>1000 qps)</li>437 </ul>438 </div>439 440 <div class="winner" style="border-left-color: #e74c3c;">441 <strong>Agentic RAG</strong> - Best for:442 <ul style="margin-left: 20px; margin-top: 10px;">443 <li>Complex multi-step reasoning</li>444 <li>Questions requiring tool use and sub-queries</li>445 <li>Scenarios where accuracy is critical (>90%)</li>446 <li>Domain-specific expert systems</li>447 </ul>448 </div>449 450 <div class="winner" style="border-left-color: #2ecc71;">451 <strong>Graph RAG</strong> - Best for:452 <ul style="margin-left: 20px; margin-top: 10px;">453 <li>Knowledge extraction from complex documents</li>454 <li>Entity and relationship-based queries</li>455 <li>Balanced latency vs. accuracy (1-2s response)</li>456 <li>Knowledge bases and expert systems</li>457 </ul>458 </div>459 </div></div>460 """461 462 def _generate_footer(self) -> str:463 """Generate footer"""464 return """465 <div class="footer">466 <p>Generated by RAG Comparison Reporter | Data-driven RAG mode selection</p>467 <p style="margin-top: 10px; font-size: 0.9rem;">468 Simple RAG • Agentic RAG • Graph RAG469 </p>470 </div>471 """472 473 def _generate_charts_script(self) -> str:474 """Generate Chart.js scripts for visualizations"""475 if not self.data.get("benchmarks"):476 return ""477 478 # Extract data for charts479 script = """480 // Charts would be generated here481 // Currently showing static data in tables above482 console.log('RAG Comparison Report loaded');483 """484 485 return script486 487 488def main():489 """Generate report from benchmark results"""490 import argparse491 492 parser = argparse.ArgumentParser(description="Generate RAG comparison HTML report")493 parser.add_argument(494 "--input",495 default="data/benchmark_results.json",496 help="Input benchmark results file",497 )498 parser.add_argument(499 "--output",500 default="rag_comparison_report.html",501 help="Output HTML file",502 )503 504 args = parser.parse_args()505 506 try:507 reporter = RAGComparisonReporter(args.input)508 output_file = reporter.generate_html(args.output)509 print(f"✅ Report generated successfully: {output_file}")510 print(f" Open in browser: open {output_file}")511 512 except Exception as e:513 print(f"❌ Error generating report: {e}")514 import traceback515 traceback.print_exc()516 return 1517 518 return 0519 520 521if __name__ == "__main__":522 exit(main())523