salim0986/graph-bug-ai
0
1import sys2from fastapi import FastAPI, BackgroundTasks, HTTPException, Request3from fastapi.responses import JSONResponse4from fastapi.middleware.cors import CORSMiddleware5from pydantic import BaseModel, Field6from typing import List, Optional7from .parser import UniversalParser, EXTENSION_MAP8from .graph_builder import GraphBuilder9from .vector_builder import VectorBuilder10from .analyzer import (11 CodeAnalyzer,12 PRAnalysisRequest,13 FileAnalysisRequest,14 DiffAnalysisRequest,15 PRAnalysisResult,16 FileAnalysisResult17)18from .context_builder import ContextBuilder, PRContext19from .workflow import CodeReviewWorkflow, WorkflowConfig, create_review_workflow20from .logger import setup_logger, LogContext21from .incremental_ingest import ingest_repo_incremental22from sentence_transformers import SentenceTransformer23from .github_client import GitHubClient, GitHubConfig24from slowapi import Limiter, _rate_limit_exceeded_handler25from slowapi.util import get_remote_address26from slowapi.errors import RateLimitExceeded27from .security_utils import (28 safe_join,29 sanitize_repo_id,30 sanitize_owner_repo,31 sanitize_file_path,32 sanitize_error_message,33 validate_request_size34)35from .rate_limiter import RateLimiter, RateLimitMiddleware36from .cleanup import DataCleanup, IngestionCheckpoint37import git38import shutil39import os40import sys41import asyncio42from concurrent.futures import ThreadPoolExecutor, as_completed43import time44import uuid45import httpx46import jwt47from datetime import datetime, timezone48 49# Initialize rate limiters50limiter = Limiter(key_func=get_remote_address)51advanced_limiter = RateLimiter()52 53app = FastAPI(title="Graph Bug AI Service", version="2.0.0")54app.state.limiter = limiter55app.add_exception_handler(RateLimitExceeded, _rate_limit_exceeded_handler)56 57# M10: OpenTelemetry instrumentation (no-op when packages are absent)58from .observability import setup_otel as _setup_otel59_setup_otel(app)60 61 62# ---------------------------------------------------------------------------63# M12: service-to-service HMAC auth (logic lives in hmac_auth.py)64# ---------------------------------------------------------------------------65 66from .hmac_auth import verify_service_hmac as _verify_service_hmac, HMAC_PROTECTED_PREFIXES as _HMAC_PROTECTED67 68 69@app.middleware("http")70async def _service_auth_middleware(request: Request, call_next):71 """Reject requests to protected endpoints that lack a valid HMAC signature."""72 secret = os.getenv("AI_SERVICE_SECRET", "")73 if secret and any(request.url.path.startswith(p) for p in _HMAC_PROTECTED):74 body_bytes = await request.body() # Starlette caches this in request._body75 sig = request.headers.get("X-Service-Signature", "")76 if not _verify_service_hmac(body_bytes, sig, secret):77 logger.warning(78 f"[M12] Invalid service signature on {request.url.path} "79 f"from {request.client}"80 )81 return JSONResponse({"error": "invalid_service_signature"}, status_code=401)82 return await call_next(request)83 84logger = setup_logger(__name__)85 86# Configure CORS87ALLOWED_ORIGINS = os.getenv("ALLOWED_ORIGINS", "http://localhost:3000").split(",")88app.add_middleware(89 CORSMiddleware,90 allow_origins=ALLOWED_ORIGINS,91 allow_credentials=True,92 allow_methods=["GET", "POST", "DELETE"],93 allow_headers=["*"],94 max_age=360095)96 97# Add advanced rate limiting middleware98app.add_middleware(RateLimitMiddleware, limiter=advanced_limiter)99 100 101# M10: request_id middleware — propagates X-Request-ID through every request102@app.middleware("http")103async def _request_id_middleware(request: Request, call_next):104 request_id = request.headers.get("X-Request-ID") or str(uuid.uuid4())105 request.state.request_id = request_id106 response = await call_next(request)107 response.headers["X-Request-ID"] = request_id108 return response109 110@app.on_event("startup")111async def startup_event():112 """Log service status on startup."""113 logger.info("=" * 80)114 logger.info("🚀 Graph Bug AI Service Starting...")115 logger.info(f" Version: 2.0.0")116 logger.info(f" GitHub Client: {'✅ Ready' if github_client else '❌ Not configured'}")117 logger.info(f" Rate Limiting: ✅ Enabled")118 logger.info(f" Data Cleanup: ✅ Enabled")119 # Fix #10: warn clearly when service-to-service auth is not configured.120 if not os.getenv("AI_SERVICE_SECRET"):121 logger.warning(122 "[M12] AI_SERVICE_SECRET is not set — service-to-service HMAC auth is DISABLED. "123 "Set this in production to prevent unauthenticated review triggers."124 )125 logger.info("=" * 80)126 127 128@app.on_event("shutdown")129async def shutdown_event():130 """Flush LangFuse traces so they are not lost on SIGTERM."""131 from .observability import get_langfuse132 lf = get_langfuse()133 if lf:134 try:135 lf.flush()136 logger.info("[M10] LangFuse traces flushed on shutdown")137 except Exception as exc:138 logger.warning(f"[M10] LangFuse flush failed: {exc}")139 140# --- CONFIGURATION ---141 142IGNORE_DIRS = {143 ".git", ".github", ".vscode", ".idea",144 "node_modules", "venv", "env",145 "dist", "build", "out", "target", "bin", "obj",146 "__pycache__", "coverage", "tmp", "temp", "migrations"147}148 149# Initialize Singletons150parser = UniversalParser()151embed_model = SentenceTransformer('all-MiniLM-L6-v2')152 153# Load config154from .config import NEO4J_URI, NEO4J_USER, NEO4J_PASSWORD, QDRANT_URL, QDRANT_API_KEY155 156# Initialize databases with config157graph_db = GraphBuilder(NEO4J_URI, (NEO4J_USER, NEO4J_PASSWORD))158vector_db = VectorBuilder(QDRANT_URL, embed_model, api_key=QDRANT_API_KEY)159 160# Initialize cleanup service161cleanup_service = DataCleanup(graph_db, vector_db)162 163# Initialize Code Analyzer and Context Builder164code_analyzer = CodeAnalyzer(graph_db, vector_db, parser)165context_builder = ContextBuilder(code_analyzer, graph_db, vector_db)166 167# Initialize GitHub Client (Phase 5.3)168try:169 logger.info("=" * 80)170 logger.info("🔑 Initializing GitHub Client...")171 logger.info("=" * 80)172 sys.stdout.flush()173 174 github_config = GitHubConfig.from_env()175 github_client = GitHubClient(github_config)176 177 # Test the token generation to ensure credentials are valid178 logger.info("Testing GitHub App credentials...")179 sys.stdout.flush()180 181 try:182 # Try to generate a JWT to verify private key is valid183 test_jwt = jwt.encode(184 {185 "iat": int(time.time()),186 "exp": int(time.time()) + 60,187 "iss": github_config.app_id188 },189 github_config.private_key,190 algorithm="RS256"191 )192 logger.info(f"✅ JWT generation successful ({len(test_jwt)} chars)")193 sys.stdout.flush()194 except Exception as jwt_error:195 logger.error(f"❌ JWT generation failed: {jwt_error}")196 logger.error(" This indicates the private key is invalid or malformed")197 sys.stdout.flush()198 raise199 200 logger.info("=" * 80)201 logger.info("✅ GitHub Client Initialized Successfully!")202 logger.info(f" App ID: {github_config.app_id}")203 logger.info(f" Private key loaded: {len(github_config.private_key)} bytes")204 logger.info(" PR comments will be posted automatically")205 logger.info(" Repository cloning will use GitHub App authentication")206 logger.info("=" * 80)207 sys.stdout.flush()208 209except Exception as e:210 logger.error("=" * 80)211 logger.error(f"❌ GITHUB CLIENT INITIALIZATION FAILED")212 logger.error(f" Error: {e}")213 logger.error(f" Error type: {type(e).__name__}")214 logger.error("=" * 80)215 logger.error("Consequences:")216 logger.error(" - PR comments will NOT be posted")217 logger.error(" - Private repositories CANNOT be cloned")218 logger.error(" - Only public repositories will work")219 logger.error("=" * 80)220 logger.error("Required environment variables:")221 logger.error(" - GITHUB_APP_ID: Your GitHub App ID")222 logger.error(" - GITHUB_PRIVATE_KEY: Your GitHub App private key (PEM format)")223 logger.error(" - Or GITHUB_PRIVATE_KEY_PATH: Path to private key file")224 logger.error("=" * 80)225 logger.error("Full traceback:", exc_info=True)226 sys.stdout.flush()227 github_client = None228 229# Initialize LangGraph Workflow (Phase 4) with dependencies230review_workflow = create_review_workflow(231 context_builder=context_builder232)233 234# --- MODELS ---235 236class RepoRequest(BaseModel):237 repo_url: str = Field(..., max_length=500, pattern=r'^https://github\.com/[\w-]+/[\w.-]+(.git)?$')238 repo_id: str = Field(..., min_length=1, max_length=200, pattern=r'^[a-zA-Z0-9_-]+$')239 installation_id: str = Field(..., pattern=r'^\d+$', max_length=20)240 incremental: Optional[bool] = False # Enable incremental ingestion241 last_commit: Optional[str] = Field(None, min_length=7, max_length=40, pattern=r'^[a-fA-F0-9]+$') # Previous commit SHA for incremental242 243class QueryRequest(BaseModel):244 repo_id: str = Field(..., min_length=1, max_length=200, pattern=r'^[a-zA-Z0-9_-]+$')245 query: str = Field(..., min_length=1, max_length=1000)246 247def process_repo_task(req: RepoRequest):248 """The Heavy Worker Function: Clones -> Parses -> Indexes"""249 logger.info("=" * 80)250 logger.info(f"🔄 STARTING FULL INGESTION")251 logger.info(f" Repo URL: {req.repo_url}")252 logger.info(f" Repo ID: {req.repo_id}")253 logger.info("=" * 80)254 sys.stdout.flush()255 256 # Sanitize repo_id to prevent path traversal257 try:258 safe_repo_id = sanitize_repo_id(req.repo_id)259 except ValueError as e:260 logger.error(f"Invalid repo_id: {e}")261 return262 263 # Use safe_join to prevent directory traversal264 try:265 local_path = safe_join("./temp_repos", safe_repo_id)266 except ValueError as e:267 logger.error(f"Path traversal attempt detected: {e}")268 return269 270 # 1. Clean & Clone271 if os.path.exists(local_path):272 shutil.rmtree(local_path)273 274 try:275 logger.info(f"Cloning repository from {req.repo_url}")276 logger.info(f"GitHub client available: {github_client is not None}")277 sys.stdout.flush()278 279 # FIX: Use GitHub App installation token for private repo access280 if github_client:281 try:282 logger.info(f"Attempting to get installation token for installation {req.installation_id}")283 sys.stdout.flush()284 285 # Get installation access token286 token = github_client.get_installation_token(int(req.installation_id))287 288 if not token or len(token) < 10:289 raise ValueError(f"Invalid token received: {len(token) if token else 0} chars")290 291 logger.info(f"✅ Successfully obtained installation token ({len(token)} chars)")292 sys.stdout.flush()293 294 # Build authenticated clone URL295 # Format: https://x-access-token:TOKEN@github.com/owner/repo.git296 auth_url = req.repo_url.replace(297 "https://github.com/",298 f"https://x-access-token:{token}@github.com/"299 )300 301 logger.info(f"Cloning with GitHub App authentication (installation {req.installation_id})")302 sys.stdout.flush()303 304 # Clone with timeout to prevent hanging305 git.Repo.clone_from(auth_url, local_path, depth=1) # Shallow clone for speed306 307 logger.info(f"✅ Successfully cloned repository with authentication")308 sys.stdout.flush()309 310 except Exception as auth_error:311 logger.error("=" * 80)312 logger.error(f"❌ GITHUB APP AUTHENTICATION FAILED")313 logger.error(f" Installation ID: {req.installation_id}")314 logger.error(f" Error: {auth_error}")315 logger.error(f" Error type: {type(auth_error).__name__}")316 logger.error("=" * 80)317 logger.error("Full traceback:", exc_info=True)318 sys.stdout.flush()319 320 # DO NOT fallback to public clone for private repos321 # This will fail with "could not read Username" error322 raise Exception(323 f"Failed to clone repository with GitHub App authentication. "324 f"This is likely a private repository. Error: {auth_error}"325 )326 else:327 # No GitHub client available - only works for public repos328 logger.error("=" * 80)329 logger.error(f"❌ GITHUB CLIENT NOT AVAILABLE")330 logger.error(f" Cannot clone private repositories without GitHub App credentials")331 logger.error(f" Set GITHUB_APP_ID and GITHUB_PRIVATE_KEY in environment")332 logger.error("=" * 80)333 sys.stdout.flush()334 335 raise Exception(336 "GitHub client not initialized. Cannot clone repository. "337 "Please set GITHUB_APP_ID and GITHUB_PRIVATE_KEY environment variables."338 )339 340 except Exception as e:341 logger.error("=" * 80)342 logger.error(f"❌ GIT CLONE FAILED")343 logger.error(f" Repo URL: {req.repo_url}")344 logger.error(f" Installation ID: {req.installation_id}")345 logger.error(f" Local path: {local_path}")346 logger.error(f" Error: {e}")347 logger.error("=" * 80)348 logger.error("Common causes:")349 logger.error(" 1. Private repo without valid GitHub App token")350 logger.error(" 2. GitHub App not installed on repository")351 logger.error(" 3. Invalid installation ID")352 logger.error(" 4. Repository doesn't exist or was deleted")353 logger.error(" 5. Network connectivity issues")354 logger.error("=" * 80)355 sys.stdout.flush()356 return357 358 start_time = time.time()359 360 # 2. Prepare Vector DB361 vector_db.ensure_collection()362 363 logger.info(f"🔍 Collecting files to process...")364 365 # Collect all valid files first366 files_to_process = []367 for root, dirs, files in os.walk(local_path):368 dirs[:] = [d for d in dirs if d not in IGNORE_DIRS]369 370 for file in files:371 file_path = os.path.join(root, file)372 filename = os.path.basename(file)373 _, ext = os.path.splitext(filename)374 is_valid = filename in EXTENSION_MAP or ext in EXTENSION_MAP375 376 if is_valid:377 files_to_process.append(file_path)378 379 total_files = len(files_to_process)380 logger.info(f"📊 Found {total_files} code files to process")381 382 if total_files == 0:383 logger.warning(f"No valid code files found in {req.repo_id}")384 shutil.rmtree(local_path)385 return386 387 # 3. Parallel Processing - Parse files in batches388 indexed_count = 0389 file_count = 0390 batch_size = 20 # Parallel workers for better throughput391 392 def parse_single_file(file_path):393 """Parse a single file and return results"""394 try:395 captures, code_bytes = parser.parse_file(file_path)396 if captures:397 rel_path = os.path.relpath(file_path, local_path)398 return (file_path, rel_path, captures, code_bytes, None)399 return None400 except Exception as e:401 return (file_path, None, None, None, str(e))402 403 logger.info(f"⚡ Starting parallel processing with {batch_size} workers...")404 405 # Process files in parallel batches406 with ThreadPoolExecutor(max_workers=batch_size) as executor:407 for i in range(0, total_files, batch_size):408 batch = files_to_process[i:i + batch_size]409 batch_start = time.time()410 411 # Submit batch with timeout protection412 futures = {}413 for fp in batch:414 future = executor.submit(parse_single_file, fp)415 futures[future] = fp416 417 # Collect results with timeout418 batch_results = []419 try:420 for future in as_completed(futures, timeout=120): # 120s timeout per batch421 try:422 result = future.result(timeout=10) # 10s to get result423 if result:424 batch_results.append(result)425 except Exception as e:426 fp = futures[future]427 logger.warning(f"Error processing {os.path.basename(fp)}: {e}")428 except Exception as batch_error:429 logger.warning(f"Batch timeout or error: {batch_error}")430 431 # Process successful parses432 vectors_batch = [] # Collect vectors for batch insert433 batch_file_count = 0434 435 for result in batch_results:436 file_path, rel_path, captures, code_bytes, error = result437 438 if error:439 logger.warning(f"Parse error in {os.path.basename(file_path)}: {error}")440 continue441 442 if not rel_path:443 continue444 445 batch_file_count += 1446 447 # Update graph448 try:449 graph_db.process_file_nodes(req.repo_id, rel_path, captures, code_bytes)450 except Exception as e:451 logger.error(f"Graph error for {rel_path}: {e}")452 continue453 454 # Collect vectors for batch insert455 for node, capture_name in captures:456 lines_of_code = node.end_point[0] - node.start_point[0]457 if lines_of_code < 3:458 continue459 460 node_type = node.type461 ALLOWED_TYPES = [462 "function_definition", "function_declaration", "function_item",463 "method_definition", "method_declaration",464 "class_definition", "class_declaration", 465 "interface_declaration", "impl_item",466 "table", "block"467 ]468 469 _, ext = os.path.splitext(rel_path)470 is_logic_node = node_type in ALLOWED_TYPES471 is_config_file = ext in [".yaml", ".yml", ".toml", ".tf", ".dockerfile"]472 473 if not (is_logic_node or (is_config_file and lines_of_code > 0)):474 continue475 476 try:477 func_code = code_bytes[node.start_byte : node.end_byte].decode("utf8", errors="ignore")478 first_line = func_code.splitlines()[0] if func_code else ""479 func_name = first_line[:100].strip()480 481 vectors_batch.append({482 "repo_id": req.repo_id,483 "func_name": func_name,484 "func_code": func_code,485 "file_path": rel_path,486 "start_line": node.start_point[0]487 })488 489 except Exception as e:490 logger.warning(f"Vector prep error: {e}")491 continue492 493 file_count += batch_file_count494 495 # Batch insert vectors496 if vectors_batch:497 try:498 logger.info(f"💾 Inserting {len(vectors_batch)} vectors into Qdrant...")499 for item in vectors_batch:500 vector_db.ingest_function_chunk(501 repo_id=item["repo_id"],502 func_name=item["func_name"],503 func_code=item["func_code"],504 file_path=item["file_path"],505 start_line=item["start_line"]506 )507 indexed_count += len(vectors_batch)508 logger.info(f"✅ Successfully inserted {len(vectors_batch)} vectors")509 except Exception as e:510 logger.error(f"Batch vector insert error: {e}")511 512 # Progress update513 batch_time = time.time() - batch_start514 progress_pct = ((i + len(batch)) / total_files) * 100515 files_per_sec = len(batch) / batch_time if batch_time > 0 else 0516 logger.info(517 f"📊 Progress: {progress_pct:.1f}% | "518 f"{file_count}/{total_files} files | "519 f"{indexed_count} functions | "520 f"{files_per_sec:.1f} files/sec"521 )522 523 # 4. Build dependencies524 logger.info(f"🔗 Building dependency graph...")525 sys.stdout.flush()526 try:527 graph_db.build_dependencies(req.repo_id)528 except Exception as e:529 logger.error(f"Error building dependencies: {e}")530 sys.stdout.flush()531 532 elapsed_time = time.time() - start_time533 logger.info("=" * 80)534 logger.info(f"✅ FULL INGESTION COMPLETE")535 logger.info(f" Time: {elapsed_time:.1f}s")536 logger.info(f" Files: {file_count}")537 logger.info(f" Functions indexed: {indexed_count}")538 logger.info(f" Speed: {file_count/elapsed_time:.1f} files/sec")539 logger.info("=" * 80)540 sys.stdout.flush()541 542 # Cleanup543 try:544 shutil.rmtree(local_path)545 except:546 pass547 548 549# --- ENDPOINTS ---550 551@app.post("/ingest")552async def ingest_repo(request: Request, req: RepoRequest, background_tasks: BackgroundTasks):553 """554 Ingest repository with optional incremental mode555 556 Modes:557 - Full ingestion: incremental=False (default) - processes all files558 - Incremental: incremental=True with last_commit - only processes changed files559 560 Performance:561 - Full: ~10-60s depending on repo size562 - Incremental: ~2-10s for typical updates563 """564 # Force immediate log output565 logger.info("=" * 80)566 logger.info(f"📥 INGESTION REQUEST RECEIVED")567 logger.info(f" Repo: {req.repo_url}")568 logger.info(f" Repo ID: {req.repo_id}")569 logger.info(f" Installation ID: {req.installation_id}")570 logger.info(f" Incremental: {req.incremental}")571 logger.info(f" Last Commit: {req.last_commit or 'N/A'}")572 logger.info("=" * 80)573 sys.stdout.flush() # Force immediate flush574 575 if req.incremental and req.last_commit:576 # Incremental mode - much faster!577 background_tasks.add_task(process_repo_incremental_task, req)578 response = {579 "status": "queued",580 "repo_id": req.repo_id,581 "mode": "incremental",582 "from_commit": req.last_commit583 }584 else:585 # Full ingestion mode586 background_tasks.add_task(process_repo_task, req)587 response = {588 "status": "queued",589 "repo_id": req.repo_id,590 "mode": "full"591 }592 593 logger.info(f"✅ Ingestion queued: {response}")594 sys.stdout.flush() # Force immediate flush595 return response596 597async def process_repo_incremental_task(req: RepoRequest):598 """599 High-performance incremental ingestion600 Only processes files that changed since last_commit601 """602 logger.info("=" * 80)603 logger.info(f"🔄 STARTING INCREMENTAL INGESTION")604 logger.info(f" Repo URL: {req.repo_url}")605 logger.info(f" Repo ID: {req.repo_id}")606 logger.info(f" From commit: {req.last_commit}")607 logger.info("=" * 80)608 sys.stdout.flush()609 610 local_path = f"./temp_repos/{req.repo_id}"611 612 try:613 # Clone or pull repository with authentication614 if os.path.exists(local_path):615 logger.info(f"Repository exists locally, pulling latest changes...")616 sys.stdout.flush()617 618 repo = git.Repo(local_path)619 620 # Update remote URL with token if GitHub client available621 if github_client:622 try:623 logger.info(f"Updating remote URL with fresh authentication token")624 sys.stdout.flush()625 626 token = github_client.get_installation_token(int(req.installation_id))627 628 if not token or len(token) < 10:629 raise ValueError(f"Invalid token received: {len(token) if token else 0} chars")630 631 auth_url = req.repo_url.replace(632 "https://github.com/",633 f"https://x-access-token:{token}@github.com/"634 )635 636 # Update remote URL637 repo.remotes.origin.set_url(auth_url)638 logger.info(f"✅ Remote URL updated with authentication")639 sys.stdout.flush()640 641 except Exception as e:642 logger.error(f"❌ Failed to update remote with auth: {e}")643 sys.stdout.flush()644 raise645 646 origin = repo.remotes.origin647 origin.pull()648 logger.info(f"✅ Successfully pulled latest changes")649 sys.stdout.flush()650 651 else:652 logger.info(f"Repository not found locally, cloning from {req.repo_url}")653 sys.stdout.flush()654 655 # Use authenticated URL for private repos656 if github_client:657 try:658 logger.info(f"Obtaining installation token for cloning")659 sys.stdout.flush()660 661 token = github_client.get_installation_token(int(req.installation_id))662 663 if not token or len(token) < 10:664 raise ValueError(f"Invalid token received: {len(token) if token else 0} chars")665 666 logger.info(f"✅ Token obtained ({len(token)} chars)")667 sys.stdout.flush()668 669 auth_url = req.repo_url.replace(670 "https://github.com/",671 f"https://x-access-token:{token}@github.com/"672 )673 674 git.Repo.clone_from(auth_url, local_path, depth=1)675 logger.info(f"✅ Successfully cloned repository with authentication")676 sys.stdout.flush()677 678 except Exception as e:679 logger.error("=" * 80)680 logger.error(f"❌ CLONE FAILED WITH AUTHENTICATION")681 logger.error(f" Installation ID: {req.installation_id}")682 logger.error(f" Error: {e}")683 logger.error("=" * 80)684 sys.stdout.flush()685 raise686 else:687 logger.error("=" * 80)688 logger.error(f"❌ GITHUB CLIENT NOT AVAILABLE")689 logger.error(f" Cannot clone private repositories")690 logger.error("=" * 80)691 sys.stdout.flush()692 raise Exception("GitHub client not initialized")693 694 # Run incremental ingestion695 stats = await ingest_repo_incremental(696 repo_id=req.repo_id,697 repo_url=req.repo_url,698 local_path=local_path,699 parser=parser,700 graph_db=graph_db,701 vector_db=vector_db,702 ignore_dirs=IGNORE_DIRS,703 last_commit=req.last_commit704 )705 706 logger.info("=" * 80)707 logger.info(f"✅ INCREMENTAL INGESTION COMPLETE")708 logger.info(f" Files processed: {stats['files_processed']}")709 logger.info(f" Files deleted: {stats.get('files_deleted', 0)}")710 logger.info(f" Nodes added: {stats['nodes_added']}")711 logger.info(f" Nodes deleted: {stats.get('nodes_deleted', 0)}")712 logger.info(f" Vectors updated: {stats['vectors_updated']}")713 logger.info("=" * 80)714 sys.stdout.flush()715 716 except Exception as e:717 logger.error("=" * 80)718 logger.error(f"❌ INCREMENTAL INGESTION FAILED")719 logger.error(f" Repo: {req.repo_url}")720 logger.error(f" Error: {e}")721 logger.error("=" * 80)722 logger.error(f"Full traceback:", exc_info=True)723 sys.stdout.flush()724 725 # Cleanup (optional - keep for next incremental update)726 # try:727 # shutil.rmtree(local_path)728 # except:729 # pass730 731# --- ENDPOINTS ---732 733@app.post("/ingest_legacy")734async def ingest_repo_legacy(req: RepoRequest, background_tasks: BackgroundTasks):735 """Legacy full ingestion endpoint (kept for backwards compatibility)"""736 background_tasks.add_task(process_repo_task, req)737 return {"status": "queued", "repo_id": req.repo_id}738 739@app.post("/query")740async def search_repo(http_request: Request, req: QueryRequest):741 """742 GraphRAG Search:743 1. Vector Search finds the relevant 'entry points'.744 2. Graph Search expands those points to find dependencies.745 """746 # 1. Vector Search (Find top 3 matches)747 results = vector_db.search_similar(req.repo_id, req.query)748 749 data = []750 for hit in results:751 payload = hit.payload752 753 # 2. Graph Expansion (Fetch context)754 # "What does this function call? What does it rely on?"755 dependencies = graph_db.get_dependencies(756 repo_id=req.repo_id, 757 file_path=payload.get("file"), 758 start_line=payload.get("start_line")759 )760 761 data.append({762 "score": hit.score,763 "type": "Primary Match",764 "name": payload.get("name"),765 "file": payload.get("file"),766 "start_line": payload.get("start_line"),767 "code": payload.get("raw_code"),768 "related_dependencies": dependencies # <--- The Graph Value769 })770 771 return {"results": data}772 773 774@app.delete("/repos/{repo_id}")775async def delete_repo_data(repo_id: str):776 """777 Cleanup Endpoint:778 Called when a user uninstalls the app or deletes a repository.779 Wipes data from both Vector DB and Graph DB.780 """781 logger.info(f"Received delete request for repo {repo_id}")782 783 # 1. Delete from Vector DB (Search Index)784 vector_db.delete_repo(repo_id)785 786 # 2. Delete from Graph DB (Structure)787 graph_db.delete_repo(repo_id)788 789 return {"status": "success", "message": f"Data for {repo_id} wiped."}790 791 792# ============================================================================793# CODE ANALYSIS ENDPOINTS (Phase 3)794# ============================================================================795 796@app.post("/analyze/pr", response_model=PRAnalysisResult)797async def analyze_pr(request: PRAnalysisRequest):798 """799 Analyze an entire Pull Request800 801 Returns comprehensive analysis including:802 - Code issues by severity and category803 - Similar code patterns found804 - Related code via graph traversal805 - Metrics for each file806 """807 try:808 logger.info(f"Received PR analysis request for PR #{request.pr_number} in repo {request.repo_id}")809 result = await code_analyzer.analyze_pr(request)810 return result811 except Exception as e:812 logger.error(f"Error analyzing PR: {e}")813 raise HTTPException(status_code=500, detail=str(e))814 815 816@app.post("/analyze/file", response_model=FileAnalysisResult)817async def analyze_file(request: FileAnalysisRequest):818 """819 Analyze a single file820 821 Returns:822 - Code issues and smells823 - Similar code patterns824 - Related code dependencies825 - File metrics826 """827 try:828 logger.info(f"Received file analysis request for {request.filename} in repo {request.repo_id}")829 result = await code_analyzer.analyze_file(request)830 return result831 except Exception as e:832 logger.error(f"Error analyzing file: {e}")833 raise HTTPException(status_code=500, detail=str(e))834 835 836@app.post("/analyze/diff", response_model=FileAnalysisResult)837async def analyze_diff(request: DiffAnalysisRequest):838 """839 Analyze a diff/patch840 841 Focuses on changed lines and their context.842 Returns issues found in new code.843 """844 try:845 logger.info(f"Received diff analysis request for {request.filename} in repo {request.repo_id}")846 result = await code_analyzer.analyze_diff(request)847 return result848 except Exception as e:849 logger.error(f"Error analyzing diff: {e}")850 raise HTTPException(status_code=500, detail=str(e))851 852 853@app.get("/health")854async def health_check():855 """Health check endpoint"""856 return {857 "status": "healthy",858 "services": {859 "api": "running",860 "neo4j": "connected" if graph_db else "not configured",861 "qdrant": "connected" if vector_db else "not configured"862 }863 }864 865 866# ========================================================================867# WEBHOOK INTEGRATION ENDPOINT (GitHub PR Review)868# ========================================================================869 870from typing import Any871 872class PRReviewWebhookRequest(BaseModel):873 """Request from frontend webhook after PR processing"""874 owner: str875 repo: str876 pr_number: int877 installation_id: str878 pull_request_id: Optional[str] = None # Optional - will be created if not provided879 repo_db_id: Optional[str] = None # Repository UUID from frontend database880 context: Optional[Any] = None # Optional - will be fetched if not provided881 gemini_api_key: Optional[str] = None # Legacy — kept for backward compat882 # M7: provider-agnostic BYO-key fields; take precedence over gemini_api_key883 api_key: Optional[str] = None884 provider: Optional[str] = "gemini" # gemini|anthropic|openai|mistral|ollama885 886 class Config:887 extra = "allow" # Allow extra fields888 889 890@app.post("/review")891async def process_pr_review_webhook(http_request: Request, request: PRReviewWebhookRequest, background_tasks: BackgroundTasks):892 """893 Main webhook endpoint for PR review processing894 895 Called by frontend webhook when PR is opened/updated.896 Orchestrates the complete AI review workflow:897 1. Fetches PR data from GitHub (if not provided)898 2. Triggers LangGraph workflow899 3. Posts results back to GitHub900 901 Runs as background task to avoid timeout.902 """903 try:904 logger.info(f"🔔 Received review webhook for PR #{request.pr_number} in {request.owner}/{request.repo}")905 logger.info(f" Installation ID: {request.installation_id}")906 logger.info(f" Pull Request ID: {request.pull_request_id or '(will generate)'}")907 logger.info(f" Context provided: {request.context is not None}")908 909 # Generate pull_request_id if not provided910 if not request.pull_request_id:911 request.pull_request_id = str(uuid.uuid4())912 logger.info(f" Generated Pull Request ID: {request.pull_request_id}")913 914 # Fetch PR data from GitHub if context not provided915 if not request.context:916 logger.info(f"📥 Context not provided, fetching PR data from GitHub...")917 try:918 repo_full_name = f"{request.owner}/{request.repo}"919 installation_id_int = int(request.installation_id)920 921 # Use global github_client922 if not github_client:923 logger.error("❌ GitHub client not initialized")924 raise HTTPException(status_code=500, detail="GitHub client not available")925 926 pr_data = await github_client.get_pull_request(927 repo_full_name=repo_full_name,928 pr_number=request.pr_number,929 installation_id=installation_id_int930 )931 932 # Build context from PR data933 context = {934 "title": pr_data["title"],935 "description": pr_data["body"],936 "files": pr_data["files"],937 "base_ref": pr_data["base"]["ref"],938 "head_ref": pr_data["head"]["ref"],939 "head_sha": pr_data["head"]["sha"], # M12: needed for inline comments940 "additions": pr_data["additions"],941 "deletions": pr_data["deletions"],942 "changed_files": pr_data["changed_files"],943 }944 945 logger.info(f"✅ Fetched PR data: {len(context['files'])} files, +{context['additions']}/-{context['deletions']}")946 except Exception as e:947 logger.error(f"❌ Failed to fetch PR data from GitHub: {e}")948 raise HTTPException(status_code=500, detail=f"Failed to fetch PR data: {str(e)}")949 else:950 context = request.context951 952 # Extract context data953 files = context.get("files", [])954 955 if not files:956 logger.warning(f"No files found in PR #{request.pr_number} context")957 return {958 "status": "skipped",959 "message": "No files to review",960 "pr_number": request.pr_number,961 "review_id": None962 }963 964 logger.info(f"📝 Processing {len(files)} files from PR #{request.pr_number}")965 966 # Use repo_db_id if provided (from frontend database), otherwise fall back to GitHub format967 repo_id = request.repo_db_id if request.repo_db_id else f"{request.owner}/{request.repo}"968 969 if not request.repo_db_id:970 logger.warning(f"⚠️ No repo_db_id provided, using GitHub format '{repo_id}' - this may cause context lookup issues")971 972 # Extract branch refs from context973 base_ref = context.get("base_ref", "main")974 head_ref = context.get("head_ref", "unknown")975 976 # Propagate request_id set by the middleware977 request_id = getattr(http_request.state, "request_id", str(uuid.uuid4()))978 979 head_sha = context.get("head_sha", "")980 981 # Queue the review workflow as a background task982 background_tasks.add_task(983 execute_review_task,984 pr_number=request.pr_number,985 repo_id=repo_id,986 pr_title=context.get("title", ""),987 pr_description=context.get("description"),988 files=files,989 base_ref=base_ref,990 head_ref=head_ref,991 head_sha=head_sha,992 installation_id=request.installation_id,993 pull_request_db_id=request.pull_request_id,994 owner=request.owner,995 repo=request.repo,996 gemini_api_key=request.gemini_api_key,997 api_key=request.api_key,998 provider=request.provider,999 request_id=request_id,1000 )1001 1002 # Return immediately with queued status1003 review_id = f"review-{request.pull_request_id}"1004 logger.info(f"✅ Review workflow queued: {review_id}")1005 1006 return {1007 "status": "queued",1008 "message": f"Review workflow started for PR #{request.pr_number}",1009 "pr_number": request.pr_number,1010 "review_id": review_id,1011 "files_count": len(files)1012 }1013 1014 except Exception as e:1015 logger.error(f"❌ Error processing review webhook: {e}", exc_info=True)1016 raise HTTPException(status_code=500, detail=str(e))1017 1018 1019# ========================================================================1020# DATABASE STORAGE HELPER1021# ========================================================================1022 1023async def store_review_in_database(1024 pull_request_db_id: str,1025 final_state: dict,1026 github_comment_id: Optional[int],1027 github_comment_url: Optional[str]1028):1029 """1030 Store review results in frontend database for analytics1031 """1032 try:1033 frontend_url = os.getenv("FRONTEND_URL", "http://localhost:3000")1034 1035 # Extract review summary data1036 review_summary = final_state.get("review_summary", {})1037 analysis = final_state.get("analysis", {})1038 route_decision = final_state.get("route_decision", {})1039 1040 # Calculate timestamps (must be UTC timezone with Z suffix for Zod validation)1041 start_time = final_state.get("start_time")1042 end_time = final_state.get("end_time")1043 1044 if start_time:1045 started_at = datetime.fromtimestamp(start_time, tz=timezone.utc).isoformat().replace('+00:00', 'Z')1046 else:1047 started_at = None1048 1049 if end_time:1050 completed_at = datetime.fromtimestamp(end_time, tz=timezone.utc).isoformat().replace('+00:00', 'Z')1051 else:1052 completed_at = datetime.now(timezone.utc).isoformat().replace('+00:00', 'Z')1053 1054 # Calculate execution time1055 execution_time_ms = int((end_time - start_time) * 1000) if (start_time and end_time) else 01056 1057 # M10: use real cost/token stats injected by execute_review_task1058 total_cost = float(final_state.get("total_cost", 0.0))1059 1060 # Determine primary model1061 model = route_decision.get("model", "gemini-2.5-flash")1062 primary_model = "flash" if "flash" in model.lower() else ("pro" if "pro" in model.lower() else "flash")1063 1064 # Build payload matching frontend's /api/reviews endpoint1065 payload = {1066 "pull_request_id": pull_request_db_id,1067 "status": final_state.get("status", "completed"),1068 "started_at": started_at,1069 "completed_at": completed_at,1070 "primary_model": primary_model,1071 "total_cost": total_cost,1072 "total_tokens_input": int(final_state.get("total_tokens_input", 0)),1073 "total_tokens_output": int(final_state.get("total_tokens_output", 0)),1074 "execution_time_ms": execution_time_ms,1075 "summary": {1076 "overallScore": 85, # Could be calculated from issues/suggestions ratio1077 "filesChanged": final_state.get("total_files", 0),1078 "issuesFound": review_summary.get("total_issues", 0),1079 "critical": review_summary.get("critical_count", 0),1080 "high": review_summary.get("high_count", 0),1081 "medium": review_summary.get("medium_count", 0),1082 "low": review_summary.get("low_count", 0),1083 "info": 0,1084 },1085 "key_changes": analysis.get("key_changes", [])[:5] if analysis else [],1086 "recommendations": [],1087 "positives": [],1088 "summary_comment_id": github_comment_id,1089 "summary_comment_url": github_comment_url,1090 "inline_comments_posted": 0,1091 }1092 1093 logger.info(f"📤 Sending review data to frontend database...")1094 logger.info(f" URL: {frontend_url}/api/reviews")1095 logger.info(f" Payload: pull_request_id={pull_request_db_id}, status={payload['status']}")1096 logger.info(f" Summary: {payload['summary']}")1097 1098 # Send to frontend database1099 async with httpx.AsyncClient() as client:1100 response = await client.post(1101 f"{frontend_url}/api/reviews",1102 json=payload,1103 timeout=10.01104 )1105 1106 if response.status_code != 200:1107 error_body = response.text1108 logger.error(f"❌ Frontend API returned {response.status_code}")1109 logger.error(f" Response: {error_body}")1110 raise Exception(f"Frontend API error: {response.status_code} - {error_body}")1111 1112 result = response.json()1113 1114 logger.info(f"✅ Stored review in database for PR {pull_request_db_id}")1115 logger.info(f" Review ID: {result.get('id')}")1116 logger.info(f" Issues: {payload['summary']['issuesFound']}, Critical: {payload['summary']['critical']}")1117 1118 except httpx.HTTPStatusError as e:1119 logger.error(f"❌ Failed to store review in database: HTTP {e.response.status_code}")1120 logger.error(f" Response: {e.response.text}")1121 except Exception as e:1122 logger.error(f"❌ Failed to store review in database: {e}", exc_info=True)1123 # Don't raise - this is non-critical, GitHub comment is primary output1124 1125 1126# ========================================================================1127# REVIEW FORMATTING HELPER1128# ========================================================================1129 1130def format_review_for_github(1131 pr_number: int,1132 pr_title: str,1133 overall_summary: str,1134 file_reviews: dict,1135 status: str1136) -> str:1137 """1138 Format review for GitHub PR comment with rich markdown1139 1140 Features:1141 - Severity badges (shields.io)1142 - Collapsible sections for long content1143 - Syntax-highlighted code blocks1144 - Color-coded severity indicators1145 1146 Args:1147 pr_number: PR number1148 pr_title: PR title1149 overall_summary: Generated review summary1150 file_reviews: Dict of file-level reviews1151 status: Workflow status1152 1153 Returns:1154 Formatted markdown for GitHub comment1155 """1156 parts = []1157 1158 # Header with badges1159 parts.append(f"# 🤖 AI Code Review - PR #{pr_number}")1160 parts.append(f"**{pr_title}**")1161 parts.append("")1162 1163 # Status badge1164 if status == "completed":1165 parts.append("")1166 else:1167 parts.append("")1168 1169 # Count severity levels in summary1170 critical_count = overall_summary.lower().count("critical")1171 high_count = overall_summary.lower().count("high priority") + overall_summary.lower().count("high:")1172 1173 if critical_count > 0:1174 parts.append("![Critical](https://img.shields.io/badge/critical-" + str(critical_count) + "-critical)")1175 if high_count > 0:1176 parts.append("![High](https://img.shields.io/badge/high-" + str(high_count) + "-orange)")1177 1178 parts.append("")1179 1180 # Overall summary with collapsible if long1181 parts.append("## 📋 Overall Review")1182 parts.append("")1183 1184 if len(overall_summary) > 3000:1185 parts.append("<details>")1186 parts.append("<summary><b>Click to expand full review</b></summary>")1187 parts.append("")1188 parts.append(overall_summary)1189 parts.append("")1190 parts.append("</details>")1191 else:1192 parts.append(overall_summary)1193 1194 parts.append("")1195 1196 # File-level reviews with rich formatting1197 if file_reviews:1198 parts.append("---")1199 parts.append("")1200 parts.append("## 📁 File-by-File Analysis")