findEthics/Atlas
0
1from fastapi import FastAPI, HTTPException, Header, Response2from fastapi.responses import HTMLResponse, StreamingResponse3from fastapi.middleware.cors import CORSMiddleware4from pydantic import BaseModel5import google.generativeai as genai6import httpx7import os8from dotenv import load_dotenv9from duckduckgo_search import DDGS10from typing import Optional, List, Dict, Any11import logging12import re13import asyncio14import threading15import time16import hashlib17import json18from functools import wraps19from collections import OrderedDict20from dataclasses import dataclass, field21 22# Load environment variables from .env file23load_dotenv()24 25 26import spacy27from rake_nltk import Rake28import nltk29from cache.chromadb_cache import ChromaDBSearchCache30from search_optimizer import (31 has_meaningful_conversation_history,32 hybrid_search_decision,33 extract_search_terms,34 format_search_context35)36 37nltk.download('stopwords')38nltk.download('punkt_tab')39 40# Configure logging41logging.basicConfig(level=logging.INFO)42logger = logging.getLogger(__name__)43 44# Initialize FastAPI app45app = FastAPI(46 title="Enhanced Chat API with Dynamic Model Selection",47 description="API with query classification and dynamic model loading for QA/Summarization",48 version="3.0.0"49)50 51# Configure CORS with restricted origins for security52app.add_middleware(53 CORSMiddleware,54 allow_origins=[55 "https://huggingface.co",56 "https://*.hf.space",57 "http://localhost:3000",58 "http://localhost:8000",59 "http://127.0.0.1:3000",60 "http://127.0.0.1:8000"61 ],62 allow_methods=["POST", "GET"],63 allow_headers=["Content-Type", "Authorization"],64)65 66# Request/Response models67class ChatRequest(BaseModel):68 prompt: str69 max_new_tokens: int = 50070 use_search: bool = True71 temperature: float = 0.772 user_id: Optional[str] = None73 history: Optional[List[Dict[str, str]]] = None74 force_search: Optional[bool] = None75 search_decision_mode: str = "balanced" # conservative, balanced, aggressive76 77class ChatResponse(BaseModel):78 response: str79 search_results: Optional[List[Dict[str, Any]]] = None80 search_decision: Optional[Dict[str, Any]] = None81 cache_info: Optional[Dict[str, Any]] = None82 83class SearchRequest(BaseModel):84 query: str85 max_results: int = 586 87 88# Configure Google AI89try:90 genai.configure(api_key=os.environ["GOOGLE_API_KEY"])91except KeyError:92 logger.error("CRITICAL: GOOGLE_API_KEY environment variable not set.")93 raise # Exit if API key is not set94 95# Initialize the Generative Model96model = genai.GenerativeModel('gemini-1.5-flash')97 98# Global NLP tools99nlp = spacy.load("en_core_web_sm")100rake = Rake()101 102# ===== Phase 3c: ChromaDB Vector Database Cache System =====103 104# Universal ChromaDB search cache - single global instance for all users105# Vector database provides better semantic matching and persistent storage106universal_search_cache = None107 108def get_universal_cache() -> ChromaDBSearchCache:109 """Get the universal ChromaDB search cache instance"""110 global universal_search_cache111 if universal_search_cache is None:112 # Initialize ChromaDB cache with environment configuration113 cache_db_path = os.getenv('CHROMADB_PATH', 'cache_db')114 cache_results_path = os.getenv('CACHE_RESULTS_PATH', 'cache_results')115 embedding_model = os.getenv('CACHE_EMBEDDING_MODEL', 'all-MiniLM-L6-v2')116 117 universal_search_cache = ChromaDBSearchCache(118 max_size=1000, # Large cache size119 default_ttl=3600, # 1 hour TTL120 cache_db_path=cache_db_path,121 cache_results_path=cache_results_path,122 embedding_model=embedding_model,123 similarity_threshold=0.7124 )125 logger.info(f"Initialized ChromaDB universal cache: {cache_db_path}")126 127 return universal_search_cache128 129# ===== End Phase 3c ChromaDB Cache System =====130 131 132 133 134def normalize_user_id(user_id: Optional[str]) -> Optional[str]:135 """Normalize user_id to None for anonymous requests"""136 if user_id is None or user_id.strip() == "":137 return None138 return user_id.strip()139 140def validate_user_id(user_id: Optional[str]) -> Optional[str]:141 """Validate user_id format and return normalized value"""142 # Normalize to None for anonymous users143 user_id = normalize_user_id(user_id)144 145 if user_id is None:146 return None # Anonymous user147 148 # Validate format: non-empty string, max 255 chars, alphanumeric + hyphens + underscores149 if len(user_id) > 255:150 raise HTTPException(status_code=400, detail="user_id must be 255 characters or less")151 152 # Check allowed characters153 if not re.match(r'^[a-zA-Z0-9_-]+$', user_id):154 raise HTTPException(status_code=400, detail="user_id can only contain alphanumeric characters, hyphens, and underscores")155 156 return user_id157 158def run_in_threadpool(func):159 """Decorator to run synchronous model inference in thread pool"""160 @wraps(func)161 async def wrapper(*args, **kwargs):162 loop = asyncio.get_event_loop()163 return await loop.run_in_executor(None, func, *args, **kwargs)164 return wrapper165 166async def search_brave(query: str, max_results: int = 5) -> List[Dict[str, Any]]:167 """Search using Brave Search API with async HTTP client"""168 try:169 # Check for API key in environment variable first, then fallback to hardcoded170 api_key = os.getenv('BRAVE_API_KEY')171 172 if not api_key:173 logger.error("No Brave API key available")174 return []175 176 headers = {177 'Accept': 'application/json',178 'Accept-Encoding': 'gzip',179 'X-Subscription-Token': api_key180 }181 182 params = {183 'q': query,184 'count': max_results,185 'safesearch': 'moderate',186 'search_lang': 'en',187 'country': 'US'188 }189 190 191 async with httpx.AsyncClient(timeout=10.0) as client:192 response = await client.get(193 'https://api.search.brave.com/res/v1/web/search',194 headers=headers,195 params=params196 )197 198 response.raise_for_status()199 200 data = response.json()201 202 web_results = data.get('web', {})203 raw_results = web_results.get('results', [])204 205 results = []206 for result in raw_results:207 results.append({208 "title": result.get("title", ""),209 "body": re.sub(r'\s+', ' ', result.get("description", "")).strip(),210 "href": result.get("url", ""),211 "source": "Brave"212 })213 214 return results[:max_results]215 216 except Exception as e:217 logger.error(f"Brave Search error: {e}")218 logger.error(f"Brave Search error type: {type(e).__name__}")219 import traceback220 logger.error(f"Brave Search traceback: {traceback.format_exc()}")221 return []222 223async def search_duckduckgo(query: str, max_results: int = 5) -> List[Dict[str, Any]]:224 """Search using DuckDuckGo with async execution and timeout handling"""225 try:226 # Run DuckDuckGo search in thread pool with timeout227 loop = asyncio.get_event_loop()228 results = await asyncio.wait_for(229 loop.run_in_executor(None, _sync_duckduckgo_search, query, max_results),230 timeout=8.0 # 8 second timeout for Hugging Face compatibility231 )232 return results233 except asyncio.TimeoutError:234 logger.warning(f"DuckDuckGo search timed out for query: {query}")235 return []236 except Exception as e:237 logger.error(f"DuckDuckGo Search error: {e}")238 return []239 240def _sync_duckduckgo_search(query: str, max_results: int) -> List[Dict[str, Any]]:241 """Synchronous DuckDuckGo search helper with retry logic"""242 max_retries = 2243 for attempt in range(max_retries):244 try:245 # Configure DDGS with more conservative settings for hosted environments246 with DDGS(timeout=5) as ddgs:247 results = []248 search_results = ddgs.text(249 query, 250 safesearch='moderate', 251 max_results=max_results,252 region='us-en' # Specify region to potentially avoid some blocks253 )254 255 for result in search_results:256 results.append({257 "title": result.get("title", ""),258 "body": re.sub(r'\s+', ' ', result.get("body", "")).strip(),259 "href": result.get("href", ""),260 "source": "DuckDuckGo"261 })262 263 logger.info(f"DuckDuckGo search successful on attempt {attempt + 1}")264 return results265 266 except Exception as e:267 logger.warning(f"DuckDuckGo attempt {attempt + 1} failed: {e}")268 if attempt == max_retries - 1: # Last attempt269 logger.error(f"DuckDuckGo search failed after {max_retries} attempts")270 raise271 # Wait briefly before retry272 import time273 time.sleep(1)274 275 return []276 277async def search_web_combined(query: str, max_results: int = 10) -> List[Dict[str, Any]]:278 """Combined web search with resilient fallback strategy"""279 try:280 logger.info(f"Combined Search: Starting search for '{query}'")281 282 # Run both searches concurrently with timeout protection283 brave_task = asyncio.create_task(search_brave(query, 6)) # Get more from Brave as primary284 duckduckgo_task = asyncio.create_task(search_duckduckgo(query, 4)) # Fewer from DDG as backup285 286 # Wait for both searches to complete with overall timeout287 try:288 brave_results, duckduckgo_results = await asyncio.wait_for(289 asyncio.gather(brave_task, duckduckgo_task, return_exceptions=True),290 timeout=12.0 # Overall timeout for both searches291 )292 except asyncio.TimeoutError:293 logger.warning("Combined search timed out, cancelling remaining tasks")294 brave_task.cancel()295 duckduckgo_task.cancel()296 brave_results, duckduckgo_results = [], []297 298 # Handle exceptions and log results299 if isinstance(brave_results, Exception):300 logger.error(f"Brave search failed: {brave_results}")301 brave_results = []302 else:303 logger.info(f"Brave search returned {len(brave_results)} results")304 305 if isinstance(duckduckgo_results, Exception):306 logger.error(f"DuckDuckGo search failed: {duckduckgo_results}")307 duckduckgo_results = []308 else:309 logger.info(f"DuckDuckGo search returned {len(duckduckgo_results)} results")310 311 # If both searches fail, return empty with warning312 if not brave_results and not duckduckgo_results:313 logger.warning(f"All search engines failed for query: {query}")314 return []315 316 # Combine results317 combined_results = brave_results + duckduckgo_results318 319 # Remove duplicates based on URL320 seen_urls = set()321 unique_results = []322 323 for result in combined_results:324 url = result.get("href", "")325 if url and url not in seen_urls:326 seen_urls.add(url)327 unique_results.append(result)328 329 # Return top results up to max_results330 final_results = unique_results[:max_results]331 logger.info(f"Combined Search: Returning {len(final_results)} total unique results")332 333 # Log search engine performance334 brave_count = len([r for r in final_results if r.get('source') == 'Brave'])335 ddg_count = len([r for r in final_results if r.get('source') == 'DuckDuckGo'])336 logger.info(f"Search distribution: Brave={brave_count}, DuckDuckGo={ddg_count}")337 338 return final_results339 340 except Exception as e:341 logger.error(f"Combined search error: {e}")342 return []343 344 345def preprocess_text(text: str) -> str:346 """Use spaCy for fast text cleaning/normalization"""347 doc = nlp(text)348 # Lemmatize and remove stopwords349 return " ".join([350 token.lemma_ for token in doc351 if not token.is_stop and not token.is_punct352 ])[:2048]353 354def format_conversation_history(history: Optional[List[Dict[str, str]]], max_entries: int = 10) -> str:355 """Format conversation history for inclusion in AI prompt"""356 if not history:357 return ""358 359 try:360 formatted_entries = []361 # Take the most recent entries up to max_entries362 recent_history = history[-max_entries:] if len(history) > max_entries else history363 364 for entry in recent_history:365 # Handle both formats: {"role": "user/assistant", "content": "..."} 366 # and {"user": "...", "assistant": "..."}367 if "role" in entry and "content" in entry:368 role = entry["role"].title() # User or Assistant369 content = entry["content"].strip()370 if content:371 formatted_entries.append(f"{role}: {content}")372 elif "user" in entry and "assistant" in entry:373 user_msg = entry["user"].strip()374 assistant_msg = entry["assistant"].strip()375 if user_msg and assistant_msg:376 formatted_entries.append(f"User: {user_msg}")377 formatted_entries.append(f"Assistant: {assistant_msg}")378 379 if formatted_entries:380 return "\n".join(formatted_entries)381 return ""382 383 except Exception as e:384 logger.warning(f"Error formatting conversation history: {e}")385 return ""386 387 388 389 390 391 392 393@app.on_event("startup")394async def startup_event():395 """Perform startup tasks like NLP model loading"""396 logger.info("Starting NLP model loading...")397 # spaCy and NLTK data are loaded implicitly on first use or by spacy.load398 399 # Test analytics database connection400 try:401 from analytics.database import test_connection402 analytics_connected = await test_connection()403 if analytics_connected:404 logger.info("Analytics database connection successful")405 else:406 logger.warning("Analytics database connection failed - analytics disabled")407 except Exception as e:408 logger.warning(f"Analytics initialization failed: {e}")409 410 logger.info("Startup complete - NLP models ready")411 412@app.post("/chat", response_model=ChatResponse)413async def chat_endpoint(request: ChatRequest, 414 response: Response,415 user_agent: str = Header(None),416 x_session_id: str = Header(None)):417 """Enhanced chat endpoint with dynamic model selection and combined search"""418 logger.info(f"Request: {request.prompt}")419 420 # Validate and extract user_id421 user_id = validate_user_id(request.user_id)422 423 # Analytics setup424 session_id = x_session_id425 message_id = None426 start_time = None427 428 try:429 # Import analytics (with fallback if not available)430 try:431 from analytics.collectors import create_session, get_session, track_message, PerformanceTimer432 analytics_available = True433 except ImportError:434 logger.warning("Analytics not available")435 analytics_available = False436 437 # Create fallback PerformanceTimer when analytics unavailable438 class PerformanceTimer:439 def __init__(self):440 self.start_time = None441 self.end_time = None442 443 def __enter__(self):444 import time445 self.start_time = time.time()446 return self447 448 def __exit__(self, exc_type, exc_val, exc_tb):449 import time450 self.end_time = time.time()451 452 @property453 def duration_ms(self) -> int:454 if self.start_time and self.end_time:455 return int((self.end_time - self.start_time) * 1000)456 return 0457 458 # Start performance timing459 with PerformanceTimer() as timer:460 # Handle session management461 if analytics_available:462 if not session_id:463 # Create new session464 session = await create_session(user_agent=user_agent, user_id=user_id)465 session_id = session.session_id466 else:467 # Get existing session or create new one if not found468 session = await get_session(session_id)469 if not session:470 session = await create_session(user_agent=user_agent, user_id=user_id)471 session_id = session.session_id472 473 search_results = []474 search_context = ""475 search_decision = None476 cache_info = None477 478 # Phase 3: Get universal cache for result caching479 search_cache = get_universal_cache()480 481 # Phase 3b: Context-Aware Request Flow Optimization482 logger.info(f"Use Search : {request.use_search}")483 has_history = has_meaningful_conversation_history(request.history)484 logger.info(f"Has meaningful conversation history: {has_history}")485 486 if request.use_search:487 # Check if search should be forced (overrides all optimizations)488 if request.force_search:489 search_decision = {490 "should_search": True,491 "reason": "Search forced by user",492 "confidence": 1.0,493 "flow_type": "forced"494 }495 perform_search = True496 logger.info(f"Using forced search flow")497 elif not has_history:498 # First Message (No History): Cache-first approach499 logger.info(f"Using cache-first flow (no conversation history)")500 search_terms = extract_search_terms(request.prompt.lower(), nlp, rake)501 logger.info(f"Extract search terms successful: {search_terms}")502 503 # Try to get from cache first (skip search decision for performance)504 cached_entry = search_cache.get(search_terms, use_semantic_matching=True, similarity_threshold=0.7)505 506 if cached_entry:507 # Cache hit! Use cached results, no search needed508 search_results = cached_entry.results509 search_context = format_search_context(search_results)510 cache_info = {511 "cache_hit": True,512 "cached_query": cached_entry.search_query,513 "cache_age_seconds": int(time.time() - cached_entry.timestamp),514 "hit_count": cached_entry.hit_count,515 "flow_type": "cache_first_hit",516 "cache_type": "chromadb_vector"517 }518 search_decision = {519 "should_search": False,520 "reason": "Cache hit in cache-first flow",521 "confidence": 1.0,522 "flow_type": "cache_first_hit",523 "cache_type": "chromadb_vector"524 }525 perform_search = False526 logger.info(f"ChromaDB Cache-first HIT: Using cached results (age: {cache_info['cache_age_seconds']}s, hits: {cached_entry.hit_count})")527 else:528 # Cache miss - perform web search without search decision overhead529 logger.info(f"ChromaDB Cache-first MISS: Performing web search")530 search_query = " ".join(search_terms) or request.prompt531 search_results = await search_web_combined(search_query, 10)532 search_context = format_search_context(search_results)533 534 # Store results in cache if search was successful535 if search_results:536 search_cache.put(search_terms, search_query, search_results)537 logger.info(f"Cached search results in universal cache for future use")538 539 cache_info = {540 "cache_hit": False,541 "stored_in_cache": len(search_results) > 0,542 "flow_type": "cache_first_miss",543 "cache_type": "chromadb_vector"544 }545 search_decision = {546 "should_search": True,547 "reason": "Cache miss in cache-first flow",548 "confidence": 1.0,549 "flow_type": "cache_first_miss",550 "cache_type": "chromadb_vector"551 }552 perform_search = True553 else:554 # Follow-up Messages (Has History): Search decision first555 logger.info(f"Using search-decision-first flow (has conversation history)")556 # Use hybrid intelligent search decision (Phase 2: AI-enhanced)557 search_decision = await hybrid_search_decision(558 request.prompt, 559 request.history, 560 request.search_decision_mode,561 nlp,562 model563 )564 search_decision["flow_type"] = "search_decision_first"565 perform_search = search_decision["should_search"]566 567 if perform_search:568 # Search needed - check cache before web search569 search_terms = extract_search_terms(request.prompt.lower(), nlp, rake)570 logger.info(f"Extract search terms successful: {search_terms}")571 572 # Try to get from cache first573 cached_entry = search_cache.get(search_terms, use_semantic_matching=True, similarity_threshold=0.7)574 575 if cached_entry:576 # Cache hit! Use cached results577 search_results = cached_entry.results578 search_context = format_search_context(search_results)579 cache_info = {580 "cache_hit": True,581 "cached_query": cached_entry.search_query,582 "cache_age_seconds": int(time.time() - cached_entry.timestamp),583 "hit_count": cached_entry.hit_count,584 "flow_type": "search_decision_cache_hit",585 "cache_type": "chromadb_vector"586 }587 logger.info(f"ChromaDB Search-decision flow Cache HIT: Using cached results (age: {cache_info['cache_age_seconds']}s, hits: {cached_entry.hit_count})")588 else:589 # Cache miss - perform web search590 logger.info(f"ChromaDB Search-decision flow Cache MISS: Performing web search")591 search_query = " ".join(search_terms) or request.prompt592 search_results = await search_web_combined(search_query, 10)593 search_context = format_search_context(search_results)594 595 # Store results in cache if search was successful596 if search_results:597 search_cache.put(search_terms, search_query, search_results)598 logger.info(f"Cached search results in universal cache for future use")599 600 cache_info = {601 "cache_hit": False,602 "stored_in_cache": len(search_results) > 0,603 "flow_type": "search_decision_cache_miss",604 "cache_type": "chromadb_vector"605 }606 else:607 # Search not needed based on conversation context608 logger.info(f"Search skipped: {search_decision['reason']}")609 cache_info = {610 "search_skipped": True,611 "flow_type": "search_decision_skip"612 }613 614 logger.info(f"Search Decision: {search_decision}")615 616 if perform_search:617 logger.info(f"Search processing complete - Results: {len(search_results)}")618 else:619 logger.info(f"Search disabled by request")620 cache_info = {"search_disabled": True}621 622 logger.info(f"Search Context: {search_context}")623 624 # Format conversation history625 conversation_history = format_conversation_history(request.history, max_entries=10)626 logger.info(f"Conversation History: {len(request.history or [])} entries")627 628 # Create a prompt for the AI model with conversation history629 if conversation_history:630 prompt_template = f"""631 You are having a conversation with a user. Here is the conversation history:632 633 Conversation History:634 ---635 {conversation_history}636 ---637 638 Based on the following context from web pages I have read and the conversation history above, please answer the user's question.639 If the context does not contain the answer and you cannot answer based on the conversation history, say that you don't have enough information.640 641 Context:642 ---643 {search_context}644 ---645 646 Question: {request.prompt}647 648 Answer:649 """650 else:651 prompt_template = f"""652 Based on the following context from web pages I have read, please answer the user's question.653 If the context does not contain the answer, say that you don't have enough information.654 655 Context:656 ---657 {search_context}658 ---659 660 Question: {request.prompt}661 662 Answer:663 """664 665 response_text = await run_gemini_inference(prompt_template)666 667 # Track message analytics668 if analytics_available and session_id:669 message = await track_message(670 session_id=session_id,671 prompt_length=len(request.prompt),672 response_length=len(response_text),673 response_time_ms=timer.duration_ms,674 used_search=request.use_search,675 max_tokens=request.max_new_tokens,676 temperature=request.temperature,677 success=True,678 user_id=user_id679 )680 if message:681 message_id = message.message_id682 683 # Add session ID to response headers684 if session_id:685 response.headers["X-Session-ID"] = session_id686 687 # Prepare response688 chat_response = ChatResponse(689 response=response_text,690 search_results=search_results,691 search_decision=search_decision,692 cache_info=cache_info693 )694 695 return chat_response696 697 except Exception as e:698 # Track failed message699 if analytics_available and session_id:700 await track_message(701 session_id=session_id,702 prompt_length=len(request.prompt),703 response_length=0,704 response_time_ms=timer.duration_ms if 'timer' in locals() else 0,705 used_search=request.use_search,706 max_tokens=request.max_new_tokens,707 temperature=request.temperature,708 success=False,709 error_message=str(e),710 user_id=user_id711 )712 713 logger.error(f"Chat error: {e}")714 raise HTTPException(status_code=500, detail=str(e))715 716@app.post("/search")717async def search_endpoint(request: SearchRequest):718 """Search endpoint with combined search engines"""719 try:720 results = await search_web_combined(request.query, request.max_results)721 return {"results": results}722 except Exception as e:723 logger.error(f"Search endpoint error: {e}")724 raise HTTPException(status_code=500, detail=str(e))725 726@app.get("/analytics/stats")727async def analytics_stats():728 """Get basic analytics statistics"""729 try:730 from analytics.dashboard import get_basic_stats731 stats = await get_basic_stats()732 return stats733 except Exception as e:734 logger.error(f"Analytics stats error: {e}")735 raise HTTPException(status_code=500, detail=str(e))736 737@app.get("/analytics/dashboard")738async def analytics_dashboard():739 """Get HTML analytics dashboard"""740 try:741 from analytics.dashboard import get_dashboard_data742 from fastapi.responses import HTMLResponse743 744 # Get dashboard data including user metrics745 data = await get_dashboard_data()746 747 # Get user statistics for the dashboard748 from analytics.dashboard import get_user_statistics, get_authenticated_vs_anonymous_metrics749 user_stats = await get_user_statistics()750 comparison_stats = await get_authenticated_vs_anonymous_metrics()751 752 # Create HTML dashboard753 html_content = f"""754 <!DOCTYPE html>755 <html lang="en">756 <head>757 <meta charset="UTF-8">758 <meta name="viewport" content="width=device-width, initial-scale=1.0">759 <title>Atlas Analytics Dashboard</title>760 <script src="https://cdn.jsdelivr.net/npm/chart.js"></script>761 <style>762 body {{763 font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;764 margin: 0;765 padding: 20px;766 background-color: #f5f5f5;767 }}768 .container {{769 max-width: 1200px;770 margin: 0 auto;771 }}772 .header {{773 background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);774 color: white;775 padding: 30px;776 border-radius: 10px;777 margin-bottom: 30px;778 text-align: center;779 }}780 .stats-grid {{781 display: grid;782 grid-template-columns: repeat(auto-fit, minmax(250px, 1fr));783 gap: 20px;784 margin-bottom: 30px;785 }}786 .stat-card {{787 background: white;788 padding: 25px;789 border-radius: 10px;790 box-shadow: 0 2px 10px rgba(0,0,0,0.1);791 text-align: center;792 }}793 .stat-number {{794 font-size: 2.5em;795 font-weight: bold;796 color: #667eea;797 margin-bottom: 10px;798 }}799 .stat-label {{800 color: #666;801 font-size: 0.9em;802 text-transform: uppercase;803 letter-spacing: 1px;804 }}805 .chart-container {{806 background: white;807 padding: 25px;808 border-radius: 10px;809 box-shadow: 0 2px 10px rgba(0,0,0,0.1);810 margin-bottom: 20px;811 }}812 .chart-title {{813 font-size: 1.2em;814 font-weight: bold;815 margin-bottom: 20px;816 color: #333;817 }}818 .refresh-btn {{819 background: #667eea;820 color: white;821 border: none;822 padding: 10px 20px;823 border-radius: 5px;824 cursor: pointer;825 font-size: 1em;826 margin-bottom: 20px;827 }}828 .refresh-btn:hover {{829 background: #5a6fd8;830 }}831 .error {{832 background: #fee;833 color: #c33;834 padding: 15px;835 border-radius: 5px;836 margin: 10px 0;837 }}838 .last-updated {{839 text-align: center;840 color: #666;841 font-size: 0.9em;842 margin-top: 20px;843 }}844 </style>845 </head>846 <body>847 <div class="container">848 <div class="header">849 <h1>🚀 Atlas Analytics Dashboard</h1>850 <p>Real-time insights into your chat application</p>851 </div>852 853 <button class="refresh-btn" onclick="location.reload()">🔄 Refresh Data</button>854 855 <div class="stats-grid">856 <div class="stat-card">857 <div class="stat-number">{data.get('basic', {}).get('total_messages', 0)}</div>858 <div class="stat-label">Total Messages</div>859 </div>860 <div class="stat-card">861 <div class="stat-number">{data.get('basic', {}).get('total_sessions', 0)}</div>862 <div class="stat-label">Total Sessions</div>863 </div>864 <div class="stat-card">865 <div class="stat-number">{data.get('basic', {}).get('active_sessions', 0)}</div>866 <div class="stat-label">Active Sessions</div>867 </div>868 <div class="stat-card">869 <div class="stat-number">{data.get('basic', {}).get('messages_today', 0)}</div>870 <div class="stat-label">Messages Today</div>871 </div>872 <div class="stat-card">873 <div class="stat-number">{data.get('basic', {}).get('search_usage_percentage', 0)}%</div>874 <div class="stat-label">Search Usage</div>875 </div>876 <div class="stat-card">877 <div class="stat-number">{data.get('basic', {}).get('average_response_time_ms', 0)}ms</div>878 <div class="stat-label">Avg Response Time</div>879 </div>880 </div>881 882 <div class="chart-container">883 <div class="chart-title">🔓 Anonymous Usage Overview</div>884 <div class="stats-grid">885 <div class="stat-card">886 <div class="stat-number">{user_stats.get('anonymous_sessions', 0)}</div>887 <div class="stat-label">Anonymous Sessions</div>888 </div>889 <div class="stat-card">890 <div class="stat-number">{user_stats.get('anonymous_messages', 0)}</div>891 <div class="stat-label">Anonymous Messages</div>892 </div>893 <div class="stat-card">894 <div class="stat-number">{round(100 - user_stats.get('authenticated_session_percentage', 0), 1)}%</div>895 <div class="stat-label">Anonymous Session %</div>896 </div>897 <div class="stat-card">898 <div class="stat-number">{round(100 - user_stats.get('authenticated_message_percentage', 0), 1)}%</div>899 <div class="stat-label">Anonymous Message %</div>900 </div>901 </div>902 </div>903 904 <div class="chart-container">905 <div class="chart-title">👥 User Analytics</div>906 <div class="stats-grid">907 <div class="stat-card">908 <div class="stat-number">{user_stats.get('unique_authenticated_users', 0)}</div>909 <div class="stat-label">Unique Users</div>910 </div>911 <div class="stat-card">912 <div class="stat-number">{user_stats.get('authenticated_sessions', 0)}</div>913 <div class="stat-label">Authenticated Sessions</div>914 </div>915 <div class="stat-card">916 <div class="stat-number">{user_stats.get('anonymous_sessions', 0)}</div>917 <div class="stat-label">Anonymous Sessions</div>918 </div>919 <div class="stat-card">920 <div class="stat-number">{user_stats.get('authenticated_session_percentage', 0)}%</div>921 <div class="stat-label">Auth Session %</div>922 </div>923 <div class="stat-card">924 <div class="stat-number">{user_stats.get('authenticated_messages', 0)}</div>925 <div class="stat-label">Authenticated Messages</div>926 </div>927 <div class="stat-card">928 <div class="stat-number">{user_stats.get('anonymous_messages', 0)}</div>929 <div class="stat-label">Anonymous Messages</div>930 </div>931 <div class="stat-card">932 <div class="stat-number">{user_stats.get('authenticated_message_percentage', 0)}%</div>933 <div class="stat-label">Auth Message %</div>934 </div>935 <div class="stat-card">936 <div class="stat-number">{user_stats.get('anonymous_messages', 0) + user_stats.get('authenticated_messages', 0)}</div>937 <div class="stat-label">Total Messages</div>938 </div>939 </div>940 </div>941 942 <div class="chart-container">943 <div class="chart-title">🔍 User Filtering</div>944 <div style="margin-bottom: 20px;">945 <input type="text" id="userIdInput" placeholder="Enter user ID to filter analytics" 946 style="padding: 10px; border: 1px solid #ddd; border-radius: 5px; width: 300px; margin-right: 10px;">947 <button onclick="filterByUser()" style="padding: 10px 20px; background: #667eea; color: white; border: none; border-radius: 5px; cursor: pointer;">948 Filter Analytics949 </button>950 <button onclick="clearFilter()" style="padding: 10px 20px; background: #6c757d; color: white; border: none; border-radius: 5px; cursor: pointer; margin-left: 10px;">951 Clear Filter952 </button>953 </div>954 <div id="userFilterResults" style="display: none;">955 <h4>User-Specific Analytics</h4>956 <div id="userStatsGrid" class="stats-grid"></div>957 </div>958 </div>959 960 <div class="chart-container">961 <div class="chart-title">📊 Hourly Message Activity (Last 24 Hours)</div>962 <canvas id="hourlyChart" width="400" height="200"></canvas>963 </div>964 965 <div class="chart-container">966 <div class="chart-title">⚡ Performance Metrics</div>967 <canvas id="performanceChart" width="400" height="200"></canvas>968 </div>969 970 <div class="chart-container">971 <div class="chart-title">👤 Authenticated vs Anonymous Comparison</div>972 <canvas id="comparisonChart" width="400" height="200"></canvas>973 </div>974 975 <div class="last-updated">976 Last updated: {data.get('generated_at', 'Unknown')}977 </div>978 </div>979 980 <script>981 // Hourly Chart982 const hourlyData = {data.get('hourly', [])};983 const hourlyLabels = hourlyData.map(d => d.hour);984 const hourlyMessages = hourlyData.map(d => d.message_count);985 const hourlySearches = hourlyData.map(d => d.search_count);986 987 new Chart(document.getElementById('hourlyChart'), {{988 type: 'line',989 data: {{990 labels: hourlyLabels,991 datasets: [{{992 label: 'Messages',993 data: hourlyMessages,994 borderColor: '#667eea',995 backgroundColor: 'rgba(102, 126, 234, 0.1)',996 tension: 0.4997 }}, {{998 label: 'With Search',999 data: hourlySearches,1000 borderColor: '#f093fb',1001 backgroundColor: 'rgba(240, 147, 251, 0.1)',1002 tension: 0.41003 }}]1004 }},1005 options: {{1006 responsive: true,1007 scales: {{1008 y: {{1009 beginAtZero: true1010 }}1011 }}1012 }}1013 }});1014 1015 // Performance Chart1016 const perfData = {data.get('performance', {})};1017 new Chart(document.getElementById('performanceChart'), {{1018 type: 'bar',1019 data: {{1020 labels: ['P50', 'P90', 'P95', 'Error Rate %'],1021 datasets: [{{1022 label: 'Performance Metrics',1023 data: [1024 perfData.response_time_p50 || 0,1025 perfData.response_time_p90 || 0,1026 perfData.response_time_p95 || 0,1027 perfData.error_rate_percentage || 01028 ],1029 backgroundColor: [1030 'rgba(102, 126, 234, 0.8)',1031 'rgba(240, 147, 251, 0.8)',1032 'rgba(255, 159, 64, 0.8)',1033 'rgba(255, 99, 132, 0.8)'1034 ]1035 }}]1036 }},1037 options: {{1038 responsive: true,1039 scales: {{1040 y: {{1041 beginAtZero: true1042 }}1043 }}1044 }}1045 }});1046 1047 // Comparison Chart1048 const comparisonData = {comparison_stats};1049 new Chart(document.getElementById('comparisonChart'), {{1050 type: 'bar',1051 data: {{1052 labels: ['Sessions', 'Messages', 'Avg Response Time (ms)', 'Search Usage %'],1053 datasets: [{{1054 label: 'Authenticated Users',1055 data: [1056 comparisonData.authenticated?.sessions || 0,1057 comparisonData.authenticated?.messages || 0,1058 comparisonData.authenticated?.avg_response_time_ms || 0,1059 comparisonData.authenticated?.search_usage_percentage || 01060 ],1061 backgroundColor: 'rgba(102, 126, 234, 0.8)'1062 }}, {{1063 label: 'Anonymous Users',1064 data: [1065 comparisonData.anonymous?.sessions || 0,1066 comparisonData.anonymous?.messages || 0,1067 comparisonData.anonymous?.avg_response_time_ms || 0,1068 comparisonData.anonymous?.search_usage_percentage || 01069 ],1070 backgroundColor: 'rgba(255, 159, 64, 0.8)'1071 }}]1072 }},1073 options: {{1074 responsive: true,1075 scales: {{1076 y: {{1077 beginAtZero: true1078 }}1079 }}1080 }}1081 }});1082 1083 // User filtering functions1084 async function filterByUser() {{1085 const userId = document.getElementById('userIdInput').value.trim();1086 if (!userId) {{1087 alert('Please enter a user ID');1088 return;1089 }}1090 1091 try {{1092 const response = await fetch(`/analytics/user/${{encodeURIComponent(userId)}}`);1093 const userData = await response.json();1094 1095 if (userData.error) {{1096 alert(`Error: ${{userData.error}}`);1097 return;1098 }}1099 1100 displayUserStats(userData);1101 }} catch (error) {{1102 alert(`Error fetching user data: ${{error.message}}`);1103 }}1104 }}1105 1106 function displayUserStats(userData) {{1107 const resultsDiv = document.getElementById('userFilterResults');1108 const statsGrid = document.getElementById('userStatsGrid');1109 1110 statsGrid.innerHTML = `1111 <div class="stat-card">1112 <div class="stat-number">${{userData.total_sessions || 0}}</div>1113 <div class="stat-label">User Sessions</div>1114 </div>1115 <div class="stat-card">1116 <div class="stat-number">${{userData.total_messages || 0}}</div>1117 <div class="stat-label">User Messages</div>1118 </div>1119 <div class="stat-card">1120 <div class="stat-number">${{userData.search_usage_percentage || 0}}%</div>1121 <div class="stat-label">Search Usage</div>1122 </div>1123 <div class="stat-card">1124 <div class="stat-number">${{userData.avg_response_time_ms || 0}}ms</div>1125 <div class="stat-label">Avg Response Time</div>1126 </div>1127 <div class="stat-card">1128 <div class="stat-number">${{userData.avg_messages_per_session || 0}}</div>1129 <div class="stat-label">Avg Msgs/Session</div>1130 </div>1131 <div class="stat-card">1132 <div class="stat-number">${{userData.active_sessions || 0}}</div>1133 <div class="stat-label">Active Sessions</div>1134 </div>1135 `;1136 1137 resultsDiv.style.display = 'block';1138 }}1139 1140 function clearFilter() {{1141 document.getElementById('userIdInput').value = '';1142 document.getElementById('userFilterResults').style.display = 'none';1143 }}1144 </script>1145 </body>1146 </html>1147 """1148 1149 return HTMLResponse(content=html_content)1150 1151 except Exception as e:1152 logger.error(f"Analytics dashboard error: {e}")1153 raise HTTPException(status_code=500, detail=str(e))1154 1155@app.get("/analytics/users")1156async def analytics_users():1157 """Get overall user statistics including authenticated vs anonymous metrics"""1158 try:1159 from analytics.dashboard import get_user_statistics1160 stats = await get_user_statistics()1161 return stats1162 except Exception as e:1163 logger.error(f"Analytics users error: {e}")1164 raise HTTPException(status_code=500, detail=str(e))1165 1166@app.get("/analytics/user/{user_id}")1167async def analytics_user(user_id: str):1168 """Get analytics for a specific user"""1169 try:1170 # Validate user_id1171 if not user_id or not isinstance(user_id, str) or len(user_id.strip()) == 0:1172 raise HTTPException(status_code=400, detail="Invalid user_id provided")1173 1174 from analytics.dashboard import get_user_analytics1175 stats = await get_user_analytics(user_id.strip())1176 return stats1177 except Exception as e:1178 logger.error(f"Analytics user error: {e}")1179 raise HTTPException(status_code=500, detail=str(e))1180 1181@app.get("/analytics/comparison")1182async def analytics_comparison():1183 """Get detailed comparison metrics between authenticated and anonymous users"""1184 try:1185 from analytics.dashboard import get_authenticated_vs_anonymous_metrics1186 stats = await get_authenticated_vs_anonymous_metrics()1187 return stats1188 except Exception as e:1189 logger.error(f"Analytics comparison error: {e}")1190 raise HTTPException(status_code=500, detail=str(e))1191 1192@app.get("/analytics/export")1193async def analytics_export(format: str = "json", days: int = 7, user_id: Optional[str] = None):1194 """Export analytics data in JSON or CSV format with optional user_id filtering"""1195 try:1196 from analytics.database import get_sessions_collection, get_messages_collection1197 from fastapi.responses import StreamingResponse1198 from datetime import datetime, timedelta1199 import json1200 import csv