findEthics/Atlas
0
1"""2Search Optimization Module for Atlas Intelligent Search Management3 4This module contains all search decision logic and optimization algorithms to reduce5unnecessary web searches by analyzing conversation context and user intent patterns.6 7Key Features:8- Rule-based search decision patterns9- AI-powered search necessity analysis 10- Conversation history context analysis11- Hybrid decision engine combining rules and AI12- Search term extraction and processing utilities13 14Authors: Atlas Development Team15Version: 1.0.016"""17 18try:19 import google.generativeai as genai20except ImportError:21 genai = None22 23try:24 import spacy25except ImportError:26 spacy = None27 28try:29 from rake_nltk import Rake30except ImportError:31 Rake = None32 33import asyncio34import logging35import time36import json37import re38from typing import Optional, List, Dict, Any, Tuple39from functools import wraps40 41# Configure logging42logger = logging.getLogger(__name__)43 44# AI Decision Cache for performance optimization45ai_decision_cache: Dict[str, Tuple[Dict[str, Any], float]] = {}46 47 48class SearchOptimizer:49 """50 Main search optimization class that encapsulates all search decision logic.51 52 This class provides a clean interface for search optimization functionality53 while maintaining state for NLP models and caching.54 """55 56 def __init__(self, nlp_model, rake_instance, gemini_model):57 """58 Initialize the SearchOptimizer with required dependencies.59 60 Args:61 nlp_model: spaCy language model instance62 rake_instance: RAKE keyword extraction instance 63 gemini_model: Google Generative AI model instance64 """65 self.nlp = nlp_model66 self.rake = rake_instance67 self.model = gemini_model68 logger.info("SearchOptimizer initialized successfully")69 70 71def extract_search_terms(text: str, nlp_model, rake_instance) -> List[str]:72 """73 Extract enhanced search terms using combined NER, syntax, and keywords.74 75 This function uses multiple NLP techniques to identify the most relevant76 search terms from user input:77 1. Named Entity Recognition (NER) for proper nouns78 2. Noun phrase extraction via syntactic analysis79 3. Keyword extraction using RAKE algorithm80 4. Question focus detection through dependency parsing81 82 Args:83 text (str): Input text to extract search terms from84 nlp_model: spaCy language model instance85 rake_instance: RAKE keyword extraction instance86 87 Returns:88 List[str]: Cleaned and deduplicated list of search terms89 90 Example:91 >>> extract_search_terms("What is machine learning?", nlp, rake)92 ['machine learning', 'machine', 'learning']93 """94 try:95 doc = nlp_model(text)96 97 # 1. Extract named entities98 entities = [ent.text for ent in doc.ents]99 100 # 2. Extract noun phrases through syntactic analysis101 noun_phrases = list(doc.noun_chunks)102 103 # 3. Extract question focus using dependency parsing104 focus_phrase = extract_focus_phrase(doc)105 106 # 4. Get keywords using RAKE107 rake_instance.extract_keywords_from_text(text)108 keywords = rake_instance.get_ranked_phrases()[:3] # Top 3 keywords109 110 # Combine and filter terms111 terms = entities + [np.text for np in noun_phrases] + keywords112 if focus_phrase:113 terms.append(focus_phrase)114 115 # Clean and deduplicate116 return clean_terms(terms, nlp_model)117 118 except Exception as e:119 logger.error(f"Error extracting search terms: {e}")120 # Fallback to simple word extraction121 words = text.lower().split()122 return [word for word in words if len(word) > 2][:5]123 124 125def extract_focus_phrase(doc) -> str:126 """127 Extract main question focus using dependency parse tree analysis.128 129 This function identifies the primary focus of a question by analyzing130 the dependency relationships in the parse tree, looking for attributes,131 subjects, and objects related to the root verb.132 133 Args:134 doc: spaCy Doc object with parsed dependencies135 136 Returns:137 str: The focused phrase or empty string if none found138 139 Example:140 For "What is machine learning?", this might return "machine learning"141 """142 try:143 for token in doc:144 if token.dep_ == "ROOT":145 for child in token.children:146 if child.dep_ in ("attr", "nsubj", "dobj"):147 return " ".join([t.text for t in child.subtree])148 return ""149 except Exception as e:150 logger.warning(f"Error extracting focus phrase: {e}")151 return ""152 153 154def clean_terms(terms: List[str], nlp_model) -> List[str]:155 """156 Remove duplicates and irrelevant terms from extracted search terms.157 158 This function performs several cleaning operations:159 1. Removes stopwords and single characters160 2. Filters out punctuation-only terms161 3. Removes redundant subphrases162 4. Deduplicates the final list163 164 Args:165 terms (List[str]): Raw list of extracted terms166 nlp_model: spaCy language model for stopword detection167 168 Returns:169 List[str]: Cleaned and deduplicated list of search terms170 """171 try:172 # Remove stopwords and single characters173 cleaned = [174 t for t in terms 175 if len(t) > 1 and not all(token.is_stop for token in nlp_model(t))176 ]177 178 # Remove redundant subphrases179 final_terms = []180 for term in sorted(cleaned, key=len, reverse=True):181 if not any(term in other for other in final_terms):182 final_terms.append(term)183 184 return final_terms[:10] # Limit to top 10 terms185 186 except Exception as e:187 logger.error(f"Error cleaning terms: {e}")188 return terms[:5] # Fallback to first 5 terms189 190 191def format_search_context(results: List[Dict[str, Any]]) -> str:192 """193 Create expanded context from combined search results with richer information.194 195 This function formats search results into a readable context string196 that can be used by the AI model for generating responses.197 198 Args:199 results (List[Dict[str, Any]]): List of search result dictionaries200 Each result should have 'source', 'title', and 'body' keys201 202 Returns:203 str: Formatted context string with source attribution204 205 Example:206 >>> results = [{"source": "Brave", "title": "AI Guide", "body": "AI is..."}]207 >>> format_search_context(results)208 '[Brave] AI Guide:\nAI is...'209 """210 try:211 if not results:212 return ""213 214 formatted_results = []215 for res in results[:10]:216 # Handle None or non-dict entries gracefully217 if not isinstance(res, dict):218 continue219 220 source = res.get('source', 'Unknown')221 title = res.get('title', 'No Title')222 body = res.get('body', 'No Content')223 224 # Ensure body is a string and truncate safely225 if body:226 body_str = str(body)[:1200]227 else:228 body_str = 'No Content'229 230 formatted_results.append(f"[{source}] {title}:\n{body_str}")231 232 return "\n\n".join(formatted_results)233 234 except Exception as e:235 logger.error(f"Error formatting search context: {e}")236 return ""237 238 239def should_perform_search(prompt: str, history: Optional[List[Dict[str, str]]], 240 search_decision_mode: str = "balanced") -> Dict[str, Any]:241 """242 Determine if web search should be performed based on conversation context and prompt patterns.243 244 This function implements rule-based search decision logic by analyzing:245 1. Conversation history presence and quality246 2. Follow-up question patterns (elaboration, clarification, referential)247 3. New information request indicators248 4. Context sufficiency for answering the question249 250 Args:251 prompt (str): User's current question/prompt252 history (Optional[List[Dict[str, str]]]): Conversation history253 search_decision_mode (str): Decision sensitivity ("conservative", "balanced", "aggressive")254 255 Returns:256 Dict[str, Any]: Dictionary containing:257 - should_search (bool): Whether to perform web search258 - reason (str): Explanation for the decision259 - confidence (float): Confidence score (0.0-1.0)260 261 Example:262 >>> should_perform_search("Tell me more about that", [{"user": "What is AI?", "assistant": "AI is..."}])263 {"should_search": False, "reason": "Follow-up question detected", "confidence": 0.8}264 """265 266 try:267 # Configuration based on search decision mode268 sensitivity_config = {269 "conservative": {270 "elaboration_threshold": 0.8,271 "referential_threshold": 0.7,272 "history_weight": 0.9273 },274 "balanced": {275 "elaboration_threshold": 0.6,276 "referential_threshold": 0.5,277 "history_weight": 0.7278 },279 "aggressive": {280 "elaboration_threshold": 0.4,281 "referential_threshold": 0.3,282 "history_weight": 0.5283 }284 }285 286 config = sensitivity_config.get(search_decision_mode, sensitivity_config["balanced"])287 prompt_lower = prompt.lower().strip()288 289 # If no history, always search (unless it's a greeting)290 if not history or len(history) == 0:291 if any(greeting in prompt_lower for greeting in ["hello", "hi", "hey", "good morning", "good afternoon"]):292 return {293 "should_search": False,294 "reason": "Simple greeting detected",295 "confidence": 0.9296 }297 return {298 "should_search": True,299 "reason": "No conversation history available",300 "confidence": 1.0301 }302 303 # Pattern detection arrays304 elaboration_patterns = [305 "elaborate", "explain more", "tell me more", "expand on", "go deeper",306 "more details", "can you explain", "give me more", "detail", "expand"307 ]308 309 clarification_patterns = [310 "what do you mean", "can you clarify", "i don't understand", "unclear",311 "confusing", "what does that mean", "could you explain", "i'm confused"312 ]313 314 referential_patterns = [315 "this", "that", "it", "the previous", "above mentioned", "earlier",316 "you said", "you mentioned", "from before", "the last"317 ]318 319 continuation_patterns = [320 "and what about", "what else", "continue", "also", "additionally",321 "furthermore", "what other", "anything else", "more on"322 ]323 324 # Score patterns325 elaboration_score = sum(1 for pattern in elaboration_patterns if pattern in prompt_lower)326 clarification_score = sum(1 for pattern in clarification_patterns if pattern in prompt_lower)327 referential_score = sum(1 for pattern in referential_patterns if pattern in prompt_lower)328 continuation_score = sum(1 for pattern in continuation_patterns if pattern in prompt_lower)329 330 # Calculate total follow-up score331 total_followup_score = elaboration_score + clarification_score + referential_score + continuation_score332 333 # Analyze recent conversation history for context relevance334 history_context_score = 0335 if history:336 recent_entries = history[-3:] # Look at last 3 exchanges337 for entry in recent_entries:338 if "role" in entry and "content" in entry:339 if entry["role"] == "assistant":340 content = entry["content"].lower()341 # Check if recent assistant responses contain substantial information342 if len(content.split()) > 20: # Substantial response343 history_context_score += 1344 elif "assistant" in entry:345 content = entry["assistant"].lower()346 if len(content.split()) > 20:347 history_context_score += 1348 349 # Decision logic350 if total_followup_score >= 2: # Strong follow-up indicators351 confidence = min(0.9, 0.5 + (total_followup_score * 0.2))352 return {353 "should_search": False,354 "reason": f"Follow-up question detected (score: {total_followup_score})",355 "confidence": confidence356 }357 358 if referential_score >= 1 and history_context_score >= 1:359 confidence = config["referential_threshold"] + (referential_score * 0.1)360 return {361 "should_search": False,362 "reason": "Referential question with sufficient context",363 "confidence": min(0.9, confidence)364 }365 366 if elaboration_score >= 1 and history_context_score >= 1:367 confidence = config["elaboration_threshold"]368 if elaboration_score >= 2:369 confidence += 0.2370 return {371 "should_search": False,372 "reason": "Elaboration request with existing context",373 "confidence": min(0.9, confidence)374 }375 376 # Check for new information requests377 new_info_patterns = [378 "what is", "who is", "when", "where", "how", "why", "latest", "recent",379 "current", "update", "news", "today", "now", "2024", "2025"380 ]381 382 new_info_score = sum(1 for pattern in new_info_patterns if pattern in prompt_lower)383 384 if new_info_score >= 2:385 return {386 "should_search": True,387 "reason": f"New information request detected (score: {new_info_score})",388 "confidence": 0.8389 }390 391 # Default: search for new topics392 return {393 "should_search": True,394 "reason": "New topic or insufficient context patterns",395 "confidence": 0.6396 }397 398 except Exception as e:399 logger.error(f"Error in rule-based search decision: {e}")400 return {401 "should_search": True,402 "reason": f"Error in analysis, defaulting to search: {str(e)[:50]}",403 "confidence": 0.5404 }405 406 407async def analyze_search_necessity(prompt: str, history: Optional[List[Dict[str, str]]] = None, 408 conversation_context: str = "", gemini_model=None) -> Dict[str, Any]:409 """410 AI-based search necessity analysis using Gemini for intelligent decision making.411 412 This function uses AI to analyze whether a web search is necessary by examining:413 1. Question type classification (new info vs clarification)414 2. Information sufficiency in conversation history415 3. Topic continuity and semantic relationships416 4. Recency requirements for the requested information417 418 Args:419 prompt (str): User's current question420 history (Optional[List[Dict[str, str]]]): Conversation history421 conversation_context (str): Formatted conversation context422 gemini_model: Google Generative AI model instance423 424 Returns:425 Dict[str, Any]: Dictionary containing:426 - should_search (bool): AI decision on search necessity427 - confidence (float): AI confidence score (0.0-1.0)428 - reason (str): Brief explanation of the decision429 - analysis (dict): Detailed analysis breakdown430 431 Example:432 >>> await analyze_search_necessity("What's the weather like?", [], "", model)433 {"should_search": True, "confidence": 0.9, "reason": "Requires current information", ...}434 """435 try:436 if not gemini_model:437 raise ValueError("Gemini model instance required for AI analysis")438 439 # Create cache key for performance optimization440 cache_key = f"{hash(prompt)}_{hash(str(history))}"441 442 # Check cache first (cache expires after 5 minutes for this session)443 current_time = time.time()444 if cache_key in ai_decision_cache:445 cached_result, timestamp = ai_decision_cache[cache_key]446 if current_time - timestamp < 300: # 5 minute cache447 logger.info("Using cached AI search decision")448 return cached_result449 450 # Format conversation history for AI analysis451 from app import format_conversation_history # Import to avoid circular dependency452 history_text = format_conversation_history(history, max_entries=5) if history else "No previous conversation"453 454 # Create AI prompt for search decision analysis455 analysis_prompt = f"""456 Analyze whether a web search is necessary for the following user question, considering the conversation history.457 458 **Conversation History:**459 {history_text}460 461 **Current Question:** {prompt}462 463 **Context:** {conversation_context[:500] if conversation_context else "No additional context"}464 465 Please analyze:466 1. **Question Type**: Is this asking for new information, clarification, elaboration, or continuation?467 2. **Information Sufficiency**: Does the conversation history contain enough information to answer this question?468 3. **Topic Continuity**: Is this question related to the previous conversation topics?469 4. **Recency Requirements**: Does this question require current/recent information that might not be in the history?470 5. **Semantic Relationship**: How semantically similar is this question to previous exchanges?471 472 Based on your analysis, determine if a web search is needed. Respond with a JSON object:473 {{474 "should_search": true/false,475 "confidence": 0.0-1.0,476 "reason": "Brief explanation of the decision",477 "analysis": {{478 "question_type": "new_information|clarification|elaboration|continuation",479 "information_sufficient": true/false,480 "topic_continuity": true/false,481 "requires_recent_info": true/false,482 "semantic_similarity": 0.0-1.0483 }}484 }}485 486 **Guidelines:**487 - If the question asks for new information not covered in history: should_search = true488 - If asking for clarification/elaboration of existing history content: should_search = false 489 - If asking for recent/current information (dates, news, updates): should_search = true490 - If question is semantically very similar to recent history: should_search = false491 - Confidence should reflect how certain you are about the decision492 """493 494 # Make AI call with timeout495 try:496 ai_response = await asyncio.wait_for(497 gemini_model.generate_content_async(analysis_prompt),498 timeout=5.0 # 5 second timeout for AI decision499 )500 501 # Parse AI response502 response_text = ai_response.text.strip()503 504 # Extract JSON from response (handle cases where AI adds extra text)505 json_match = re.search(r'\{.*\}', response_text, re.DOTALL)506 if json_match:507 json_str = json_match.group()508 result = json.loads(json_str)509 510 # Validate required fields511 required_fields = ['should_search', 'confidence', 'reason']512 if all(field in result for field in required_fields):513 # Ensure confidence is within valid range514 result['confidence'] = max(0.0, min(1.0, float(result['confidence'])))515 516 # Cache the result517 ai_decision_cache[cache_key] = (result, current_time)518 519 logger.info(f"AI search decision: {result['should_search']} (confidence: {result['confidence']:.2f}) - {result['reason']}")520 return result521 else:522 raise ValueError("Missing required fields in AI response")523 else:524 raise ValueError("No valid JSON found in AI response")525 526 except asyncio.TimeoutError:527 logger.warning("AI search decision timed out")528 raise529 except Exception as e:530 logger.error(f"AI search decision parsing error: {e}")531 raise532 533 except Exception as e:534 logger.error(f"AI search analysis failed: {e}")535 # Return fallback decision indicating AI failure536 return {537 "should_search": True, # Conservative fallback538 "confidence": 0.3,539 "reason": f"AI analysis failed: {str(e)[:100]}",540 "analysis": {"ai_failed": True}541 }542 543 544def has_meaningful_conversation_history(history: Optional[List[Dict[str, str]]] = None) -> bool:545 """546 Detect if request has meaningful conversation history for context-aware flow routing.547 548 This function analyzes conversation history to determine if there's sufficient549 context for making informed search decisions. It handles both conversation550 formats and validates content quality.551 552 Args:553 history (Optional[List[Dict[str, str]]]): Conversation history list554 Supports formats: [{"role": "user/assistant", "content": "..."}]555 or [{"user": "...", "assistant": "..."}]556 557 Returns:558 bool: True if conversation has meaningful history, False otherwise559 560 Example:561 >>> has_meaningful_conversation_history([{"user": "Hi", "assistant": "Hello there!"}])562 True563 >>> has_meaningful_conversation_history([])564 False565 """566 try:567 if not history or len(history) == 0:568 return False569 570 # Check for malformed history entries571 meaningful_entries = 0572 for entry in history:573 if isinstance(entry, dict):574 # Handle both formats: {"role": "user/assistant", "content": "..."} 575 # and {"user": "...", "assistant": "..."}576 if ("role" in entry and "content" in entry and 577 entry.get("content", "").strip() and 578 len(entry["content"].strip()) > 5): # Minimum meaningful content579 meaningful_entries += 1580 elif ("user" in entry and "assistant" in entry and581 entry.get("user", "").strip() and entry.get("assistant", "").strip() and582 len(entry["user"].strip()) > 5 and len(entry["assistant"].strip()) > 5):583 meaningful_entries += 1584 585 # Consider history meaningful if we have at least one substantive exchange586 return meaningful_entries >= 1587 588 except Exception as e:589 logger.warning(f"Error detecting conversation history: {e}")590 return False # Conservative fallback591 592 593def analyze_conversation_context(prompt: str, history: Optional[List[Dict[str, str]]] = None, 594 nlp_model=None) -> Dict[str, Any]:595 """596 Analyze conversation context for semantic similarity and topic continuity.597 598 This function performs sophisticated context analysis using NLP techniques:599 1. Semantic similarity analysis using spaCy word vectors600 2. Topic continuity assessment through keyword overlap601 3. Information coverage evaluation based on history richness602 4. Context quality scoring for decision confidence603 604 Args:605 prompt (str): Current user question606 history (Optional[List[Dict[str, str]]]): Conversation history607 nlp_model: spaCy language model for semantic analysis608 609 Returns:610 Dict[str, Any]: Dictionary containing context analysis metrics:611 - topic_continuity (float): Topic overlap score (0.0-1.0)612 - semantic_similarity (float): Average semantic similarity (0.0-1.0)613 - information_coverage (float): History coverage score (0.0-1.0)614 - context_richness (float): Overall context quality (0.0-1.0)615 616 Example:617 >>> analyze_conversation_context("Tell me more", [{"assistant": "AI is..."}], nlp)618 {"topic_continuity": 0.7, "semantic_similarity": 0.8, ...}619 """620 try:621 if not nlp_model:622 raise ValueError("NLP model required for context analysis")623 624 if not history or len(history) == 0:625 return {626 "topic_continuity": 0.0,627 "semantic_similarity": 0.0,628 "information_coverage": 0.0,629 "context_richness": 0.0630 }631 632 # Use spaCy to analyze semantic similarity633 prompt_doc = nlp_model(prompt.lower())634 635 # Analyze recent conversation entries636 recent_entries = history[-3:] if len(history) > 3 else history637 638 similarity_scores = []639 topic_keywords = set()640 total_context_length = 0641 642 for entry in recent_entries:643 if "role" in entry and "content" in entry and entry["role"] == "assistant":644 content = entry["content"]645 content_doc = nlp_model(content.lower())646 647 # Calculate semantic similarity648 similarity = prompt_doc.similarity(content_doc)649 similarity_scores.append(similarity)650 651 # Extract topic keywords652 for token in content_doc:653 if not token.is_stop and not token.is_punct and len(token.text) > 2:654 topic_keywords.add(token.lemma_)655 656 total_context_length += len(content.split())657 658 elif "assistant" in entry:659 content = entry["assistant"]660 content_doc = nlp_model(content.lower())661 662 similarity = prompt_doc.similarity(content_doc)663 similarity_scores.append(similarity)664 665 for token in content_doc:666 if not token.is_stop and not token.is_punct and len(token.text) > 2:667 topic_keywords.add(token.lemma_)668 669 total_context_length += len(content.split())670 671 # Calculate metrics672 avg_similarity = sum(similarity_scores) / len(similarity_scores) if similarity_scores else 0.0673 674 # Topic continuity based on keyword overlap675 prompt_keywords = set()676 for token in prompt_doc:677 if not token.is_stop and not token.is_punct and len(token.text) > 2:678 prompt_keywords.add(token.lemma_)679 680 topic_overlap = len(prompt_keywords.intersection(topic_keywords)) / max(len(prompt_keywords), 1)681 682 # Information coverage (how much context is available)683 context_richness = min(1.0, total_context_length / 100) # Normalize to 0-1684 685 return {686 "topic_continuity": topic_overlap,687 "semantic_similarity": avg_similarity,688 "information_coverage": len(recent_entries) / 3.0, # Normalized to max 3 entries689 "context_richness": context_richness690 }691 692 except Exception as e:693 logger.error(f"Context analysis failed: {e}")694 return {695 "topic_continuity": 0.0,696 "semantic_similarity": 0.0,697 "information_coverage": 0.0,698 "context_richness": 0.0699 }700 701 702async def hybrid_search_decision(prompt: str, history: Optional[List[Dict[str, str]]] = None,703 search_decision_mode: str = "balanced", nlp_model=None, 704 gemini_model=None) -> Dict[str, Any]:705 """706 Hybrid search decision combining rule-based and AI-based analysis.707 708 This function implements the core hybrid decision engine that combines:709 1. Fast rule-based pattern matching for obvious cases710 2. AI analysis for ambiguous scenarios requiring deeper understanding711 3. Context analysis for semantic relationship assessment712 4. Confidence-based decision weighting and fallback mechanisms713 714 Args:715 prompt (str): User's current question716 history (Optional[List[Dict[str, str]]]): Conversation history717 search_decision_mode (str): Decision mode ("conservative", "balanced", "aggressive")718 nlp_model: spaCy language model for context analysis719 gemini_model: Google Generative AI model for intelligent analysis720 721 Returns:722 Dict[str, Any]: Comprehensive decision dictionary containing:723 - should_search (bool): Final search decision724 - confidence (float): Overall confidence score725 - reason (str): Explanation of decision logic726 - rule_decision (dict): Rule-based analysis results727 - ai_decision (dict, optional): AI analysis results if used728 - context_analysis (dict): Semantic context metrics729 - decision_method (str): Method used ("rule_based", "hybrid", "fallback_rule")730 731 Example:732 >>> await hybrid_search_decision("What else can you tell me?", history, "balanced", nlp, ai)733 {"should_search": False, "confidence": 0.85, "reason": "Rule-based: Follow-up detected", ...}734 """735 try:736 # Step 1: Get rule-based decision737 rule_decision = should_perform_search(prompt, history, search_decision_mode)738 739 # Step 2: Analyze conversation context740 context_analysis = analyze_conversation_context(prompt, history, nlp_model)741 742 # Step 3: Determine if AI analysis is needed743 ai_threshold = {744 "conservative": 0.8,745 "balanced": 0.6, 746 "aggressive": 0.4747 }.get(search_decision_mode, 0.6)748 749 # Use AI for ambiguous cases (low confidence rule decisions)750 if rule_decision["confidence"] < ai_threshold:751 logger.info(f"Rule confidence {rule_decision['confidence']:.2f} below threshold {ai_threshold}, using AI analysis")752 753 # Get AI decision754 from app import format_conversation_history # Import to avoid circular dependency755 conversation_context = format_conversation_history(history, max_entries=3)756 ai_decision = await analyze_search_necessity(prompt, history, conversation_context, gemini_model)757 758 # Combine decisions with weighted confidence759 rule_weight = rule_decision["confidence"]760 ai_weight = ai_decision["confidence"]761 total_weight = rule_weight + ai_weight762 763 if total_weight > 0:764 # Weighted decision765 final_should_search = (766 (rule_decision["should_search"] * rule_weight + 767 ai_decision["should_search"] * ai_weight) / total_weight768 ) > 0.5769 770 final_confidence = (rule_decision["confidence"] + ai_decision["confidence"]) / 2771 else:772 # Fallback to rule decision773 final_should_search = rule_decision["should_search"]774 final_confidence = rule_decision["confidence"]775 776 return {777 "should_search": final_should_search,778 "confidence": final_confidence,779 "reason": f"Hybrid: Rule={rule_decision['reason'][:50]}..., AI={ai_decision['reason'][:50]}...",780 "rule_decision": rule_decision,781 "ai_decision": ai_decision,782 "context_analysis": context_analysis,783 "decision_method": "hybrid"784 }785 else:786 # High confidence rule decision, no need for AI787 logger.info(f"Rule confidence {rule_decision['confidence']:.2f} above threshold, using rule-based decision")788 return {789 "should_search": rule_decision["should_search"],790 "confidence": rule_decision["confidence"],791 "reason": f"Rule-based: {rule_decision['reason']}",792 "rule_decision": rule_decision,793 "context_analysis": context_analysis,794 "decision_method": "rule_based"795 }796 797 except Exception as e:798 logger.error(f"Hybrid search decision failed: {e}")799 # Fallback to rule-based decision800 rule_decision = should_perform_search(prompt, history, search_decision_mode)801 return {802 "should_search": rule_decision["should_search"],803 "confidence": rule_decision["confidence"],804 "reason": f"Fallback to rules due to error: {str(e)[:50]}",805 "rule_decision": rule_decision,806 "decision_method": "fallback_rule",807 "error": str(e)808 }809 810 811# Convenience functions for maintaining backward compatibility812def get_search_optimizer_instance(nlp_model, rake_instance, gemini_model) -> SearchOptimizer:813 """814 Factory function to create SearchOptimizer instance with dependencies.815 816 Args:817 nlp_model: spaCy language model instance818 rake_instance: RAKE keyword extraction instance819 gemini_model: Google Generative AI model instance820 821 Returns:822 SearchOptimizer: Configured optimizer instance823 """824 return SearchOptimizer(nlp_model, rake_instance, gemini_model)