MMo4/csit-ned-chatbot
0
1"""
2Graph-enhanced retrieval system for CSIT RAG.
3Combines FAISS vector search with knowledge graph context.
4"""
5
6import logging
7import re
8from typing import List, Dict, Any, Optional, Tuple
9from langchain_core.documents import Document
10import numpy as np
11from src.utils.fuzzy_matcher import get_fuzzy_matcher
12
13logger = logging.getLogger(__name__)
14
15class GraphEnhancedRetriever:
16 """Retriever that combines FAISS vector search with graph-based context weighting."""
17
18 def __init__(self,
19 faiss_manager,
20 knowledge_graph,
21 context_weighter,
22 embedding_manager):
23
24 self.faiss_manager = faiss_manager
25 self.knowledge_graph = knowledge_graph
26 self.context_weighter = context_weighter
27 self.embedding_manager = embedding_manager
28
29 # Configuration
30 self.base_search_k = 15 # Search more documents initially
31 self.final_k = 5 # Return top K after reranking
32 self.context_weight = 0.4 # Weight for context vs similarity
33 self.similarity_weight = 0.6 # Weight for vector similarity
34
35 # Fuzzy matching for names
36 self.fuzzy_matcher = get_fuzzy_matcher(threshold=65.0) # Lower threshold for more matches
37 self._faculty_names_cache = None
38
39 def retrieve(self,
40 query: str,
41 processed_query,
42 conversation_session,
43 k: int = 5,
44 metadata_filter: Optional[Dict[str, Any]] = None) -> List[Tuple[Document, float, Dict[str, Any]]]:
45 """
46 Retrieve documents using graph-enhanced approach.
47
48 Returns:
49 List of (Document, combined_score, explanation) tuples
50 """
51
52 try:
53 logger.info(f"Starting graph-enhanced retrieval for query: {query[:100]}...")
54
55 # 🔹 STEP 0: Apply fuzzy matching for faculty queries
56 enhanced_query = query
57 if processed_query.detected_intent == 'faculty_info':
58 enhanced_query = self._apply_fuzzy_matching(query)
59 logger.info(f"Applied fuzzy matching: '{query}' -> '{enhanced_query}'")
60
61 # 🔹 STEP 0.5: KEYWORD-BASED DOCUMENT RETRIEVAL
62 # Get documents that contain exact keywords from query (critical for accuracy)
63 keyword_matched_docs = self._get_keyword_matched_documents(query)
64 if keyword_matched_docs:
65 logger.info(f"🔑 Found {len(keyword_matched_docs)} documents via keyword matching")
66
67 # 🔹 SMART METADATA FILTERING based on query intent
68 if metadata_filter is None:
69 metadata_filter = {}
70
71 # If academic query, filter to academic category ONLY
72 if processed_query.detected_intent == 'academic_info':
73 metadata_filter['category'] = 'academics'
74 logger.info("🎓 Academic query detected - filtering to category='academics'")
75
76 # If specific departments mentioned, filter by those
77 if processed_query.detected_departments:
78 # Add department filter to metadata_filter
79 metadata_filter['departments'] = processed_query.detected_departments
80 logger.info(f"📚 Filtering to departments: {processed_query.detected_departments}")
81
82 # 🔹 NEW: If specific specializations mentioned, filter by those
83 # BUT: If empty list (meaning user asked about 3rd/4th year without specifying),
84 # DON'T filter - let them see all options
85 if processed_query.detected_specializations:
86 metadata_filter['specialization'] = processed_query.detected_specializations
87 logger.info(f"🎯 Filtering to specializations: {processed_query.detected_specializations}")
88 else:
89 # No specialization filter - increase retrieval to get ALL specialization tracks
90 self.base_search_k = 25 # Increase to get AI, Data Science, Cyber, Gaming, General
91 logger.info(f"📚 No specialization filter - increased retrieval to show all tracks")
92
93 # If faculty query, filter to faculty category
94 elif processed_query.detected_intent == 'faculty_info':
95 metadata_filter['category'] = 'faculty'
96 logger.info("👨🏫 Faculty query detected - filtering to category='faculty'")
97
98 # Check if this is a comprehensive faculty listing query
99 # (vs asking about a specific person)
100 comprehensive_patterns = [
101 'all faculty', 'faculty members', 'list of faculty', 'who are',
102 'faculty list', 'tell me about faculty', 'csit faculty',
103 'department faculty', 'faculty at', 'faculty in'
104 ]
105 is_comprehensive = any(pattern in query.lower() for pattern in comprehensive_patterns)
106
107 if is_comprehensive:
108 # For comprehensive queries, retrieve MORE documents
109 # to ensure we get the main listing + individual profiles
110 self.base_search_k = 20 # Temporarily increase
111 logger.info("📋 Comprehensive faculty query - increased retrieval to 20 docs")
112
113 # If achievements/alumni query, filter to achievements category (handle both spellings)
114 elif processed_query.detected_intent == 'achievements':
115 # Note: Some files have "acheivements" (misspelled), others "achievements"
116 # We'll search without filtering but boost achievements documents
117 logger.info("🏆 Achievements/Alumni query detected - will prioritize achievements category")
118 self.base_search_k = 20 # Increase to get more results including achievements
119
120 # Step 1: Generate query embedding (use enhanced query for faculty)
121 query_for_embedding = enhanced_query if processed_query.detected_intent == 'faculty_info' else processed_query.enhanced_query
122 query_embedding = self.embedding_manager.embed_query(query_for_embedding)
123 if query_embedding.size == 0:
124 logger.error("Failed to generate query embedding")
125 return []
126
127 # Step 2: Initial vector search (with smart filtering)
128 initial_results = self.faiss_manager.search(
129 query_embedding,
130 k=self.base_search_k,
131 metadata_filter=metadata_filter
132 )
133
134 # Reset base_search_k after use
135 if processed_query.detected_intent == 'faculty_info':
136 self.base_search_k = 15 # Reset to default
137
138 # 🔹 STEP 2.1: MERGE keyword-matched documents with vector search results
139 # This ensures critical keyword matches are ALWAYS included
140 if keyword_matched_docs:
141 existing_chunk_ids = {doc.metadata.get('chunk_id') for doc, _ in initial_results}
142 for doc in keyword_matched_docs:
143 if doc.metadata.get('chunk_id') not in existing_chunk_ids:
144 # Add with high score (low distance) to prioritize keyword matches
145 initial_results.insert(0, (doc, 0.3)) # Very high priority
146 logger.info(f"🔑 Force-included keyword-matched doc: {doc.metadata.get('title', 'Unknown')[:50]}")
147
148 if not initial_results:
149 logger.warning("No initial vector search results found")
150 return []
151
152 # 🔹 STEP 2.5: For comprehensive faculty queries, ensure main faculty listing is included
153 if processed_query.detected_intent == 'faculty_info':
154 comprehensive_patterns = [
155 'all faculty', 'faculty members', 'list of faculty', 'who are',
156 'faculty list', 'tell me about faculty', 'csit faculty',
157 'department faculty', 'faculty at', 'faculty in'
158 ]
159 is_comprehensive = any(pattern in query.lower() for pattern in comprehensive_patterns)
160
161 if is_comprehensive:
162 # Check if main faculty listing (chunk_id: "faculty_listing_cct_neduet") is in results
163 chunk_ids = [doc.metadata.get('chunk_id') for doc, _ in initial_results]
164
165 if "faculty_listing_cct_neduet" not in chunk_ids:
166 # Force include the main faculty listing document
167 all_docs = self.faiss_manager.get_all_documents()
168 for doc in all_docs:
169 if doc.metadata.get('chunk_id') == "faculty_listing_cct_neduet":
170 # Add it with a high score (low distance) to ensure it's included
171 initial_results.insert(0, (doc, 0.5)) # Insert at beginning with good score
172 logger.info("📋 Force-included main faculty listing document for comprehensive query")
173 break
174
175 # 🔹 STEP 2.6: For comprehensive student clubs queries, ensure main 7 active clubs are included
176 if processed_query.detected_intent == 'student_activities':
177 comprehensive_club_patterns = [
178 'student clubs', 'what clubs', 'all clubs', 'clubs at', 'clubs in',
179 'tell me about clubs', 'list of clubs', 'csit clubs', 'department clubs'
180 ]
181 is_comprehensive_clubs = any(pattern in query.lower() for pattern in comprehensive_club_patterns)
182
183 if is_comprehensive_clubs:
184 # Ensure all 7 main active clubs are in results
185 main_active_clubs = [
186 "koderz_club", # Priority 1 - Main coding club
187 "cybersents_club_overview", # Priority 2 - Cybersecurity
188 "ai_alliance", # Priority 3 - AI
189 "data_insight_student_club", # Priority 4 - Data Science
190 "csit_gameverse_club", # Priority 5 - Gaming & Animation
191 "ledger_league", # Priority 6 - Blockchain
192 "qubit_qrew" # Priority 7 - Quantum Technologies
193 ]
194
195 chunk_ids = [doc.metadata.get('chunk_id') for doc, _ in initial_results]
196 all_docs = self.faiss_manager.get_all_documents()
197
198 # Force include any missing main clubs
199 for club_id in main_active_clubs:
200 if club_id not in chunk_ids:
201 for doc in all_docs:
202 if doc.metadata.get('chunk_id') == club_id:
203 initial_results.append((doc, 0.5)) # Add with good score
204 logger.info(f"🎯 Force-included {club_id} for comprehensive clubs query")
205 break
206
207 # 🔹 STEP 2.8: For events queries, ensure major events are included
208 if processed_query.detected_intent == 'events':
209 # Check if this is a general events query (not specific event)
210 general_event_patterns = [
211 'events', 'activities', 'what.*happen', 'upcoming',
212 'csit events', 'department events'
213 ]
214 is_general_events = any(re.search(pattern, query.lower()) for pattern in general_event_patterns)
215
216 if is_general_events:
217 # Ensure major events are included
218 major_events = [
219 "techfest_fall_25", # Latest TechFest
220 "sports_fest_csit_2025", # SportsFest
221 # ICONICS is in i_event.md, not separate file
222 ]
223
224 chunk_ids = [doc.metadata.get('chunk_id') for doc, _ in initial_results]
225 all_docs = self.faiss_manager.get_all_documents()
226
227 for event_id in major_events:
228 if event_id not in chunk_ids:
229 for doc in all_docs:
230 if doc.metadata.get('chunk_id') == event_id:
231 initial_results.insert(0, (doc, 0.4)) # High priority
232 logger.info(f"📅 Force-included major event: {event_id}")
233 break
234
235 # Step 3: Extract concepts from all candidate documents
236 candidate_concepts = self._extract_document_concepts(initial_results)
237
238 # Step 4: Calculate context weights using graph
239 context_weights = self.context_weighter.calculate_context_weights(
240 query_concepts=processed_query.extracted_concepts,
241 conversation_session=conversation_session,
242 all_document_concepts=candidate_concepts
243 )
244
245 # Step 5: Rerank results using combined scoring
246 enhanced_results = self._rerank_with_context(
247 initial_results,
248 context_weights,
249 processed_query,
250 conversation_session
251 )
252
253 # Step 6: Apply event-specific sorting and filtering
254 if processed_query.detected_intent == 'events':
255 enhanced_results = self._sort_events_by_importance_and_recency(enhanced_results)
256 logger.info("📅 Sorted events by importance and recency")
257
258 # Step 6.25: BOOST documents that contain exact query keywords (highest priority)
259 # This ensures that if user asks "what is iconics", ICONICS docs come first
260 enhanced_results = self._boost_exact_keyword_matches(enhanced_results, query)
261
262 # Step 6.5: Apply student clubs sorting and filtering
263 if processed_query.detected_intent == 'student_activities':
264 enhanced_results = self._sort_student_clubs_by_priority(enhanced_results)
265 logger.info("🎯 Sorted student clubs by priority")
266
267 # Step 7: Apply final filtering and return top K
268 # For comprehensive faculty queries, return more results
269 if processed_query.detected_intent == 'faculty_info':
270 comprehensive_patterns = ['all faculty', 'faculty members', 'list of', 'who are', 'faculty list', 'tell me about faculty']
271 is_comprehensive = any(pattern in query.lower() for pattern in comprehensive_patterns)
272 if is_comprehensive:
273 # Return top 10 for comprehensive faculty queries (instead of 5)
274 final_results = enhanced_results[:min(10, len(enhanced_results))]
275 logger.info(f"📋 Comprehensive faculty query - returning {len(final_results)} results for complete listing")
276 else:
277 final_results = enhanced_results[:k]
278 # For comprehensive student clubs queries, return more results
279 elif processed_query.detected_intent == 'student_activities':
280 comprehensive_club_patterns = ['student clubs', 'what clubs', 'all clubs', 'clubs at', 'list of clubs']
281 is_comprehensive_clubs = any(pattern in query.lower() for pattern in comprehensive_club_patterns)
282 if is_comprehensive_clubs:
283 # Return top 10 for comprehensive clubs queries (7 main clubs + competitions)
284 final_results = enhanced_results[:min(10, len(enhanced_results))]
285 logger.info(f"🎯 Comprehensive clubs query - returning {len(final_results)} results for all main clubs")
286 else:
287 final_results = enhanced_results[:k]
288 # For academic queries without specialization, return more results to cover all tracks
289 elif processed_query.detected_intent == 'academic_info' and not processed_query.detected_specializations:
290 # Return top 10 to include all specialization tracks (AI, DS, Cyber, Gaming, General)
291 final_results = enhanced_results[:min(10, len(enhanced_results))]
292 logger.info(f"📚 Multi-track academic query - returning {len(final_results)} results for all specializations")
293 # For achievements queries, return more results
294 elif processed_query.detected_intent == 'achievements':
295 final_results = enhanced_results[:min(10, len(enhanced_results))]
296 logger.info(f"🏆 Achievements query - returning {len(final_results)} results")
297 else:
298 final_results = enhanced_results[:k]
299
300 logger.info(f"Retrieved {len(final_results)} documents with graph enhancement")
301 return final_results
302
303 except Exception as e:
304 logger.error(f"Error in graph-enhanced retrieval: {e}")
305 return []
306
307 def _extract_document_concepts(self, search_results: List[Tuple[Document, float]]) -> List[str]:
308 """Extract all concepts from candidate documents."""
309
310 concepts = set()
311
312 for document, _ in search_results:
313 metadata = document.metadata
314
315 # Extract from metadata
316 if 'extracted_concepts' in metadata:
317 concepts.update(metadata['extracted_concepts'])
318
319 if 'topics' in metadata:
320 topics = metadata['topics']
321 if isinstance(topics, list):
322 concepts.update(topics)
323
324 if 'departments' in metadata:
325 departments = metadata['departments']
326 if isinstance(departments, list):
327 concepts.update(departments)
328
329 # Extract from content (simple approach)
330 content_concepts = self._simple_concept_extraction(document.page_content)
331 concepts.update(content_concepts)
332
333 return list(concepts)
334
335 def _simple_concept_extraction(self, content: str) -> List[str]:
336 """Simple concept extraction from document content."""
337
338 concepts = []
339 content_lower = content.lower()
340
341 # Key concepts to look for
342 concept_keywords = [
343 'bcit', 'se', 'cis', 'software engineering', 'computer science',
344 'career', 'job', 'salary', 'employment', 'company',
345 'theory', 'theoretical', 'practical', 'hands-on',
346 'curriculum', 'course', 'lab', 'project',
347 'admission', 'merit', 'requirement',
348 'anxiety', 'worry', 'confidence', 'concern'
349 ]
350
351 for keyword in concept_keywords:
352 if keyword in content_lower:
353 concepts.append(keyword)
354
355 return concepts
356
357 def _rerank_with_context(self,
358 initial_results: List[Tuple[Document, float]],
359 context_weights: Dict[str, float],
360 processed_query,
361 conversation_session) -> List[Tuple[Document, float, Dict[str, Any]]]:
362 """Rerank initial results using graph-based context."""
363
364 enhanced_results = []
365
366 for document, similarity_distance in initial_results:
367
368 # Calculate context relevance score
369 context_score = self._calculate_document_context_score(
370 document, context_weights, processed_query
371 )
372
373 # Convert FAISS distance to similarity (lower distance = higher similarity)
374 # Normalize to 0-1 range
375 similarity_score = 1.0 / (1.0 + similarity_distance)
376
377 # Combine scores
378 combined_score = (
379 self.similarity_weight * similarity_score +
380 self.context_weight * context_score
381 )
382
383 # Create explanation
384 explanation = {
385 'similarity_score': similarity_score,
386 'similarity_distance': similarity_distance,
387 'context_score': context_score,
388 'combined_score': combined_score,
389 'contributing_concepts': self._get_contributing_concepts(document, context_weights),
390 'emotional_relevance': self._check_emotional_relevance(document, processed_query),
391 'conversation_relevance': self._check_conversation_relevance(document, conversation_session)
392 }
393
394 enhanced_results.append((document, combined_score, explanation))
395
396 # Sort by combined score (higher is better)
397 enhanced_results.sort(key=lambda x: x[1], reverse=True)
398
399 return enhanced_results
400
401 def _calculate_document_context_score(self,
402 document: Document,
403 context_weights: Dict[str, float],
404 processed_query) -> float:
405 """Calculate context relevance score for a document."""
406
407 # Extract document concepts
408 doc_concepts = self._get_document_concepts(document)
409
410 if not doc_concepts or not context_weights:
411 return 0.0
412
413 # Calculate weighted relevance
414 total_weight = 0.0
415 matched_concepts = 0
416
417 for concept in doc_concepts:
418 if concept in context_weights:
419 total_weight += context_weights[concept]
420 matched_concepts += 1
421
422 # Average weight with coverage penalty
423 if matched_concepts == 0:
424 return 0.0
425
426 avg_weight = total_weight / matched_concepts
427
428 # Apply coverage boost (more matched concepts = higher score)
429 coverage_ratio = matched_concepts / len(doc_concepts)
430 coverage_boost = 0.8 + (0.4 * coverage_ratio) # 0.8 to 1.2 range
431
432 context_score = avg_weight * coverage_boost
433
434 # Additional scoring factors
435
436 # Boost for department match
437 query_departments = processed_query.detected_departments
438 doc_departments = document.metadata.get('departments', [])
439 if query_departments and doc_departments:
440 if any(dept in doc_departments for dept in query_departments):
441 context_score += 0.2
442
443 # Boost for emotional context match
444 query_emotions = processed_query.detected_emotions
445 doc_emotional_context = document.metadata.get('emotional_context', '')
446 if 'worried' in query_emotions or 'anxious' in query_emotions:
447 if doc_emotional_context == 'reassuring':
448 context_score += 0.3
449
450 return min(context_score, 1.0) # Cap at 1.0
451
452 def _get_document_concepts(self, document: Document) -> List[str]:
453 """Extract all concepts from a document."""
454
455 concepts = []
456 metadata = document.metadata
457
458 # From metadata
459 if 'extracted_concepts' in metadata:
460 concepts.extend(metadata['extracted_concepts'])
461
462 if 'topics' in metadata:
463 topics = metadata['topics']
464 if isinstance(topics, list):
465 concepts.extend(topics)
466
467 if 'departments' in metadata:
468 departments = metadata['departments']
469 if isinstance(departments, list):
470 concepts.extend(departments)
471
472 # From category and subcategory
473 if 'category' in metadata:
474 concepts.append(metadata['category'])
475
476 if 'subcategory' in metadata:
477 concepts.append(metadata['subcategory'])
478
479 # From content
480 content_concepts = self._simple_concept_extraction(document.page_content)
481 concepts.extend(content_concepts)
482
483 return list(set(concepts)) # Remove duplicates
484
485 def _get_contributing_concepts(self, document: Document, context_weights: Dict[str, float]) -> List[Tuple[str, float]]:
486 """Get concepts that contributed to the document's context score."""
487
488 doc_concepts = self._get_document_concepts(document)
489 contributing = []
490
491 for concept in doc_concepts:
492 if concept in context_weights and context_weights[concept] > 0.1:
493 contributing.append((concept, context_weights[concept]))
494
495 # Sort by weight
496 contributing.sort(key=lambda x: x[1], reverse=True)
497 return contributing[:5] # Top 5 contributing concepts
498
499 def _check_emotional_relevance(self, document: Document, processed_query) -> Dict[str, Any]:
500 """Check emotional relevance of document to query."""
501
502 query_emotions = processed_query.detected_emotions
503 doc_emotional_context = document.metadata.get('emotional_context', '')
504
505 relevance = {
506 'query_emotions': query_emotions,
507 'document_emotional_context': doc_emotional_context,
508 'is_relevant': False,
509 'relevance_type': None
510 }
511
512 # Check for emotional matches
513 if 'worried' in query_emotions or 'anxious' in query_emotions:
514 if doc_emotional_context in ['reassuring', 'supportive']:
515 relevance['is_relevant'] = True
516 relevance['relevance_type'] = 'reassuring_for_anxiety'
517
518 elif 'curious' in query_emotions:
519 if doc_emotional_context in ['informative', 'detailed']:
520 relevance['is_relevant'] = True
521 relevance['relevance_type'] = 'informative_for_curiosity'
522
523 elif 'confused' in query_emotions:
524 if doc_emotional_context in ['clarifying', 'comparative']:
525 relevance['is_relevant'] = True
526 relevance['relevance_type'] = 'clarifying_for_confusion'
527
528 return relevance
529
530 def _check_conversation_relevance(self, document: Document, conversation_session) -> Dict[str, Any]:
531 """Check how relevant document is to conversation flow."""
532
533 conv_context = conversation_session.get_current_context()
534
535 relevance = {
536 'matches_primary_interests': False,
537 'continues_recent_topics': False,
538 'addresses_decision_stage': False,
539 'fills_information_gap': False
540 }
541
542 doc_concepts = self._get_document_concepts(document)
543
544 # Check against primary interests
545 primary_interests = conv_context.get('primary_interests', [])
546 if any(interest in doc_concepts for interest in primary_interests):
547 relevance['matches_primary_interests'] = True
548
549 # Check against recent topics
550 recent_topics = conv_context.get('recent_topics', [])
551 if any(topic in doc_concepts for topic in recent_topics):
552 relevance['continues_recent_topics'] = True
553
554 # Check decision stage relevance
555 decision_stage = conv_context.get('decision_stage', 'exploration')
556 doc_category = document.metadata.get('category', '')
557
558 if decision_stage == 'comparison' and doc_category == 'comparisons':
559 relevance['addresses_decision_stage'] = True
560 elif decision_stage == 'decision' and doc_category in ['career_outcomes', 'student_concerns']:
561 relevance['addresses_decision_stage'] = True
562
563 # Check for information gaps
564 missing_info = conversation_session.get_missing_information(self.knowledge_graph)
565 if any(missing in doc_concepts for missing in missing_info):
566 relevance['fills_information_gap'] = True
567
568 return relevance
569
570 def get_retrieval_explanation(self,
571 results: List[Tuple[Document, float, Dict[str, Any]]],
572 processed_query,
573 conversation_session) -> Dict[str, Any]:
574 """Generate explanation of retrieval decisions."""
575
576 if not results:
577 return {'status': 'no_results'}
578
579 conv_context = conversation_session.get_current_context()
580
581 explanation = {
582 'total_results': len(results),
583 'query_analysis': {
584 'intent': processed_query.detected_intent,
585 'departments': processed_query.detected_departments,
586 'emotions': processed_query.detected_emotions,
587 'concepts': processed_query.extracted_concepts
588 },
589 'conversation_context': {
590 'emotional_state': conv_context.get('emotional_state'),
591 'decision_stage': conv_context.get('decision_stage'),
592 'primary_interests': conv_context.get('primary_interests', [])[:3]
593 },
594 'top_results_analysis': [],
595 'scoring_weights': {
596 'similarity_weight': self.similarity_weight,
597 'context_weight': self.context_weight
598 }
599 }
600
601 # Analyze top 3 results
602 for i, (document, score, doc_explanation) in enumerate(results[:3]):
603 result_analysis = {
604 'rank': i + 1,
605 'title': document.metadata.get('title', 'Untitled'),
606 'category': document.metadata.get('category', 'unknown'),
607 'combined_score': round(score, 3),
608 'similarity_score': round(doc_explanation['similarity_score'], 3),
609 'context_score': round(doc_explanation['context_score'], 3),
610 'key_contributing_concepts': [concept for concept, _ in doc_explanation['contributing_concepts'][:3]],
611 'emotional_relevance': doc_explanation['emotional_relevance']['is_relevant'],
612 'conversation_relevance': any(doc_explanation['conversation_relevance'].values())
613 }
614 explanation['top_results_analysis'].append(result_analysis)
615
616 return explanation
617
618 def _apply_fuzzy_matching(self, query: str) -> str:
619 """
620 Apply fuzzy matching to expand faculty names in query.
621 E.g., "Dr. Mubashir" -> "Dr. Mubashir Prof. Dr. Muhammad Mubashir Khan"
622 """
623 try:
624 # Get faculty names from knowledge base
625 faculty_names = self._get_faculty_names()
626
627 if not faculty_names:
628 return query
629
630 # Use fuzzy matcher to expand query
631 expanded_query = self.fuzzy_matcher.expand_query_with_matches(query, faculty_names)
632
633 return expanded_query
634
635 except Exception as e:
636 logger.error(f"Error applying fuzzy matching: {e}")
637 return query
638
639 def _get_faculty_names(self) -> List[str]:
640 """
641 Extract all faculty names from the knowledge base.
642 Caches results for performance.
643 """
644 if self._faculty_names_cache is not None:
645 return self._faculty_names_cache
646
647 try:
648 faculty_names = []
649
650 # Get all documents from FAISS
651 all_docs = self.faiss_manager.get_all_documents()
652
653 for doc in all_docs:
654 metadata = doc.metadata
655 category = metadata.get('category', '')
656
657 # Look for faculty documents
658 if category == 'faculty':
659 content = doc.page_content
660
661 # Extract faculty names from content
662 # Look for patterns like "**Prof. Dr. Name**" or "**Dr. Name**"
663 import re
664 name_patterns = [
665 r'\*\*([^*]+)\.\*\*', # **Name.**
666 r'\*\*([^*]+Dr\.[^*]+)\*\*', # **Title Dr. Name**
667 r'Prof\.\s+Dr\.\s+([A-Z][a-z]+(?:\s+[A-Z][a-z]+)+)', # Prof. Dr. Full Name
668 r'Dr\.\s+([A-Z][a-z]+(?:\s+[A-Z][a-z]+)+)', # Dr. Full Name
669 r'Engr\.\s+Dr\.\s+([A-Z][a-z]+(?:\s+[A-Z][a-z]+)+)', # Engr. Dr. Full Name
670 ]
671
672 for pattern in name_patterns:
673 matches = re.findall(pattern, content)
674 for match in matches:
675 # Clean the name
676 name = match.strip().rstrip('.')
677 if len(name) > 5 and name not in faculty_names: # Filter out short strings
678 faculty_names.append(name)
679
680 # Cache the results
681 self._faculty_names_cache = list(set(faculty_names))
682
683 logger.info(f"Extracted {len(self._faculty_names_cache)} faculty names from knowledge base")
684
685 return self._faculty_names_cache
686
687 except Exception as e:
688 logger.error(f"Error extracting faculty names: {e}")
689 return []
690
691 def _sort_events_by_importance_and_recency(self, results: List[Tuple[Document, float, Dict]]) -> List[Tuple[Document, float, Dict]]:
692 """
693 Sort event documents by importance and recency.
694
695 Priority Order:
696 1. Major events (TechFest, ICONICS, SportsFest, Alumni Reunion) - Latest first
697 2. Other events - Latest first
698
699 Args:
700 results: List of (document, score, explanation) tuples
701
702 Returns:
703 Sorted list with major recent events first
704 """
705 import re
706 from datetime import datetime
707
708 def get_event_priority(doc: Document) -> Tuple[int, str]:
709 """
710 Return (priority_level, date_string) for sorting.
711 Lower priority number = higher importance
712 Later date = more recent
713 """
714 metadata = doc.metadata
715 subcategory = metadata.get('subcategory', '').lower()
716 chunk_id = metadata.get('chunk_id', '').lower()
717 title = metadata.get('title', '').lower()
718
719 # Extract year from various metadata fields
720 date_str = "2000-01-01" # Default old date
721
722 # Try to extract from data_freshness
723 if 'data_freshness' in metadata:
724 date_str = metadata['data_freshness']
725 # Try to extract from chunk_id or title
726 else:
727 year_match = re.search(r'20\d{2}', chunk_id + ' ' + title)
728 if year_match:
729 year = year_match.group()
730 # Try to extract month
731 month = "01"
732 if 'fall' in chunk_id or 'fall' in title:
733 month = "10" # Fall = October
734 elif 'spring' in chunk_id or 'spring' in title:
735 month = "03" # Spring = March
736 date_str = f"{year}-{month}-01"
737
738 # Check content for keyword matching
739 content_lower = doc.page_content.lower()[:1000] # Check first 1000 chars
740
741 # Determine priority level
742 # Priority 1: TechFest (most important, happens twice a year)
743 if 'techfest' in subcategory or 'techfest' in chunk_id or 'tech_fest' in subcategory:
744 return (1, date_str)
745
746 # Priority 2: ICONICS (major international conference)
747 elif 'iconics' in chunk_id or 'iconics' in title or 'iconics' in content_lower:
748 return (2, date_str)
749
750 # Priority 3: SportsFest (department-wide event)
751 elif 'sports' in subcategory or 'sportsfest' in chunk_id or 's_fest' in chunk_id:
752 return (3, date_str)
753
754 # Priority 4: Alumni Reunion (new major event)
755 elif 'alumni reunion' in title.lower() or 'alumni reunion' in content_lower:
756 return (4, date_str)
757
758 # Priority 5: Other events
759 else:
760 return (5, date_str)
761
762 # Add priority and date to each result
763 results_with_priority = []
764 for doc, score, explanation in results:
765 priority, date_str = get_event_priority(doc)
766 results_with_priority.append((doc, score, explanation, priority, date_str))
767
768 # Sort by: 1) Priority (lower = higher), 2) Date (newer = higher), 3) Original score
769 results_with_priority.sort(
770 key=lambda x: (x[3], x[4], -x[1]), # priority asc, date asc, score desc
771 reverse=False # Lower priority number comes first
772 )
773
774 # But we want NEWER dates first, so let's fix this
775 results_with_priority.sort(
776 key=lambda x: (x[3], x[4]), # priority asc, date desc (we'll reverse date)
777 reverse=False
778 )
779
780 # Actually, let me redo this more carefully
781 # Group by priority
782 priority_groups = {}
783 for doc, score, explanation, priority, date_str in results_with_priority:
784 if priority not in priority_groups:
785 priority_groups[priority] = []
786 priority_groups[priority].append((doc, score, explanation, date_str))
787
788 # Sort each priority group by date (newest first) and then by score
789 sorted_results = []
790 for priority in sorted(priority_groups.keys()):
791 group = priority_groups[priority]
792 # Sort by date DESC (newest first), then by score DESC (highest first)
793 group.sort(key=lambda x: (x[3], x[1]), reverse=True)
794 # Add to final results (without priority and date)
795 sorted_results.extend([(doc, score, explanation) for doc, score, explanation, _ in group])
796
797 logger.info(f"📅 Event sorting applied - prioritized {len(sorted_results)} events")
798 return sorted_results
799
800 def _sort_student_clubs_by_priority(self, results: List[Tuple[Document, float, Dict]]) -> List[Tuple[Document, float, Dict]]:
801 """
802 Sort student clubs by priority.
803
804 Priority Order:
805 1. Koderz Club (Main coding club - Priority 1)
806 2. CyberSENTS (Cybersecurity - Priority 2)
807 3. AI Alliance (AI - Priority 3)
808 4. Data Insight (Data Science - Priority 4)
809 5. Gameverse (Gaming & Animation - Priority 5)
810 6. Ledger League (Blockchain - Priority 6)
811 7. Qubits Qrew (Quantum Technologies - Priority 7)
812 8. Other clubs/competitions (WebKode, MLSA - Priority 8+)
813
814 Args:
815 results: List of (document, score, explanation) tuples
816
817 Returns:
818 Sorted list with main active clubs first in priority order
819 """
820
821 def get_club_priority(doc: Document) -> int:
822 """
823 Return priority level for sorting.
824 Lower number = higher priority
825 """
826 chunk_id = doc.metadata.get('chunk_id', '').lower()
827 title = doc.metadata.get('title', '').lower()
828
829 # Priority 1: Koderz Club (main coding club)
830 if 'koderz_club' in chunk_id or chunk_id == 'koderz_club':
831 return 1
832
833 # Priority 2: CyberSENTS (cybersecurity)
834 elif 'cybersents_club' in chunk_id or 'cybersents' in chunk_id:
835 return 2
836
837 # Priority 3: AI Alliance
838 elif 'ai_alliance' in chunk_id or chunk_id == 'ai_alliance':
839 return 3
840
841 # Priority 4: Data Insight (Data Science)
842 elif 'data_insight' in chunk_id:
843 return 4
844
845 # Priority 5: Gameverse (Gaming & Animation)
846 elif 'gameverse' in chunk_id:
847 return 5
848
849 # Priority 6: Ledger League (Blockchain)
850 elif 'ledger_league' in chunk_id or 'ledger' in chunk_id:
851 return 6
852
853 # Priority 7: Qubits Qrew (Quantum)
854 elif 'qubit' in chunk_id or 'qrew' in chunk_id:
855 return 7
856
857 # Priority 8: WebKode (competition by Koderz Club)
858 elif 'webkode' in chunk_id or 'web_kode' in chunk_id:
859 return 8
860
861 # Priority 9: MLSA (inactive club)
862 elif 'mlsa' in chunk_id:
863 return 9
864
865 # Priority 10: Other competitions/events
866 elif 'koderz_kombat' in chunk_id or 'kombat' in chunk_id:
867 return 10
868
869 # Priority 99: Other
870 else:
871 return 99
872
873 # Add priority to each result
874 results_with_priority = []
875 for doc, score, explanation in results:
876 priority = get_club_priority(doc)
877 results_with_priority.append((doc, score, explanation, priority))
878
879 # Sort by: 1) Priority (lower = higher), 2) Original score (higher = better)
880 results_with_priority.sort(key=lambda x: (x[3], -x[1]))
881
882 # Remove priority from tuples
883 sorted_results = [(doc, score, explanation) for doc, score, explanation, _ in results_with_priority]
884
885 logger.info(f"🎯 Student clubs sorting applied - prioritized {len(sorted_results)} clubs")
886 return sorted_results
887
888 def _boost_exact_keyword_matches(self, results: List[Tuple[Document, float, Dict]], query: str) -> List[Tuple[Document, float, Dict]]:
889 """
890 Boost documents that contain exact keywords from the query to the TOP of results.
891 This ensures that if user asks "what is iconics", ICONICS docs come first, not TechFest.
892 """
893 query_lower = query.lower()
894
895 # Extract key terms from query (ignore common words)
896 stop_words = {'what', 'is', 'the', 'a', 'an', 'about', 'tell', 'me', 'can', 'you', 'give', 'info', 'information'}
897 query_terms = [word for word in query_lower.split() if word not in stop_words and len(word) > 2]
898
899 if not query_terms:
900 return results
901
902 # Separate exact matches from others
903 exact_matches = []
904 other_results = []
905
906 for doc, score, explanation in results:
907 doc_content_lower = doc.page_content.lower()
908 doc_title_lower = doc.metadata.get('title', '').lower()
909 doc_chunk_id = doc.metadata.get('chunk_id', '').lower()
910
911 # Check if document contains any of the key query terms
912 has_exact_match = False
913 for term in query_terms:
914 if (term in doc_title_lower or
915 term in doc_chunk_id or
916 term in doc_content_lower[:2000]): # Check first 2000 chars
917 has_exact_match = True
918 break
919
920 if has_exact_match:
921 # Boost score significantly
922 boosted_score = score * 1.5 # Increase score by 50%
923 exact_matches.append((doc, boosted_score, explanation))
924 else:
925 other_results.append((doc, score, explanation))
926
927 # Exact matches come first (already sorted by score within each group)
928 if exact_matches:
929 logger.info(f"🎯 Boosted {len(exact_matches)} documents with exact query keyword matches")
930
931 # Return exact matches first, then others
932 return exact_matches + other_results
933
934 def _get_keyword_matched_documents(self, query: str) -> List[Document]:
935 """
936 Find documents that contain critical keywords from the query.
937 This is a BM25-style keyword matching to complement semantic search.
938
939 Critical for:
940 - Exact event names (SportsFest, ICONICS, TechFest)
941 - Specific faculty names
942 - Program names
943 - Club names
944 """
945 import re
946
947 query_lower = query.lower()
948 matched_docs = []
949
950 # Define critical keywords and their variations
951 critical_keywords = {
952 # Events
953 'sportsfest': ['sportsfest', 'sports fest', 'sports_fest', 's_fest'],
954 'iconics': ['iconics', 'international conference'],
955 'techfest': ['techfest', 'tech fest', 'tech_fest'],
956 'alumni reunion': ['alumni reunion', 'alumni meet'],
957
958 # Clubs
959 'koderz': ['koderz', 'koderz club', 'koderz klub'],
960 'cybersents': ['cybersents', 'cyber sents', 'cybersecurity club'],
961 'ai alliance': ['ai alliance', 'artificial intelligence club'],
962 'data insight': ['data insight', 'data insights', 'data science club'],
963 'gameverse': ['gameverse', 'gaming club', 'game verse'],
964 'ledger league': ['ledger league', 'blockchain club'],
965 'qubits qrew': ['qubits qrew', 'qubit qrew', 'quantum club'],
966
967 # Academic
968 'webkode': ['webkode', 'web kode', 'web code'],
969 'koderz kombat': ['koderz kombat', 'koders kombat'],
970
971 # Achievements
972 'shining stars': ['shining stars', 'alumni', 'notable alumni', 'achievements'],
973 }
974
975 # Check which critical keywords are present in query
976 matched_keywords = []
977 for keyword_group, variations in critical_keywords.items():
978 for variation in variations:
979 if variation in query_lower:
980 matched_keywords.append(keyword_group)
981 break
982
983 if not matched_keywords:
984 return []
985
986 logger.info(f"🔑 Detected critical keywords: {matched_keywords}")
987
988 # Get all documents and check for keyword matches
989 all_docs = self.faiss_manager.get_all_documents()
990
991 for doc in all_docs:
992 doc_content_lower = doc.page_content.lower()
993 doc_title_lower = doc.metadata.get('title', '').lower()
994 doc_chunk_id = doc.metadata.get('chunk_id', '').lower()
995 search_keywords = doc.metadata.get('search_keywords', [])
996 if isinstance(search_keywords, list):
997 search_keywords_str = ' '.join(search_keywords).lower()
998 else:
999 search_keywords_str = str(search_keywords).lower()
1000
1001 # Check if document contains the matched keywords
1002 for keyword_group in matched_keywords:
1003 variations = critical_keywords[keyword_group]
1004 for variation in variations:
1005 # Check in title, chunk_id, content, or search_keywords
1006 if (variation in doc_title_lower or
1007 variation in doc_chunk_id or
1008 variation in search_keywords_str or
1009 variation in doc_content_lower[:500]): # Check first 500 chars
1010
1011 if doc not in matched_docs:
1012 matched_docs.append(doc)
1013 logger.info(f"🔑 Keyword match: '{variation}' found in '{doc.metadata.get('title', 'Unknown')[:40]}'")
1014 break
1015
1016 return matched_docs