hina625/agent-Backend
0
1import os2import json3import re4import requests5from bs4 import BeautifulSoup6from pypdf import PdfReader7import logging8from dotenv import load_dotenv9 10# Load env variables11load_dotenv()12 13logger = logging.getLogger("dynamic-rag")14logger.setLevel(logging.INFO)15 16# Optional dependencies check for Hybrid Search17try:18 from sentence_transformers import SentenceTransformer19 HAS_SEMANTIC = True20except ImportError:21 HAS_SEMANTIC = False22 logger.warning("sentence-transformers not found. Will use Hugging Face API fallback for embeddings.")23 24class DynamicRAG:25 def __init__(self):26 self.has_semantic = HAS_SEMANTIC27 self.dense_model = None28 29 self.supabase_url = os.getenv("SUPABASE_URL", "")30 self.supabase_key = os.getenv("SUPABASE_ANON_KEY", "")31 32 if self.has_semantic:33 try:34 logger.info("Initializing SentenceTransformer all-MiniLM-L6-v2 local model...")35 self.dense_model = SentenceTransformer('all-MiniLM-L6-v2')36 logger.info("SentenceTransformer model loaded successfully.")37 except Exception as e:38 logger.error(f"Failed to load SentenceTransformer model: {e}. Using Hugging Face API fallback.")39 self.has_semantic = False40 41 if not self.supabase_url or not self.supabase_key:42 logger.error("Supabase environment variables (SUPABASE_URL / SUPABASE_ANON_KEY) are missing!")43 44 # Auto-initialize database with default portfolio if empty45 self.check_and_init_db()46 47 def check_and_init_db(self):48 if not self.supabase_url or not self.supabase_key:49 return50 51 headers = {52 "apikey": self.supabase_key,53 "Authorization": f"Bearer {self.supabase_key}"54 }55 56 try:57 # Check if any chunk exists58 res = requests.get(59 f"{self.supabase_url}/rest/v1/document_chunks?select=id&limit=1",60 headers=headers,61 timeout=1062 )63 if res.status_code == 200:64 data = res.json()65 if len(data) == 0:66 logger.info("Supabase database is empty. Uploading default portfolio data...")67 self.load_default_portfolio()68 else:69 logger.error(f"Failed to check Supabase database: {res.status_code} - {res.text}")70 except Exception as e:71 logger.error(f"Error checking Supabase db: {e}")72 73 def generate_embedding(self, text: str) -> list:74 clean_text = re.sub(r'\s+', ' ', text).strip()75 if not clean_text:76 return [0.0] * 38477 78 # 1. Try local SentenceTransformer model (if installed)79 if self.has_semantic and self.dense_model:80 try:81 embedding = self.dense_model.encode([clean_text], show_progress_bar=False)82 return embedding[0].tolist()83 except Exception as e:84 logger.error(f"Local embedding generation failed: {e}. Falling back to OpenAI API.")85 86 # 2. Fallback: OpenAI Embeddings API (text-embedding-3-large with 384 dimensions)87 try:88 api_key = os.getenv("OPENAI_API_KEY", "")89 if not api_key:90 logger.error("OpenAI API Key (OPENAI_API_KEY) is missing in environment variables!")91 return [0.0] * 38492 93 # If it's a Groq key (gsk_...), Groq does not support embeddings, so warn and fallback94 if api_key.startswith("gsk_"):95 logger.error("OPENAI_API_KEY is configured with a Groq key. Groq does not support embeddings. Please use a real OpenAI API Key.")96 return [0.0] * 38497 98 headers = {99 "Content-Type": "application/json",100 "Authorization": f"Bearer {api_key}"101 }102 payload = {103 "model": "text-embedding-3-large",104 "input": clean_text,105 "dimensions": 384106 }107 108 response = requests.post(109 "https://api.openai.com/v1/embeddings",110 headers=headers,111 json=payload,112 timeout=10113 )114 if response.status_code == 200:115 result = response.json()116 if "data" in result and len(result["data"]) > 0:117 return result["data"][0]["embedding"]118 logger.error(f"OpenAI embedding API failed: {response.status_code} - {response.text}")119 except Exception as e:120 logger.error(f"OpenAI embedding API error: {e}")121 122 # Final fallback: return zero vector123 return [0.0] * 384124 125 def upload_chunks(self, chunks: list) -> bool:126 if not chunks:127 return True128 if not self.supabase_url or not self.supabase_key:129 return False130 131 headers = {132 "Content-Type": "application/json",133 "apikey": self.supabase_key,134 "Authorization": f"Bearer {self.supabase_key}"135 }136 137 try:138 res = requests.post(139 f"{self.supabase_url}/rest/v1/document_chunks",140 headers=headers,141 json=chunks,142 timeout=15143 )144 if res.status_code in [200, 201]:145 logger.info(f"Successfully uploaded {len(chunks)} chunks to Supabase.")146 return True147 else:148 logger.error(f"Supabase upload failed: {res.status_code} - {res.text}")149 return False150 except Exception as e:151 logger.error(f"Failed to upload to Supabase: {e}")152 return False153 154 def load_default_portfolio(self):155 base_dir = os.path.dirname(os.path.abspath(__file__))156 json_path = os.path.join(base_dir, "portfolio_data.json")157 if not os.path.exists(json_path):158 return159 160 try:161 with open(json_path, "r", encoding="utf-8") as f:162 data = json.load(f)163 164 raw_texts = []165 166 # Add profile167 profile = data.get("agent_profile", {})168 raw_texts.append((169 f"{profile.get('name')} profile: {profile.get('role')}. {profile.get('purpose')} Built by {profile.get('creator')}.",170 "Default Portfolio Profile"171 ))172 173 # Add pillars174 for pillar in data.get("cognitive_pillars", []):175 raw_texts.append((176 f"Cognitive Pillar '{pillar.get('title')}': {pillar.get('description')}",177 "Portfolio Cognitive Core"178 ))179 180 # Add projects181 for project in data.get("projects", []):182 raw_texts.append((183 f"Project '{project.get('name')}' (Tech: {project.get('tech_stack')}, Success Rate: {project.get('success_rate')}, Complexity: {project.get('complexity')}): {project.get('description')}",184 "Portfolio Projects"185 ))186 187 # Add capabilities188 for cap in data.get("capabilities", []):189 modules = ", ".join(cap.get("modules", []))190 raw_texts.append((191 f"Technical Capability '{cap.get('name')}' (Category: {cap.get('category')}, Reliability: {cap.get('reliability')}): {cap.get('description')}. Modules: {modules}",192 "Portfolio Capabilities"193 ))194 195 # Add execution lifecycle196 for step in data.get("execution_lifecycle", []):197 raw_texts.append((198 f"Execution Lifecycle Step {step.get('step')} '{step.get('title')}': {step.get('description')}",199 "Portfolio Lifecycle"200 ))201 202 # Chunk, embed, and prepare upload list203 chunks = []204 for text, source in raw_texts:205 cleaned_text = re.sub(r'\s+', ' ', text).strip()206 if not cleaned_text:207 continue208 209 chunk_size = 1000210 overlap = 200211 start = 0212 while start < len(cleaned_text):213 end = start + chunk_size214 chunk = cleaned_text[start:end]215 216 embedding = self.generate_embedding(chunk)217 chunks.append({218 "content": chunk,219 "source": source,220 "embedding": embedding221 })222 start += chunk_size - overlap223 224 self.upload_chunks(chunks)225 except Exception as e:226 logger.error(f"Error loading default portfolio data: {e}")227 228 def add_pdf(self, file_path: str) -> bool:229 if not os.path.isabs(file_path):230 base_dir = os.path.dirname(os.path.abspath(__file__))231 file_path = os.path.join(base_dir, file_path)232 233 if not os.path.exists(file_path):234 logger.error(f"PDF path not found: {file_path}")235 return False236 try:237 reader = PdfReader(file_path)238 full_text = []239 for page in reader.pages:240 page_text = page.extract_text()241 if page_text:242 full_text.append(page_text)243 244 combined_text = "\n".join(full_text)245 if not combined_text.strip():246 logger.error(f"PDF is empty or has non-extractable text: {file_path}")247 return False248 249 source_name = f"PDF File: {os.path.basename(file_path)}"250 251 # Chunk and generate embeddings252 cleaned_text = re.sub(r'\s+', ' ', combined_text).strip()253 chunks = []254 chunk_size = 1000255 overlap = 200256 start = 0257 while start < len(cleaned_text):258 end = start + chunk_size259 chunk = cleaned_text[start:end]260 embedding = self.generate_embedding(chunk)261 chunks.append({262 "content": chunk,263 "source": source_name,264 "embedding": embedding265 })266 start += chunk_size - overlap267 268 return self.upload_chunks(chunks)269 except Exception as e:270 logger.error(f"Error reading PDF {file_path}: {e}")271 return False272 273 def add_url(self, url: str) -> bool:274 try:275 headers = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64)"}276 response = requests.get(url, headers=headers, timeout=10)277 if response.status_code != 200:278 logger.error(f"Failed to fetch URL {url}, status code: {response.status_code}")279 return False280 281 soup = BeautifulSoup(response.content, "html.parser")282 283 # Remove script, style, nav, header, footer elements284 for element in soup(["script", "style", "nav", "header", "footer"]):285 element.decompose()286 287 text = soup.get_text()288 cleaned_text = "\n".join([line.strip() for line in text.splitlines() if line.strip()])289 290 if not cleaned_text.strip():291 logger.error(f"Scraped text from URL is empty: {url}")292 return False293 294 source_name = f"Web URL: {url}"295 296 # Chunk and generate embeddings297 cleaned_text = re.sub(r'\s+', ' ', cleaned_text).strip()298 chunks = []299 chunk_size = 1000300 overlap = 200301 start = 0302 while start < len(cleaned_text):303 end = start + chunk_size304 chunk = cleaned_text[start:end]305 embedding = self.generate_embedding(chunk)306 chunks.append({307 "content": chunk,308 "source": source_name,309 "embedding": embedding310 })311 start += chunk_size - overlap312 313 return self.upload_chunks(chunks)314 except Exception as e:315 logger.error(f"Error fetching URL {url}: {e}")316 return False317 318 def search(self, query: str, top_n: int = 4) -> str:319 if not self.supabase_url or not self.supabase_key:320 return "Supabase credentials are not configured."321 322 try:323 query_clean = re.sub(r'[^\w\s]', '', query.lower()).strip()324 if not query_clean:325 return ""326 327 # 1. Generate query embedding vector328 query_emb = self.generate_embedding(query)329 330 # 2. Query similarity search (match_documents) function in Supabase331 headers = {332 "Content-Type": "application/json",333 "apikey": self.supabase_key,334 "Authorization": f"Bearer {self.supabase_key}"335 }336 337 payload = {338 "query_embedding": query_emb,339 "match_threshold": 0.25,340 "match_count": top_n341 }342 343 res = requests.post(344 f"{self.supabase_url}/rest/v1/rpc/match_documents",345 headers=headers,346 json=payload,347 timeout=10348 )349 350 if res.status_code == 200:351 matches = res.json()352 results = []353 for m in matches:354 if m.get("similarity", 0) > 0.15: # Extra threshold safety filter355 results.append(f"[{m.get('source')}]: {m.get('content')}")356 357 if not results:358 return "No matching context found. Proceed with standard portfolio details."359 360 return "\n\n".join(results)361 else:362 logger.error(f"Supabase RPC search failed: {res.status_code} - {res.text}")363 return "Error querying search index."364 except Exception as e:365 logger.error(f"Error performing search: {e}")366 return "Error searching documents."367