Revathi2006/document_ai_pdf
0
1import os2import uuid3import pdfplumber4import chromadb5from fastapi import FastAPI, UploadFile, HTTPException, Form, Request6from fastapi.responses import JSONResponse, HTMLResponse7from fastapi.middleware.cors import CORSMiddleware8from fastapi.staticfiles import StaticFiles9from fastapi.templating import Jinja2Templates10from sentence_transformers import SentenceTransformer11from typing import List, Optional12from dotenv import load_dotenv13import logging14from pathlib import Path15import io16import re17import requests18import json19import time20 21# Configure logging22logging.basicConfig(level=logging.INFO)23logger = logging.getLogger(__name__)24 25# Load environment variables26load_dotenv()27 28# Configuration29CHROMA_DB_PATH = "./chroma_db"30EMBEDDING_MODEL = "all-MiniLM-L6-v2"31HUGGINGFACE_API_KEY = os.getenv("HUGGINGFACE_API_KEY")32CHUNK_SIZE = 50033 34# Create necessary directories35Path("static").mkdir(exist_ok=True)36Path("templates").mkdir(exist_ok=True)37 38app = FastAPI(title="DOC AI", description="AI-powered document analysis tool")39 40# Serve static files and templates41app.mount("/static", StaticFiles(directory="static"), name="static")42templates = Jinja2Templates(directory="templates")43 44# CORS Setup45app.add_middleware(46 CORSMiddleware,47 allow_origins=["*"],48 allow_methods=["*"],49 allow_headers=["*"],50)51 52def chunk_text(text: str, chunk_size: int = CHUNK_SIZE) -> List[str]:53 """Split text into chunks while preserving sentence boundaries"""54 sentences = re.split(r'(?<=[.!?])\s+', text)55 56 chunks = []57 current_chunk = ""58 59 for sentence in sentences:60 if len(current_chunk) + len(sentence) < chunk_size:61 current_chunk += " " + sentence if current_chunk else sentence62 else:63 if current_chunk:64 chunks.append(current_chunk.strip())65 current_chunk = sentence66 67 if current_chunk.strip():68 chunks.append(current_chunk.strip())69 70 return chunks71 72# Initialize services73try:74 chroma_client = chromadb.PersistentClient(path=CHROMA_DB_PATH)75 document_collection = chroma_client.get_or_create_collection(76 name="documents",77 metadata={"hnsw:space": "cosine"}78 )79 embedder = SentenceTransformer(EMBEDDING_MODEL)80 logger.info("Services initialized successfully")81except Exception as e:82 logger.error(f"Service Init Error: {e}")83 raise84 85class DictionaryService:86 def __init__(self):87 self.session = requests.Session()88 self.session.headers.update({89 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'90 })91 92 def fetch_free_dictionary_api(self, word: str) -> List[str]:93 """Fetch definitions from Free Dictionary API (free, no key required)"""94 try:95 url = f"https://api.dictionaryapi.dev/api/v2/entries/en/{word}"96 logger.info(f"Fetching definition from Free Dictionary API for: {word}")97 98 response = self.session.get(url, timeout=10)99 100 if response.status_code == 200:101 data = response.json()102 definitions = []103 104 if isinstance(data, list) and len(data) > 0:105 for meaning in data[0].get('meanings', []):106 part_of_speech = meaning.get('partOfSpeech', '')107 for definition in meaning.get('definitions', []):108 def_text = definition.get('definition', '').strip()109 if def_text:110 if part_of_speech:111 definitions.append(f"({part_of_speech}) {def_text}")112 else:113 definitions.append(def_text)114 115 logger.info(f"Found {len(definitions)} definitions for {word}")116 return definitions[:8] # Return first 8 definitions117 118 elif response.status_code == 404:119 logger.warning(f"Word '{word}' not found in Free Dictionary API")120 return []121 else:122 logger.warning(f"Free Dictionary API returned status: {response.status_code}")123 return []124 125 except Exception as e:126 logger.warning(f"Free Dictionary API failed: {e}")127 return []128 129 def fetch_urbandictionary(self, word: str) -> List[str]:130 """Fetch definitions from Urban Dictionary (good for modern/slang terms)"""131 try:132 url = f"https://api.urbandictionary.com/v0/define?term={word}"133 logger.info(f"Fetching definition from Urban Dictionary for: {word}")134 135 response = self.session.get(url, timeout=10)136 137 if response.status_code == 200:138 data = response.json()139 definitions = []140 141 for item in data.get('list', [])[:3]: # Get first 3 definitions142 def_text = item.get('definition', '').strip()143 # Clean up Urban Dictionary formatting144 def_text = re.sub(r'\[.*?\]', '', def_text) # Remove [word] links145 if def_text and len(def_text) > 10:146 definitions.append(def_text)147 148 logger.info(f"Found {len(definitions)} definitions from Urban Dictionary")149 return definitions150 else:151 return []152 153 except Exception as e:154 logger.warning(f"Urban Dictionary failed: {e}")155 return []156 157 def fetch_merriam_webster(self, word: str) -> List[str]:158 """Fetch definitions from Merriam-Webster (web scraping fallback)"""159 try:160 url = f"https://www.merriam-webster.com/dictionary/{word}"161 logger.info(f"Fetching definition from Merriam-Webster for: {word}")162 163 response = self.session.get(url, timeout=10)164 165 if response.status_code == 200:166 # Simple regex to extract definitions167 html = response.text168 definitions = []169 170 # Look for definition patterns171 patterns = [172 r'<span class="dt-text">([^<]+)</span>',173 r'<p class="definition-inner-item"[^>]*>([^<]+)</p>',174 r'<div class="vg">([^<]+)</div>'175 ]176 177 for pattern in patterns:178 matches = re.findall(pattern, html)179 for match in matches:180 def_text = match.strip()181 if def_text and len(def_text) > 10 and word.lower() in def_text.lower():182 definitions.append(def_text)183 184 logger.info(f"Found {len(definitions)} definitions from Merriam-Webster")185 return definitions[:5]186 else:187 return []188 189 except Exception as e:190 logger.warning(f"Merriam-Webster failed: {e}")191 return []192 193 def get_definitions(self, word: str) -> List[str]:194 """Get definitions from multiple reliable sources"""195 logger.info(f"Looking up definitions for: {word}")196 197 # Try multiple sources198 sources = [199 self.fetch_free_dictionary_api(word), # Primary source200 self.fetch_urbandictionary(word), # For modern terms201 self.fetch_merriam_webster(word) # Fallback202 ]203 204 all_definitions = []205 seen_definitions = set()206 207 for definitions in sources:208 for definition in definitions:209 # Clean and deduplicate210 clean_def = definition.strip()211 if (len(clean_def) > 10 and 212 clean_def not in seen_definitions and 213 not clean_def.startswith('http')):214 seen_definitions.add(clean_def)215 all_definitions.append(clean_def)216 217 # Remove near-duplicates218 unique_definitions = []219 for definition in all_definitions:220 is_duplicate = False221 for existing in unique_definitions:222 similarity = len(set(definition.lower().split()) & set(existing.lower().split()))223 if similarity > 3: # If more than 3 words overlap224 is_duplicate = True225 break226 if not is_duplicate:227 unique_definitions.append(definition)228 229 logger.info(f"Total unique definitions found: {len(unique_definitions)}")230 return unique_definitions[:10] # Return up to 10 definitions231 232# Initialize dictionary service233dict_service = DictionaryService()234 235def create_local_summary(text: str, max_sentences: int = 8) -> str:236 """Create a simple summary by extracting key sentences"""237 sentences = re.split(r'(?<=[.!?])\s+', text)238 sentences = [s.strip() for s in sentences if len(s.strip()) > 20]239 240 if not sentences:241 return "Unable to generate summary from the provided text."242 243 # Score sentences (simple heuristic)244 scored_sentences = []245 for i, sentence in enumerate(sentences):246 score = len(sentence)247 if i < len(sentences) * 0.3:248 score *= 1.2249 key_terms = ['summary', 'conclusion', 'important', 'key', 'main', 'primary', 'result']250 if any(term in sentence.lower() for term in key_terms):251 score *= 1.5252 scored_sentences.append((sentence, score))253 254 scored_sentences.sort(key=lambda x: x[1], reverse=True)255 top_sentences = [s[0] for s in scored_sentences[:max_sentences]]256 257 final_sentences = []258 for sentence in sentences:259 if sentence in top_sentences:260 final_sentences.append(sentence)261 262 summary = " ".join(final_sentences)263 264 return summary265 266@app.get("/", response_class=HTMLResponse)267async def serve_frontend(request: Request):268 return templates.TemplateResponse("index.html", {"request": request})269 270@app.post("/upload")271async def upload_file(file: UploadFile):272 if not file.filename.lower().endswith('.pdf'):273 raise HTTPException(400, "Only PDF files are accepted.")274 275 try:276 file_content = await file.read()277 278 with pdfplumber.open(io.BytesIO(file_content)) as pdf:279 text = "\n".join(page.extract_text() or "" for page in pdf.pages)280 281 if not text.strip():282 raise HTTPException(400, "PDF contains no readable text")283 284 chunks = chunk_text(text)285 if not chunks:286 raise HTTPException(400, "Could not extract meaningful text from PDF")287 288 embeddings = embedder.encode(chunks)289 file_id = str(uuid.uuid4())290 291 document_collection.add(292 ids=[f"{file_id}_{i}" for i in range(len(chunks))],293 embeddings=embeddings.tolist(),294 documents=chunks,295 metadatas=[{"file_id": file_id, "filename": file.filename} for _ in chunks]296 )297 298 return {299 "file_id": file_id,300 "filename": file.filename,301 "status": "success",302 "chunks": len(chunks)303 }304 except Exception as e:305 logger.error(f"Upload error: {e}")306 raise HTTPException(500, f"Failed to process PDF: {str(e)}")307 308@app.post("/ask")309async def ask_question(question: str = Form(...), file_id: Optional[str] = Form(None)):310 try:311 if not question.strip():312 raise HTTPException(400, "Question cannot be empty")313 314 question_embedding = embedder.encode(question).tolist()315 316 query_params = {"query_embeddings": [question_embedding], "n_results": 3}317 if file_id:318 query_params["where"] = {"file_id": file_id}319 320 results = document_collection.query(321 **query_params,322 include=["documents", "distances", "metadatas"]323 )324 325 answers = []326 if results['documents']:327 for doc, dist, meta in zip(results['documents'][0], results['distances'][0], results['metadatas'][0]):328 answers.append({329 "text": doc,330 "similarity": float(1 - dist),331 "source": "document",332 "filename": meta.get("filename", "Unknown")333 })334 335 # For definition questions, use dictionary service336 if not answers and ("definition" in question.lower() or "define" in question.lower() or "what is" in question.lower()):337 # Extract word from question338 words = re.findall(r"['\"](.*?)['\"]", question)339 if not words:340 # Try to extract word from "what is X" pattern341 match = re.search(r'(?:what is|define|definition of)\s+([a-zA-Z]+)', question.lower())342 if match:343 words = [match.group(1)]344 345 if words:346 word = words[0].lower()347 definitions = dict_service.get_definitions(word)348 if definitions:349 answers.append({350 "text": "\n".join([f"{i+1}. {defn}" for i, defn in enumerate(definitions)]),351 "similarity": 1.0,352 "source": "dictionary",353 "filename": f"Definition of {word}"354 })355 356 if not answers:357 answers.append({358 "text": "No relevant information found in the documents. Try rephrasing your question or check the dictionary feature for definitions.",359 "similarity": 0.0,360 "source": "system",361 "filename": "Help"362 })363 364 return {"answers": answers}365 except Exception as e:366 logger.error(f"Question error: {e}")367 raise HTTPException(500, f"Failed to get answer: {str(e)}")368 369@app.get("/dictionary/{word}")370async def dictionary_lookup(word: str):371 try:372 if not word.strip():373 raise HTTPException(400, "Word cannot be empty")374 375 logger.info(f"Dictionary lookup for: {word}")376 377 # Get definitions from reliable sources378 definitions = dict_service.get_definitions(word)379 380 if definitions:381 return {382 "word": word,383 "definitions": definitions,384 "source": "Free Dictionary API + Merriam-Webster",385 "sources_used": len(definitions)386 }387 else:388 # Provide helpful fallback for common words389 common_words = {390 "document": [391 "A written or printed record that provides information or serves as an official record",392 "A computer file containing text, images, or other data",393 "To record or report something in detail, typically in writing"394 ],395 "computer": [396 "An electronic device for storing and processing data, typically in binary form",397 "A machine that can be instructed to carry out sequences of arithmetic or logical operations automatically"398 ],399 "python": [400 "A high-level programming language known for its readability and versatility, used for web development, data analysis, and artificial intelligence",401 "A large constricting snake found in tropical regions of Africa, Asia, and Australia"402 ],403 "ai": [404 "Artificial Intelligence: The simulation of human intelligence processes by machines, especially computer systems",405 "The capability of a machine to imitate intelligent human behavior and perform tasks that typically require human intelligence"406 ],407 "cat": [408 "A small domesticated carnivorous mammal with soft fur, a short snout, and retractable claws",409 "Any member of the family Felidae, including lions, tigers, and leopards"410 ],411 "dog": [412 "A domesticated carnivorous mammal that typically has a long snout, an acute sense of smell, and a barking, howling, or whining voice",413 "A member of the canine family, often kept as a pet or used for hunting, guarding, or assisting"414 ]415 }416 417 if word.lower() in common_words:418 return {419 "word": word,420 "definitions": common_words[word.lower()],421 "source": "Common Definitions",422 "sources_used": 1423 }424 else:425 return {426 "word": word,427 "definitions": [428 f"No definitions found for '{word}'.",429 "This might be because:",430 "• The word is very uncommon or specialized",431 "• There might be a temporary network issue",432 "• The word might be misspelled",433 "Please try checking the spelling or try a different word."434 ],435 "source": "No definitions available",436 "sources_used": 0437 }438 439 except Exception as e:440 logger.error(f"Dictionary error: {e}")441 return {442 "word": word,443 "definitions": [f"Error retrieving definitions: {str(e)}"],444 "source": "Error",445 "sources_used": 0446 }447 448@app.get("/summarize/{file_id}")449async def summarize_document(file_id: str):450 try:451 results = document_collection.get(452 where={"file_id": file_id},453 include=["documents", "metadatas"]454 )455 456 if not results['documents']:457 raise HTTPException(404, "Document not found")458 459 full_text = " ".join(results['documents'])460 filename = results['metadatas'][0]['filename'] if results['metadatas'] else "Unknown"461 462 summary = create_local_summary(full_text)463 464 return {465 "file_id": file_id,466 "filename": filename,467 "summary": summary.strip(),468 "source": "Local Processing"469 }470 except Exception as e:471 logger.error(f"Summarization error: {e}")472 raise HTTPException(500, f"Failed to summarize document: {str(e)}")473 474@app.get("/files")475async def list_files():476 try:477 results = document_collection.get()478 files = {}479 for metadata in results['metadatas']:480 file_id = metadata['file_id']481 if file_id not in files:482 files[file_id] = metadata['filename']483 484 return {"files": [{"id": k, "name": v} for k, v in files.items()]}485 except Exception as e:486 logger.error(f"File list error: {e}")487 return {"files": []}488 489@app.get("/health")490async def health_check():491 return {492 "status": "healthy",493 "dictionary_service": "active",494 "embedding_model": EMBEDDING_MODEL,495 "message": "Dictionary service is running with Free Dictionary API"496 }497 498@app.get("/test-dictionary/{word}")499async def test_dictionary(word: str):500 """Test the dictionary service"""501 definitions = dict_service.get_definitions(word)502 return {503 "word": word,504 "definitions_found": len(definitions),505 "definitions": definitions,506 "status": "success" if definitions else "no_definitions"507 }508if __name__ == "__main__":509 import uvicorn510 port = int(os.environ.get("PORT", 7860)) # Use HF Space port511 uvicorn.run("backend:app", host="0.0.0.0", port=port)512