Neerajkadari/Context-Aware_Conversational_Intelligence_System

1"""2Document Loader Module3======================4Handles parsing and chunking of uploaded PDF and DOCX files.5Supports: PDF, DOCX, TXT, CSV6"""7 8import os9from typing import List10import PyPDF211import docx12 13 14class DocumentLoader:15    """Handles parsing and chunking of uploaded documents."""16 17    def __init__(self, chunk_size: int = 500, chunk_overlap: int = 50):18        self.chunk_size = chunk_size19        self.chunk_overlap = chunk_overlap20 21    def process_file(self, file_path: str, filename: str) -> List[str]:22        """Reads a file and returns overlapping text chunks."""23        text = ""24        ext = os.path.splitext(filename)[1].lower()25 26        try:27            if ext == '.pdf':28                text = self._read_pdf(file_path)29            elif ext == '.docx':30                text = self._read_docx(file_path)31            elif ext in ['.txt', '.csv', '.json']:32                with open(file_path, 'r', encoding='utf-8', errors='ignore') as f:33                    text = f.read()34            else:35                raise ValueError(f"Unsupported file format: {ext}. Supported: PDF, DOCX, TXT")36        except Exception as e:37            raise RuntimeError(f"Failed to read file {filename}: {str(e)}")38 39        text = " ".join(text.split())40        if not text.strip():41            raise ValueError(f"No text could be extracted from {filename}")42        return self._chunk_text(text)43 44    def _read_pdf(self, file_path: str) -> str:45        """Extract text from PDF files."""46        text = ""47        with open(file_path, 'rb') as f:48            reader = PyPDF2.PdfReader(f)49            for page in reader.pages:50                extracted = page.extract_text()51                if extracted:52                    text += extracted + " "53        return text54 55    def _read_docx(self, file_path: str) -> str:56        """Extract text from DOCX files."""57        doc = docx.Document(file_path)58        paragraphs = []59        for para in doc.paragraphs:60            if para.text.strip():61                paragraphs.append(para.text)62        # Also extract text from tables63        for table in doc.tables:64            for row in table.rows:65                row_text = " | ".join(cell.text.strip() for cell in row.cells if cell.text.strip())66                if row_text:67                    paragraphs.append(row_text)68        return "\n".join(paragraphs)69 70    def _chunk_text(self, text: str) -> List[str]:71        """Split text into overlapping chunks by word count."""72        words = text.split()73        chunks = []74        step = max(1, self.chunk_size - self.chunk_overlap)75 76        for i in range(0, len(words), step):77            chunk = " ".join(words[i: i + self.chunk_size])78            if chunk.strip():79                chunks.append(chunk)80 81        return chunks if chunks else [text]82