NavRBot/knowledge-agent-platform

1"""2ingest.py3Multi-format loaders -> plain text chunks.4 5Covers Phase-1 formats: pdf, docx, pptx, xlsx/csv, json, txt, images (OCR), web URLs.6PowerBI (.pbix) and Tableau (.twbx) are binary/proprietary container formats without7a free Python parser worth trusting for text extraction — Phase-2 note at bottom8of README explains the workaround (export to CSV/PDF first, or use their REST APIs).9"""10import json11import io12from pathlib import Path13 14import pandas as pd15import requests16from bs4 import BeautifulSoup17from pypdf import PdfReader18from docx import Document as DocxDocument19from pptx import Presentation20from PIL import Image21import pytesseract22 23 24def load_pdf(path):25    reader = PdfReader(path)26    return "\n".join(page.extract_text() or "" for page in reader.pages)27 28 29def load_docx(path):30    doc = DocxDocument(path)31    return "\n".join(p.text for p in doc.paragraphs)32 33 34def load_pptx(path):35    prs = Presentation(path)36    text = []37    for slide in prs.slides:38        for shape in slide.shapes:39            if hasattr(shape, "text"):40                text.append(shape.text)41    return "\n".join(text)42 43 44def load_spreadsheet(path):45    # Handles .xlsx and .csv46    if str(path).lower().endswith(".csv"):47        df = pd.read_csv(path)48    else:49        df = pd.read_excel(path, sheet_name=None)  # all sheets50        if isinstance(df, dict):51            frames = [f"### Sheet: {name}\n{d.to_string(index=False)}" for name, d in df.items()]52            return "\n\n".join(frames)53    return df.to_string(index=False)54 55 56def load_json(path):57    data = json.loads(Path(path).read_text())58    return json.dumps(data, indent=2)59 60 61def load_txt(path):62    return Path(path).read_text(errors="ignore")63 64 65def load_image(path):66    img = Image.open(path)67    return pytesseract.image_to_string(img)68 69 70def load_url(url):71    resp = requests.get(url, timeout=15, headers={"User-Agent": "Mozilla/5.0"})72    soup = BeautifulSoup(resp.text, "html.parser")73    for tag in soup(["script", "style", "nav", "footer"]):74        tag.decompose()75    return soup.get_text(separator="\n", strip=True)76 77 78LOADERS = {79    ".pdf": load_pdf,80    ".docx": load_docx,81    ".pptx": load_pptx,82    ".xlsx": load_spreadsheet,83    ".csv": load_spreadsheet,84    ".json": load_json,85    ".txt": load_txt,86    ".png": load_image,87    ".jpg": load_image,88    ".jpeg": load_image,89}90 91 92def load_file(path):93    ext = Path(path).suffix.lower()94    loader = LOADERS.get(ext)95    if not loader:96        raise ValueError(f"Unsupported file type: {ext}")97    return loader(path)98 99 100def chunk_text(text, chunk_size=800, overlap=100):101    """Simple sliding-window chunker (no extra dependency)."""102    words = text.split()103    chunks = []104    i = 0105    while i < len(words):106        chunk = " ".join(words[i:i + chunk_size])107        if chunk.strip():108            chunks.append(chunk)109        i += chunk_size - overlap110    return chunks111