evanderpool/rag-knowledge-base

# 1. Install dependencies
pip install --user -r requirements.txt

# 2. Add your Groq API key (free at console.groq.com)
cp .env.example .env   # Windows: copy .env.example .env

# 3a. Web UI (Phase 2)
streamlit run app.py

# 3b. CLI (Phase 1)
python ingest.py company_policy.pdf
python query.py

Document
  → loader.load_document()
      → PyMuPDF page-by-page extraction     # preserves page numbers for citations
      → _clean()                             # strip artifacts, fix hyphenation
      → _recursive_split() + _merge()        # 1000-char boundary-aware chunks
      → _apply_overlap()                     # 150-char overlap between chunks
      → list[Chunk]                          # text + source + page + file_hash + token_estimate

  → kb.ingest(chunks)                        # KnowledgeBase in embedder.py
      → is_ingested(file_hash)?              # skip if already in store (dedup)
      → SentenceTransformer.encode()         # all-MiniLM-L6-v2, batch=32
      → ChromaDB.add()                       # cosine similarity, persistent local storage

  → engine.ask(question)                     # QueryEngine in query.py
      → kb.retrieve(question, top_k=5)       # embed query → cosine similarity search
      → _apply_token_budget()                # fit chunks within 4000-token context limit
      → Groq(Llama 3.3 70B)                  # structured prompt with context + citation rules
      → QueryResult(answer, sources, ...)    # answer + deduplicated source list

# Ingest
python ingest.py policy.pdf                  # single file
python ingest.py doc1.pdf doc2.txt           # multiple files
python ingest.py docs/                       # whole folder
python ingest.py --list                      # show what's in the knowledge base
python ingest.py --delete policy.pdf         # remove a document

# Query
python query.py                              # interactive Q&A with streaming