RustSa/linux_documentation_support_chatbot

1import os2from langchain_community.document_loaders import PyPDFLoader, TextLoader3from langchain.text_splitter import CharacterTextSplitter4from langchain_openai import OpenAIEmbeddings5from langchain_community.vectorstores import Chroma6 7def load_and_index_documents(data_dir="data"):8    """Load documents from `data_dir`, split into chunks, and create a Chroma vector store."""9    docs = []10    # Load PDF and text files11    for filename in os.listdir(data_dir):12        path = os.path.join(data_dir, filename)13        if filename.lower().endswith(".pdf"):14            loader = PyPDFLoader(path)15            pages = loader.load_and_split()  # each page has metadata with 'page' info:contentReference[oaicite:15]{index=15}16            docs.extend(pages)17        elif filename.lower().endswith(".txt"):18            loader = TextLoader(path, encoding='utf-8')19            docs.extend(loader.load())  # single Document20        # (Add other formats if needed)21    # Split documents into chunks with overlap22    splitter = CharacterTextSplitter(chunk_size=1000, chunk_overlap=100)23    chunks = splitter.split_documents(docs)24    # Create embeddings and vector store (Chroma)25    embeddings = OpenAIEmbeddings()26    vector_store = Chroma.from_documents(chunks, embeddings)27    return vector_store28 29# Example usage (called at app startup)30# vectordb = load_and_index_documents()