ank52/logic_stream

1import os2import json3 4try:5    from sentence_transformers import SentenceTransformer6    import faiss7    import numpy as np8except ImportError:9    print("Please run `pip install sentence-transformers faiss-cpu` to build the Vector DB.")10    exit(1)11 12def build_vector_database():13    print("🚀 Starting Big Data Vector Knowledge Base Construction...")14 15    # 1. Load Big Data Corpus16    # In reality, this would be thousands of pages of Docs.17    # We will simulate the corpus here:18    corpus = [19        {"id": 1, "text": "Billing Subscriptions: How to change your billing date and payment method."},20        {"id": 2, "text": "Refunds: I was double charged for my subscription, how do I get my money back?"},21        {"id": 3, "text": "Account Security: How to reset your password and enable 2FA."},22        {"id": 4, "text": "Technical Issues: iOS App keeps crashing on startup when opening images."}23    ]24 25    # 2. Extract texts to embed26    documents = [doc["text"] for doc in corpus]27    28    # 3. Load Embedding Model29    print("🧠 Loading SentenceTransformer Embedding Model (all-MiniLM-L6-v2) ...")30    embedder = SentenceTransformer('all-MiniLM-L6-v2')31 32    # 4. Generate High-Dimensional Vector Embeddings33    print(f"🧬 Converting {len(documents)} documents into Semantic Vectors...")34    embeddings = embedder.encode(documents, show_progress_bar=True)35    embeddings = np.array(embeddings).astype('float32')36 37    # 5. Build FAISS Index (The actual Vector Database)38    dimension = embeddings.shape[1]39    index = faiss.IndexFlatL2(dimension)40    41    print("🗄️ Inserting Vectors into FAISS Space...")42    index.add(embeddings)43 44    # 6. Save the Index to Disk45    os.makedirs("../vector_db", exist_ok=True)46    faiss.write_index(index, "../vector_db/knowledge_base.index")47    48    # Save the id-mapping separately so the Django server can retrieve the text49    with open("../vector_db/corpus_mapping.json", "w") as f:50        json.dump(corpus, f)51 52    print("✅ Vector Database successfully built and saved to ../vector_db/knowledge_base.index!")53 54if __name__ == "__main__":55    build_vector_database()56