Vicky2692834/FEM-r1

1import json2import numpy as np3import random4import streamlit as st5from sentence_transformers import SentenceTransformer6 7@st.cache_resource8def load_youtube_data(base_path, embedding_model_name, chunk_tokens, overlap_tokens):9    embedding_space_file_name = f'{base_path}/yt_embedding_space_{embedding_model_name}_tpc{chunk_tokens}_o{overlap_tokens}.json'10    with open(embedding_space_file_name, 'r') as json_file:11        loaded_data = json.load(json_file)12    13    embedding_space = np.array(loaded_data['embedding_space'])14    return loaded_data['chunks'], embedding_space15 16@st.cache_resource17def load_book_data(base_path, embedding_model_name, chunk_tokens, overlap_tokens):18    embedding_space_file_name = f'{base_path}/latex_embedding_space_by_sections_{embedding_model_name}_tpc{chunk_tokens}_o{overlap_tokens}.json'19    with open(embedding_space_file_name, 'r') as json_file:20        loaded_data = json.load(json_file)21    22    embedding_space = np.array(loaded_data['embedding_space'])23    return loaded_data['chunks'], embedding_space24 25@st.cache_resource26def load_summary(file_path):27    with open(file_path, 'r') as file:28        transcripts = json.load(file)29    return transcripts30 31def embed_question_sentence_transformer(texts, model_name="sentence-transformers/all-MiniLM-L6-v2"):32    model = SentenceTransformer(model_name)33    embeddings = model.encode(texts)34    35    return embeddings.tolist()36 37def fixed_knn_retrieval(question_embedding, context_embeddings, top_k=5, min_k=1):38    39    question_embedding = np.array(question_embedding)40 41    # Normalize 42    question_embedding = question_embedding / np.linalg.norm(question_embedding)43    context_embeddings = context_embeddings / np.linalg.norm(context_embeddings, axis=1, keepdims=True)44    45    # Calculate cosine similarities between the question embedding and all context embeddings.46    similarities = np.dot(context_embeddings, question_embedding)47    # Sort the similarities in descending order and get the corresponding indices.48    sorted_indices = np.argsort(similarities)[::-1]49    # Select the top_k most similar contexts, ensuring at least min_k contexts are selected.50    selected_indices = sorted_indices[:max(top_k, min_k)].tolist()51    return selected_indices52 53 54def get_random_question(text_file):55    with open(text_file, "r") as file:56        questions = [line.strip() for line in file]57    return random.choice(questions)