yamanavijayavardhan/answer-grading-app

1import fitz  # PyMuPDF2import re3import chromadb4import sys5import os6import uuid7sys.path.append(os.path.dirname(os.path.dirname(__file__)))8from all_models import models9 10def clean_text(text):11    # Keep only letters, numbers, punctuation, whitespace, and newlines12    cleaned_text = re.sub(r"[^a-zA-Z0-9\s.,!?;:'\"()\-]", "", text)13    return cleaned_text14 15def extract_text_from_pdf(pdf_path):16    text = ""17    with fitz.open(pdf_path) as doc:18        for page in doc:19            page_text = page.get_text()20            cleaned_text = clean_text(page_text)21            text += cleaned_text22    return text23 24def clean_data(text):25    cleaned_text = re.sub(r'\n{2,}', '. \n', text)  # Replace multiple newlines with a single newline26    cleaned_text = re.sub(r' {2,}', '. \n', cleaned_text)  # Replace multiple spaces with a newline27    return cleaned_text.strip()  # Strip leading/trailing whitespace28 29def combine_list(strings):30    combined_list = [] 31    current_combined = ""32    for string in strings:33        word_count = len(string.split())34        35        if len(current_combined.split()) < 20:36            current_combined += " " + string.strip()  # Adding space before new string37            38        # If the combined string reaches at least 20 words, add it to the final list39        if len(current_combined.split()) >= 20:40            combined_list.append(current_combined)  # Strip to remove leading/trailing whitespace41            current_combined = ""  # Reset for the next round42    if current_combined:43        combined_list.append(current_combined.strip())44    return combined_list45 46def create_databse(data, name):47    # Initialize the Persistent Client48    client = chromadb.PersistentClient(path="correct_answer_generation/chroma_db")49    50    collection_names = client.list_collections()51    if name in collection_names:52        client.delete_collection(name)  # Delete the old collection53    54    # Create a Collection55    collection = client.create_collection(name)56 57    # Generate embeddings using the singleton model58    embeddings = models.similarity_model.encode(data, batch_size=32, convert_to_tensor=True)59 60    # Create documents and add them to the collection61    unique_id = [str(uuid.uuid4()) for _ in range(len(embeddings))]62    63    collection.add(64        documents=data,65        ids=unique_id66    )67 68def create_database_main(path):69    pdf_path = path70    pdf_text = extract_text_from_pdf(pdf_path)71    data = clean_data(pdf_text)72    data = data.split('. \n')73    for i in range(len(data)):74        data[i] = re.sub(r' \n', ' ', data[i])75        data[i] = re.sub(r'\s+', ' ', data[i])76    data = [text for text in data if len(text) >= 2]77    data = combine_list(data)78    79    path = path.replace("/", "_")80    create_databse(data, path)81    82