Anti-Gamer/TextInsightGPT-using-SBERT

1import spacy
2import unicodedata
3import re
4import streamlit as st
5from sentence_transformers import SentenceTransformer, util
6
7# Preprocess the text
8def clean_text(text):
9    # Normalize Unicode characters
10    text = unicodedata.normalize('NFKC', text)
11
12    # Replace non-breaking or zero-width spaces with regular spaces
13    text = text.replace('\u200a', ' ').replace('\u00a0', ' ')
14
15    return text
16
17# Load the SBERT model
18model = SentenceTransformer('all-MiniLM-L6-v2')
19
20# Load and clean the text file
21with open('long-story.txt', encoding='utf-8') as f:
22    text = f.read()
23
24cleaned_text = clean_text(text)
25
26# Define the regular expression to match titles and the content between them
27pattern = r"\n\n([A-Z\s]+)\n([^\n]+(?:\n[^\n]+)*)"
28
29# Use findall to capture all title-content pairs
30sections = re.findall(pattern, cleaned_text)
31
32# Store the sections in a dictionary with embeddings for content
33sections_dict = {}
34embeddings_dict = {}
35
36for title, content in sections:
37    # Clean the title and content by stripping unnecessary newlines
38    cleaned_title = title.strip().lower()  # Convert the title to lowercase for case-insensitive matching
39    cleaned_content = content.strip()
40    
41    # Create embeddings for each section content
42    content_embedding = model.encode(cleaned_content, convert_to_tensor=True)
43    
44    sections_dict[cleaned_title] = cleaned_content
45    embeddings_dict[cleaned_title] = content_embedding
46
47# Function to retrieve content based on user input (semantic matching with SBERT)
48def get_section(user_input):
49    # Normalize user input to lowercase for matching
50    user_input = user_input.lower()
51    
52    # Generate the embedding for the user input
53    user_input_embedding = model.encode(user_input, convert_to_tensor=True)
54    
55    best_match = None
56    best_score = -1
57
58    # Iterate through the sections to find the best match based on cosine similarity
59    for title, section_embedding in embeddings_dict.items():
60        cosine_score = util.pytorch_cos_sim(user_input_embedding, section_embedding)[0][0].item()
61        
62        if cosine_score > best_score:
63            best_score = cosine_score
64            best_match = title
65
66    # Return the best matching section
67    if best_score > 0.5:  # You can adjust the threshold based on your needs
68        return sections_dict[best_match]
69    else:
70        return "No matching section found."
71
72# Streamlit UI
73def chatbot_ui():
74    st.title("Text-Based Chatbot")
75
76    # Display instructions
77    st.write("Ask the chatbot for specific sections from the document by typing keywords like 'productivity', 'life hacks', 'communication skills', 'skill development', 'personal development', 'goal setting' ")
78
79    # Input field for the user
80    user_input = st.text_input("Enter a keyword", "")
81
82    # If the user enters a keyword, get the matching section and display it
83    if user_input:
84        section_content = get_section(user_input)
85        st.write(section_content)
86
87# Run the Streamlit app
88if __name__ == "__main__":
89    chatbot_ui()
90