awacke1/Transcript-EDA-NLTK

1import streamlit as st2from sklearn.feature_extraction.text import TfidfVectorizer3from sklearn.cluster import KMeans4from sklearn.metrics.pairwise import linear_kernel, cosine_similarity5import nltk6from nltk.corpus import stopwords7from nltk import FreqDist8import re9import os10import base6411from graphviz import Digraph12from io import BytesIO13import networkx as nx14import matplotlib.pyplot as plt15 16st.set_page_config(17    page_title="📺Transcript📜EDA🔍NLTK",18    page_icon="🌠",19    layout="wide",20    initial_sidebar_state="expanded",21    menu_items={22        'Get Help': 'https://huggingface.co/awacke1',23        'Report a bug': "https://huggingface.co/awacke1",24        'About': "https://huggingface.co/awacke1"25    }26)27 28st.markdown('''291. 🔍 **Transcript Insights Using Exploratory Data Analysis (EDA)** 📊 - Unveil hidden patterns 🕵️‍♂️ and insights 🧠 in your transcripts. 🏆.302. 📜 **Natural Language Toolkit (NLTK)** 🛠️:- your compass 🧭 in the vast landscape of NLP.313. 📺 **Transcript Analysis** 📈:Speech recognition 🎙️ and thematic extraction 🌐, audiovisual content to actionable insights 🔑.32''')33 34# 📥 Download NLTK data35@st.cache_resource36def download_nltk_data():37    try:38        nltk.data.find('tokenizers/punkt')39        nltk.data.find('corpora/stopwords')40    except LookupError:41        with st.spinner('Downloading required NLTK data...'):42            nltk.download('punkt')43            nltk.download('stopwords')44    st.success('NLTK data is ready!')45 46download_nltk_data()47 48# 🕰️ Remove timestamps49def remove_timestamps(text):50    return re.sub(r'\d{1,2}:\d{2}\n.*\n', '', text)51 52# 📊 Extract high information words53def extract_high_information_words(text, top_n=10):54    try:55        words = [word.lower() for word in nltk.word_tokenize(text) if word.isalpha()]56        stop_words = set(stopwords.words('english'))57        filtered_words = [word for word in words if word not in stop_words]58        return [word for word, _ in FreqDist(filtered_words).most_common(top_n)]59    except Exception as e:60        st.error(f"Error in extract_high_information_words: {str(e)}")61        return []62 63# 🔗 Create relationship graph64def create_relationship_graph(words):65    graph = Digraph()66    for i, word in enumerate(words):67        graph.node(str(i), word)68        if i > 0:69            graph.edge(str(i-1), str(i), label=word)70    return graph71 72# 📈 Display relationship graph73def display_relationship_graph(words):74    try:75        graph = create_relationship_graph(words)76        st.graphviz_chart(graph)77    except Exception as e:78        st.error(f"Error displaying relationship graph: {str(e)}")79 80# 🔍 Extract context words81def extract_context_words(text, high_information_words):82    words = nltk.word_tokenize(text)83    return [(words[i-1] if i > 0 else None, word, words[i+1] if i < len(words)-1 else None)84            for i, word in enumerate(words) if word.lower() in high_information_words]85 86# 📊 Create context graph87def create_context_graph(context_words):88    graph = Digraph()89    for i, (before, high, after) in enumerate(context_words):90        if before:91            graph.node(f'before{i}', before, shape='box')92            graph.edge(f'before{i}', f'high{i}', label=before)93        graph.node(f'high{i}', high, shape='ellipse')94        if after:95            graph.node(f'after{i}', after, shape='diamond')96            graph.edge(f'high{i}', f'after{i}', label=after)97    return graph98 99# 📈 Display context graph100def display_context_graph(context_words):101    try:102        graph = create_context_graph(context_words)103        st.graphviz_chart(graph)104    except Exception as e:105        st.error(f"Error displaying context graph: {str(e)}")106 107# 📊 Display context table108def display_context_table(context_words):109    table = "| Before | High Info Word | After |\n|--------|----------------|-------|\n"110    table += "\n".join(f"| {b if b else ''} | {h} | {a if a else ''} |" for b, h, a in context_words)111    st.markdown(table)112 113# 📁 Load example files114def load_example_files():115    excluded_files = {'freeze.txt', 'requirements.txt', 'packages.txt', 'pre-requirements.txt'}116    example_files = [f for f in os.listdir() if f.endswith('.txt') and f not in excluded_files]117    if example_files:118        selected_file = st.selectbox("📄 Select an example file:", example_files)119        if st.button(f"📂 Load {selected_file}"):120            with open(selected_file, 'r', encoding="utf-8") as file:121                return file.read()122    else:123        st.write("No suitable example files found.")124    return None125 126# 🧠 Cluster sentences127def cluster_sentences(sentences, num_clusters):128    sentences = [s for s in sentences if len(s) > 10]129    num_clusters = min(num_clusters, len(sentences))130    vectorizer = TfidfVectorizer()131    X = vectorizer.fit_transform(sentences)132    kmeans = KMeans(n_clusters=num_clusters, random_state=42)133    kmeans.fit(X)134    clustered_sentences = [[] for _ in range(num_clusters)]135    for i, label in enumerate(kmeans.labels_):136        similarity = linear_kernel(kmeans.cluster_centers_[label:label+1], X[i:i+1]).flatten()[0]137        clustered_sentences[label].append((similarity, sentences[i]))138    return [[s for _, s in sorted(cluster, reverse=True)] for cluster in clustered_sentences]139 140# 💾 Get text file download link141def get_text_file_download_link(text_to_download, filename='Output.txt', button_label="💾 Save"):142    b64 = base64.b64encode(text_to_download.encode()).decode()143    return f'<a href="data:file/txt;base64,{b64}" download="{filename}" style="margin-top:20px;">{button_label}</a>'144 145# 📊 Get high info words per cluster146def get_high_info_words_per_cluster(cluster_sentences, num_words=5):147    return [extract_high_information_words(" ".join(cluster), num_words) for cluster in cluster_sentences]148 149# 📊 Plot cluster words150def plot_cluster_words(cluster_sentences):151    for i, cluster in enumerate(cluster_sentences):152        words = re.findall(r'\b[a-z]{4,}\b', " ".join(cluster))153        word_freq = FreqDist(words)154        top_words = [word for word, _ in word_freq.most_common(20)]155        vectorizer = TfidfVectorizer()156        X = vectorizer.fit_transform(top_words)157        similarity_matrix = cosine_similarity(X.toarray())158        G = nx.from_numpy_array(similarity_matrix)159        pos = nx.spring_layout(G, k=0.5)160        plt.figure(figsize=(8, 6))161        nx.draw_networkx(G, pos, node_size=500, font_size=12, font_weight='bold', with_labels=True, 162                         labels={i: word for i, word in enumerate(top_words)}, 163                         node_color='skyblue', edge_color='gray')164        plt.axis('off')165        plt.title(f"Cluster {i+1} Word Arrangement")166        st.pyplot(plt)167        st.markdown(f"**Cluster {i+1} Details:**")168        st.markdown(f"Top Words: {', '.join(top_words)}")169        st.markdown(f"Number of Sentences: {len(cluster)}")170        st.markdown("---")171 172# Main code for UI173uploaded_file = st.file_uploader("📁 Choose a .txt file", type=['txt'])174 175example_text = load_example_files()176 177if example_text:178    file_text = example_text179elif uploaded_file:180    file_text = uploaded_file.read().decode("utf-8")181else:182    file_text = ""183 184if file_text:185    text_without_timestamps = remove_timestamps(file_text)186    top_words = extract_high_information_words(text_without_timestamps, 10)187 188    with st.expander("📊 Top 10 High Information Words"):189        st.write(top_words)190 191    with st.expander("📈 Relationship Graph"):192        display_relationship_graph(top_words)193 194    context_words = extract_context_words(text_without_timestamps, top_words)195 196    with st.expander("🔗 Context Graph"):197        display_context_graph(context_words)198 199    with st.expander("📑 Context Table"):200        display_context_table(context_words)201 202    sentences = [line.strip() for line in file_text.split('\n') if len(line.strip()) > 10]203 204    num_sentences = len(sentences)205    st.write(f"Total Sentences: {num_sentences}")206 207    num_clusters = st.slider("Number of Clusters", min_value=2, max_value=10, value=5)208    clustered_sentences = cluster_sentences(sentences, num_clusters)209 210    col1, col2 = st.columns(2)211 212    with col1:213        st.subheader("Original Text")214        original_text = "\n".join(sentences)215        st.text_area("Original Sentences", value=original_text, height=400)216 217    with col2:218        st.subheader("Clustered Text")219        clusters = ""220        clustered_text = ""221        cluster_high_info_words = get_high_info_words_per_cluster(clustered_sentences)222 223        for i, cluster in enumerate(clustered_sentences):224            cluster_text = "\n".join(cluster)225            high_info_words = ", ".join(cluster_high_info_words[i])226            clusters += f"Cluster {i+1} (High Info Words: {high_info_words})\n"227            clustered_text += f"Cluster {i+1} (High Info Words: {high_info_words}):\n{cluster_text}\n\n"228 229        st.text_area("Clusters", value=clusters, height=200)230        st.text_area("Clustered Sentences", value=clustered_text, height=200)231 232        clustered_sentences_flat = [sentence for cluster in clustered_sentences for sentence in cluster]233        if set(sentences) == set(clustered_sentences_flat):234            st.write("✅ All sentences are accounted for in the clustered output.")235        else:236            st.write("❌ Some sentences are missing in the clustered output.")237    238    plot_cluster_words(clustered_sentences)239 240st.markdown("For more information and updates, visit our [help page](https://huggingface.co/awacke1).")