sashank812/multi-document-summarization

1import re2import docx3from bs4 import BeautifulSoup4from PyPDF2 import PdfReader5 6from sentence_transformers import SentenceTransformer, util7 8import warnings9import hdbscan10 11import numpy as np12import seaborn as sns13 14from transformers import BartTokenizer, BartForConditionalGeneration, BartConfig15 16import torch17from transformers import LongformerTokenizer, EncoderDecoderModel18 19import nltk20from nltk.corpus import stopwords21from nltk.tokenize import word_tokenize, sent_tokenize22from sklearn.feature_extraction.text import TfidfVectorizer23import matplotlib.pyplot as plt24 25nltk.download("punkt_tab")26nltk.download("stopwords")27nltk.download("punkt")28 29import matplotlib.pyplot as plt30import scipy.cluster.hierarchy as sch31from sklearn.metrics.pairwise import cosine_similarity32 33import plotly.express as px34from sklearn.manifold import TSNE35 36from wordcloud import WordCloud37import matplotlib.pyplot as plt38 39import pandas as pd40import json41import xml.etree.ElementTree as ET42import os43import warnings44import pptx45 46import io47from PIL import Image48 49warnings.filterwarnings("ignore")50 51 52def clean_text(text):53    text = re.sub(r"http\S+|www\S+|https\S+", "", text)54    text = re.sub(r"\s+", " ", text).strip()55    text = re.sub(r"[^\w\s,.]", "", text)56    return text57 58 59def extract_and_clean_text(file_path):60    text = ""61    if file_path.endswith(".docx"):62        doc = docx.Document(file_path)63        for paragraph in doc.paragraphs:64            text += paragraph.text + " "65    elif file_path.endswith(".txt"):66        with open(file_path, "r", encoding="utf-8") as f:67            text = f.read()68    elif file_path.endswith((".html", ".htm")):69        with open(file_path, "r", encoding="utf-8") as f:70            html_content = f.read()71        soup = BeautifulSoup(html_content, "html.parser")72        text = soup.get_text(separator=" ", strip=True)73    elif file_path.endswith(".pdf"):74        reader = PdfReader(file_path)75        for page in reader.pages:76            text += page.extract_text() + " "77    elif file_path.endswith(".csv"):78        df = pd.read_csv(file_path)79        text = " ".join(df.astype(str).agg(" ".join, axis=1))80    elif file_path.endswith(".xlsx"):81        df = pd.read_excel(file_path)82        text = " ".join(df.astype(str).agg(" ".join, axis=1))83    elif file_path.endswith(".json"):84        with open(file_path, "r", encoding="utf-8") as f:85            data = json.load(f)86        text = " ".join([str(item) for item in data])87    elif file_path.endswith(".xml"):88        tree = ET.parse(file_path)89        root = tree.getroot()90        text = " ".join([elem.text for elem in root.iter() if elem.text])91    elif file_path.endswith(".pptx"):92        from pptx import Presentation93 94        prs = Presentation(file_path)95        for slide in prs.slides:96            for shape in slide.shapes:97                if hasattr(shape, "text"):98                    text += shape.text + " "99    else:100        raise ValueError("Unsupported file type: {}".format(file_path))101    cleaned_text = clean_text(text)102    return cleaned_text103 104 105def clean_files(file_list):106    cleaned_files = []107    for file in file_list:108        cleaned_files.append(extract_and_clean_text(file))109    return cleaned_files110 111 112def get_embeddings(text):113    model = SentenceTransformer("all-mpnet-base-v2")114    embeddings = model.encode(text)115    return embeddings116 117 118def clustering_labels(embeddings):119    warnings.filterwarnings("ignore")120    embeddings = np.array(embeddings)121    if len(embeddings) < 2:122        raise ValueError(123            "Not enough data points for clustering. At least 2 are required."124        )125    min_cluster_size = min(2, len(embeddings))126    cluster = hdbscan.HDBSCAN(127        min_cluster_size=min_cluster_size,128        metric="euclidean",129        cluster_selection_method="eom",130    ).fit(embeddings)131    return cluster.labels_132 133 134def bart_summarizer(text):135    model_name_bart = "facebook/bart-large-cnn"136    tokenizer = BartTokenizer.from_pretrained(model_name_bart)137    model = BartForConditionalGeneration.from_pretrained(model_name_bart)138    tokenize_inputs = tokenizer.encode(139        text, return_tensors="pt", max_length=1024, truncation=True140    )141    ids_summarization = model.generate(142        tokenize_inputs, num_beams=4, max_length=150, early_stopping=True143    )144    summary_decoded = tokenizer.decode(ids_summarization[0], skip_special_tokens=True)145    return summary_decoded146 147 148def longformer_summarizer(text):149    tokenizer = LongformerTokenizer.from_pretrained("allenai/longformer-base-4096")150    model = EncoderDecoderModel.from_pretrained(151        "patrickvonplaten/longformer2roberta-cnn_dailymail-fp16"152    )153    inputs = tokenizer(154        text, return_tensors="pt", padding="longest", truncation=True155    ).input_ids156    ids_summarization = model.generate(inputs)157    summary_decoded = tokenizer.decode(ids_summarization[0], skip_special_tokens=True)158    return summary_decoded159 160 161def longformer_summarizer_long_text(162    text, max_chunk_length=4000, overlap=200, max_summary_length=1024163):164    tokenizer = LongformerTokenizer.from_pretrained("allenai/longformer-base-4096")165    model = EncoderDecoderModel.from_pretrained(166        "patrickvonplaten/longformer2roberta-cnn_dailymail-fp16"167    )168    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")169    model = model.to(device)170    tokens = tokenizer.encode(text)171    if len(tokens) <= max_chunk_length:172        inputs = tokenizer(text, return_tensors="pt", padding="longest").input_ids.to(173            device174        )175        summary_ids = model.generate(inputs, max_length=max_summary_length)176        summary = tokenizer.decode(summary_ids[0], skip_special_tokens=True)177        return summary178    chunk_summaries = []179    for i in range(0, len(tokens), max_chunk_length - overlap):180        chunk_tokens = tokens[i : i + max_chunk_length]181        if len(chunk_tokens) < 100:182            continue183        chunk_text = tokenizer.decode(chunk_tokens, skip_special_tokens=True)184        inputs = tokenizer(185            chunk_text, return_tensors="pt", padding="longest"186        ).input_ids.to(device)187        summary_ids = model.generate(inputs, max_length=max_summary_length // 2)188        chunk_summary = tokenizer.decode(summary_ids[0], skip_special_tokens=True)189        chunk_summaries.append(chunk_summary)190    final_summary = " ".join(chunk_summaries)191    return final_summary192 193 194def summarize_text(text):195    bart_tokenizer = BartTokenizer.from_pretrained("facebook/bart-large-cnn")196    input_length = len(bart_tokenizer.encode(text))197    if input_length < 1024:198        summary = bart_summarizer(text)199    elif input_length < 4096:200        summary = longformer_summarizer(text)201    else:202        summary = longformer_summarizer_long_text(text)203    return summary204 205 206def summarize(embeddings, labels, cleaned_files):207    no_of_clusters = max(labels) + 1208    clusters_embeddings = []209    clusters_text = [""] * no_of_clusters210    for i in range(no_of_clusters):211        clusters_embeddings.append(embeddings[labels == i])212    noise_docs = []213    for label, text_chunk in zip(labels, cleaned_files):214        if label != -1:215            clusters_text[label] += text_chunk216        else:217            noise_docs.append(text_chunk)218    clusters_text.extend(noise_docs)219    cluster_texts_combined = ["".join(cluster) for cluster in clusters_text]220    final_summaries = [221        summarize_text(cluster_text) for cluster_text in cluster_texts_combined222    ]223    return final_summaries224 225 226def tfidf_plot(all_text):227    tokens = word_tokenize(all_text.lower())228    stop_words = set(stopwords.words("english"))229    filtered_tokens = [w for w in tokens if not w in stop_words and w.isalnum()]230    vectorizer = TfidfVectorizer()231    tfidf_matrix = vectorizer.fit_transform([" ".join(filtered_tokens)])232    feature_names = vectorizer.get_feature_names_out()233    tfidf_scores = tfidf_matrix.toarray()[0]234    top_n = 25235    top_indices = tfidf_scores.argsort()[-top_n:]236    top_words = [feature_names[i] for i in top_indices]237    top_scores = [tfidf_scores[i] for i in top_indices]238    fig, ax = plt.subplots(figsize=(10, 5))239    ax.barh(top_words, top_scores, color="skyblue")240    ax.set_xlabel("TF-IDF Score")241    ax.set_ylabel("Words")242    ax.set_title("Top {} Important Words (TF-IDF)".format(top_n))243    ax.invert_yaxis()244    return fig245 246 247def dendrogram_plot(embeddings, labels):248    similarity_matrix = cosine_similarity(embeddings)249    distance_matrix = 1 - similarity_matrix250    linkage_matrix = sch.linkage(distance_matrix, method="ward")251    dendrogram_labels = [252        f"Doc {i} (Cluster {labels[i]})" if labels[i] != -1 else f"Doc {i} (Noise)"253        for i in range(len(labels))254    ]255    fig, ax = plt.subplots(figsize=(12, 8))256    sch.dendrogram(257        linkage_matrix,258        labels=dendrogram_labels,259        orientation="right",260        leaf_font_size=10,261        ax=ax,262    )263    ax.set_title("Hierarchical Dendrogram of Document Clusters", fontsize=14)264    ax.set_xlabel("Distance", fontsize=12)265    ax.set_ylabel("Documents", fontsize=12)266    unique_labels = set(labels)267    legend_labels = [268        f"Cluster {label}" if label != -1 else "Noise" for label in unique_labels269    ]270    ax.legend(legend_labels, loc="upper right", title="Clusters", fontsize=10)271    plt.tight_layout()272    return fig273 274 275def tsne_plot(embeddings, labels):276    n_samples = len(embeddings)277    if n_samples < 2:278        fig, ax = plt.subplots(figsize=(6, 4))279        ax.text(280            0.5,281            0.5,282            "t-SNE plot is not applicable for a single document.",283            fontsize=12,284            ha="center",285            va="center",286            wrap=True,287        )288        ax.axis("off")289        return fig290    perplexity = min(30, n_samples - 1)291    tsne = TSNE(n_components=2, perplexity=perplexity, random_state=42)292    reduced_embeddings = tsne.fit_transform(embeddings)293    fig, ax = plt.subplots(figsize=(8, 6))294    scatter = ax.scatter(295        reduced_embeddings[:, 0],296        reduced_embeddings[:, 1],297        c=labels,298        cmap="viridis",299        s=50,300        alpha=0.8,301    )302    ax.set_title("t-SNE Visualization of Document Clusters", fontsize=14)303    ax.set_xlabel("t-SNE Dimension 1", fontsize=12)304    ax.set_ylabel("t-SNE Dimension 2", fontsize=12)305    unique_labels = set(labels)306    for label in unique_labels:307        ax.scatter([], [], label=f"Cluster {label}" if label != -1 else "Noise", s=50)308    ax.legend(loc="upper right", title="Clusters", fontsize=10)309    cbar = plt.colorbar(scatter, ax=ax)310    cbar.set_label("Cluster Labels", fontsize=12)311    return fig312 313 314def wordcloud_plot(all_text):315    wordcloud = WordCloud(width=800, height=400, background_color="white").generate(316        all_text317    )318    fig, ax = plt.subplots(figsize=(10, 5), facecolor=None)319    ax.imshow(wordcloud)320    ax.axis("off")321    plt.tight_layout(pad=0)322    buf = io.BytesIO()323    fig.savefig(buf, format="png")324    buf.seek(0)325    img = Image.open(buf)326    img_array = np.array(img)327    buf.close()328    plt.close(fig)329    return img_array330 331 332def summarize_docs(files_text):333    if files_text:334        cleaned_files = clean_files(files_text)335        if len(cleaned_files) == 1:336            summary = summarize_text(cleaned_files[0])337            return (338                f"Summary for the uploaded document:\n{summary}",339                None,340                None,341                None,342                None,343            )344        embeddings = get_embeddings(cleaned_files)345        if len(embeddings) < 2:346            return (347                "Not enough documents for clustering. Please upload more files.",348                None,349                None,350                None,351                None,352            )353        labels = clustering_labels(embeddings)354        summaries = summarize(embeddings, labels, cleaned_files)355        summary_output = "\n".join(356            [357                f"• Summary for cluster/doc {i+1}:\n{summary}"358                for i, summary in enumerate(summaries)359            ]360        )361        all_text = " ".join(cleaned_files)362        tfidf_fig = tfidf_plot(all_text)  # Get the tfidf plot figure363        dendrogram_fig = dendrogram_plot(364            embeddings, labels365        )  # Get the dendrogram plot figure366        tsne_fig = tsne_plot(embeddings, labels)  # Get the t-sne plot figure367        wordcloud_fig = wordcloud_plot(all_text)  # Get the wordcloud plot figure368        return summary_output, tfidf_fig, dendrogram_fig, tsne_fig, wordcloud_fig369    else:370        return "No files uploaded.", None, None, None, None371 372 373import gradio as gr374 375with gr.Blocks() as demo:376    gr.Markdown("# 📰 Multi-Document Summarization")377 378    with gr.Row():379        with gr.Column():380            file_upload = gr.Files(label="Upload Your Files")381            gr.Markdown(382                "### Supported File Types: 📄 `.docx` 📝 `.txt` 🌐 `.html` 📑 `.pdf` 📊 `.csv` 📈 `.xlsx` 🗂 `.json` 🗃 `.xml` 🎞 `.pptx`",383                elem_id="file-types-info",384            )385            summarize_btn = gr.Button("Summarize")386 387        with gr.Column():388            summary_output = gr.Textbox(label="• Bullet List of Summaries", lines=10)389 390    gr.Markdown("## 📊 Visualizations")391 392    with gr.Row():393        dendro = gr.Plot(label="Dendrogram")394        tsne = gr.Plot(label="t-SNE")395 396    with gr.Row():397        tfidf = gr.Plot(label="TF-IDF")398 399    with gr.Row():400        wordcloud = gr.Image(label="Word Cloud")401 402    summarize_btn.click(403        summarize_docs,404        inputs=file_upload,405        outputs=[summary_output, tfidf, dendro, tsne, wordcloud],406    )407 408demo.launch(share=True)409