sberhe/software-release-notes-classification

1import tensorflow as tf2import streamlit as st3import pandas as pd4import numpy as np5from datasets import load_dataset6from transformers import AutoTokenizer, TFAutoModel7from sklearn.feature_extraction.text import TfidfVectorizer8from sklearn.cluster import KMeans9from sklearn.decomposition import PCA10 11# Load the dataset12dataset = load_dataset("sberhe/2023-1000-software-release-notes")13 14# Load a pre-trained model and tokenizer (TensorFlow version)15model_name = "bert-base-uncased"16tokenizer = AutoTokenizer.from_pretrained(model_name)17model = TFAutoModel.from_pretrained(model_name)18 19# Tokenize the data20def tokenize_function(examples):21    return tokenizer(examples["text"], padding="max_length", truncation=True, max_length=512)22 23tokenized_datasets = dataset.map(tokenize_function, batched=True)24 25# Function to extract embeddings26def extract_embeddings(batch):27    inputs = {k: tf.convert_to_tensor(v) for k, v in batch.items() if k in tokenizer.model_input_names}28    outputs = model(**inputs)29    # Use the embeddings of the [CLS] token ([0])30    return {"embeddings": outputs.last_hidden_state[:, 0].numpy()}31 32# Apply the function to extract embeddings in batches33embeddings_dataset = tokenized_datasets.map(extract_embeddings, batched=True)34 35# Flatten the embeddings and reduce dimensionality using PCA36embeddings = np.vstack(embeddings_dataset['train']['embeddings'])37pca = PCA(n_components=2)  # Using 2 components for better visualization38embeddings_2d = pca.fit_transform(embeddings)39 40# Perform unsupervised clustering (K-Means)41num_clusters = 5042kmeans = KMeans(n_clusters=num_clusters)43cluster_labels = kmeans.fit_predict(embeddings_2d)44 45# Create a DataFrame with cluster labels and original texts46original_texts = [example['text'] for example in dataset['train']]47df = pd.DataFrame({'text': original_texts, 'Cluster': cluster_labels})48 49# ...50 51# TF-IDF calculation and finding representative terms for each cluster52vectorizer = TfidfVectorizer(stop_words='english')53X_tfidf = vectorizer.fit_transform(df['text'])54feature_names = vectorizer.get_feature_names_out()55 56cluster_names = []57for i in range(num_clusters):58    indices = df[df['Cluster'] == i].index59    # Aggregate the TF-IDF scores for each feature in cluster i60    aggregated_tfidf = np.mean(X_tfidf[indices], axis=0)61    # Convert to array (if it's not already an array) and get the index of the max tf-idf score62    aggregated_tfidf_array = np.array(aggregated_tfidf).flatten()63    max_tfidf_index = aggregated_tfidf_array.argmax()64    cluster_names.append(feature_names[max_tfidf_index])65 66# Count the size of each cluster67cluster_sizes = df['Cluster'].value_counts().sort_index()68 69# Output cluster names and sizes using Streamlit70for i in range(num_clusters):71    cluster_name = cluster_names[i]72    cluster_size = cluster_sizes.get(i, 0)  # Get size with a default of 0 if cluster is empty73    print(f"Cluster {i+1} (Name: {cluster_name}, Size: {cluster_size})")74 75# ...