rajistics/News_Topic_Clustering

1from bertopic import BERTopic2import streamlit as st3import streamlit.components.v1 as components4#from datasets import load_dataset5import pandas as pd6from datasets import load_dataset7import json8 9##Load Dataset from HF Hub10#dataset = load_dataset("rshah/million-headlines")11#news = pd.DataFrame.from_dict(dataset["train"])12 13#Load dataset locally - faster for demo14news = pd.read_parquet("topic_10000.par")15news['date'] = pd.to_datetime(news['publish_date'], format='%Y%m%d')16timestamps = news.date.to_list()17tweets = news.headline_text.to_list()18 19#Load topics20with open("topics", "r") as fp:21    topics = json.load(fp)22 23option_n = 524 25st.set_page_config(page_title="News Topic Clustering")26st.title("News Topic Clustering")27st.caption("By Rajiv Shah")28st.caption("")29st.caption("This is a simple example of using identifying topics in the [one million ABC news headline dataset](https://huggingface.co/datasets/rshah/million-headlines).  \30    If you look at the code for this app, you will see how it uses just a few lines of [BERTopic](https://maartengr.github.io/BERTopic/index.html) to \31         build the topics and create the visualizations")32st.caption("The preloaded existing model provides the more interesting results.  However, this app can be run live by building a new model, but \33    is limited to a small number of rows. I also limited topics over time to the existing model.")34 35 36form = st.sidebar.form("Main Settings")37form.header("Main Settings")38option = form.selectbox(39    'What model would you like to run',40    ('Load existing model', 'Build new model'),index=0)41 42option_n = form.number_input(43    'What topic would you like to get terms for?',44    min_value=0,max_value=10,value=5)45 46submitted = form.form_submit_button(label = 'Select Model')47 48if option == 'Load existing model':49    ##Load existing model50    topic_model = BERTopic.load("topic_10000.model")51    #topics, _ = topic_model.transform(tweets)52else:53    ##Builds Topic Model54    #news_sample = news[(news['date'] > '2015-06-01')]55    news_sample = news[(news['date'] > '2017-01-01') & (news['date'] < '2019-01-01') ]56    news_sample = news_sample.sample(200,random_state=123)57    tweets = news_sample.headline_text.to_list()58    topic_model = BERTopic(min_topic_size=5, verbose=True)59    topics, _ = topic_model.fit_transform(tweets)60 61 62#Get top topics63freq = topic_model.get_topic_info()64freq = freq.iloc[1: , :]   ##drop -1 row65freq.head(10)66st.header("The Main Topic Clusters")67st.write(freq)68 69 70topic_nr = freq.iloc[option_n]["Topic"]  # We select a frequent topic71st.caption("")72st.write('Top words in topic cluster: ',option_n)73#st.caption(option_n)74mytuple = (topic_model.get_topic(topic_nr))75for item in mytuple:76    st.write(str(item[0]))77 78st.header("Relationships between clusters ")79st.plotly_chart(topic_model.visualize_hierarchy())80 81 82if option == 'Load existing model':83    st.header("Topics over time for Existing Model")84    topics_over_time = topic_model.topics_over_time(docs=tweets, 85                                                topics=topics, 86                                                timestamps=timestamps, 87                                                global_tuning=True, 88                                                evolution_tuning=True, 89                                                nr_bins=20)90 91    st.plotly_chart(topic_model.visualize_topics_over_time(topics_over_time, top_n_topics=20))