ldarriba/topicmodelingbpd
0
1from bertopic import BERTopic2import pickle3import numba4from st_aggrid import AgGrid5import streamlit as st6import pandas as pd7import datetime8import matplotlib.pyplot as plt9import numpy as np10import plotly.express as px11import plotly.graph_objects as go12 13st.set_option('deprecation.showfileUploaderEncoding', False)14st.title("Análisis de tópicos de reviews")15 16#----------------------------------------------------------------------17#Function definitions18#----------------------------------------------------------------------19 20#File names21# BANK = "ICBC"22# DATE = "11-10-2022"23BANK = "BPD"24DATE = "24-10-2022"25MODEL_FILENAME = f"BERTopic_{BANK}_{DATE}.model"26# TOPICS_FILENAME = r"topics_ICBC.txt"27TOPICS_FILENAME = f"topics_{BANK}_GoogleStore_{DATE}.txt"28PROBS_FILENAME = f"topics_{BANK}_GoogleStore_{DATE}.txt"29INPUTFILE = f"Google Play Store Review {BANK}.csv"30model = None31 32 33@st.cache(allow_output_mutation=True)34def load_model():35 return BERTopic.load(MODEL_FILENAME) 36 37# def load_topics(model):38# # with open(TOPICS_FILENAME, 'rb') as fp:39# # topics = pickle.load(fp)40# topics, probs = model.transform(docs)41 42 # return topics, probs43 44@st.cache(allow_output_mutation=True)45def load_data():46 global model47 #Read CSV file with reviews48 df = pd.read_csv(INPUTFILE, encoding="utf-8", engine="python", quotechar='"', sep="¬", on_bad_lines='warn')49 50 #Keep only those reviews whose rating is bad (<= stars)51 # df = df[df['score'] <= 2]52 53 #Create a new Dataframe with the review, date and topic54 # df = df[['content', 'at']]55 df.rename(columns={'content': 'review', 'at': 'date'}, inplace=True)56 57 #Drop rows with NaN values in 'review'58 df = df[df['review'].notna()]59 60 docs = df['review'].values.tolist()61 62 with open(TOPICS_FILENAME, 'rb') as fp:63 topics = pickle.load(fp)64 with open(PROBS_FILENAME, 'rb') as fp:65 probs = pickle.load(fp)66 # topics, probs = model.transform(docs)67 68 df['topic'] = topics69 70 71 #Generate a list of each topic number and their "name", created from the most significant word72 topics_list = list(model.topics.keys())73 74 topics_names = {}75 for topic in topics_list:76 if topic == -1:77 topics_names[-1] = "Unclassified"78 else:79 topics_names[topic] = model.get_topic(topic)[0][0]80 81 # df['Topic Number'] = topics82 # df['Topic Name'] = df['Topic Number'].apply(lambda x: topics_names[x])83 84 #Generate a list of each topic number and their "name", created from the most significant word85 topics_list = list(model.topics.keys())86 87 topics_names = {}88 for topic in topics_list:89 # for topic in [0]:90 if topic == -1:91 topics_names[-1] = ("Unclassified", "Unclassified")92 else:93 # topics_names[topic] = model.get_topic(topic)[0][0]94 topics_names[topic] = get_topic_name(topic)95 96 df['Topic Number'] = topics97 df['Topic Name'] = df['Topic Number'].apply(lambda x: topics_names[x][0])98 df['Subtopic Name'] = df['Topic Number'].apply(lambda x: topics_names[x][1])99 100 return df, topics, probs101 102def get_topic_name(tnum):103 ''' 104 This function takes the topic number and generates105 a dictionary with the most relevant words, together with their score106 (the score is obtained adding all the c-TF-IDF scores of the 'words' in 107 which the word appears)108 109 returns: (1) the most relevant word as topic name110 (2) the five most relevant words as subtopic name, separated by an '_'111 '''112 topic_name_dict = {}113 114 for item in model.get_topic(tnum):115 for subitem in item[0].split():116 if subitem not in topic_name_dict.keys():117 topic_name_dict[subitem] = 0118 119 topic_name_dict[subitem] += item[1]120 121 topic_name_dict = dict(sorted(topic_name_dict.items(), key = lambda x: x[1], reverse = True))122 topics_list = list(topic_name_dict.keys())123 most_relevant_word = topics_list[0]124 subtopic_name = str(tnum) + "_" + "_".join(topics_list[:5])125 print(subtopic_name)126 return most_relevant_word, subtopic_name 127 128@st.cache(allow_output_mutation=True)129def load_periods_data():130 return pd.read_csv('periods.csv')131 132def compute_nr_bins(days_per_bin=30):133 return int((pd.to_datetime(max(df['date'])) - pd.to_datetime(min(df['date']))).days / days_per_bin)134 135def build_new_df_time_window(df, days):136 yend = datetime.datetime(2022,9,27)137 # yend = df['date'].max()[:10]138 139 if days == -1:140 ystart = df['date'].min()[:10]141 else:142 ystart = str(yend - datetime.timedelta(days))143 yend = str(yend)144 145 df_time_window = df.loc[(df['date'] >= ystart) & (df['date'] <= yend)]146 df_time_window.reset_index(inplace=True, drop=True)147 148 return df_time_window149 150def plot_sentiment(rat):151 ratings = {1: 0, 2:0, 3:0, 4:0, 5:0}152 for k, v in rat.items():153 ratings[k] = v154 score_dict = {155 'Positivos': 0,156 'Neutros': 0,157 'Negativos': 0,158 }159 160 if ratings[4] + ratings[5] > 0:161 score_dict['Positivos'] = ratings[4] + ratings[5]162 if ratings[3] > 0:163 score_dict['Neutros'] = ratings[3]164 if ratings[1] + ratings[2] > 0:165 score_dict['Negativos'] = ratings[1] + ratings[2]166 167 data = np.array(list(score_dict.values()))168 data_cum = data.cumsum()169 starts = data_cum - data170 171 st.write("Distribución porcentual de sentimiento para este tópico:")172 st.markdown('<p style="font-family:sans-serif; color:Green;display:inline">Positivo </p> <p style="font-family:sans-serif; color:Yellow;display:inline">Neutro </p> <p style="font-family:sans-serif; color:Red;display:inline">Negativo</p>', unsafe_allow_html=True)173 174 175 fig, ax = plt.subplots(figsize=(9.2, 1))176 ax.barh(y= 0, width=data, left=starts, height=1, color=('green', 'yellow', 'red'))177 plt.axis('off')178 st.pyplot(fig)179 180def plot_sunburst(docs):181 topic_distribution = docs[docs['Topic Number'] != -1][['Topic Name', 'Subtopic Name']].copy()182 topic_distribution['days'] = option183 184 fig = px.sunburst(topic_distribution,185 path=["days", "Topic Name", "Subtopic Name",],186 # values='Total',187 width=750, height=750,188 title=f"Reviews {option}",189 )190 191 st.plotly_chart(fig)192 193#----------------------------------------------------------------------194# Model and topics loading195#----------------------------------------------------------------------196 197with st.spinner('Cargando modelo en memoria ...'):198 model = load_model()199 200with st.spinner('Cargando datos ...'):201 df, topics, probs = load_data()202 203#----------------------------------------------------------------------204# Topics search205#----------------------------------------------------------------------206 207texto = st.text_input("Ingrese el término/tópico a buscar", '')208if texto != '':209 with st.spinner('Detectando ...'):210 similar_topics, similarity = model.find_topics(texto, top_n=3)211 st.plotly_chart(model.visualize_barchart(similar_topics, top_n_topics=3))212 213#----------------------------------------------------------------------214# Topics time evolution plot215#----------------------------------------------------------------------216 217df_interval = load_periods_data()218 219periods_list = df_interval['period'].tolist()220 221option = st.selectbox(222 label='Seleccione el intervalo de tiempo a graficar',223 options=df_interval['period'])224 225df_period = df_interval.loc[df_interval['period'] == option]226period_days = df_period['days'].tolist()[0]227period_freq = df_period['frequency'].tolist()[0]228 229if option:230 with st.spinner('Construyendo diagrama ...'):231 nr_bins = compute_nr_bins(days_per_bin=int(period_freq))232 233 # docs_time_window, timestamps_time_window, topics_time_window = build_new_df_time_window(df, period_days)234 docs_time_window = build_new_df_time_window(df, period_days)235 236 topics_over_time = model.topics_over_time(237 docs=docs_time_window['review'], 238 topics=docs_time_window['topic'], 239 timestamps=docs_time_window['date'],240 nr_bins=nr_bins,241 global_tuning=False,242 evolution_tuning=True)243 244 st.plotly_chart(model.visualize_topics_over_time(245 topics_over_time=topics_over_time, 246 top_n_topics=5,247 topics=None,248 normalize_frequency=False))249 250#----------------------------------------------------------------------251# Plot the sunburst chart252#----------------------------------------------------------------------253 plot_sunburst(docs_time_window)254 255 #----------------------------------------------------------------------256 # Show all reviews corresponding to a given topic257 #----------------------------------------------------------------------258 topico_para_listar = st.number_input(f"Ingrese el número de tópico para desplegar los reviews asociados (entre 0 y {max(topics)})", min_value=-1, max_value=max(topics))259 pd.set_option('display.max_colwidth', None)260 261 if topico_para_listar >= -1:262 # new_df = df[df['Topic Number'] == topico_para_listar]263 new_df = docs_time_window[docs_time_window['Topic Number'] == topico_para_listar]264 rating_dict = new_df['score'].value_counts()265 266 #Plot pie chart with percentage of positive, neutral and negative ratings267 # print("Rating:")268 # print(rating_dict)269 plot_sentiment(rating_dict)270 271 # df_topic = df[df['topic'] == topico_para_listar][['date', 'review']]272 df_topic = new_df[['date', 'score', 'review']]273 df_topic.reset_index(inplace=True, drop=True)274 df_topic.rename(columns={'date': 'Fecha', 'review': 'Comentario'}, inplace=True)275 276 AgGrid(df_topic, height=500, fit_columns_on_grid_load=True)277 278 279