dafqi/indo_twitter_sentiment_app

1import pandas as pd2import numpy as np 3from PIL import Image4import plotly.express as px5from wordcloud import WordCloud6import matplotlib.pyplot as plt7import string 8import re #regex library9#umap 10import umap11import hdbscan12import plotly.graph_objects as go13from bertopic import BERTopic14from sklearn.feature_extraction.text import CountVectorizer15 16# import word_tokenize from NLTK17from transformers import AutoTokenizer18from script.plotting import visualize_barchart19 20def load_stopwords():21    stopwords = pd.read_csv("assets/stopwordbahasa.csv", header=None)22    stopwords = stopwords[0].tolist()23    stopwords = stopwords + list(string.punctuation)24    return stopwords25 26def tokenisasi(df):27    stopwords = load_stopwords()28    tokenizer = AutoTokenizer.from_pretrained('indobert')29    tokens = df.content.apply(lambda x: tokenizer.tokenize(x))30    tokens = tokens.apply(lambda x: [x for x in x if (not x.startswith('##') and x not in stopwords and len(x) > 4)])31    return tokens32 33def get_wordcloud(df,kelas_sentiment):34    mask = np.array(Image.open('./assets/twitter.png'))35    cmap_dict = {'positif': 'YlGn', 'negatif': 'OrRd', 'netral': 'GnBu'}36    tokens = tokenisasi(df[df.sentiment == kelas_sentiment])37    tokens = tokens.apply(lambda x: ' '.join(x))38    text = ' '.join(tokens)39    # check if text empty or not 40    try :41        wordcloud = WordCloud(width = 800, height = 800,42                        background_color ='black',43                        min_font_size = 10,44                        colormap = cmap_dict[kelas_sentiment],45                        mask = mask).generate(text)46    except: 47        wordcloud = WordCloud(width = 800, height = 800,48                        background_color ='black',49                        min_font_size = 10,50                        colormap = cmap_dict[kelas_sentiment],51                        mask = mask).generate("None")52    return wordcloud53 54def plot_text(df,kelas,embedding_model):55    df = df[df.sentiment == kelas]56    data = embedding_model.encode(df.values.tolist())57    umap_model = umap.UMAP(n_neighbors=min(df.shape[0],5),random_state = 42) 58    umap_data = umap_model.fit_transform(data)59    clusterer = hdbscan.HDBSCAN(min_cluster_size=round((df.shape[0])**(0.5)-1),min_samples=3)60    clusterer.fit(umap_data)61 62    labels = ['cluster ' + str(i) for i in clusterer.labels_]63    # replace cluster -1 with outlier 64    labels = ["outlier" if i == "cluster -1" else i for i in labels ]65    text = df["content"].str.wrap(50).apply(lambda x: x.replace('\n', '<br>'))66    67    fig = px.scatter(x=umap_data[:,0], y=umap_data[:,1],color = clusterer.labels_)68    # remove legend69    fig = px.scatter(x=umap_data[:,0], y=umap_data[:,1],color = labels,text = text)70    #set text color 71    fig.update_traces(textfont_color='rgba(0,0,0,0)',marker_size = 8)72    # set background color73    fig.update_layout(plot_bgcolor='rgba(0,0,0,0)')74    # set margin 75    fig.update_layout(margin=dict(l=40, r=5, t=0, b=40))76    # set axis color to grey77    fig.update_xaxes(showgrid=False, zeroline=False, linecolor='rgb(200,200,200)')78    fig.update_yaxes( zeroline=False, linecolor='rgb(200,200,200)')79    # set font sans-serif80    fig.update_layout(font_family="sans-serif")81    # remove legend82    fig.update_layout(showlegend=False)83 84    # set legend title to cluster85    return df["content"],data,fig86 87def topic_modelling(df,embed_df):88    data = df.apply(lambda x: ' '.join([w for w in x.split() if len(w)>3]))89    stopwords = load_stopwords()90    # remove empty data 91    topic_model = BERTopic(92        calculate_probabilities=True,93        # cluster model 94        hdbscan_model = hdbscan.HDBSCAN(min_cluster_size=5,prediction_data=True),95        vectorizer_model=CountVectorizer(stop_words=stopwords),96        language="indonesian",97    )98    topics, probs = topic_model.fit_transform(data,embed_df)99    topic_labels = topic_model.generate_topic_labels(100        topic_prefix = False,101        separator = ", ",102    )103    topic_model.set_topic_labels(topic_labels)104    fig = visualize_barchart(topic_model)105    # set title to Kata Kunci tiap Topic 106    # fig.update_layout(title_text="Topic yang sering muncul")107    return fig,topic_model