22-028andre/PPE

1import streamlit as st2import pandas as pd3import networkx as nx4import matplotlib.pyplot as plt5import seaborn as sns6import re7import string8import fitz  # PyMuPDF9import nltk10from nltk.corpus import stopwords11from nltk.util import ngrams12from collections import Counter13 14# ===============================15# 1. KONFIGURASI HALAMAN16# ===============================17st.set_page_config(18    page_title="Keyword Extraction - Dashboard UAS",19    layout="wide",20    initial_sidebar_state="expanded"21)22 23st.title("📊 Keyword Extraction Berbasis Graph & Centrality")24st.markdown("Dashboard ini mengekstraksi kata kunci penting menggunakan perpaduan **Unigram**, **Bigram**, dan berbagai algoritma **Centrality**.")25 26# ===============================27# 2. DOWNLOAD RESOURCE NLTK28# ===============================29@st.cache_resource30def download_nltk_data():31    try:32        nltk.download('punkt')33        nltk.download('stopwords')34        nltk.download('punkt_tab')35    except:36        pass37 38download_nltk_data()39 40# ===============================41# 3. FUNGSI PREPROCESSING TEKS42# ===============================43def preprocess_text(text):44    # Simpan versi asli45    original_sample = text.replace('◼', '')46    47    # 1. Cleaning & Normalisasi48    clean = original_sample.lower()49    clean = re.sub(r'[^\x00-\x7f]', r'', clean) # Hapus karakter non-ASCII50    clean = clean.translate(str.maketrans('', '', string.punctuation))51    clean = re.sub(r'\d+', '', clean)52    53    # 2. Tokenisasi54    tokens = nltk.word_tokenize(clean)55    56    # 3. Stopword Removal57    stop_words = set(stopwords.words('indonesian'))58    59    # Ekstraksi Unigram60    unigrams = [t for t in tokens if t not in stop_words and len(t) > 2]61    62    # Ekstraksi Bigram63    bi_gen = list(ngrams(tokens, 2))64    bigrams = [65        f"{b[0]} {b[1]}" for b in bi_gen 66        if b[0] not in stop_words and b[1] not in stop_words 67        and len(b[0]) > 2 and len(b[1]) > 268    ]69    70    cleaned_sentence = " ".join(unigrams)71    return unigrams, bigrams, cleaned_sentence, original_sample72 73# ===============================74# 4. SIDEBAR & FILE UPLOAD75# ===============================76with st.sidebar:77    st.header("Konfigurasi")78    uploaded_file = st.file_uploader("Upload PDF atau TXT", type=["pdf", "txt"])79    st.divider()80    window_size = st.slider("Graph Window Size", 1, 5, 2)81    st.info("Window size menentukan jarak antar kata untuk dianggap berhubungan dalam graph.")82 83# ===============================84# 5. MAIN LOGIC85# ===============================86if uploaded_file is not None:87    if uploaded_file.type == "application/pdf":88        with fitz.open(stream=uploaded_file.read(), filetype="pdf") as doc:89            raw_text = "".join([page.get_text() for page in doc])90    else:91        raw_text = uploaded_file.read().decode("utf-8")92 93    if not raw_text.strip():94        st.error("Dokumen tidak memiliki teks.")95    else:96        unigrams, bigrams, cleaned_text, original_text = preprocess_text(raw_text)97        combined_all = unigrams + bigrams98        99        tab1, tab2, tab3, tab4 = st.tabs([100            "📄 Hasil Preprocessing", 101            "🔢 Matriks Ko-okurensi", 102            "🕸️ Network Graph", 103            "🏆 Centrality Analysis"104        ])105 106        # --- TAB 1: PREPROCESSING & STATISTIK ---107        with tab1:108            st.subheader("Perbandingan Teks")109            col_a, col_b = st.columns(2)110            with col_a:111                st.markdown("**Teks Asli (Fragmen):**")112                st.text_area("Original", original_text[:1000] + "...", height=200, label_visibility="collapsed")113            with col_b:114                st.markdown("**Teks Hasil Preprocessing:**")115                st.text_area("Cleaned", cleaned_text[:1000] + "...", height=200, label_visibility="collapsed")116            117            st.divider()118            st.subheader("Statistik Frekuensi")119            c1, c2 = st.columns(2)120            with c1:121                st.write("**Top 15 Unigrams**")122                u_df = pd.DataFrame(Counter(unigrams).most_common(15), columns=["Kata", "Freq"])123                st.bar_chart(u_df.set_index("Kata"))124            with c2:125                st.write("**Top 15 Bigrams**")126                b_df = pd.DataFrame(Counter(bigrams).most_common(15), columns=["Frase", "Freq"])127                st.bar_chart(b_df.set_index("Frase"), color="#ffaa00")128 129        # --- TAB 2: MATRIKS KO-OKURENSI ---130        with tab2:131            st.subheader("Matriks Hubungan (Top 15 Keywords)")132            top_15 = [item for item, count in Counter(combined_all).most_common(15)]133            matrix_df = pd.DataFrame(0, index=top_15, columns=top_15)134            for i in range(len(combined_all) - 1):135                for j in range(i + 1, min(i + window_size + 1, len(combined_all))):136                    w1, w2 = combined_all[i], combined_all[j]137                    if w1 in top_15 and w2 in top_15:138                        matrix_df.loc[w1, w2] += 1139                        matrix_df.loc[w2, w1] += 1140 141            fig_m, ax_m = plt.subplots(figsize=(10, 8))142            sns.heatmap(matrix_df, annot=True, cmap="YlGnBu", fmt="d", ax=ax_m)143            st.pyplot(fig_m)144 145        # --- TAB 3: NETWORK GRAPH ---146        with tab3:147            st.subheader("Representasi Graph Kata")148            G = nx.Graph()149            for i in range(len(combined_all) - 1):150                for j in range(i + 1, min(i + window_size + 1, len(combined_all))):151                    G.add_edge(combined_all[i], combined_all[j])152 153            num_nodes = st.slider("Jumlah node di graph", 10, 50, 20)154            top_nodes = [n for n, c in Counter(combined_all).most_common(num_nodes)]155            sub = G.subgraph(top_nodes)156 157            fig_g, ax_g = plt.subplots(figsize=(12, 7))158            pos = nx.kamada_kawai_layout(sub)159            nx.draw(sub, pos, with_labels=True, node_color="#00b4d8", 160                    node_size=2000, font_size=10, edge_color="#dddddd", width=1.5, ax=ax_g)161            st.pyplot(fig_g)162 163        # --- TAB 4: CENTRALITY RESULTS (DENGAN DEGREE/BETWEENNESS/CLOSENESS) ---164        with tab4:165            st.subheader("Hasil Analisis Centrality")166            167            # Perhitungan Centrality168            pr_scores = nx.pagerank(G)169            dc_scores = nx.degree_centrality(G)170            bc_scores = nx.betweenness_centrality(G)171            cc_scores = nx.closeness_centrality(G)172            173            # Penggabungan ke DataFrame174            centrality_results = []175            for word in pr_scores.keys():176                centrality_results.append({177                    "Keyword": word,178                    "PageRank": pr_scores[word],179                    "Degree": dc_scores[word],180                    "Betweenness": bc_scores[word],181                    "Closeness": cc_scores[word]182                })183            184            full_df = pd.DataFrame(centrality_results)185            186            # Tampilkan Top 20 berdasarkan PageRank187            top_20_df = full_df.sort_values(by="PageRank", ascending=False).head(20)188            top_20_df.index = range(1, 21)189            190            st.markdown("### Top 20 Kata Berdasarkan Berbagai Centrality")191            st.dataframe(top_20_df, use_container_width=True)192 193            # --- Visualisasi Tambahan ---194            st.divider()195            st.subheader("Visualisasi Metrik")196            metric_choice = st.selectbox("Pilih Metrik untuk Grafik:", ["PageRank", "Degree", "Betweenness", "Closeness"])197            198            fig_v, ax_v = plt.subplots(figsize=(10, 6))199            v_data = full_df.sort_values(by=metric_choice, ascending=False).head(15)200            sns.barplot(x=metric_choice, y="Keyword", data=v_data, palette="magma", ax=ax_v)201            ax_v.set_title(f"Top 15 Keywords - {metric_choice}")202            st.pyplot(fig_v)203            204            # Download205            csv = full_df.to_csv(index=False).encode('utf-8')206            st.download_button("Download Full CSV", csv, "centrality_analysis.csv", "text/csv")207 208else:209    st.info("Silakan unggah dokumen PDF atau TXT untuk memulai ekstraksi.")