TwinklData/Community_Collections_App
0
1################################2# CONFIGURATION3################################4 5import streamlit as st6import pandas as pd7import hashlib8import joblib9from io import BytesIO10from umap import UMAP11from hdbscan import HDBSCAN12 13import os14 15from streamlit_extras.metric_cards import style_metric_cards16from streamlit_extras.add_vertical_space import add_vertical_space17 18# ---- FUNCTIONS ----19 20from src.extract_usage import extract_usage21from src.necessity_index import compute_necessity, index_scaler, qcut_labels22from src.column_detection import detect_freeform_col, detect_id_col, detect_school_type_col23from src.shortlist import shortlist_applications24from src.twinkl_originals import find_book_candidates25from src.preprocess_text import normalise_text 26import src.models.topic_modeling_pipeline as topic_modeling_pipeline27from src.px_charts import plot_histogram, plot_topic_countplot28from typing import Tuple29 30style_metric_cards(box_shadow=False, border_left_color='#E7F4FF',background_color='#E7F4FF', border_size_px=0, border_radius_px=6)31 32##################################33# CACHED PROCESSING FUNCTIONS34##################################35 36# -----------------------------------------------------------------------------37# Heavy processing (IO + NLP) is cached to avoid re‑executing when the UI state38# changes. The function only re‑runs if the **file contents** change.39# -----------------------------------------------------------------------------40 41@st.cache_resource42def load_heartfelt_predictor():43 model_path = os.path.join("src", "models", "heartfelt_pipeline.joblib")44 return joblib.load(model_path)45 46@st.cache_resource47def load_embeddings_model():48 return topic_modeling_pipeline.load_embedding_model('all-MiniLM-L12-v2')49 50@st.cache_data(show_spinner=True)51def load_and_process(raw_csv: bytes) -> Tuple[pd.DataFrame, str]:52 """53 Load CSV from raw bytes, detect freeform column, compute necessity scores,54 and extract usage items. Returns processed DataFrame and freeform column name.55 """56 # Read Uploaded Data 57 df_orig = pd.read_csv(BytesIO(raw_csv))58 59 # Detect freeform column60 freeform_col = detect_freeform_col(df_orig)61 id_col = detect_id_col(df_orig)62 school_type_col = detect_school_type_col(df_orig)63 print(id_col)64 65 df_orig = df_orig[df_orig[freeform_col].notna()]66 67 #Word Count68 df_orig['word_count'] = df_orig[freeform_col].fillna('').str.split().str.len()69 70 # Compute Necessity Scores71 scored = df_orig.join(df_orig[freeform_col].apply(compute_necessity))72 scored['necessity_index'] = index_scaler(scored['necessity_index'].values)73 scored['priority'] = qcut_labels(scored['necessity_index'])74 75 # Find Twinkl Originals Candidates76 scored['book_candidates'] = find_book_candidates(scored, freeform_col, school_type_col)77 78 # Label Heartfelt Applications79 scored['clean_text'] = scored[freeform_col].map(normalise_text)80 model = load_heartfelt_predictor()81 scored['is_heartfelt'] = model.predict(scored['clean_text'].astype(str))82 83 84 85 # Usage Extraction86 docs = df_orig[freeform_col].to_list()87 scored['usage'] = extract_usage(docs)88 89 return scored, freeform_col, id_col90 91# -----------------------------------------------------------------------------92# Derivative computations that rely only on the processed DataFrame are also93# cached. These are lightweight but still benefit from caching because this94# function might be called multiple times during widget interaction.95# -----------------------------------------------------------------------------96 97 98@st.cache_data(show_spinner=True)99def compute_shortlist(df: pd.DataFrame) -> pd.DataFrame:100 """Pre‑compute shortlist_score for all rows (used for both modes)."""101 return shortlist_applications(df, k=len(df))102 103@st.cache_resource(show_spinner=True)104def run_topic_modeling():105 106 try:107 108 # ------- 1. Tokenize texts into sentences -------109 nlp = topic_modeling_pipeline.load_spacy_model(model_name='en_core_web_sm')110 111 sentences = []112 mappings = []113 114 for idx, application_text in df[freeform_col].dropna().items():115 for sentence in topic_modeling_pipeline.spacy_sent_tokenize(application_text):116 sentences.append(sentence)117 mappings.append(idx)118 119 120 # -------- 2. Generate embeddings -------121 122 embeddings_model = load_embeddings_model()123 embeddings = embeddings_model.encode(sentences, show_progress_bar=True)124 125 # -------- 3. Topic Modeling --------126 127 umap_model = UMAP(n_neighbors=7, n_components=5, min_dist=0.0, metric='cosine', random_state=42)128 hdbscan_model = HDBSCAN(min_cluster_size=10, metric='euclidean', cluster_selection_method='eom', prediction_data=True)129 130 # --------- 4. Perform Topic Modeling ---------131 132 topic_model, topics, probs = topic_modeling_pipeline.bertopic_model(sentences, embeddings, embeddings_model, umap_model, hdbscan_model)133 134 topic_modeling_pipeline.ai_labels_to_custom_name(topic_model) 135 136 return topic_model, topics, probs, mappings137 138 except Exception as e:139 st.error(f"Topic modeling failed: {e}")140 st.code(traceback.format_exc()) # Shows the full error in a nice code box141 return None, None, None, None142 143 144 145################################146# MAIN APP SCRIPT147################################148 149st.title("🪷 Community Collections Helper")150 151uploaded_file = st.file_uploader("Upload grant applications file for analysis", type='csv', label_visibility='collapsed')152 153 154# ========== FINGERPRINTING CURRENT FILE ==========155# This helps avoid reruns of certain functions as156# long as the file stays the same157 158if uploaded_file is not None:159 raw = uploaded_file.read()160 file_hash = hashlib.md5(raw).hexdigest()161 st.session_state["current_file_hash"] = file_hash162else:163 raw = None164 st.session_state.pop("current_file_hash", None)165 166if raw is None:167 st.stop()168 169## ====== DATA PROCESSING ======170 171df, freeform_col, id_col = load_and_process(raw) # from cached function172topic_model, topics, probs, mappings = run_topic_modeling() # from cached function173 174if topic_model is not None:175 label_map = (topic_model176 .get_topic_info()177 .set_index("Topic")["CustomName"]178 .to_dict())179 df = topic_modeling_pipeline.attach_topics(df, mappings, topics, label_map, col="topics")180else:181 st.warning("Topics could not be generated; continuing without them.")182 183 184topics_df = topic_model.get_topic_info()185topics_df = topics_df[topics_df['Topic'] > -1]186topics_df.drop(columns=['Name', 'OpenAI'], inplace=True)187cols_to_move = ['Topic','CustomName']188topics_df = topics_df[cols_to_move + [col for col in topics_df.columns if col not in cols_to_move]]189topics_df.rename(columns={'CustomName':'Topic Name', 'Topic':'Topic Nr.'}, inplace=True)190 191 192 193book_candidates_df = df[df['book_candidates'] == True]194 195###############################196# SIDE PANNEL #197###############################198 199with st.sidebar:200 st.title("Shortlist Mode")201 202 203 quantile_map = {"strict": 0.75, "generous": 0.5}204 mode = st.segmented_control(205 "Select one option",206 options=["strict", "generous"],207 default="strict",208 )209 210 scored_full = compute_shortlist(df)211 threshold_score = scored_full["shortlist_score"].quantile(quantile_map[mode])212 auto_short_df = scored_full[scored_full["shortlist_score"] >= threshold_score]213 214 st.title("Filters")215 216 ## --- Dataframe To Filter ---217 options = ['All applications', 'Not shortlisted'] 218 selected_view = st.pills('Choose data to filter', options, default='Not shortlisted')219 st.write("")220 221 ## --- Necessity Index Filtering ---222 min_idx = float(df['necessity_index'].min())223 max_idx = float(df['necessity_index'].max())224 filter_range = st.slider(225 "Necessity Index Range", min_value=min_idx, max_value=max_idx, value=(min_idx, max_idx)226 )227 228 def filter_all_applications(df, auto_short_df, filter_range):229 return df[df['necessity_index'].between(filter_range[0], filter_range[1])]230 231 def filter_not_shortlisted(df, auto_short_df, filter_range):232 return df[233 (~df.index.isin(auto_short_df.index)) &234 (df['necessity_index'].between(filter_range[0], filter_range[1]))235 ]236 237 filter_map = {238 'All applications': filter_all_applications,239 'Not shortlisted': filter_not_shortlisted,240 }241 242 filtered_df = filter_map[selected_view](df, auto_short_df, filter_range)243 244 ## -------- Topic Filtering -------245 246 topic_options = sorted(topics_df['Topic Name'].unique())247 selected_topics = st.multiselect("Filter by Topic(s)", options=topic_options, default=[])248 249 if selected_topics:250 selected_set = set(selected_topics)251 filtered_df = filtered_df[252 filtered_df['topics'].apply(lambda topic_list: selected_set.issubset(set(topic_list)))253 ]254 255 st.markdown(f"**Total Applications:** {len(df)}")256 st.markdown(f"**Filtered Applications:** {len(filtered_df)}")257 258 manual_keys = [k for k in st.session_state.keys() if k.startswith("shortlist_")]259 manually_shortlisted = [int(k.split("_")[1]) for k in manual_keys if st.session_state[k]]260 261 st.markdown(f"**Manually Shortlisted:** {len(manually_shortlisted)}")262 if manually_shortlisted:263 csv = df.loc[manually_shortlisted].to_csv(index=False).encode("utf-8")264 st.download_button(265 "Download Manual Shortlist",266 data=csv,267 file_name="manual_shortlist.csv",268 mime="text/csv",269 icon="⬇️",270 )271 272 273 add_vertical_space(4)274 st.divider()275 st.badge("Version 1.0.0", icon=':material/category:',color='violet')276 st.markdown("""277 :grey[Made with 🩷 by the AI Innovation Team278 Contact: lynn.perez@twinkl.com]279 """)280 281 282 283## ====== CREATE TAB SECTIONS =======284tab1, tab2 = st.tabs(["Shortlist Manager","Insights"])285 286 287##################################################288# SHORTLIST MANAGER TAB # 289##################################################290 291with tab1:292 293 ## =========== AUTOMATIC SHORTLIST =========294 295 st.header("Automatic Shortlist")296 297 csv_auto = auto_short_df.to_csv(index=False).encode("utf-8")298 all_processed_data = df.to_csv(index=False).encode("utf-8")299 book_candidates = book_candidates_df.to_csv(index=False).encode("utf-8")300 topic_descriptions_csv = topics_df.to_csv(index=False).encode("utf-8")301 302 303 csv_options = {304 "Shortlist": (csv_auto, "shortlist.csv"),305 "All Processed Data": (all_processed_data, "all_processed.csv"),306 "Book Candidates": (book_candidates, "book_candidates.csv"),307 "Topic Descriptions": (topic_descriptions_csv, "topic_descriptions.csv"),308 }309 310 choice = st.selectbox("Select a file for download", list(csv_options.keys()))311 312 csv_data, file_name = csv_options[choice]313 314 315 st.download_button(316 label=f"Download {choice}",317 data=csv_data,318 file_name=file_name,319 mime="text/csv",320 help="This button will download the selected file from above",321 icon="⬇️"322 323 )324 325 326 st.write("")327 total_col, shortlistCounter_col, mode_col = st.columns(3)328 329 total_col.metric("Applications Submitted", len(df))330 shortlistCounter_col.metric("Shorlist Length", len(auto_short_df))331 mode_col.metric("Mode", mode)332 333 shorltist_cols_to_show = [334 id_col,335 freeform_col,336 'book_candidates',337 'usage',338 'necessity_index',339 'urgency_score',340 'severity_score',341 'vulnerability_score',342 'shortlist_score',343 'is_heartfelt',344 'topics',345 ]346 347 st.dataframe(auto_short_df.loc[:, shorltist_cols_to_show], hide_index=True)348 349 ## ====== APPLICATIONS REVIEW =======350 351 add_vertical_space(2)352 st.header("Manual Filtering")353 st.info("Use the **side panel** filters to more easily sort through applications that you'd like to review.", icon=':material/info:')354 355 st.write("")356 if len(filtered_df) > 0:357 st.markdown("#### Filtered Applications")358 for idx, row in filtered_df.iterrows():359 with st.expander(f"Application {int(row[id_col])}"):360 st.write("")361 col1, col2, col3, col4 = st.columns(4)362 col1.metric("Necessity", f"{row['necessity_index']:.1f}")363 col2.metric("Urgency", f"{int(row['urgency_score'])}")364 col3.metric("Severity", f"{int(row['severity_score'])}")365 col4.metric("Vulnerability", f"{int(row['vulnerability_score'])}")366 367 st.markdown("##### Excerpt")368 st.write(row[freeform_col])369 370 # HTML for clean usage items 371 usage_items = [item for item in row['usage'] if item and item.lower() != 'none']372 if usage_items:373 st.markdown("##### Usage")374 pills_html = "".join(375 f"<span style='display:inline-block;background-color:#E7F4FF;color:#125E9E;border-radius:20px;padding:4px 10px;margin:2px;font-size:0.95rem;'>{item}</span>"376 for item in usage_items377 )378 st.html(pills_html)379 else:380 st.caption("*No usage found*")381 382 topic_items = [item for item in row['topics'] if item and item.lower() != 'none']383 if topic_items:384 st.markdown("##### Topics")385 topic_boxes_html= "".join(386 f"<span style='display:inline-block;background-color:#ECE0FC;color:#6741B9;border-radius:5px;padding:4px 10px;margin:2px;font-size:0.95rem;'>{item}</span>"387 for item in topic_items388 )389 st.html(topic_boxes_html)390 else:391 st.caption("_No topics assigned for this application_")392 393 394 st.checkbox(395 "Add to shortlist",396 key=f"shortlist_{idx}"397 )398 399 else:400 st.markdown(401 """402 <br>403 <div style="text-align: center; font-size: 1.2em">404 🍂 <span style="color: grey;">No applications matched these filters...</span>405 </div>406 """,407 unsafe_allow_html=True,408 )409 410 411#########################################412# INSIGHTS TAB #413#########################################414 415with tab2:416 417 418 ## =========== DATA OVERVIEW ==========419 420 st.header("General Insights")421 add_vertical_space(1)422 423 col1, col2, col3 = st.columns(3)424 col1.metric("Applications Submitted", len(df))425 col2.metric("Median N.I", df['necessity_index'].median().round(2))426 col3.metric("Avg. Word Count", f"{df['word_count'].mean().round(1)}")427 428 ## --- NI Distribution Plot ---429 ni_distribution_plt = plot_histogram(df, col_to_plot='necessity_index', bins=50, title='Necessity Index Histogram')430 st.plotly_chart(ni_distribution_plt)431 432 433 ## ============= TOPIC MODELING ============434 435 st.header("Topic Modeling")436 add_vertical_space(1)437 438 ## ------- Display Topics Dataframe ------439 440 with st.popover("How are topic extracted?", icon="🌱"):441 442 st.write("""443 **About Topic Modeling**444 445 We use BERTopic to :primary[**dynamically**] extract the most common topics from the natural language data.446 447 BERTopic is a machine learning technique that allows us to group documents (in this case, sentences within application letters) based on their semantic similarity and other patterns such as word frequency and placement.448 449 The table you see below shows you the extracted topics, alongside their top 10 extracted keywords and a small sample of real texts from the applications that demonstrate where the topics came from.450 451 **Table Info**452 - **Topic Nr.:** The 'id' of the topic.453 - **Topic Name:** This is an AI-generated label based on a few samples of application responses alongside their corresponding keywords.454 - **Representation:** Top 10 keywords that best represent a topic455 - **Representative Docs**: Sample sentences contributing to the topic456 """)457 st.dataframe(topics_df, hide_index=True)458 459 ## -------- Plot Topics Chart ----------460 461 topic_count_plot = plot_topic_countplot(topics_df, topic_id_col='Topic Nr.', topic_name_col='Topic Name', representation_col='Representation', height=500, title='Topic Frequency Chart')462 st.plotly_chart(topic_count_plot, use_container_width=True)463 464 ## --------- User Updates -----------465 466 if st.session_state.get("topic_toast_shown_for") != st.session_state["current_file_hash"]:467 st.toast(468 """469 **Topic modeling is ready!** View the results on the _Insights_ tab470 """,471 icon='🎉'472 )473 474 st.session_state["topic_toast_shown_for"] = st.session_state["current_file_hash"]475