BHAVIKBANKER/BERTopic_AG_final
1
1"""2app.py — Gradio UI entry point.3ORIGINAL structure and all tabs preserved.4NEW: second file upload for methodology CSV, technique sheets 1-4,5 journal cross-tabulation chart + table, technique optimisation log.6"""7import os, json8import re9import pandas as pd, numpy as np10import gradio as gr11import plotly.express as px12import plotly.graph_objects as go13from agent import run_pipeline, METHODOLOGY_PATTERNS, TECHNIQUE_PATTERNS14 15# ── CSV preview ──────────────────────────────────────────────────────────────16def _preview(file):17 if not file: return "Upload a Scopus CSV to begin."18 df = pd.read_csv(file.name)19 df.columns = df.columns.str.lower()20 has_t = "title" in df.columns21 has_a = "abstract" in df.columns22 n = len(df)23 blanks_t = int(df["title"].isna().sum()) if has_t else n24 blanks_a = int(df["abstract"].isna().sum()) if has_a else n25 ok = "✅" if has_t and has_a and blanks_t < n and blanks_a < n else "❌"26 return (f"## {ok} CSV loaded — {n} entries\n\n"27 f"| Column | Present | Blank rows |\n|---|---|---|\n"28 f"| title | {'✅' if has_t else '❌'} | {blanks_t} |\n"29 f"| abstract | {'✅' if has_a else '❌'} | {blanks_a} |\n\n"30 f"**Usable papers:** {n - max(blanks_t, blanks_a)} / {n}")31 32 33def _preview_methodology(file):34 if not file: return "Upload methodology CSV (title, doi, methodology) to enable technique analysis."35 df = pd.read_csv(file.name)36 df.columns = df.columns.str.lower()37 has_t = "title" in df.columns38 has_m = "methodology" in df.columns39 has_d = "doi" in df.columns40 n = len(df)41 ok = "✅" if has_t and has_m else "❌"42 return (f"## {ok} Methodology CSV — {n} papers\n\n"43 f"| Column | Present |\n|---|---|\n"44 f"| title | {'✅' if has_t else '❌'} |\n"45 f"| doi | {'✅' if has_d else '⚠ optional'} |\n"46 f"| methodology | {'✅' if has_m else '❌'} |\n\n"47 f"Journals will be auto-detected from DOI + title.")48 49 50# ── Original helper builders ─────────────────────────────────────────────────51def _top_papers_df(top_papers: dict) -> pd.DataFrame:52 rows = []53 for cid in sorted(top_papers.keys()):54 for p in top_papers[cid]:55 rows.append({"Cluster": cid, "Label": p["cluster_label"],56 "Rank": p["rank"], "Title": p["title"],57 "Abstract Snippet": p["abstract_snippet"]})58 return pd.DataFrame(rows)59 60 61def _methodology_summary_df(methodology_data: dict, interps: dict) -> pd.DataFrame:62 rows = []63 for cid in sorted(methodology_data.keys()):64 md = methodology_data[cid]65 label = interps.get(cid, {}).get("label", f"Cluster {cid}")66 rows.append({67 "Cluster": cid,68 "Label": label,69 "Dominant Method": md.get("dominant_method", "—"),70 "Dominant Technique": md.get("dominant_technique", "—"),71 "Empirical %": md.get("empirical_pct", 0),72 "Theoretical %": md.get("theoretical_pct", 0),73 "Mixed %": md.get("mixed_pct", 0),74 "Methods (≥2 LLMs)": ", ".join(75 f"{m['name']} ({m['pct']}%, {m['agreement']})"76 for m in md.get("methodologies", [])),77 "Techniques (≥2 LLMs)": ", ".join(78 f"{t['name']} ({t['pct']}%, {t['agreement']})"79 for t in md.get("techniques", [])),80 "Regex Confirmed": ", ".join(md.get("regex_confirmed_consensus", [])) or "—",81 "Regex Rejected": ", ".join(md.get("regex_rejected_consensus", [])) or "—",82 })83 return pd.DataFrame(rows)84 85 86def _extraction_pipeline_df(methodology_data: dict, interps: dict) -> pd.DataFrame:87 rows = []88 for cid in sorted(methodology_data.keys()):89 md = methodology_data[cid]90 label = interps.get(cid, {}).get("label", f"Cluster {cid}")91 scan = md.get("regex_scan", {})92 for item in md.get("methodologies", []) + md.get("techniques", []):93 name = item["name"]94 regex_hits= scan.get("methods",{}).get(name,[]) or scan.get("techniques",{}).get(name,[])95 matched = ", ".join(dict.fromkeys(h["match"] for h in regex_hits))[:80] if regex_hits else "—"96 rows.append({"Cluster": cid, "Label": label, "Item": name,97 "Type": "Method" if item in md.get("methodologies",[]) else "Technique",98 "Regex Match":matched, "Regex Fired": "✅" if regex_hits else "❌",99 "LLM Votes": item["llm_votes"], "Agreement": item["agreement"],100 "Avg Pct (%)":item["pct"], "Evidence": item.get("evidence","—"),101 "Gate Passed":"✅ ACCEPTED"})102 for item in md.get("rejected_methods",[]) + md.get("rejected_techniques",[]):103 name = item["name"]104 regex_hits= scan.get("methods",{}).get(name,[]) or scan.get("techniques",{}).get(name,[])105 matched = ", ".join(dict.fromkeys(h["match"] for h in regex_hits))[:80] if regex_hits else "—"106 rows.append({"Cluster": cid, "Label": label, "Item": name,107 "Type": "Method" if item in md.get("rejected_methods",[]) else "Technique",108 "Regex Match":matched, "Regex Fired": "✅" if regex_hits else "❌",109 "LLM Votes": item["llm_votes"], "Agreement": item["agreement"],110 "Avg Pct (%)":item["pct"], "Evidence": item.get("evidence","—"),111 "Gate Passed":"❌ REJECTED (single LLM)"})112 return pd.DataFrame(rows) if rows else pd.DataFrame()113 114 115def _per_llm_methodology_df(methodology_data: dict, interps: dict) -> pd.DataFrame:116 rows = []117 for cid in sorted(methodology_data.keys()):118 md = methodology_data[cid]119 label = interps.get(cid,{}).get("label", f"Cluster {cid}")120 raw = md.get("llm_raw",{})121 def _fmt(r, key):122 return " | ".join(f"{i['name']} ({i.get('pct',0)}%)" for i in r.get(key,[])) or "—"123 rows.append({"Cluster": cid, "Label": label,124 "Groq Methods": _fmt(raw.get("groq",{}), "methodologies"),125 "Mistral Methods": _fmt(raw.get("mistral",{}), "methodologies"),126 "Gemini Methods": _fmt(raw.get("gemini",{}), "methodologies"),127 "Groq Techniques": _fmt(raw.get("groq",{}), "techniques"),128 "Mistral Techniques": _fmt(raw.get("mistral",{}), "techniques"),129 "Gemini Techniques": _fmt(raw.get("gemini",{}), "techniques"),130 "Groq E/T/M": f"{raw.get('groq',{}).get('empirical_pct',0)}/"131 f"{raw.get('groq',{}).get('theoretical_pct',0)}/"132 f"{raw.get('groq',{}).get('mixed_pct',0)}",133 "Mistral E/T/M": f"{raw.get('mistral',{}).get('empirical_pct',0)}/"134 f"{raw.get('mistral',{}).get('theoretical_pct',0)}/"135 f"{raw.get('mistral',{}).get('mixed_pct',0)}",136 "Gemini E/T/M": f"{raw.get('gemini',{}).get('empirical_pct',0)}/"137 f"{raw.get('gemini',{}).get('theoretical_pct',0)}/"138 f"{raw.get('gemini',{}).get('mixed_pct',0)}",139 })140 return pd.DataFrame(rows)141 142 143def _regex_hits_df(methodology_data: dict, interps: dict) -> pd.DataFrame:144 rows = []145 for cid in sorted(methodology_data.keys()):146 md = methodology_data[cid]147 label = interps.get(cid,{}).get("label", f"Cluster {cid}")148 scan = md.get("regex_scan",{})149 for category, hits in scan.get("methods",{}).items():150 for h in hits:151 rows.append({"Cluster": cid, "Label": label, "Bank": "Methodology",152 "Pattern Category": category, "Matched Text": h["match"],153 "Paper #": h["doc"], "Char Span": f"{h['span'][0]}–{h['span'][1]}"})154 for category, hits in scan.get("techniques",{}).items():155 for h in hits:156 rows.append({"Cluster": cid, "Label": label, "Bank": "Technique",157 "Pattern Category": category, "Matched Text": h["match"],158 "Paper #": h["doc"], "Char Span": f"{h['span'][0]}–{h['span'][1]}"})159 return pd.DataFrame(rows) if rows else pd.DataFrame()160 161 162def _methodology_bar_chart(methodology_data: dict, interps: dict) -> go.Figure:163 labels_list, empirical, theoretical, mixed = [], [], [], []164 for cid in sorted(methodology_data.keys()):165 md = methodology_data[cid]166 labels_list.append(interps.get(cid,{}).get("label", f"C{cid}")[:30])167 empirical.append(md.get("empirical_pct", 0))168 theoretical.append(md.get("theoretical_pct", 0))169 mixed.append(md.get("mixed_pct", 0))170 fig = go.Figure()171 fig.add_trace(go.Bar(name="Empirical %", x=labels_list, y=empirical, marker_color="#3dba7a"))172 fig.add_trace(go.Bar(name="Theoretical %", x=labels_list, y=theoretical, marker_color="#5b9cf6"))173 fig.add_trace(go.Bar(name="Mixed %", x=labels_list, y=mixed, marker_color="#f5a623"))174 fig.update_layout(barmode="stack", template="plotly_dark", height=420,175 paper_bgcolor="#0d1117", plot_bgcolor="#161b22",176 title="Research Orientation per Cluster — Averaged across Groq + Mistral + Gemini",177 xaxis_title="Cluster", yaxis_title="Percentage (%)",178 font=dict(size=11), legend=dict(orientation="h", y=1.12), xaxis_tickangle=-35)179 return fig180 181 182def _refinement_df(rl: list) -> pd.DataFrame:183 if not rl:184 return pd.DataFrame(columns=["Cluster","Iteration","Old Label","New Label",185 "Issues","Improvement","Hallucination Detected"])186 return pd.DataFrame([{187 "Cluster": r["cluster"], "Iteration": r["iteration"],188 "Old Label": r["old_label"], "New Label": r["new_label"],189 "Issues": "; ".join(r.get("issues",[])),190 "Improvement": r["improvement_score"],191 "Hallucination Detected": r["hallucination_detected"],192 } for r in rl])193 194 195def _regex_pattern_info() -> str:196 m_list = "\n".join(f"- **{k}**: `{v.pattern}`" for k,v in METHODOLOGY_PATTERNS.items())197 t_list = "\n".join(f"- **{k}**: `{v.pattern}`" for k,v in TECHNIQUE_PATTERNS.items())198 return (199 "### How Cluster Methodology Extraction Works\n\n"200 "**Step 1 — Regex Pre-Scan:** Two compiled pattern banks run against representative "201 "abstracts. Every match recorded with exact character span, matched text, paper number.\n\n"202 "**Step 2 — 3-LLM Council:** Groq, Mistral, Gemini each receive regex evidence + abstracts. "203 "Each LLM confirms/rejects regex hits and adds any missed methods/techniques.\n\n"204 "**Step 3 — ≥2-LLM Gate:** Only items named by ≥2 LLMs survive. Percentages averaged.\n\n"205 "**Step 4 — Orientation:** Empirical/Theoretical/Mixed averaged across 3 LLMs.\n\n"206 "---\n\n#### Methodology Bank\n" + m_list +207 "\n\n#### Technique Bank\n" + t_list)208 209 210# ── NEW helpers for methodology-CSV pipeline ─────────────────────────────────211def _tech_sheet_df(sheet_rows: list) -> pd.DataFrame:212 return pd.DataFrame(sheet_rows) if sheet_rows else pd.DataFrame()213 214 215def _tech_llm_pct_chart(comp_sheets: dict) -> go.Figure:216 """217 Grouped bar: for each technique, show the % of papers it was found in218 by each of the 3 LLMs (Groq, Mistral, Gemini) + Consolidated.219 """220 s1 = comp_sheets.get(1, [])221 s2 = comp_sheets.get(2, [])222 s3 = comp_sheets.get(3, [])223 s4 = comp_sheets.get(4, [])224 225 def _freq(rows):226 counts = {}227 n = len(rows) or 1228 for row in rows:229 for t in (row.get("techniques","") or "").split(", "):230 t = t.strip().title()231 if t and t != "—":232 counts[t] = counts.get(t,0) + 1233 return {k: round(v/n*100) for k,v in counts.items()}234 235 f1 = _freq(s1); f2 = _freq(s2); f3 = _freq(s3); f4 = _freq(s4)236 all_techs = sorted(set(f1)|set(f2)|set(f3)|set(f4))237 238 fig = go.Figure()239 fig.add_trace(go.Bar(name="Groq", x=all_techs, y=[f1.get(t,0) for t in all_techs], marker_color="#5b9cf6"))240 fig.add_trace(go.Bar(name="Mistral", x=all_techs, y=[f2.get(t,0) for t in all_techs], marker_color="#f5a623"))241 fig.add_trace(go.Bar(name="Gemini", x=all_techs, y=[f3.get(t,0) for t in all_techs], marker_color="#a855f7"))242 fig.add_trace(go.Bar(name="Consolidated", x=all_techs, y=[f4.get(t,0) for t in all_techs], marker_color="#3dba7a"))243 fig.update_layout(barmode="group", template="plotly_dark", height=480,244 paper_bgcolor="#0d1117", plot_bgcolor="#161b22",245 title="Computational Technique Frequency — % of Papers per LLM (Groq / Mistral / Gemini / Consolidated)",246 xaxis_title="Technique", yaxis_title="% of papers",247 font=dict(size=10), legend=dict(orientation="h", y=1.12), xaxis_tickangle=-40)248 return fig249 250 251def _journal_crosstab_chart(journal_crosstab: dict) -> go.Figure:252 """253 Grouped bar: for each technique, show % usage per journal.254 Journals on x-axis, techniques as bar groups.255 """256 ct = journal_crosstab.get("consolidated", {})257 journals = journal_crosstab.get("journals", [])258 techniques= journal_crosstab.get("techniques", [])259 260 if not journals or not techniques:261 fig = go.Figure()262 fig.update_layout(template="plotly_dark", title="No journal data available",263 paper_bgcolor="#0d1117")264 return fig265 266 COLORS = ["#5b9cf6","#3dba7a","#f5a623","#e04d4d","#a855f7","#06b6d4",267 "#f97316","#84cc16","#ec4899","#14b8a6","#8b5cf6","#ef4444"]268 269 fig = go.Figure()270 for i, tech in enumerate(techniques[:15]): # cap at 15 techniques for readability271 pcts = [ct.get(j,{}).get(tech, 0) for j in journals]272 fig.add_trace(go.Bar(name=tech, x=journals, y=pcts,273 marker_color=COLORS[i % len(COLORS)]))274 275 fig.update_layout(barmode="group", template="plotly_dark", height=500,276 paper_bgcolor="#0d1117", plot_bgcolor="#161b22",277 title="Computational Technique Usage — Cross-Tabulation by Journal (%)",278 xaxis_title="Journal", yaxis_title="% of papers using technique",279 font=dict(size=10), legend=dict(orientation="h", y=1.15), xaxis_tickangle=-20)280 return fig281 282 283def _journal_crosstab_df(journal_crosstab: dict) -> pd.DataFrame:284 ct = journal_crosstab.get("consolidated", {})285 journals = journal_crosstab.get("journals", [])286 techniques= journal_crosstab.get("techniques", [])287 paper_counts = journal_crosstab.get("journal_paper_counts", {})288 rows = []289 for j in journals:290 row = {"Journal": j, "N Papers": paper_counts.get(j,0)}291 for t in techniques:292 row[t] = f"{ct.get(j,{}).get(t,0)}%"293 rows.append(row)294 return pd.DataFrame(rows)295 296 297def _tech_opt_df(opt_log: list) -> pd.DataFrame:298 if not opt_log:299 return pd.DataFrame(columns=["Technique","Refined Name","Hallucination",300 "High Variance","Groq %","Mistral %","Gemini %",301 "Suggestion","Split Into","Merge With"])302 return pd.DataFrame([{303 "Technique": r["technique"],304 "Refined Name": r["refined_name"],305 "Hallucination": r["is_hallucination"],306 "High Variance": r["high_variance"],307 "Groq %": r["pct_groq"],308 "Mistral %": r["pct_mistral"],309 "Gemini %": r["pct_gemini"],310 "Suggestion": r["suggestion"],311 "Split Into": r["split_into"],312 "Merge With": r["merge_with"],313 } for r in opt_log])314 315 316def _per_llm_freq_df(journal_crosstab: dict) -> pd.DataFrame:317 """Per-LLM technique frequency across all papers in methodology CSV."""318 per_llm = journal_crosstab.get("per_llm_freq", {})319 techniques = sorted(set(t for d in per_llm.values() for t in d.keys()))320 rows = []321 for t in techniques:322 rows.append({323 "Technique": t,324 "Groq %": per_llm.get("Groq",{}).get(t, 0),325 "Mistral %": per_llm.get("Mistral",{}).get(t, 0),326 "Gemini %": per_llm.get("Gemini",{}).get(t, 0),327 "Variance": round(max(328 per_llm.get("Groq",{}).get(t,0),329 per_llm.get("Mistral",{}).get(t,0),330 per_llm.get("Gemini",{}).get(t,0),331 ) - min(332 per_llm.get("Groq",{}).get(t,0),333 per_llm.get("Mistral",{}).get(t,0),334 per_llm.get("Gemini",{}).get(t,0),335 )),336 })337 return pd.DataFrame(rows).sort_values("Groq %", ascending=False)338 339 340# ── NEW: Cluster Sizes bar chart (what supervisor pointed to) ────────────────341def _cluster_sizes_chart(interps: dict, disc: dict) -> go.Figure:342 """343 Bar chart: Papers per Cluster — coloured by discipline rule status.344 Green = passes both constraints (mass ≤ 25%, size ≥ 5).345 Yellow = exceeds 25% mass cap (dominant cluster warning).346 Red = below min-size of 5 (too small).347 Number label shown on top of each bar, exactly like supervisor's image.348 """349 cluster_sizes = disc.get("cluster_sizes", {})350 n_docs = sum(cluster_sizes.values()) or 1351 max_allowed = int(0.25 * n_docs)352 353 labels, sizes, colors, texts = [], [], [], []354 for cid in sorted(interps.keys()):355 label = interps[cid]["label"]356 size = cluster_sizes.get(cid, interps[cid].get("strong",0) + interps[cid].get("weak",0))357 mass_pct = size / n_docs358 359 color = "#3dba7a" # green — PASS360 if mass_pct > 0.25:361 color = "#f5c518" # yellow — mass violation (like supervisor image)362 elif size < 5:363 color = "#e04d4d" # red — too small364 365 labels.append(label)366 sizes.append(size)367 colors.append(color)368 texts.append(str(size))369 370 fig = go.Figure(go.Bar(371 x=labels, y=sizes,372 marker_color=colors,373 text=texts,374 textposition="outside",375 textfont=dict(size=11, color="#c9d1d9"),376 ))377 fig.add_hline(y=max_allowed, line_dash="dash", line_color="#f5a623",378 annotation_text=f"25% cap ({max_allowed} papers)",379 annotation_font_color="#f5a623")380 fig.update_layout(381 template="plotly_dark", height=520,382 paper_bgcolor="#0d1117", plot_bgcolor="#161b22",383 title="Cluster Sizes (Papers per Cluster) — Green=PASS · Yellow=Mass>25% · Red=Size<5",384 xaxis_title="Cluster", yaxis_title="Number of Papers",385 font=dict(size=10), xaxis_tickangle=-40,386 showlegend=False,387 margin=dict(t=80, b=200),388 )389 return fig390 391 392# ── NEW: Reproducibility panel ────────────────────────────────────────────────393def _reproducibility_df(td: dict, interps: dict) -> pd.DataFrame:394 """395 Shows what the supervisor means by 'run again and again, topic list is same'.396 Pulls the stability ARI (already computed across 3 seeds in tools.py) and397 shows per-cluster persistence as a proxy for how stable each cluster is.398 High persistence = cluster survives across seeds = reproducible.399 Low persistence = cluster may disappear or merge on re-run.400 """401 cluster_persistence = td.get("cluster_persistence", {})402 overall_stability = td["metrics"].get("stability", 0.0)403 rows = []404 for cid in sorted(interps.keys()):405 pers = cluster_persistence.get(cid, 0.0)406 label = interps[cid]["label"]407 size = interps[cid].get("strong",0) + interps[cid].get("weak",0)408 stable_verdict = "✅ Stable" if pers >= 0.7 else \409 "⚠ Borderline" if pers >= 0.4 else \410 "❌ Fragile"411 rows.append({412 "Cluster": cid,413 "Label": label,414 "Cluster Persistence": round(pers, 4),415 "Strong Members": interps[cid].get("strong", 0),416 "Weak Members": interps[cid].get("weak", 0),417 "Total Papers": size,418 "Stability Verdict": stable_verdict,419 "Note": ("Likely same label on re-run" if pers >= 0.7 else420 "Label may shift slightly" if pers >= 0.4 else421 "May merge/split on re-run — consider merging with adjacent cluster"),422 })423 df = pd.DataFrame(rows).sort_values("Cluster Persistence", ascending=False)424 # Prepend overall ARI row425 overall_row = pd.DataFrame([{426 "Cluster": "ALL",427 "Label": f"Overall ARI Stability across 3 seeds = {round(overall_stability,4)}",428 "Cluster Persistence": overall_stability,429 "Strong Members": "—", "Weak Members": "—", "Total Papers": "—",430 "Stability Verdict": "✅ Stable" if overall_stability >= 0.8 else431 "⚠ Borderline" if overall_stability >= 0.5 else "❌ Unstable",432 "Note": "ARI close to 1.0 → running the pipeline again will produce the same clusters",433 }])434 return pd.concat([overall_row, df], ignore_index=True)435 436 437def _reproducibility_chart(td: dict, interps: dict) -> go.Figure:438 """Horizontal bar of cluster persistence — shows which clusters are stable."""439 cluster_persistence = td.get("cluster_persistence", {})440 labels, persis, colors = [], [], []441 for cid in sorted(interps.keys(), key=lambda c: cluster_persistence.get(c,0)):442 p = cluster_persistence.get(cid, 0.0)443 labels.append(interps[cid]["label"][:35])444 persis.append(round(p, 4))445 colors.append("#3dba7a" if p >= 0.7 else "#f5a623" if p >= 0.4 else "#e04d4d")446 447 fig = go.Figure(go.Bar(448 x=persis, y=labels, orientation="h",449 marker_color=colors,450 text=[str(v) for v in persis],451 textposition="outside",452 ))453 fig.add_vline(x=0.7, line_dash="dot", line_color="#3dba7a",454 annotation_text="Stable threshold (0.7)")455 fig.add_vline(x=0.4, line_dash="dot", line_color="#f5a623",456 annotation_text="Borderline (0.4)")457 fig.update_layout(458 template="plotly_dark", height=max(400, len(labels)*28),459 paper_bgcolor="#0d1117", plot_bgcolor="#161b22",460 title="Cluster Persistence — Proxy for Reproducibility\n"461 "Green ≥ 0.7 (stable) · Orange 0.4–0.7 (borderline) · Red < 0.4 (fragile)",462 xaxis_title="Persistence Score", yaxis_title="",463 font=dict(size=10), margin=dict(l=260),464 )465 return fig466 467 468# ── NEW: Human interpretability check ────────────────────────────────────────469def _interpretability_df(interps: dict) -> pd.DataFrame:470 """471 Flags what supervisor called 'human interpretable topic list'.472 Checks two things:473 1. Label overlap — pairs of cluster labels that share ≥2 significant words474 (e.g. 'Cybersecurity and Privacy' vs 'Cyber-Risk Management and Online Security').475 2. Vagueness — labels containing generic terms like 'systems', 'digital', 'data'476 as the ONLY meaningful content.477 Output is a table the supervisor can review to confirm distinctiveness.478 """479 import itertools480 NOISE = {"the","and","for","with","using","based","from","that","are","this",481 "in","of","a","to","an","on","at","by","or","as","is","its","via",482 "systems","digital","information","management","based","driven"}483 VAGUE_SINGLES = {"systems","digital","data","information","analysis","research",484 "study","approach","framework","model","methods","technology"}485 486 def _sig_words(label: str) -> set:487 words = set(re.findall(r"\b[a-z]{4,}\b", label.lower()))488 return words - NOISE489 490 rows = []491 cids = sorted(interps.keys())492 labels_map = {cid: interps[cid]["label"] for cid in cids}493 494 # Check every pair495 seen_pairs = set()496 for cid_a, cid_b in itertools.combinations(cids, 2):497 la, lb = labels_map[cid_a], labels_map[cid_b]498 wa, wb = _sig_words(la), _sig_words(lb)499 overlap = wa & wb500 if len(overlap) >= 2:501 pair_key = tuple(sorted([cid_a, cid_b]))502 if pair_key not in seen_pairs:503 seen_pairs.add(pair_key)504 rows.append({505 "Issue": "⚠ Label Overlap",506 "Cluster A": cid_a,507 "Label A": la,508 "Cluster B": cid_b,509 "Label B": lb,510 "Shared Words": ", ".join(sorted(overlap)),511 "Severity": "HIGH — consider merging" if len(overlap) >= 3512 else "MEDIUM — review distinctiveness",513 "Action": "Check if these two clusters cover the same research theme. "514 "If yes, increase min_cluster_size to force a merge.",515 })516 517 # Check each label for vagueness518 for cid in cids:519 label = labels_map[cid]520 sig = _sig_words(label)521 vague = sig & VAGUE_SINGLES522 specific = sig - VAGUE_SINGLES523 if len(specific) == 0:524 rows.append({525 "Issue": "❌ Too Vague",526 "Cluster A": cid,527 "Label A": label,528 "Cluster B": "—",529 "Label B": "—",530 "Shared Words": ", ".join(vague),531 "Severity": "HIGH — label is not human interpretable",532 "Action": "Run optimization pass to refine the label, "533 "or manually inspect keyphrases for more specific terms.",534 })535 536 if not rows:537 rows.append({538 "Issue": "✅ All Clear",539 "Cluster A": "—", "Label A": "All labels are distinct and specific",540 "Cluster B": "—", "Label B": "—",541 "Shared Words": "—", "Severity": "NONE",542 "Action": "Topic list is human interpretable and non-overlapping.",543 })544 545 return pd.DataFrame(rows)546 547 548# ── Pipeline runner ──────────────────────────────────────────────────────────549def _run(corpus_file, method_file, gk, mk, gek, n_trials, n_optimize,550 progress=gr.Progress(track_tqdm=True)):551 if not corpus_file: raise gr.Error("Upload a Scopus corpus CSV first.")552 gk = gk.strip() or os.getenv("GROQ_API_KEY","")553 mk = mk.strip() or os.getenv("MISTRAL_API_KEY","")554 gek = gek.strip() or os.getenv("GEMINI_API_KEY","")555 if not all([gk,mk,gek]): raise gr.Error("All 3 API keys required.")556 557 method_path = method_file.name if method_file else None558 559 progress(0.05, desc="📥 Loading CSV…")560 progress(0.10, desc="🔬 Embedding corpus with SPECTER-2…")561 r = run_pipeline(corpus_file.name, gk, mk, gek,562 int(n_trials), int(n_optimize), method_path)563 if r.get("error"): raise gr.Error(r["error"])564 progress(0.85, desc="📊 Building outputs…")565 566 td, interps = r["topic_data"], r.get("interpretations",{})567 disc, met = td["discipline"], td["metrics"]568 ar = r.get("agreement_rates",{})569 rl = r.get("refinement_log", [])570 571 def _s(ok): return "✅ PASS" if ok else "❌ FAIL"572 summary = (573 f"## Pipeline Complete — {disc['n_clusters']} clusters discovered\n\n"574 f"| Criterion | Value | Status |\n|---|---|---|\n"575 f"| Max cluster mass | {round(disc['max_mass_pct']*100,1)}% | {_s(disc['max_mass_ok'])} |\n"576 f"| Min cluster size | {disc['min_size']} | {_s(disc['min_size_ok'])} |\n"577 f"| Persistence (mean) | {round(met['persistence'],4)} | — |\n"578 f"| DBCV | {round(met['dbcv'],4)} | — |\n"579 f"| Stability (3 seeds) | {round(met['stability'],4)} | — |\n\n"580 f"**Trials:** {td['n_trials_run']} (best #{td['best_trial']}) · "581 f"**Agreement:** Triple {ar.get('triple',0)}% · Two+ {ar.get('two_or_more',0)}% · "582 f"**Optimization passes:** {n_optimize} · **Labels refined:** {len(rl)}"583 )584 585 # UMAP scatter586 u2d = np.array(td["umap_2d"])587 sdf = pd.DataFrame({"UMAP-1":u2d[:,0],"UMAP-2":u2d[:,1],588 "Cluster":[str(l) for l in td["labels"]],589 "Doc":[d[:60] for d in td["documents"]]})590 fig = px.scatter(sdf, x="UMAP-1", y="UMAP-2", color="Cluster",591 hover_data=["Doc"], opacity=0.75,592 title="2-D UMAP visualisation of SPECTER-2 embeddings")593 fig.update_layout(template="plotly_dark", height=500,594 paper_bgcolor="#0d1117", plot_bgcolor="#161b22", font=dict(size=11))595 596 # Trial log + Pareto597 tl = pd.DataFrame(td["trial_log"])598 tl_cols = [c for c in ["trial","discipline_pass","n_clusters","persistence",599 "dbcv","max_mass_pct","min_size","n_noise"] if c in tl.columns]600 tl_show = tl[tl_cols] if not tl.empty else pd.DataFrame()601 602 pfig = go.Figure()603 if not tl.empty:604 for passed, color, name in [(True,"#3dba7a","PASS"),(False,"#e04d4d","FAIL")]:605 sub = tl[tl["discipline_pass"]==passed]606 if not sub.empty:607 pfig.add_trace(go.Scatter(x=sub["max_mass_pct"],y=sub["persistence"],608 mode="markers",marker=dict(size=8,color=color),name=name,609 text=sub["trial"],hovertemplate="Trial %{text}<br>Mass: %{x:.0%}<br>Pers: %{y:.3f}"))610 pfig.add_vline(x=0.25,line_dash="dash",line_color="#5a6480",annotation_text="25% rule")611 pfig.update_layout(template="plotly_dark",height=400,612 paper_bgcolor="#0d1117",plot_bgcolor="#161b22",613 title="Pareto front — Persistence vs Max cluster mass",614 xaxis_title="Max cluster mass",yaxis_title="Persistence",font=dict(size=11))615 616 cdf_rows = []617 for cid in sorted(interps.keys()):618 v = interps[cid]619 cdf_rows.append({"Cluster":cid,"Label":v["label"],"Agreement":v["agreement"],620 "Strong":v["strong"],"Weak":v["weak"],621 "Persistence":round(v.get("persistence",0),4),622 "Keyphrases":", ".join(v.get("keyphrases",[]))})623 cdf = pd.DataFrame(cdf_rows)624 625 sheets = r.get("sheets",{})626 s1 = pd.DataFrame(sheets.get(1,[])); s2 = pd.DataFrame(sheets.get(2,[]))627 s3 = pd.DataFrame(sheets.get(3,[])); s4 = pd.DataFrame(sheets.get(4,[]))628 sp = r.get("sheet_paths",{})629 mdf = pd.DataFrame(r.get("mismatch_table",[]))630 631 md_data = r.get("methodology_data",{})632 top_papers_df = _top_papers_df(r.get("top_papers",{}))633 method_sum_df = _methodology_summary_df(md_data, interps)634 method_chart = _methodology_bar_chart(md_data, interps)635 extraction_df = _extraction_pipeline_df(md_data, interps)636 per_llm_meth_df = _per_llm_methodology_df(md_data, interps)637 regex_hits_df = _regex_hits_df(md_data, interps)638 pattern_info = _regex_pattern_info()639 refine_df = _refinement_df(rl)640 641 # ── NEW: methodology-CSV outputs ─────────────────────────────────────────642 comp_sheets = r.get("comp_technique_sheets", {1:[], 2:[], 3:[], 4:[]})643 jct = r.get("journal_crosstab", {})644 tech_opt_log = r.get("technique_opt_log", [])645 646 tech_s1 = _tech_sheet_df(comp_sheets.get(1,[]))647 tech_s2 = _tech_sheet_df(comp_sheets.get(2,[]))648 tech_s3 = _tech_sheet_df(comp_sheets.get(3,[]))649 tech_s4 = _tech_sheet_df(comp_sheets.get(4,[]))650 651 tech_llm_chart = _tech_llm_pct_chart(comp_sheets)652 jct_chart = _journal_crosstab_chart(jct)653 jct_df = _journal_crosstab_df(jct)654 per_llm_freq_df = _per_llm_freq_df(jct)655 tech_opt_df = _tech_opt_df(tech_opt_log)656 657 # ── NEW: cluster sizes, reproducibility, interpretability ─────────────────658 cluster_sizes_fig = _cluster_sizes_chart(interps, disc)659 repro_chart = _reproducibility_chart(td, interps)660 repro_df = _reproducibility_df(td, interps)661 interpretability_df = _interpretability_df(interps)662 663 progress(1.0, desc="✅ Done!")664 dl_files = [f for f in [sp.get(1),sp.get(2),sp.get(3),sp.get(4),r.get("json_path")] if f]665 666 return (667 # ── original outputs (order preserved) ───────────────────────────────668 summary, fig, pfig, tl_show, cdf,669 top_papers_df,670 method_chart, method_sum_df, extraction_df, per_llm_meth_df,671 regex_hits_df, pattern_info,672 refine_df,673 s1, s2, s3, s4,674 dl_files if dl_files else None,675 mdf,676 # ── new outputs ───────────────────────────────────────────────────────677 tech_llm_chart,678 tech_s1, tech_s2, tech_s3, tech_s4,679 per_llm_freq_df,680 jct_chart,681 jct_df,682 tech_opt_df,683 # ── supervisor additions ──────────────────────────────────────────────684 cluster_sizes_fig,685 repro_chart,686 repro_df,687 interpretability_df,688 )689 690 691# ── UI ────────────────────────────────────────────────────────────────────────692css = ".gradio-container{background:#0d1117!important;color:#c9d1d9!important}" \693 "footer{display:none!important}"694 695with gr.Blocks(theme=gr.themes.Base(primary_hue="blue", neutral_hue="slate"),696 css=css, title="SPECTER-2 Topic Analyzer") as demo:697 gr.Markdown("# 📐 SPECTER-2 Topic Analyzer")698 699 with gr.Row():700 # ── Left sidebar ─────────────────────────────────────────────────────701 with gr.Column(scale=1):702 gr.Markdown("### 📄 Corpus CSV")703 file_in = gr.File(label="Upload Scopus CSV (title + abstract)",704 file_types=[".csv"])705 preview_out = gr.Markdown("Upload a CSV to see stats.")706 707 gr.Markdown("### 🔬 Methodology CSV *(optional)*")708 method_file_in = gr.File(label="Upload Methodology CSV (title, doi, methodology)",709 file_types=[".csv"])710 method_preview = gr.Markdown("Upload methodology CSV to enable technique analysis.")711 712 gr.Markdown("### 🔑 API Keys")713 groq_in = gr.Textbox(label="Groq API Key", type="password",714 placeholder="or set GROQ_API_KEY env var")715 mistral_in = gr.Textbox(label="Mistral API Key", type="password",716 placeholder="or set MISTRAL_API_KEY env var")717 gemini_in = gr.Textbox(label="Gemini API Key", type="password",718 placeholder="or set GEMINI_API_KEY env var")719 720 gr.Markdown("### ⚙ Parameters")721 trials_in = gr.Slider(10, 100, 50, step=5, label="Optuna Trials")722 optimize_in = gr.Slider(1, 5, 1, step=1,723 label="🔁 Optimization Passes",724 info="Pass 1 = no refinement. 2–5 = LLM critic audits topic labels "725 "AND technique labels for hallucinations + improvements.")726 run_btn = gr.Button("▶ Run Full Pipeline", variant="primary", size="lg")727 728 # ── Main panel ────────────────────────────────────────────────────────729 with gr.Column(scale=3):730 with gr.Tabs():731 732 # ── original tabs (order / content unchanged) ─────────────────733 with gr.Tab("Summary"):734 summary_out = gr.Markdown()735 736 with gr.Tab("2-D UMAP"):737 scatter_out = gr.Plot()738 739 with gr.Tab("Pareto Front"):740 pareto_out = gr.Plot()741 742 with gr.Tab("Trial Log"):743 trial_out = gr.Dataframe()744 745 with gr.Tab("Clusters"):746 cluster_out = gr.Dataframe()747 748 with gr.Tab("🗞 Top 3 Papers"):749 gr.Markdown("### Top 3 Representative Papers per Cluster\n"750 "Ranked by cosine similarity to cluster centroid "751 "in SPECTER-2 embedding space.")752 top_papers_out = gr.Dataframe(753 headers=["Cluster","Label","Rank","Title","Abstract Snippet"],754 wrap=True)755 756 with gr.Tab("🔬 Cluster Methodology"):757 gr.Markdown("### Cluster-Level Methodology — 3-LLM Council\n"758 "Derived from representative abstracts per cluster. "759 "≥2-LLM gate applied.")760 method_chart_out = gr.Plot()761 method_summary_out = gr.Dataframe(wrap=True)762 763 with gr.Tab("⚙ Cluster Extraction Pipeline"):764 gr.Markdown("### Full Regex + LLM Extraction Trace (per cluster)")765 extraction_out = gr.Dataframe(wrap=True)766 767 with gr.Tab("🤖 Cluster Per-LLM Votes"):768 gr.Markdown("### Raw Per-LLM Methodology Votes (per cluster)")769 per_llm_out = gr.Dataframe(wrap=True)770 771 with gr.Tab("🔍 Cluster Regex Hits"):772 gr.Markdown("### Regex Pattern Matches (per cluster)\n"773 "Every match with exact character span and paper number.")774 regex_hits_out = gr.Dataframe(wrap=True)775 regex_info_out = gr.Markdown()776 777 with gr.Tab("🔁 Refinement Log"):778 gr.Markdown("### Topic Label Optimization Log\n"779 "Changes made by LLM critic per optimization pass.")780 refine_out = gr.Dataframe(wrap=True)781 782 with gr.Tab("Sheet 1 — Groq"): s1_out = gr.Dataframe()783 with gr.Tab("Sheet 2 — Mistral"): s2_out = gr.Dataframe()784 with gr.Tab("Sheet 3 — Gemini"): s3_out = gr.Dataframe()785 with gr.Tab("Sheet 4 — Consolidated"): s4_out = gr.Dataframe()786 with gr.Tab("RQ Mismatch"): mismatch_out = gr.Dataframe()787 with gr.Tab("Downloads"):788 dl_out = gr.File(label="All sheet CSVs + topics.json",789 file_count="multiple")790 791 # ── NEW tabs: methodology CSV pipeline ────────────────────────792 with gr.Tab("💻 Comp. Techniques — LLM % Chart"):793 gr.Markdown("### Computational Technique Frequency — Methodology CSV\n"794 "For each technique, shows the % of papers it was extracted "795 "from by each of the 3 LLMs independently + the consolidated "796 "result (≥2-LLM gate). Bars grouped by technique.")797 tech_llm_chart_out = gr.Plot()798 799 with gr.Tab("💻 Tech Sheet 1 — Groq"):800 gr.Markdown("### Groq raw technique extraction — one row per paper")801 tech_s1_out = gr.Dataframe(wrap=True)802 803 with gr.Tab("💻 Tech Sheet 2 — Mistral"):804 gr.Markdown("### Mistral raw technique extraction — one row per paper")805 tech_s2_out = gr.Dataframe(wrap=True)806 807 with gr.Tab("💻 Tech Sheet 3 — Gemini"):808 gr.Markdown("### Gemini raw technique extraction — one row per paper")809 tech_s3_out = gr.Dataframe(wrap=True)810 811 with gr.Tab("💻 Tech Sheet 4 — Consolidated"):812 gr.Markdown("### Consolidated techniques — ≥2-LLM agreement, one row per paper")813 tech_s4_out = gr.Dataframe(wrap=True)814 815 with gr.Tab("📊 Tech Frequency by LLM"):816 gr.Markdown("### Per-LLM Technique Frequency Table\n"817 "% of all papers where each LLM extracted each technique. "818 "High variance = LLMs disagree → optimization flag.")819 per_llm_freq_out = gr.Dataframe(wrap=True)820 821 with gr.Tab("🗂 Journal Cross-Tabulation"):822 gr.Markdown("### Technique × Journal Cross-Tabulation\n"823 "Rows = journals auto-detected from DOI/title. "824 "Columns = consolidated techniques. "825 "Values = % of papers in that journal using the technique.\n\n"826 "**Journals detected:** MISQ, JAIS, ISR, JMIS, PAJAIS, "827 "ECIS, ICIS, Other.")828 jct_chart_out = gr.Plot()829 jct_df_out = gr.Dataframe(wrap=True)830 831 with gr.Tab("🔧 Technique Optimization"):832 gr.Markdown("### Technique Label Improvement Suggestions\n"833 "Groq critic flags: hallucination, high inter-LLM variance "834 "(>15% gap), split/merge recommendations.\n"835 "Only runs when Optimization Passes ≥ 2.")836 tech_opt_out = gr.Dataframe(wrap=True)837 838 # ── Supervisor-requested additions ────────────────────────────839 with gr.Tab("📊 Cluster Sizes"):840 gr.Markdown(841 "### Cluster Sizes (Papers per Cluster)\n"842 "Exact chart your supervisor highlighted. "843 "**Green** = passes both discipline rules (mass ≤ 25%, size ≥ 5). "844 "**Yellow** = cluster exceeds 25% mass cap — dominant cluster warning. "845 "**Red** = cluster has fewer than 5 papers — too small.\n\n"846 "The orange dashed line marks the 25% cap. Any bar above it "847 "will fail the discipline check and the pipeline will re-optimise."848 )849 cluster_sizes_out = gr.Plot()850 851 with gr.Tab("🔄 Reproducibility"):852 gr.Markdown(853 "### Reproducibility — 'Run Again and Again, Topic List is the Same'\n\n"854 "Your supervisor wants proof that running the pipeline multiple times "855 "produces the **same clusters**. This tab shows two measures:\n\n"856 "**Overall ARI Stability** (top row) — Adjusted Rand Index averaged "857 "across 3 random seeds. ARI = 1.0 means identical clusters every run. "858 "ARI ≥ 0.8 is considered stable for publication.\n\n"859 "**Cluster Persistence** (per row) — how strongly each cluster's "860 "structure is preserved in the condensed HDBSCAN tree. "861 "High persistence → cluster survives parameter variation → "862 "same label will appear on re-run. "863 "Low persistence → cluster may split or merge → label may change.\n\n"864 "🟢 ≥ 0.7 Stable · 🟡 0.4–0.7 Borderline · 🔴 < 0.4 Fragile"865 )866 repro_chart_out = gr.Plot()867 repro_df_out = gr.Dataframe(wrap=True)868 869 with gr.Tab("🧠 Interpretability Check"):870 gr.Markdown(871 "### Human Interpretability Check — 'Topic List Must Be Distinct'\n\n"872 "Your supervisor flagged that labels like "873 "*'Cybersecurity and Privacy'* and *'Cyber-Risk Management and Online Security'* "874 "look like the same topic. This tab automatically detects:\n\n"875 "**⚠ Label Overlap** — pairs of cluster labels sharing ≥ 2 significant "876 "words (noise words like 'and', 'for', 'in' excluded). "877 "Overlapping labels suggest the two clusters may cover the same theme "878 "and should be reviewed for merging.\n\n"879 "**❌ Too Vague** — labels where all meaningful words are generic "880 "('systems', 'digital', 'data') with no domain-specific content. "881 "These need the optimization pass to refine them.\n\n"882 "**Action column** tells you exactly what to do for each flag."883 )884 interpretability_out = gr.Dataframe(wrap=True)885 886 # ── Wire callbacks ────────────────────────────────────────────────────────887 file_in.change(_preview, inputs=[file_in], outputs=[preview_out])888 method_file_in.change(_preview_methodology, inputs=[method_file_in], outputs=[method_preview])889 890 run_btn.click(891 _run,892 inputs=[file_in, method_file_in, groq_in, mistral_in, gemini_in,893 trials_in, optimize_in],894 outputs=[895 # original896 summary_out, scatter_out, pareto_out, trial_out, cluster_out,897 top_papers_out,898 method_chart_out, method_summary_out, extraction_out, per_llm_out,899 regex_hits_out, regex_info_out,900 refine_out,901 s1_out, s2_out, s3_out, s4_out,902 dl_out, mismatch_out,903 # new904 tech_llm_chart_out,905 tech_s1_out, tech_s2_out, tech_s3_out, tech_s4_out,906 per_llm_freq_out,907 jct_chart_out,908 jct_df_out,909 tech_opt_out,910 # supervisor additions911 cluster_sizes_out,912 repro_chart_out,913 repro_df_out,914 interpretability_out,915 ],916 )917 918if __name__ == "__main__":919 demo.launch(server_name="0.0.0.0", server_port=7860)