sampstad/narrative_detector
0
1"""Narrative Extractor — Gradio front-end for HuggingFace Spaces (ZeroGPU)."""2 3import html4import json5import logging6import os7import urllib.parse8import urllib.request9 10import gradio as gr11import spaces12import spacy13from sentence_transformers import SentenceTransformer14from transformers import pipeline as hf_pipeline15 16from pipeline import extract_narrative17 18logging.getLogger("fastcoref").setLevel(logging.ERROR)19logging.getLogger("transformers").setLevel(logging.ERROR)20 21# ── Model cache (populated on first GPU call) ─────────────────────────────────22 23MODELS: dict = {}24 25 26def _ensure_models() -> None:27 """Load all models onto GPU if not already loaded. Called inside @spaces.GPU."""28 if MODELS:29 return30 31 nlp = spacy.load("en_core_web_lg")32 spacy.prefer_gpu()33 34 try:35 from fastcoref import FCoref36 coref = FCoref(device="cuda")37 except Exception as exc:38 print(f"FCoref unavailable ({exc}) — coreference disabled.")39 coref = None40 41 sentiment = hf_pipeline(42 "text-classification",43 model="distilbert-base-uncased-finetuned-sst-2-english",44 device=0,45 )46 47 embedder = SentenceTransformer("all-mpnet-base-v2", device="cuda")48 49 nli = hf_pipeline(50 "text-classification",51 model="cross-encoder/nli-deberta-v3-base",52 device=0,53 top_k=None,54 )55 56 summariser = hf_pipeline(57 "summarization",58 model="sshleifer/distilbart-cnn-12-6",59 device=0,60 )61 62 MODELS.update({63 "nlp": nlp, "coref": coref, "sentiment": sentiment,64 "embedder": embedder, "nli": nli, "summariser": summariser,65 })66 67# ── Wikidata lookup ─────────────────────────────────────────────────────────68 69_WD_CACHE: dict[str, dict] = {}70 71 72def _wikidata_lookup(name: str) -> dict:73 """Return {description, lat, lon} for *name* from Wikidata. Cached."""74 if name in _WD_CACHE:75 return _WD_CACHE[name]76 out: dict = {"description": "", "lat": None, "lon": None}77 try:78 # Step 1: search — fetch multiple candidates so we can pick the best one79 params = urllib.parse.urlencode({80 "action": "wbsearchentities",81 "search": name,82 "language": "en",83 "limit": 5,84 "format": "json",85 })86 req = urllib.request.Request(87 f"https://www.wikidata.org/w/api.php?{params}",88 headers={"User-Agent": "NarrativeDetector/1.0"},89 )90 with urllib.request.urlopen(req, timeout=4) as resp:91 hits = json.loads(resp.read().decode()).get("search", [])92 if not hits:93 _WD_CACHE[name] = out94 return out95 96 # Disambiguation: for all-caps / short abbreviations, skip hits whose97 # description suggests a linguistic / code / letter entry.98 _LINGUISTIC_SKIP = (99 "language", "letter", "characters", "alphabet", "phoneme",100 "code", "symbol", "abbreviation used", "romanization",101 )102 is_abbrev = name.isupper() and len(name) <= 6103 best = hits[0]104 if is_abbrev:105 for h in hits:106 desc_lower = h.get("description", "").lower()107 if not any(skip in desc_lower for skip in _LINGUISTIC_SKIP):108 best = h109 break110 111 qid = best.get("id", "")112 out["description"] = best.get("description", "")113 114 # Step 2: fetch P625 (coordinate location) via wbgetentities115 if qid:116 params2 = urllib.parse.urlencode({117 "action": "wbgetentities",118 "ids": qid,119 "props": "claims",120 "format": "json",121 })122 req2 = urllib.request.Request(123 f"https://www.wikidata.org/w/api.php?{params2}",124 headers={"User-Agent": "NarrativeDetector/1.0"},125 )126 with urllib.request.urlopen(req2, timeout=4) as resp2:127 claims = (128 json.loads(resp2.read().decode())129 .get("entities", {}).get(qid, {}).get("claims", {})130 )131 p625 = claims.get("P625", [])132 if p625:133 coords = p625[0]["mainsnak"]["datavalue"]["value"]134 out["lat"] = coords.get("latitude")135 out["lon"] = coords.get("longitude")136 except Exception:137 pass138 _WD_CACHE[name] = out139 return out140 141 142def _wikidata_description(name: str) -> str:143 return _wikidata_lookup(name)["description"]144 145 146# ── Formatting helpers ────────────────────────────────────────────────────────147 148_SENTIMENT_COLORS = {149 "POSITIVE": ("#d4edda", "#155724", "🟢"),150 "NEGATIVE": ("#f8d7da", "#721c24", "🔴"),151 "NEUTRAL": ("#e2e3e5", "#383d41", "⚪"),152}153 154 155def _fmt_sentiment(s: dict) -> str:156 bg, fg, icon = _SENTIMENT_COLORS[s["label"]]157 return (158 f'<span style="background:{bg};color:{fg};padding:5px 14px;'159 f'border-radius:20px;font-size:0.9rem;font-weight:700;letter-spacing:0.02em;">'160 f'{icon} {s["label"]} · {s["score"]:.0%}</span>'161 f'<p style="margin:8px 0 0;font-size:0.75em;color:#6b7280">'162 f'Classified by <strong style="color:#374151">DistilBERT-SST-2</strong> '163 f'(binary positive/negative fine-tune on SST-2; '164 f'neutral = low-confidence prediction)</p>'165 )166 167 168def _entity_table(label: str, icon: str, names: list) -> str:169 if not names:170 return ""171 rows = ""172 for name in names:173 desc = _wikidata_description(name)174 desc_cell = f'<span style="color:#6b7280;font-size:0.88em">{html.escape(desc)}</span>' if desc else '<span style="color:#d1d5db;font-size:0.85em"><em>No Wikidata entry found</em></span>'175 rows += (176 f'<tr style="border-bottom:1px solid #f3f4f6"><td style="padding:5px 10px 5px 4px;font-weight:600;color:#111827">'177 f'{html.escape(name)}</td>'178 f'<td style="padding:5px 4px 5px 10px;color:#374151">{desc_cell}</td></tr>'179 )180 return (181 f'<p style="margin:6px 0 4px;font-size:0.72rem;font-weight:700;text-transform:uppercase;letter-spacing:0.07em;color:#6b7280">{icon} {label}</p>'182 f'<table style="border-collapse:collapse;width:100%;font-size:0.88rem">'183 f'<thead><tr style="background:#f9fafb">'184 f'<th style="padding:5px 10px 5px 4px;text-align:left;border-bottom:1px solid #e5e7eb;color:#6b7280;font-size:0.72rem;text-transform:uppercase;letter-spacing:0.06em">Name</th>'185 f'<th style="padding:5px 4px 5px 10px;text-align:left;border-bottom:1px solid #e5e7eb;color:#6b7280;font-size:0.72rem;text-transform:uppercase;letter-spacing:0.06em">'186 f'Description <span style="font-weight:400;font-size:0.9em;text-transform:none;color:#9ca3af">(via Wikidata)</span></th>'187 f'</tr></thead>'188 f'<tbody>{rows}</tbody></table>'189 )190 191 192def _fmt_who(ents: dict) -> str:193 parts = [194 _entity_table("People", "👤", ents.get("people", [])),195 _entity_table("Organisations", "🏢", ents.get("organisations", [])),196 ]197 result = "".join(p for p in parts if p)198 return result if result else '<p style="color:#1a1a1a"><em>None identified.</em></p>'199 200 201def _fmt_where_map(ents: dict) -> str:202 places = ents.get("places", [])203 if not places:204 return '<p style="color:#1a1a1a"><em>None identified.</em></p>'205 206 try:207 import folium208 _folium_ok = True209 except ImportError:210 _folium_ok = False211 212 if not _folium_ok:213 return "<p>📍 " + " · ".join(html.escape(p) for p in places) + "</p>"214 215 located, unlocated = [], []216 for p in places:217 info = _wikidata_lookup(p)218 if info["lat"] is not None:219 located.append(220 {"name": p, "lat": info["lat"], "lon": info["lon"], "desc": info["description"]})221 else:222 unlocated.append(p)223 224 parts = []225 if located:226 lats = [p["lat"] for p in located]227 lons = [p["lon"] for p in located]228 center = [sum(lats) / len(lats), sum(lons) / len(lons)]229 m = folium.Map(location=center, zoom_start=4, tiles="CartoDB positron")230 for p in located:231 popup_html = f'<b>{html.escape(p["name"])}</b>'232 if p["desc"]:233 popup_html += f'<br><span style="font-size:0.85em">{html.escape(p["desc"])}</span>'234 folium.Marker(235 location=[p["lat"], p["lon"]],236 popup=folium.Popup(popup_html, max_width=220),237 tooltip=p["name"],238 ).add_to(m)239 if len(located) > 1:240 m.fit_bounds([[min(lats), min(lons)], [max(lats), max(lons)]])241 parts.append(m._repr_html_())242 243 if unlocated:244 parts.append(245 "<p style='margin-top:6px;color:#1a1a1a'>📍 Also mentioned (no coordinates found): "246 + " · ".join(html.escape(p) for p in unlocated)247 + "</p>"248 )249 elif not located:250 parts.append("<p style='color:#1a1a1a'>📍 " + " · ".join(html.escape(p)251 for p in places) + "</p>")252 253 return "".join(parts)254 255 256def _fmt_temporal_html(items: list) -> str:257 if not items:258 return '<p style="color:#1a1a1a"><em>None found.</em></p>'259 tags = ""260 for t in items:261 if t["normalized"]:262 norm = f' <span style="font-size:0.78em;color:#555;font-weight:400">({html.escape(t["normalized"])})</span>'263 else:264 norm = ""265 tags += (266 f'<span style="display:inline-block;margin:3px 4px;padding:4px 12px;'267 f'background:#f0f4ff;color:#1e3a8a;border:1px solid #c7d0f8;'268 f'border-radius:20px;font-size:0.82em;font-weight:600">'269 f'{html.escape(t["text"])}{norm}</span>'270 )271 return f'<div style="margin:4px 0">{tags}</div>'272 273 274def _fmt_causal(items: list) -> str:275 if not items:276 return '<p style="color:#1a1a1a;font-size:0.9em"><em>None found.</em></p>'277 parts = []278 for c in items:279 pct = int(c["confidence"] * 100)280 if c.get("connective"):281 meta = f'via <em>{c["connective"]}</em>'282 else:283 meta = "NLI entailment"284 bar_w = min(pct, 100)285 parts.append(286 f'<div style="border:1px solid #e5e7eb;border-left:3px solid #4f6ef7;'287 f'padding:10px 14px;margin:8px 0;border-radius:0 8px 8px 0;background:#fafafa">'288 f'<div style="display:flex;align-items:center;gap:8px;margin-bottom:6px">'289 f'<span style="font-size:0.72rem;color:#6b7280;font-weight:600;text-transform:uppercase;letter-spacing:0.05em">{meta}</span>'290 f'<div style="flex:1;height:4px;background:#e5e7eb;border-radius:2px">'291 f'<div style="width:{bar_w}%;height:4px;background:#4f6ef7;border-radius:2px"></div></div>'292 f'<span style="font-size:0.72rem;font-weight:700;color:#4f6ef7">{pct}%</span></div>'293 f'<span style="color:#111827;font-size:0.9rem"><strong>{html.escape(c["cause"])}</strong>'294 f'<span style="color:#4f6ef7;margin:0 8px">→</span>'295 f'{html.escape(c["effect"])}</span>'296 f'</div>'297 )298 return "".join(parts)299 300 301def _fmt_svo_table(actions: list) -> str:302 if not actions:303 return '<p style="color:#1a1a1a"><em>No key actions identified.</em></p>'304 rows = "".join(305 f"<tr style='border-bottom:1px solid #f3f4f6'>"306 f"<td style='padding:7px 10px;font-weight:600;color:#111827;font-size:0.9rem'>{html.escape(a['subject'])}</td>"307 f"<td style='padding:7px 10px;color:#6b7280;font-style:italic;font-size:0.9rem'>{html.escape(a['verb'])}</td>"308 f"<td style='padding:7px 10px;color:#111827;font-size:0.9rem'>{html.escape(a['object'])}</td>"309 f"</tr>"310 for a in actions311 )312 return (313 f'<table style="border-collapse:collapse;width:100%;margin:8px 0;font-size:0.9rem">'314 f'<thead><tr style="background:#f9fafb">'315 f'<th style="padding:7px 10px;text-align:left;border-bottom:1px solid #e5e7eb;color:#6b7280;font-size:0.72rem;text-transform:uppercase;letter-spacing:0.06em">Subject</th>'316 f'<th style="padding:7px 10px;text-align:left;border-bottom:1px solid #e5e7eb;color:#6b7280;font-size:0.72rem;text-transform:uppercase;letter-spacing:0.06em">Verb</th>'317 f'<th style="padding:7px 10px;text-align:left;border-bottom:1px solid #e5e7eb;color:#6b7280;font-size:0.72rem;text-transform:uppercase;letter-spacing:0.06em">Object</th>'318 f'</tr></thead>'319 f'<tbody>{rows}</tbody></table>'320 )321 322 323def _tab_label(thread: dict, idx: int) -> str:324 """Short tab label: truncated summary, or lede sentence, or Thread N."""325 summary = thread.get("summary", "")326 if summary and len(summary.split()) > 3:327 return summary[:55].rsplit(" ", 1)[0] + "…" if len(summary) > 55 else summary328 lede = thread.get("lede", "")329 if lede and len(lede.split()) > 3:330 short = lede[:55].rsplit(" ", 1)[0] + "…" if len(lede) > 55 else lede331 return short332 return f"Thread {idx + 1}"333 334 335def _fmt_thread_body(thread: dict) -> str:336 """Render the 5W content sections for one thread (used by tabs and single-thread view)."""337 return _fmt_thread_card(thread, multi=False)338 339 340def _fmt_thread_card(thread: dict, multi: bool) -> str:341 tid = thread["id"] + 1342 s = thread["sentiment"]343 icon = {"POSITIVE": "🟢", "NEGATIVE": "🔴", "NEUTRAL": "⚪"}[s["label"]]344 n = len(thread["sentences"])345 plural = "s" if n != 1 else ""346 title_prefix = f"Thread {tid} — " if multi else ""347 348 sections = [349 ("🔗 Summary", f'<p style="margin:4px 0;color:#1a1a1a">{html.escape(thread["summary"])}</p>' if thread.get("summary") else None),350 ("👤 Who", _fmt_who(thread["entities"])),351 ("⚡ What — Tone", _fmt_sentiment(s)),352 ("⚡ What — Key Actions", _fmt_svo_table(thread["actions"])),353 ("🕐 When", _fmt_temporal_html(thread["temporal"])),354 ("📍 Where", _fmt_where_map(thread["entities"])),355 ("🔗 Why — Direct Causal Links", _fmt_causal(thread["causal"])),356 ]357 body = "".join(358 f'<details open style="margin:8px 0;border:1px solid #e5e7eb;'359 f'border-radius:10px;overflow:hidden">'360 f'<summary style="font-size:0.82rem;font-weight:700;letter-spacing:0.06em;'361 f'text-transform:uppercase;cursor:pointer;padding:10px 16px;'362 f'background:#f9fafb;color:#374151;user-select:none">'363 f'{sec_title}</summary>'364 f'<div style="padding:14px 16px;color:#111827;background:#fff">{sec_html}</div>'365 f'</details>'366 for sec_title, sec_html in sections if sec_html is not None367 )368 lede_html = (369 f'<p style="margin:0 0 14px;padding:10px 14px;background:#f9fafb;'370 f'border-radius:8px;color:#6b7280;font-size:0.88em;line-height:1.5">'371 f'{html.escape(thread["lede"])}</p>'372 ) if multi else ""373 _SENT_ACCENT = {"POSITIVE": "#16a34a", "NEGATIVE": "#dc2626", "NEUTRAL": "#6b7280"}374 accent = _SENT_ACCENT[s["label"]]375 return (376 f'<div style="border:1px solid #e5e7eb;border-top:3px solid {accent};'377 f'border-radius:12px;padding:18px 22px;margin:12px 0;background:#fff;'378 f'box-shadow:0 1px 4px rgba(0,0,0,.05)">'379 f'<p style="margin:0 0 4px;font-size:0.7rem;font-weight:700;letter-spacing:0.08em;'380 f'text-transform:uppercase;color:{accent}">'381 f'{title_prefix or "Thread"} · {icon} {s["label"]} · {n} sentence{plural}</p>'382 f'{lede_html}{body}</div>'383 )384 385 386# ── Loading spinner helper ────────────────────────────────────────────────────387 388_SPINNER_CSS = (389 "@keyframes nt-spin{to{transform:rotate(360deg)}}"390 ".nt-spin{display:inline-block;width:48px;height:48px;"391 "border:5px solid #e0e0e0;border-top-color:#4A90E2;"392 "border-radius:50%;animation:nt-spin 0.8s linear infinite}"393)394 395 396def _loading_html(stage: str) -> str:397 return (398 f'<style>{_SPINNER_CSS}</style>'399 '<div style="text-align:center;padding:64px 24px">'400 '<div class="nt-spin"></div>'401 f'<p style="margin:20px 0 0;font-size:1.05rem;color:#555;font-style:italic">'402 f'{html.escape(stage)}</p></div>'403 )404 405 406def _show_spinner(text: str):407 """Immediately returns spinner HTML (no GPU, fires instantly on click)."""408 text = text.strip()409 if len(text) < 30:410 return "<p><em>Please enter at least a sentence or two of text.</em></p>"411 return _loading_html("Loading models… (may take ~30 s on first run)")412 413 414# ── Main analysis function ────────────────────────────────────────────────────415 416@spaces.GPU(duration=120)417def analyse(text: str, progress=gr.Progress()):418 text = text.strip()419 if len(text) < 30:420 return "<p><em>Please enter at least a sentence or two of text.</em></p>"421 422 progress(0.0, desc="Loading models…")423 _ensure_models()424 425 result = extract_narrative(text, MODELS, _progress=progress)426 threads = result["threads"]427 if not threads:428 return "<p><em>No narrative structure could be extracted.</em></p>"429 430 overall_summary = result.get("summary", "")431 summary_html = (432 f'<div style="background:#f8faff;border:1px solid #d0daf5;border-radius:12px;'433 f'padding:16px 20px;margin:0 0 20px;">'434 f'<p style="margin:0 0 4px;font-size:0.7rem;font-weight:700;letter-spacing:0.08em;'435 f'text-transform:uppercase;color:#6b7280">Overall Summary</p>'436 f'<p style="margin:0;color:#111827;font-size:0.97rem;line-height:1.55">{html.escape(overall_summary)}</p></div>'437 ) if overall_summary else ""438 439 # Build data-URI download link for JSON440 import base64441 json_bytes = json.dumps(result, ensure_ascii=False, indent=2).encode("utf-8")442 b64 = base64.b64encode(json_bytes).decode("ascii")443 download_link = (444 f'<div style="margin:0 0 20px;text-align:right">'445 f'<a href="data:application/json;base64,{b64}" download="narrative.json" '446 f'style="font-size:0.8rem;color:#4f6ef7;text-decoration:none;'447 f'background:#eef1ff;border:1px solid #c7d0ff;padding:5px 14px;'448 f'border-radius:20px;font-weight:600">↓ Download JSON</a></div>'449 )450 451 n = len(threads)452 if n == 1:453 return summary_html + download_link + _fmt_thread_body(threads[0])454 455 # CSS-only tabs (radio + label trick)456 panel_css = "@import url('https://fonts.googleapis.com/css2?family=Inter:wght@400;600;700&display=swap');" + "".join(457 f'#nt-r{i}:checked~.nt-panels #nt-p{i}{{display:block}}'458 f'#nt-r{i}:checked~.nt-bar label[for="nt-r{i}"]{{background:#4f6ef7;color:#fff;border-color:#4f6ef7;box-shadow:0 2px 8px rgba(79,110,247,.25)}}'459 for i in range(n)460 )461 style = f'<style>.nt-panel{{display:none}}{panel_css}</style>'462 463 radios = "".join(464 f'<input type="radio" name="nt" id="nt-r{i}" {"checked" if i == 0 else ""} '465 f'style="position:absolute;opacity:0;width:0;height:0">'466 for i in range(n)467 )468 469 bar_labels = ""470 for i, t in enumerate(threads):471 s = t["sentiment"]472 icon = {"POSITIVE": "🟢", "NEGATIVE": "🔴", "NEUTRAL": "⚪"}[s["label"]]473 label = _tab_label(t, i)474 bar_labels += (475 f'<label for="nt-r{i}" style="cursor:pointer;padding:7px 16px;'476 f'border-radius:20px;border:1.5px solid #e2e5ed;background:#fff;'477 f'color:#374151;font-size:0.82rem;font-weight:600;display:inline-block;'478 f'max-width:240px;overflow:hidden;text-overflow:ellipsis;white-space:nowrap;'479 f'transition:all .15s" '480 f'title="{html.escape(label)}">{icon} {html.escape(label)}</label>'481 )482 bar = f'<div class="nt-bar" style="display:flex;flex-wrap:wrap;gap:8px;margin-bottom:24px;padding-bottom:16px;border-bottom:1px solid #e5e7eb">{bar_labels}</div>'483 484 panels = '<div class="nt-panels">' + "".join(485 f'<div class="nt-panel" id="nt-p{i}">{_fmt_thread_body(t)}</div>'486 for i, t in enumerate(threads)487 ) + '</div>'488 489 return summary_html + download_link + style + radios + bar + panels490 491 492# ── Gradio UI ─────────────────────────────────────────────────────────────────493 494with gr.Blocks(495 title="Narrative Detector",496 theme=gr.themes.Base(497 primary_hue="slate",498 neutral_hue="slate",499 font=[gr.themes.GoogleFont("Inter"), "ui-sans-serif", "sans-serif"],500 ),501 css="""502 .gradio-container { max-width: 860px !important; margin: 0 auto; }503 footer { display: none !important; }504 #input-col textarea { font-size: 0.95rem; }505 .gr-button-primary { border-radius: 8px !important; font-weight: 600 !important; }506 """,507) as demo:508 gr.Markdown(509 """# Narrative Detector510*Paste any news text to extract the five W's — who, what, when, where and why — across narrative threads.*"""511 )512 513 text_input = gr.Textbox(514 lines=8,515 label="Input text",516 placeholder="Write or paste your text here…",517 )518 519 gr.Examples(520 examples=[521 ["Iran's state media reported Sunday that a drone strike hit an oil facility near Tehran, killing two workers and injuring six others. The Islamic Revolutionary Guard Corps blamed the attack on Israeli operatives, vowing retaliation. Israel declined to comment. The incident sent oil prices up 3 percent in early Asian trading."],522 ["The United Nations Security Council convened an emergency session Friday after North Korea launched what officials described as an intermediate-range ballistic missile that flew over Japan before splashing down in the Pacific Ocean. South Korean and US forces raised their alert level. Japanese Prime Minister Fumio Kishida condemned the launch and called for calm while diplomats scrambled to draft a formal response."],523 ["Hurricane Milton made landfall near Siesta Key, Florida on Wednesday as a Category 3 storm, knocking out power to more than three million homes. Rescue teams from FEMA and the National Guard deployed across Tampa Bay, where storm surge reached six feet in low-lying neighbourhoods. Florida Governor Ron DeSantis declared a state of emergency and urged residents to stay indoors while crews cleared downed trees from major highways."],524 ],525 inputs=text_input,526 label="Try an example",527 cache_examples=False,528 )529 530 submit_btn = gr.Button("Analyse ✨", variant="primary")531 532 gr.Markdown("---")533 534 output_html = gr.HTML()535 536 submit_btn.click(537 fn=_show_spinner,538 inputs=text_input,539 outputs=output_html,540 api_name=False,541 queue=False,542 ).then(543 fn=analyse,544 inputs=text_input,545 outputs=output_html,546 api_name="analyse",547 )548 549 gr.Markdown(550 "---\n<sub>Powered by spaCy · FastCoref · DistilBERT · mpnet · HDBSCAN · DistilBART · ZeroGPU</sub>")551 552if __name__ == "__main__":553 demo.launch()554