jbeek123/FDAMAUDE
0
1# maude_app.py — Streamlit app: find variants → fetch MAUDE → dashboard + CSV2# Includes: variant discovery, country/state filters, progress per variant,3# keyword filter across product_problems + mdr_text_concat,4# "Download filtered CSV" (standard MAUDE columns) and "Download all CSV".5 6import re7import json8import requests9import pandas as pd10import streamlit as st11import altair as alt12from datetime import date, timedelta13 14OPENFDA_URL = "https://api.fda.gov/device/event.json"15 16# ===============================17# Helpers: quoting, queries, fetch18# ===============================19 20def quote_term(s):21 """22 Safely quote a term for the openFDA Lucene query.23 - If the term contains anything besides [A-Za-z0-9_], put it in "double quotes"24 and escape any internal quotes.25 - Else, return as-is (unquoted).26 """27 if not s:28 return '""'29 if re.search(r'[^A-Za-z0-9_]', s):30 escaped = s.replace('"', r'\"')31 return '"' + escaped + '"'32 return s33 34 35def build_search(manufacturer, start_yyyymmdd, end_yyyymmdd, country=None, state=None, extra_terms=None):36 """37 Build the openFDA /device/event.json search= query string.38 """39 parts = []40 parts.append(f'device.manufacturer_d_name:{quote_term(manufacturer)}')41 parts.append(f'date_received:[{start_yyyymmdd} TO {end_yyyymmdd}]')42 43 if country:44 parts.append(45 f'(reporter_country_code:{quote_term(country)} OR manufacturer_country:{quote_term(country)})'46 )47 48 if state:49 parts.append(50 f'(manufacturer_state:{quote_term(state)} OR '51 f'manufacturer_contact_state:{quote_term(state)} OR '52 f'distributor_state:{quote_term(state)} OR '53 f"reporter_state:{quote_term(state)})"54 )55 56 if extra_terms:57 parts.extend(extra_terms)58 59 return " AND ".join(parts)60 61 62def fetch_count_variants(parent_hint, start_yyyymmdd, end_yyyymmdd, limit=1000):63 """64 Find manufacturer name variants using count=device.manufacturer_d_name.exact.65 Returns a list of dicts: {'term': name, 'count': int}66 """67 params = {68 "search": f"date_received:[{start_yyyymmdd} TO {end_yyyymmdd}]",69 "count": "device.manufacturer_d_name.exact",70 "limit": str(limit),71 }72 r = requests.get(OPENFDA_URL, params=params, timeout=40)73 r.raise_for_status()74 rows = (r.json().get("results") or [])75 if parent_hint:76 pat = re.compile(parent_hint, re.I)77 rows = [x for x in rows if pat.search(x.get("term", ""))]78 rows.sort(key=lambda x: int(x.get("count", 0)), reverse=True)79 return rows80 81 82def fetch_events_for_manufacturer(manufacturer, start, end, country=None, state=None,83 page_limit=100, max_pages=1000):84 """85 Page through /device/event.json for a single manufacturer until no more results.86 Returns a list of event dicts.87 """88 all_results = []89 skip = 090 pages = 091 while pages < max_pages:92 search = build_search(manufacturer, start, end, country=country, state=state)93 params = {94 "search": search,95 "limit": str(page_limit),96 "skip": str(skip),97 }98 r = requests.get(OPENFDA_URL, params=params, timeout=45)99 if r.status_code == 404:100 break101 r.raise_for_status()102 payload = r.json()103 batch = payload.get("results") or []104 if not batch:105 break106 all_results.extend(batch)107 if len(batch) < page_limit:108 break # last page109 skip += page_limit110 pages += 1111 return all_results112 113 114# These are the columns you asked to export for the "filtered CSV"115CSV_STD_ORDER = [116 "mdr_report_key","report_number","date_received","date_of_event","event_type",117 "manufacturer_d_name","brand_name","generic_name","device_report_product_code",118 "model_number","udi_di","product_problems","source_type","report_source_code",119 "patient_sex","patient_age","patient_problems","mdr_text_concat"120]121 122def normalize_events_to_df(events):123 """124 Flatten raw event JSON list -> DataFrame.125 Adds/derives the fields required by CSV_STD_ORDER (blank if missing).126 """127 if not events:128 return pd.DataFrame()129 130 df = pd.json_normalize(events)131 132 # Top-level simple fields133 for col in ["mdr_report_key","report_number","date_received","date_of_event",134 "event_type","manufacturer_d_name","source_type","report_source_code"]:135 if col not in df.columns:136 df[col] = ""137 138 # Patient info (these are often nested; fill blanks if missing)139 # Note: openFDA event format varies; if patient dict exists:140 if "patient" in df.columns:141 # patient can be a dict or list; be defensive142 def _sex(x):143 if isinstance(x, dict):144 return x.get("patient_sex", "")145 if isinstance(x, list) and x:146 return x[0].get("patient_sex", "")147 return ""148 def _age(x):149 if isinstance(x, dict):150 return x.get("patient_age", "")151 if isinstance(x, list) and x:152 return x[0].get("patient_age", "")153 return ""154 def _probs(x):155 if isinstance(x, dict):156 return ", ".join(x.get("patient_problems", []) or [])157 if isinstance(x, list) and x:158 return ", ".join(x[0].get("patient_problems", []) or [])159 return ""160 df["patient_sex"] = df["patient"].apply(_sex)161 df["patient_age"] = df["patient"].apply(_age)162 df["patient_problems"] = df["patient"].apply(_probs)163 else:164 for col in ["patient_sex","patient_age","patient_problems"]:165 df[col] = ""166 167 # First device fields168 if "device" in df.columns:169 dseries = df["device"]170 df["brand_name"] = dseries.apply(lambda x: x[0].get("brand_name", "") if isinstance(x, list) and x else "")171 df["generic_name"] = dseries.apply(lambda x: x[0].get("generic_name", "") if isinstance(x, list) and x else "")172 df["device_report_product_code"] = dseries.apply(lambda x: x[0].get("device_report_product_code", "") if isinstance(x, list) and x else "")173 df["model_number"] = dseries.apply(lambda x: x[0].get("model_number", "") if isinstance(x, list) and x else "")174 df["udi_di"] = dseries.apply(lambda x: x[0].get("udi_di", "") if isinstance(x, list) and x else "")175 else:176 for col in ["brand_name","generic_name","device_report_product_code","model_number","udi_di"]:177 df[col] = ""178 179 # Problems list -> string180 if "product_problems" in df.columns:181 df["product_problems"] = df["product_problems"].apply(182 lambda v: ", ".join(v) if isinstance(v, list) else (v or "")183 )184 else:185 df["product_problems"] = ""186 187 # Text blobs concat188 if "mdr_text" in df.columns:189 df["mdr_text_concat"] = df["mdr_text"].apply(190 lambda v: " | ".join([t.get("text", "") for t in v]) if isinstance(v, list) else ""191 )192 else:193 df["mdr_text_concat"] = ""194 195 # Ensure all standard columns exist196 for c in CSV_STD_ORDER:197 if c not in df.columns:198 df[c] = ""199 200 return df201 202 203# ===============================204# Streamlit UI205# ===============================206 207st.set_page_config(page_title="FDA MAUDE Dashboard", layout="wide")208st.title("FDA MAUDE Dashboard")209 210# ---- Step 0: inputs211col1, col2 = st.columns(2)212with col1:213 parent_name = st.text_input("Enter parent company name (e.g., GE, NuVasive, Philips)", value="")214with col2:215 end_default = date.today()216 start_default = end_default - timedelta(days=730)217 sdate = st.date_input("Start date", value=start_default, format="YYYY/MM/DD")218 edate = st.date_input("End date", value=end_default, format="YYYY/MM/DD")219 220col3, col4 = st.columns(2)221with col3:222 country = st.text_input("Country (e.g., US)", value="US")223with col4:224 state = st.text_input("State (optional, e.g., CA)", value="")225 226start_str = sdate.strftime("%Y%m%d")227end_str = edate.strftime("%Y%m%d")228 229st.divider()230 231# ---- Step 1: discover variants232if "variants" not in st.session_state:233 st.session_state["variants"] = []234if "selected_names" not in st.session_state:235 st.session_state["selected_names"] = []236 237if st.button("Find company variants", type="primary"):238 if not parent_name.strip():239 st.warning("Please type a parent company name first.")240 st.stop()241 with st.spinner("Searching FDA MAUDE for manufacturer name variants..."):242 try:243 rows = fetch_count_variants(parent_name.strip(), start_str, end_str, limit=1000)244 except requests.HTTPError as e:245 st.error(f"Fetch error: {e}")246 rows = []247 st.session_state["variants"] = rows248 st.session_state["selected_names"] = [r["term"] for r in rows[:6]] # preselect a few249 250# Show variants table + multiselect251if st.session_state["variants"]:252 st.subheader("Step 2 — Select manufacturer variants")253 var_df = pd.DataFrame(st.session_state["variants"])254 var_df.columns = ["manufacturer", "count"]255 st.dataframe(var_df, hide_index=True, use_container_width=True)256 257 st.session_state["selected_names"] = st.multiselect(258 "Which names should be included?",259 options=list(var_df["manufacturer"]),260 default=st.session_state["selected_names"],261 )262 263st.divider()264 265# ---- Step 2: fetch + dashboard266if st.session_state.get("selected_names"):267 if st.button("Fetch reports & build dashboard", type="primary"):268 names = st.session_state["selected_names"]269 st.subheader("Fetching data")270 all_rows = []271 for name in names:272 with st.status(f"Fetching: {name}", expanded=False) as status:273 try:274 events = fetch_events_for_manufacturer(275 manufacturer=name,276 start=start_str,277 end=end_str,278 country=country.strip() or None,279 state=state.strip() or None,280 )281 all_rows.extend(events)282 status.update(label=f"Fetched {len(events)} events — {name}",283 state="complete", expanded=False)284 except requests.HTTPError as e:285 status.update(label=f"HTTP error for {name}: {e}", state="error", expanded=True)286 except Exception as e:287 status.update(label=f"Error for {name}: {e}", state="error", expanded=True)288 289 if not all_rows:290 st.info("No records found for the chosen inputs.")291 st.stop()292 293 df = normalize_events_to_df(all_rows)294 # Deduplicate by FDA report key295 before = len(df)296 if "mdr_report_key" in df.columns:297 df = df.drop_duplicates(subset=["mdr_report_key"])298 st.caption(f"Deduplicated {before} → {len(df)} rows.")299 300 st.session_state["df"] = df301 302# ---- Dashboard (render if data is present)303if "df" in st.session_state and not st.session_state["df"].empty:304 df = st.session_state["df"].copy()305 306 st.subheader("Filters")307 colf1, colf2, colf3 = st.columns([2, 2, 2])308 309 with colf1:310 event_opts = sorted([x for x in df["event_type"].dropna().unique() if x != ""])311 selected_events = st.multiselect("Filter by Event Type", options=event_opts, default=event_opts)312 313 with colf2:314 model_query = st.text_input("Search models (substring)", value="")315 316 # NEW: Keyword search across product_problems & mdr_text_concat317 with colf3:318 kw_text = st.text_input("Keyword(s) (comma-separated)", placeholder="e.g., ssd, nvme, power supply")319 kw_mode = st.radio("Match", options=["Any keyword", "All keywords"], horizontal=True, index=0)320 321 if st.button("Clear filters"):322 selected_events = event_opts323 model_query = ""324 kw_text = ""325 326 # Apply filters327 fdf = df.copy()328 if selected_events:329 fdf = fdf[fdf["event_type"].isin(selected_events)]330 if model_query.strip():331 mq = model_query.strip().lower()332 fdf = fdf[fdf["model_number"].fillna("").str.lower().str.contains(mq)]333 334 # Keyword filter (product_problems + mdr_text_concat)335 if kw_text.strip():336 keywords = [k.strip().lower() for k in kw_text.split(",") if k.strip()]337 if keywords:338 text_space = (339 fdf["product_problems"].fillna("").str.lower() + " " +340 fdf["mdr_text_concat"].fillna("").str.lower()341 )342 if kw_mode.startswith("All"):343 mask = pd.Series(True, index=text_space.index)344 for k in keywords:345 mask = mask & text_space.str.contains(re.escape(k))346 else:347 pattern = "|".join([re.escape(k) for k in keywords])348 mask = text_space.str.contains(pattern)349 fdf = fdf[mask]350 351 # Top models chart (by selected events)352 st.subheader("Top Models (by selected Event Type)")353 top = (354 fdf.groupby("model_number", dropna=False, as_index=False)355 .size()356 .rename(columns={"size":"count"})357 .sort_values("count", ascending=False)358 .head(25)359 )360 if not top.empty:361 chart = (362 alt.Chart(top)363 .mark_bar()364 .encode(365 x=alt.X("count:Q", title="Count"),366 y=alt.Y("model_number:N", sort="-x", title="Model Number"),367 tooltip=["model_number", "count"]368 )369 .properties(height=420)370 )371 st.altair_chart(chart, use_container_width=True)372 else:373 st.info("No rows after current filters.")374 375 # Summary table (event_type × model_number)376 st.subheader("Summary Table")377 if not fdf.empty:378 summary = (379 fdf.groupby(["event_type", "model_number"], dropna=False, as_index=False)380 .size()381 .rename(columns={"size":"count"})382 .sort_values(["event_type", "count"], ascending=[True, False])383 )384 st.dataframe(summary, use_container_width=True, hide_index=True)385 else:386 st.info("No rows to summarize.")387 388 st.subheader("Underlying Events")389 cols_show = [390 "date_received", "event_type", "brand_name", "generic_name",391 "model_number", "product_problems", "report_number"392 ]393 for c in cols_show:394 if c not in fdf.columns:395 fdf[c] = ""396 st.dataframe(fdf[cols_show], use_container_width=True, hide_index=True)397 398 # =============================399 # Download buttons (side-by-side)400 # =============================401 cdl, cdr = st.columns(2)402 403 # Left: Download filtered CSV (standard column order)404 with cdl:405 # Make sure all standard columns are present in current filtered DF406 filt_export = fdf.copy()407 for c in CSV_STD_ORDER:408 if c not in filt_export.columns:409 filt_export[c] = ""410 filt_export = filt_export[CSV_STD_ORDER]411 st.download_button(412 "Download filtered CSV (standard columns)",413 filt_export.to_csv(index=False).encode("utf-8"),414 file_name="maude_filtered_standard.csv",415 mime="text/csv",416 )417 418 # Right: Download ALL CSV (full, deduped dataset)419 with cdr:420 all_export = df.copy()421 st.download_button(422 "Download ALL CSV (full dataset)",423 all_export.to_csv(index=False).encode("utf-8"),424 file_name="maude_all_full.csv",425 mime="text/csv",426 )427 428