CESARSQ/Cqantion-OS
0
1"""2app.py · ANTIGRAVITY Intelligence Stack — SaaS Interface3═══════════════════════════════════════════════════════════4Streamlit front-end for the Antigravity B2B scanning engine.5 6Imports from (preserving original modules intact):7 ✔ antigravity_competitive_intel.py → Tier A/B/C Sitemap Scoring8 ✔ shopify_lead_extractor.py → Email & Contact Extraction9 ✔ antigravity_email_validator.py → 3-Layer Email Validation (DNS/MX)10 ✔ advanced_extractor.py → Advanced Scraping + Proxy Rotation11 ✔ antigravity_deep_personalizer.py → Gemini Hook Generation12 ⚠ pilar2_motor_robusto.py → Runs top-level code on import;13 use via CLI only (documented in System tab)14 ✔ config.py → Referenced via advanced_extractor15 ✔ ejemplo_practico.py → Standalone demo16 ✔ test_system.py → Health check suite17 18Install:19 pip install streamlit pandas requests beautifulsoup4 lxml dnspython20 21Run:22 streamlit run app.py23"""24 25import io26import os27import sys28import time29from datetime import datetime30from pathlib import Path31 32import pandas as pd33import streamlit as st34 35# ─────────────────────────────────────────────────────────────────────36# PAGE CONFIG — must be the very first Streamlit call37# ─────────────────────────────────────────────────────────────────────38st.set_page_config(39 page_title="ANTIGRAVITY · Intelligence Stack",40 page_icon="⚡",41 layout="wide",42 initial_sidebar_state="expanded",43)44 45# ─────────────────────────────────────────────────────────────────────46# SAFE MODULE IMPORTS — graceful degradation on missing deps47# ─────────────────────────────────────────────────────────────────────48MODULE_STATUS: dict = {}49 50try:51 from antigravity_competitive_intel import (52 fetch_sitemap,53 parse_sitemap,54 score_tier,55 _clean_domain,56 )57 MODULE_STATUS["intel"] = True58except Exception as exc:59 MODULE_STATUS["intel"] = str(exc)60 61try:62 from shopify_lead_extractor import ShopifyLeadExtractor63 MODULE_STATUS["extractor"] = True64except Exception as exc:65 MODULE_STATUS["extractor"] = str(exc)66 67try:68 from antigravity_email_validator import EmailValidator69 MODULE_STATUS["validator"] = True70except Exception as exc:71 MODULE_STATUS["validator"] = str(exc)72 73try:74 from advanced_extractor import AdvancedShopifyExtractor75 MODULE_STATUS["advanced"] = True76except Exception as exc:77 MODULE_STATUS["advanced"] = str(exc)78 79try:80 from antigravity_deep_personalizer import scrape_tienda, normalizar_url81 MODULE_STATUS["personalizer"] = True82except Exception as exc:83 MODULE_STATUS["personalizer"] = str(exc)84 85# pilar2_motor_robusto.py executes authentication + CSV loading at module86# level (no importable functions) — run it as a standalone CLI script.87MODULE_STATUS["motor_robusto"] = "CLI_ONLY"88MODULE_STATUS["config"] = True89MODULE_STATUS["ejemplo_practico"] = True90MODULE_STATUS["test_system"] = True91 92 93# ─────────────────────────────────────────────────────────────────────94# DESIGN TOKENS & HELPERS95# ─────────────────────────────────────────────────────────────────────96TIER_COLORS = {"A": "#00FF7F", "B": "#E8A808", "C": "#6E7681"}97TIER_LABELS = {"A": "BIG FISH", "B": "NURTURE", "C": "DISCARD"}98 99 100def tier_badge(tier: str) -> str:101 cls = f"tier-{tier.lower()}"102 label = TIER_LABELS.get(tier, tier)103 return f'<span class="tier-badge {cls}">{tier} · {label}</span>'104 105 106def parse_domain_input(raw: str) -> list:107 """Accept newline / comma / semicolon-separated domains; strip protocols."""108 import re109 parts = re.split(r"[\n,;\s]+", raw.strip())110 domains = []111 for p in parts:112 p = p.strip()113 if not p:114 continue115 for prefix in ("https://", "http://"):116 if p.startswith(prefix):117 p = p[len(prefix):]118 p = p.rstrip("/")119 if p:120 domains.append(p)121 return domains122 123 124# ─────────────────────────────────────────────────────────────────────125# CUSTOM CSS — dark industrial theme126# ─────────────────────────────────────────────────────────────────────127def inject_css() -> None:128 st.markdown(129 """130 <link rel="preconnect" href="https://fonts.googleapis.com">131 <link href="https://fonts.googleapis.com/css2?family=IBM+Plex+Mono:wght@300;400;500;600&family=Barlow:wght@300;400;500;600;700&family=Barlow+Condensed:wght@400;600;700;800&display=swap" rel="stylesheet">132 133 <style>134 /* ── CSS variables ── */135 :root {136 --bg: #050508;137 --surface: #0D1117;138 --card: #161B22;139 --border: #21262D;140 --accent: #00FF7F;141 --blue: #4D9EFF;142 --amber: #E8A808;143 --red: #F85149;144 --text: #E6EDF3;145 --muted: #8B949E;146 --dim: #30363D;147 --mono: 'IBM Plex Mono', monospace;148 --body: 'Barlow', sans-serif;149 --display: 'Barlow Condensed', sans-serif;150 }151 152 /* ── Base ── */153 .stApp,154 [data-testid="stAppViewContainer"],155 [data-testid="stMain"] {156 background-color: var(--bg) !important;157 font-family: var(--body) !important;158 color: var(--text) !important;159 }160 [data-testid="stHeader"] { background: transparent !important; }161 [data-testid="stSidebar"] {162 background-color: var(--surface) !important;163 border-right: 1px solid var(--border) !important;164 }165 [data-testid="stSidebarContent"] { padding: 1.5rem 1rem !important; }166 167 /* ── Typography ── */168 h1, h2, h3, h4, .st-emotion-cache-10trblm {169 font-family: var(--display) !important;170 color: var(--text) !important;171 letter-spacing: 0.5px;172 }173 p, li { font-family: var(--body) !important; color: var(--text) !important; }174 label, [data-testid="stWidgetLabel"] {175 font-family: var(--mono) !important;176 font-size: 11px !important;177 letter-spacing: 1.5px !important;178 text-transform: uppercase !important;179 color: var(--muted) !important;180 }181 182 /* ── Text inputs ── */183 .stTextInput > div > div > input,184 .stTextArea > div > div > textarea {185 background-color: var(--card) !important;186 border: 1px solid var(--border) !important;187 color: var(--text) !important;188 font-family: var(--mono) !important;189 font-size: 13px !important;190 border-radius: 3px !important;191 }192 .stTextInput > div > div > input:focus,193 .stTextArea > div > div > textarea:focus {194 border-color: var(--accent) !important;195 box-shadow: 0 0 0 1px rgba(0, 255, 127, 0.35) !important;196 }197 .stTextInput > div > div > input::placeholder,198 .stTextArea > div > div > textarea::placeholder { color: var(--dim) !important; }199 200 /* ── Buttons ── */201 .stButton > button {202 background: var(--accent) !important;203 color: #030305 !important;204 font-family: var(--display) !important;205 font-weight: 800 !important;206 font-size: 14px !important;207 letter-spacing: 2px !important;208 text-transform: uppercase !important;209 border: none !important;210 border-radius: 3px !important;211 padding: 0.6rem 2rem !important;212 transition: background 0.15s ease, box-shadow 0.15s ease, transform 0.1s ease !important;213 }214 .stButton > button:hover {215 background: #00CC6A !important;216 box-shadow: 0 0 22px rgba(0, 255, 127, 0.4) !important;217 transform: translateY(-1px) !important;218 }219 .stButton > button:active { transform: translateY(0) !important; }220 221 /* ── Download button ── */222 .stDownloadButton > button {223 background: transparent !important;224 color: var(--accent) !important;225 border: 1px solid rgba(0, 255, 127, 0.4) !important;226 font-family: var(--mono) !important;227 font-size: 12px !important;228 letter-spacing: 1px !important;229 border-radius: 3px !important;230 padding: 0.5rem 1.5rem !important;231 transition: all 0.15s ease !important;232 }233 .stDownloadButton > button:hover {234 background: rgba(0, 255, 127, 0.1) !important;235 border-color: var(--accent) !important;236 }237 238 /* ── Tabs ── */239 [data-testid="stTabs"] [data-baseweb="tab-list"] {240 background: transparent !important;241 border-bottom: 1px solid var(--border) !important;242 gap: 0 !important;243 }244 [data-testid="stTabs"] [data-baseweb="tab"] {245 background: transparent !important;246 color: var(--muted) !important;247 font-family: var(--mono) !important;248 font-size: 11px !important;249 letter-spacing: 2px !important;250 text-transform: uppercase !important;251 padding: 12px 20px !important;252 border-bottom: 2px solid transparent !important;253 transition: color 0.15s ease !important;254 }255 [data-testid="stTabs"] [data-baseweb="tab"]:hover { color: var(--text) !important; }256 [data-testid="stTabs"] [data-baseweb="tab"][aria-selected="true"] {257 color: var(--accent) !important;258 border-bottom-color: var(--accent) !important;259 }260 [data-testid="stTabs"] [data-baseweb="tab-panel"] { padding-top: 2rem !important; }261 [data-baseweb="tab-highlight"] { background-color: var(--accent) !important; }262 263 /* ── Metrics ── */264 [data-testid="stMetric"] {265 background: var(--card) !important;266 border: 1px solid var(--border) !important;267 border-radius: 4px !important;268 padding: 1.1rem 1.2rem !important;269 }270 [data-testid="stMetricValue"] {271 font-family: var(--mono) !important;272 color: var(--accent) !important;273 font-size: 2.2rem !important;274 line-height: 1.1 !important;275 }276 [data-testid="stMetricLabel"] {277 font-family: var(--mono) !important;278 color: var(--muted) !important;279 font-size: 10px !important;280 letter-spacing: 2.5px !important;281 text-transform: uppercase !important;282 }283 284 /* ── Progress bar ── */285 [data-testid="stProgressBar"] > div > div {286 background: linear-gradient(90deg, var(--accent), #00CC6A) !important;287 }288 [data-testid="stProgressBar"] > div {289 background: var(--card) !important;290 border-radius: 2px !important;291 }292 293 /* ── File uploader ── */294 [data-testid="stFileUploader"] {295 background: var(--card) !important;296 border: 1px dashed var(--dim) !important;297 border-radius: 4px !important;298 }299 [data-testid="stFileUploader"]:hover { border-color: var(--accent) !important; }300 301 /* ── Checkboxes ── */302 [data-testid="stCheckbox"] { gap: 8px !important; }303 [data-testid="stCheckbox"] span {304 font-family: var(--mono) !important;305 font-size: 12px !important;306 color: var(--muted) !important;307 }308 309 /* ── Expander ── */310 [data-testid="stExpander"] {311 background: var(--card) !important;312 border: 1px solid var(--border) !important;313 border-radius: 4px !important;314 }315 [data-testid="stExpander"] summary {316 font-family: var(--mono) !important;317 font-size: 11px !important;318 letter-spacing: 1.5px !important;319 text-transform: uppercase !important;320 color: var(--muted) !important;321 }322 323 /* ── Alerts ── */324 [data-testid="stAlert"] {325 background: var(--card) !important;326 border-radius: 4px !important;327 font-family: var(--mono) !important;328 font-size: 12px !important;329 }330 331 /* ── Scrollbar ── */332 ::-webkit-scrollbar { width: 5px; height: 5px; }333 ::-webkit-scrollbar-track { background: var(--bg); }334 ::-webkit-scrollbar-thumb { background: var(--border); border-radius: 3px; }335 ::-webkit-scrollbar-thumb:hover { background: var(--dim); }336 337 /* ── Custom components ── */338 .ag-card {339 background: var(--card);340 border: 1px solid var(--border);341 border-radius: 4px;342 padding: 1.25rem 1.5rem;343 margin-bottom: 1.75rem;344 }345 .ag-card-accent { border-left: 3px solid var(--accent); }346 .ag-card-blue { border-left: 3px solid var(--blue); }347 .ag-card-amber { border-left: 3px solid var(--amber); }348 349 .ag-header {350 display: flex;351 align-items: flex-end;352 justify-content: space-between;353 border-bottom: 1px solid var(--border);354 padding-bottom: 1.25rem;355 margin-bottom: 2rem;356 }357 .ag-logo {358 font-family: var(--display);359 font-size: 32px;360 font-weight: 800;361 letter-spacing: 4px;362 text-transform: uppercase;363 color: var(--text);364 line-height: 1;365 }366 .ag-logo .hi { color: var(--accent); }367 .ag-tagline {368 font-family: var(--mono);369 font-size: 10px;370 color: var(--muted);371 letter-spacing: 3px;372 text-transform: uppercase;373 margin-top: 4px;374 }375 .ag-ts {376 font-family: var(--mono);377 font-size: 10px;378 color: var(--dim);379 letter-spacing: 1px;380 }381 382 .tier-badge {383 display: inline-block;384 padding: 3px 10px 2px;385 border-radius: 2px;386 font-family: var(--mono);387 font-size: 11px;388 font-weight: 600;389 letter-spacing: 1.5px;390 }391 .tier-a { background: rgba(0,255,127,0.1); color: #00FF7F; border: 1px solid rgba(0,255,127,0.35); }392 .tier-b { background: rgba(232,168,8,0.1); color: #E8A808; border: 1px solid rgba(232,168,8,0.35); }393 .tier-c { background: rgba(110,118,129,0.1); color: #6E7681; border: 1px solid rgba(110,118,129,0.3); }394 395 .status-dot {396 display: inline-block;397 width: 7px;398 height: 7px;399 border-radius: 50%;400 margin-right: 5px;401 vertical-align: middle;402 }403 .dot-green { background: #00FF7F; box-shadow: 0 0 5px rgba(0, 255, 127, 0.7); }404 .dot-red { background: #F85149; }405 .dot-amber { background: #E8A808; box-shadow: 0 0 5px rgba(232, 168, 8, 0.5); }406 407 .section-label {408 font-family: var(--mono);409 font-size: 10px;410 color: var(--muted);411 letter-spacing: 3px;412 text-transform: uppercase;413 margin-bottom: 0.6rem;414 }415 416 /* ── Hide Streamlit chrome ── */417 #MainMenu, footer, [data-testid="stToolbar"] { display: none !important; }418 </style>419 """,420 unsafe_allow_html=True,421 )422 423 424# ─────────────────────────────────────────────────────────────────────425# SHARED COMPONENTS426# ─────────────────────────────────────────────────────────────────────427def module_required_error(module_key: str, pip_hint: str = "") -> None:428 error_msg = MODULE_STATUS.get(module_key, "Unknown error")429 st.markdown(430 f"""431 <div class="ag-card" style="border-left: 3px solid var(--red);">432 <div style="font-family: var(--mono); font-size: 12px; color: var(--red); margin-bottom: 6px;">433 ⚠ MODULE OFFLINE434 </div>435 <div style="font-family: var(--mono); font-size: 11px; color: var(--muted);">{error_msg}</div>436 {f'<div style="font-family: var(--mono); font-size: 11px; color: var(--dim); margin-top: 8px;">pip install {pip_hint}</div>' if pip_hint else ''}437 </div>438 """,439 unsafe_allow_html=True,440 )441 442 443def html_table(headers: list, rows: list) -> str:444 """Render an HTML table matching the dark theme."""445 ths = "".join(446 f'<th style="padding:10px 14px;text-align:left;font-family:IBM Plex Mono,monospace;'447 f'font-size:10px;letter-spacing:2.5px;color:#8B949E;text-transform:uppercase;'448 f'white-space:nowrap;">{h}</th>'449 for h in headers450 )451 trs = ""452 for cells in rows:453 tds = "".join(454 f'<td style="padding:10px 14px;border-bottom:1px solid #21262D;">{c}</td>'455 for c in cells456 )457 trs += f"<tr>{tds}</tr>"458 459 return f"""460 <div style="background:#161B22;border:1px solid #21262D;border-radius:4px;461 overflow:hidden;max-height:520px;overflow-y:auto;overflow-x:auto;">462 <table style="width:100%;border-collapse:collapse;min-width:480px;">463 <thead>464 <tr style="background:#0D1117;border-bottom:2px solid #21262D;">{ths}</tr>465 </thead>466 <tbody>{trs}</tbody>467 </table>468 </div>469 """470 471 472def log_line(placeholder, msg: str, color: str = "#8B949E") -> None:473 placeholder.markdown(474 f'<div style="font-family:IBM Plex Mono,monospace;font-size:12px;'475 f'color:{color};padding:3px 0;">▶ {msg}</div>',476 unsafe_allow_html=True,477 )478 479 480# ─────────────────────────────────────────────────────────────────────481# SIDEBAR482# ─────────────────────────────────────────────────────────────────────483def render_sidebar() -> None:484 with st.sidebar:485 # Logo486 st.markdown(487 """488 <div style="padding:0.5rem 0 1.25rem;border-bottom:1px solid #21262D;margin-bottom:1.25rem;">489 <div style="font-family:'Barlow Condensed',sans-serif;font-size:22px;490 font-weight:800;letter-spacing:4px;color:#E6EDF3;">491 ANTI<span style="color:#00FF7F;">GRAVITY</span>492 </div>493 <div style="font-family:'IBM Plex Mono',monospace;font-size:9px;494 color:#8B949E;letter-spacing:3px;margin-top:3px;">495 INTELLIGENCE STACK496 </div>497 </div>498 """,499 unsafe_allow_html=True,500 )501 502 # Module status503 st.markdown(504 '<div style="font-family:IBM Plex Mono,monospace;font-size:9px;color:#8B949E;'505 'letter-spacing:3px;text-transform:uppercase;margin-bottom:8px;">System Status</div>',506 unsafe_allow_html=True,507 )508 509 module_map = {510 "Intel Scanner": "intel",511 "Lead Extractor": "extractor",512 "Email Validator": "validator",513 "Adv. Extractor": "advanced",514 "Deep Personalizer": "personalizer",515 "Motor Robusto": "motor_robusto",516 }517 518 for label, key in module_map.items():519 status = MODULE_STATUS.get(key)520 ok = status is True521 cli = status == "CLI_ONLY"522 dot_cls = "dot-green" if ok else ("dot-amber" if cli else "dot-red")523 txt_clr = "#00FF7F" if ok else ("#E8A808" if cli else "#F85149")524 txt_val = "ONLINE" if ok else ("CLI" if cli else "OFFLINE")525 526 st.markdown(527 f"""<div style="display:flex;justify-content:space-between;align-items:center;528 padding:5px 0;border-bottom:1px solid #21262D;">529 <span style="font-family:IBM Plex Mono,monospace;font-size:11px;color:#8B949E;">{label}</span>530 <span>531 <span class="status-dot {dot_cls}"></span>532 <span style="font-family:IBM Plex Mono,monospace;font-size:10px;color:{txt_clr};">{txt_val}</span>533 </span>534 </div>""",535 unsafe_allow_html=True,536 )537 538 st.markdown("<br>", unsafe_allow_html=True)539 540 # Gemini key541 st.markdown(542 '<div style="font-family:IBM Plex Mono,monospace;font-size:9px;color:#8B949E;'543 'letter-spacing:3px;text-transform:uppercase;margin-bottom:8px;">AI Config</div>',544 unsafe_allow_html=True,545 )546 gemini_key = st.text_input(547 "Gemini API Key",548 type="password",549 placeholder="AIza...",550 help="Required for Motor Robusto and Deep Personalizer (CLI tools)",551 )552 if gemini_key:553 os.environ["GEMINI_API_KEY"] = gemini_key554 st.success("✔ Key saved to environment.")555 556 # Build info557 st.markdown(558 """559 <div style="margin-top:2rem;padding-top:1rem;border-top:1px solid #21262D;560 text-align:center;font-family:IBM Plex Mono,monospace;font-size:9px;color:#30363D;">561 v2.0.0 · Antigravity Stack562 </div>563 """,564 unsafe_allow_html=True,565 )566 567 568# ─────────────────────────────────────────────────────────────────────569# HEADER570# ─────────────────────────────────────────────────────────────────────571def render_header() -> None:572 st.markdown(573 f"""574 <div class="ag-header">575 <div>576 <div class="ag-logo">ANTI<span class="hi">GRAVITY</span></div>577 <div class="ag-tagline">B2B Intelligence Stack · Shopify Market Scanner</div>578 </div>579 <div class="ag-ts">{datetime.now().strftime("%Y-%m-%d %H:%M")}</div>580 </div>581 """,582 unsafe_allow_html=True,583 )584 585 586# ─────────────────────────────────────────────────────────────────────587# TAB 1 — INTELLIGENCE SCANNER588# ─────────────────────────────────────────────────────────────────────589def render_intel_scanner() -> None:590 st.markdown(591 """592 <div class="ag-card ag-card-accent">593 <div style="font-family:'Barlow Condensed',sans-serif;font-size:22px;594 font-weight:700;letter-spacing:2px;color:#E6EDF3;margin-bottom:3px;">595 INTELLIGENCE SCANNER596 </div>597 <div style="font-family:'IBM Plex Mono',monospace;font-size:11px;color:#8B949E;">598 Shopify Sitemap Analysis · Tier Classification (A = Hot · B = Nurture · C = Discard)599 </div>600 </div>601 """,602 unsafe_allow_html=True,603 )604 605 if MODULE_STATUS.get("intel") is not True:606 module_required_error("intel", "requests pandas beautifulsoup4 lxml")607 return608 609 col_in, col_up = st.columns([3, 2])610 with col_in:611 domain_input = st.text_area(612 "Target Domains",613 placeholder="gymshark.com\nalphalete.com\nnobullproject.com",614 height=130,615 key="intel_domains",616 help="One domain per line · commas or semicolons also work",617 )618 with col_up:619 uploaded_csv = st.file_uploader(620 "or upload CSV (needs TIENDA column)",621 type=["csv"],622 key="intel_csv",623 )624 625 run_btn = st.button(626 "⚡ Execute Intel Scanner",627 use_container_width=True,628 key="run_intel",629 )630 631 if not run_btn:632 return633 634 # ── Gather domains ──────────────────────────────────────────────635 domains = []636 if uploaded_csv:637 try:638 df_up = pd.read_csv(639 uploaded_csv, sep=None, engine="python",640 dtype=str, on_bad_lines="skip",641 )642 df_up.columns = [c.strip() for c in df_up.columns]643 if "TIENDA" not in df_up.columns:644 st.error("CSV must have a 'TIENDA' column.")645 return646 domains = df_up["TIENDA"].dropna().str.strip().tolist()647 except Exception as exc:648 st.error(f"CSV error: {exc}")649 return650 elif domain_input.strip():651 domains = parse_domain_input(domain_input)652 else:653 st.warning("Enter at least one domain.")654 return655 656 if not domains:657 st.warning("No valid domains found.")658 return659 660 # ── Run scan ────────────────────────────────────────────────────661 st.markdown("<br>", unsafe_allow_html=True)662 status_ph = st.empty()663 prog = st.progress(0)664 total = len(domains)665 results = []666 667 for i, domain in enumerate(domains):668 clean = _clean_domain(domain)669 log_line(670 status_ph,671 f"Scanning <b style='color:#E6EDF3'>{clean}</b> "672 f"<span style='color:#4D9EFF'>[{i+1}/{total}]</span>",673 )674 prog.progress(i / max(total, 1))675 676 xml_text = fetch_sitemap(domain)677 if xml_text:678 prods, last_dt = parse_sitemap(xml_text)679 else:680 prods, last_dt = 0, None681 682 tier = score_tier(prods, last_dt)683 last_str = last_dt.strftime("%Y-%m-%d") if last_dt else "N/A"684 685 results.append({686 "Domain": clean,687 "Tier": tier,688 "Products": prods,689 "Last Update": last_str,690 })691 692 prog.progress(1.0)693 log_line(status_ph, f"✓ Scan complete · {total} domains processed", "#00FF7F")694 695 # ── Metrics ─────────────────────────────────────────────────────696 df_r = pd.DataFrame(results)697 st.markdown("<br>", unsafe_allow_html=True)698 tc = df_r["Tier"].value_counts().to_dict()699 c1, c2, c3, c4 = st.columns(4)700 c1.metric("SCANNED", total)701 c2.metric("TIER A · HOT", tc.get("A", 0))702 c3.metric("TIER B · WARM", tc.get("B", 0))703 c4.metric("TIER C · COLD", tc.get("C", 0))704 705 # ── Results table ────────────────────────────────────────────────706 st.markdown("<br><div class='section-label'>Scan Results</div>", unsafe_allow_html=True)707 rows = []708 for _, row in df_r.sort_values(709 "Tier", key=lambda x: x.map({"A": 0, "B": 1, "C": 2})710 ).iterrows():711 t = row["Tier"]712 clr = TIER_COLORS.get(t, "#8B949E")713 rows.append([714 f'<span style="font-family:IBM Plex Mono,monospace;font-size:13px;color:#E6EDF3;">{row["Domain"]}</span>',715 tier_badge(t),716 f'<span style="font-family:IBM Plex Mono,monospace;font-size:13px;color:{clr};font-weight:600;">{row["Products"]}</span>',717 f'<span style="font-family:IBM Plex Mono,monospace;font-size:12px;color:#8B949E;">{row["Last Update"]}</span>',718 ])719 st.markdown(html_table(["Domain", "Tier", "Products", "Last Updated"], rows), unsafe_allow_html=True)720 721 # ── Download ─────────────────────────────────────────────────────722 st.markdown("<br>", unsafe_allow_html=True)723 csv_bytes = df_r.to_csv(index=False, encoding="utf-8-sig").encode()724 st.download_button(725 "↓ Download Results CSV",726 data=csv_bytes,727 file_name=f"ag_intel_{datetime.now().strftime('%Y%m%d_%H%M%S')}.csv",728 mime="text/csv",729 )730 731 732# ─────────────────────────────────────────────────────────────────────733# TAB 2 — LEAD EXTRACTOR734# ─────────────────────────────────────────────────────────────────────735def render_lead_extractor() -> None:736 st.markdown(737 """738 <div class="ag-card ag-card-blue">739 <div style="font-family:'Barlow Condensed',sans-serif;font-size:22px;740 font-weight:700;letter-spacing:2px;color:#E6EDF3;margin-bottom:3px;">741 LEAD EXTRACTOR742 </div>743 <div style="font-family:'IBM Plex Mono',monospace;font-size:11px;color:#8B949E;">744 Contact Page Scraper · Email Harvesting · Generic Address Filter745 </div>746 </div>747 """,748 unsafe_allow_html=True,749 )750 751 if MODULE_STATUS.get("extractor") is not True:752 module_required_error("extractor", "requests")753 return754 755 domain_input = st.text_area(756 "Target Domains",757 placeholder="gymshark.com\nalphalete.com",758 height=110,759 key="extractor_domains",760 help="Crawler checks /, /pages/contact, /pages/about and similar paths",761 )762 col_a, col_b = st.columns(2)763 with col_a:764 filter_generic = st.checkbox("Filter generic emails", value=True, key="ext_filter")765 with col_b:766 pass # reserved for future options767 768 run_btn = st.button("⚡ Extract Leads", use_container_width=True, key="run_extractor")769 770 if not run_btn:771 return772 773 domains = parse_domain_input(domain_input)774 if not domains:775 st.warning("Enter at least one domain.")776 return777 778 st.markdown("<br>", unsafe_allow_html=True)779 prog = st.progress(0)780 status_ph = st.empty()781 782 extractor = ShopifyLeadExtractor(max_stores=len(domains))783 all_leads = []784 total = len(domains)785 786 for i, domain in enumerate(domains):787 url = f"https://{domain}" if not domain.startswith("http") else domain788 log_line(789 status_ph,790 f"Extracting from <b style='color:#E6EDF3'>{domain}</b> "791 f"<span style='color:#4D9EFF'>[{i+1}/{total}]</span>",792 )793 prog.progress(i / max(total, 1))794 795 contact = extractor._extract_contact_info(url)796 emails = contact.get("emails", [])797 if filter_generic:798 emails = [e for e in emails if not extractor._is_generic_email(e)]799 800 for email in emails:801 all_leads.append({802 "Domain": domain,803 "Store Name": contact.get("store_name", ""),804 "Email": email,805 "Extracted": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),806 })807 808 prog.progress(1.0)809 log_line(810 status_ph,811 f"✓ Done · {len(all_leads)} emails found across {total} domains",812 "#00FF7F",813 )814 815 if not all_leads:816 st.info("No emails found. Try disabling the generic filter, or verify the domains are reachable.")817 return818 819 df_leads = pd.DataFrame(all_leads)820 st.markdown("<br>", unsafe_allow_html=True)821 c1, c2 = st.columns(2)822 c1.metric("EMAILS FOUND", len(all_leads))823 c2.metric("DOMAINS HIT", df_leads["Domain"].nunique())824 825 st.markdown("<br><div class='section-label'>Extracted Leads</div>", unsafe_allow_html=True)826 rows = [827 [828 f'<span style="font-family:IBM Plex Mono,monospace;font-size:12px;color:#8B949E;">{r["Domain"]}</span>',829 f'<span style="font-family:IBM Plex Mono,monospace;font-size:13px;color:#4D9EFF;">{r["Email"]}</span>',830 f'<span style="font-family:IBM Plex Mono,monospace;font-size:12px;color:#8B949E;">{r["Store Name"]}</span>',831 ]832 for _, r in df_leads.iterrows()833 ]834 st.markdown(html_table(["Domain", "Email", "Store Name"], rows), unsafe_allow_html=True)835 836 st.markdown("<br>", unsafe_allow_html=True)837 csv_bytes = df_leads.to_csv(index=False, encoding="utf-8-sig").encode()838 st.download_button(839 "↓ Download Leads CSV",840 data=csv_bytes,841 file_name=f"ag_leads_{datetime.now().strftime('%Y%m%d_%H%M%S')}.csv",842 mime="text/csv",843 )844 845 846# ─────────────────────────────────────────────────────────────────────847# TAB 3 — EMAIL VALIDATOR848# ─────────────────────────────────────────────────────────────────────849def render_email_validator() -> None:850 st.markdown(851 """852 <div class="ag-card ag-card-amber">853 <div style="font-family:'Barlow Condensed',sans-serif;font-size:22px;854 font-weight:700;letter-spacing:2px;color:#E6EDF3;margin-bottom:3px;">855 EMAIL VALIDATOR856 </div>857 <div style="font-family:'IBM Plex Mono',monospace;font-size:11px;color:#8B949E;">858 3-Layer Pipeline · Syntax → DNS/MX Lookup → Disposable Blacklist · Zero SMTP859 </div>860 </div>861 """,862 unsafe_allow_html=True,863 )864 865 if MODULE_STATUS.get("validator") is not True:866 module_required_error("validator", "dnspython pandas")867 return868 869 col_in, col_up = st.columns([3, 2])870 with col_in:871 email_input = st.text_area(872 "Emails to validate",873 placeholder="founder@brand.com\nceo@startup.io\ninfo@shop.com",874 height=140,875 key="val_emails",876 )877 with col_up:878 uploaded_csv = st.file_uploader(879 "or upload CSV (needs EMAIL or CORREO column)",880 type=["csv"],881 key="val_csv",882 )883 884 run_btn = st.button("⚡ Run Validation Pipeline", use_container_width=True, key="run_validator")885 886 if not run_btn:887 return888 889 emails: list = []890 if uploaded_csv:891 try:892 df_up = pd.read_csv(uploaded_csv, sep=None, engine="python", dtype=str, on_bad_lines="skip")893 df_up.columns = [c.strip().upper() for c in df_up.columns]894 col_e = next((c for c in ["EMAIL", "CORREO"] if c in df_up.columns), None)895 if not col_e:896 st.error("CSV must have an 'EMAIL' or 'CORREO' column.")897 return898 emails = df_up[col_e].dropna().str.strip().tolist()899 except Exception as exc:900 st.error(f"CSV error: {exc}")901 return902 elif email_input.strip():903 emails = [e.strip() for e in email_input.strip().splitlines() if e.strip()]904 else:905 st.warning("Enter emails or upload a CSV.")906 return907 908 if not emails:909 st.warning("No emails to validate.")910 return911 912 st.markdown("<br>", unsafe_allow_html=True)913 prog = st.progress(0)914 status_ph = st.empty()915 916 validator = EmailValidator()917 results: list = []918 total = len(emails)919 920 for i, email in enumerate(emails):921 log_line(922 status_ph,923 f"Validating <b style='color:#E6EDF3'>{email}</b> "924 f"<span style='color:#4D9EFF'>[{i+1}/{total}]</span>",925 )926 prog.progress(i / max(total, 1))927 status = validator.validate(email)928 results.append({"Email": email, "Status": status})929 930 prog.progress(1.0)931 s = validator.stats932 pct = (s["valid"] / s["total"] * 100) if s["total"] > 0 else 0933 log_line(status_ph, f"✓ Complete · {s['valid']}/{total} valid ({pct:.1f}%)", "#00FF7F")934 935 # Metrics936 df_r = pd.DataFrame(results)937 st.markdown("<br>", unsafe_allow_html=True)938 c1, c2, c3, c4 = st.columns(4)939 c1.metric("TOTAL", s["total"])940 c2.metric("VALID ✓", s["valid"])941 c3.metric("INVALID ✗", s["total"] - s["valid"])942 c4.metric("DNS ERRORS", s["dns_errors"])943 944 # Breakdown945 with st.expander("Validation Breakdown"):946 st.markdown(947 f"""948 <div style="font-family:IBM Plex Mono,monospace;font-size:12px;949 display:grid;grid-template-columns:1fr 60px;gap:6px 16px;padding:0.5rem 0;">950 <span style="color:#8B949E;">Invalid Syntax</span>951 <span style="color:#F85149;">{s['invalid_syntax']}</span>952 <span style="color:#8B949E;">Invalid DNS/MX</span>953 <span style="color:#F85149;">{s['invalid_dns']}</span>954 <span style="color:#8B949E;">Disposable Domain</span>955 <span style="color:#F85149;">{s['invalid_disposable']}</span>956 <span style="color:#8B949E;">Unique Domains Cached</span>957 <span style="color:#E6EDF3;">{len(validator.mx_cache)}</span>958 </div>959 """,960 unsafe_allow_html=True,961 )962 963 # Table964 st.markdown("<br><div class='section-label'>Validation Results</div>", unsafe_allow_html=True)965 rows = []966 for _, row in df_r.iterrows():967 valid = row["Status"] == "Valid"968 clr = "#00FF7F" if valid else "#F85149"969 icon = "✓" if valid else "✗"970 rows.append([971 f'<span style="font-family:IBM Plex Mono,monospace;font-size:13px;color:#E6EDF3;">{row["Email"]}</span>',972 f'<span style="font-family:IBM Plex Mono,monospace;font-size:12px;color:{clr};font-weight:600;">{icon} {row["Status"]}</span>',973 ])974 st.markdown(html_table(["Email", "Status"], rows), unsafe_allow_html=True)975 976 st.markdown("<br>", unsafe_allow_html=True)977 col_dl1, col_dl2 = st.columns(2)978 with col_dl1:979 st.download_button(980 "↓ Full Results CSV",981 data=df_r.to_csv(index=False, encoding="utf-8-sig").encode(),982 file_name=f"ag_validation_full_{datetime.now().strftime('%Y%m%d_%H%M%S')}.csv",983 mime="text/csv",984 )985 with col_dl2:986 df_valid = df_r[df_r["Status"] == "Valid"]987 st.download_button(988 "↓ Valid Only CSV",989 data=df_valid.to_csv(index=False, encoding="utf-8-sig").encode(),990 file_name=f"ag_validation_valid_{datetime.now().strftime('%Y%m%d_%H%M%S')}.csv",991 mime="text/csv",992 )993 994 995# ─────────────────────────────────────────────────────────────────────996# TAB 4 — FULL PIPELINE997# ─────────────────────────────────────────────────────────────────────998def render_full_pipeline() -> None:999 st.markdown(1000 """1001 <div class="ag-card ag-card-accent">1002 <div style="font-family:'Barlow Condensed',sans-serif;font-size:22px;1003 font-weight:700;letter-spacing:2px;color:#E6EDF3;margin-bottom:3px;">1004 FULL B2B PIPELINE1005 </div>1006 <div style="font-family:'IBM Plex Mono',monospace;font-size:11px;color:#8B949E;">1007 Intel Scan → Email Extract → Validate · End-to-End in One Click1008 </div>1009 </div>1010 """,1011 unsafe_allow_html=True,1012 )1013 1014 # Check all required modules1015 required = ["intel", "extractor", "validator"]1016 offline = [k for k in required if MODULE_STATUS.get(k) is not True]1017 if offline:1018 st.error(f"Required modules offline: {', '.join(offline)}")1019 return1020 1021 domain_input = st.text_area(1022 "Target Domains",1023 placeholder="gymshark.com\nalphalete.com\nnobullproject.com",1024 height=120,1025 key="pipe_domains",1026 )1027 col_a, col_b = st.columns(2)1028 with col_a:1029 skip_tier_c = st.checkbox(1030 "Skip Tier C for email extraction",1031 value=True,1032 key="pipe_skip_c",1033 help="Saves time by not scraping stores with 0 products / no sitemap",1034 )1035 with col_b:1036 filter_generic = st.checkbox("Filter generic emails", value=True, key="pipe_filter")1037 1038 run_btn = st.button("⚡ Run Full B2B Pipeline", use_container_width=True, key="run_pipeline")1039 1040 if not run_btn:1041 return1042 1043 domains = parse_domain_input(domain_input)1044 if not domains:1045 st.warning("Enter at least one domain.")1046 return1047 1048 total = len(domains)1049 st.markdown("<br>", unsafe_allow_html=True)1050 1051 # Live log area (single row, updates in place)1052 log_ph = st.empty()1053 prog = st.progress(0)1054 1055 extractor = ShopifyLeadExtractor(max_stores=total)1056 validator = EmailValidator()1057 tier_map: dict = {}1058 email_map: dict = {}1059 1060 # ── Phase 1: Intel ──────────────────────────────────────────────1061 log_line(log_ph, "PHASE 1 / 3 — Intelligence Scan", "#4D9EFF")1062 for i, domain in enumerate(domains):1063 log_line(log_ph, f"[INTEL] {domain} [{i+1}/{total}]")1064 prog.progress(i / (total * 3))1065 1066 xml = fetch_sitemap(domain)1067 prods, last_dt = (parse_sitemap(xml) if xml else (0, None))1068 tier = score_tier(prods, last_dt)1069 tier_map[domain] = {1070 "tier": tier,1071 "products": prods,1072 "last_update": last_dt.strftime("%Y-%m-%d") if last_dt else "N/A",1073 }1074 1075 a_cnt = sum(1 for v in tier_map.values() if v["tier"] == "A")1076 b_cnt = sum(1 for v in tier_map.values() if v["tier"] == "B")1077 c_cnt = sum(1 for v in tier_map.values() if v["tier"] == "C")1078 log_line(log_ph, f"Intel done · A={a_cnt} B={b_cnt} C={c_cnt}", "#00FF7F")1079 1080 # ── Phase 2: Extract ────────────────────────────────────────────1081 log_line(log_ph, "PHASE 2 / 3 — Lead Extraction", "#4D9EFF")1082 to_scrape = [1083 d for d in domains1084 if not (skip_tier_c and tier_map.get(d, {}).get("tier") == "C")1085 ]1086 1087 for i, domain in enumerate(to_scrape):1088 log_line(log_ph, f"[EXTRACT] {domain} [{i+1}/{len(to_scrape)}]")1089 prog.progress((total + i) / (total * 3))1090 1091 url = f"https://{domain}" if not domain.startswith("http") else domain1092 contact = extractor._extract_contact_info(url)1093 emails = contact.get("emails", [])1094 if filter_generic:1095 emails = [e for e in emails if not extractor._is_generic_email(e)]1096 email_map[domain] = emails1097 1098 all_emails_flat = [e for v in email_map.values() for e in v]1099 log_line(log_ph, f"Extraction done · {len(all_emails_flat)} emails found", "#00FF7F")1100 1101 # ── Phase 3: Validate ───────────────────────────────────────────1102 log_line(log_ph, "PHASE 3 / 3 — Email Validation", "#4D9EFF")1103 email_validity: dict = {}1104 for i, email in enumerate(all_emails_flat):1105 log_line(log_ph, f"[VALIDATE] {email} [{i+1}/{len(all_emails_flat)}]")1106 prog.progress((total * 2 + i) / max(total * 3, 1))1107 email_validity[email] = validator.validate(email)1108 1109 prog.progress(1.0)1110 v_count = sum(1 for v in email_validity.values() if v == "Valid")1111 log_line(log_ph, f"✓ Pipeline complete · {v_count}/{len(all_emails_flat)} emails valid", "#00FF7F")1112 1113 # ── Build output DataFrame ───────────────────────────────────────1114 final: list = []1115 for domain in domains:1116 intel = tier_map.get(domain, {"tier": "C", "products": 0, "last_update": "N/A"})1117 emails = email_map.get(domain, [])1118 if emails:1119 for email in emails:1120 final.append({1121 "Domain": domain,1122 "Tier": intel["tier"],1123 "Products": intel["products"],1124 "Last Update": intel["last_update"],1125 "Email": email,1126 "Email Status": email_validity.get(email, "—"),1127 })1128 else:1129 final.append({1130 "Domain": domain,1131 "Tier": intel["tier"],1132 "Products": intel["products"],1133 "Last Update": intel["last_update"],1134 "Email": "—",1135 "Email Status": "—",1136 })1137 1138 df_final = pd.DataFrame(final)1139 1140 # Metrics1141 st.markdown("<br>", unsafe_allow_html=True)1142 c1, c2, c3, c4 = st.columns(4)1143 c1.metric("DOMAINS", total)1144 c2.metric("TIER A LEADS", a_cnt)1145 c3.metric("EMAILS FOUND", len(all_emails_flat))1146 c4.metric("EMAILS VALID", v_count)1147 1148 # Table1149 st.markdown("<br><div class='section-label'>Pipeline Output</div>", unsafe_allow_html=True)1150 rows = []1151 for _, row in df_final.sort_values(1152 "Tier", key=lambda x: x.map({"A": 0, "B": 1, "C": 2})1153 ).iterrows():1154 t = row["Tier"]1155 clr = TIER_COLORS.get(t, "#8B949E")1156 email_clr = "#4D9EFF" if row["Email"] != "—" else "#30363D"1157 valid_clr = ("#00FF7F" if row["Email Status"] == "Valid"1158 else "#F85149" if row["Email Status"] == "Invalid" else "#30363D")1159 rows.append([1160 f'<span style="font-family:IBM Plex Mono,monospace;font-size:12px;color:#8B949E;">{row["Domain"]}</span>',1161 tier_badge(t),1162 f'<span style="font-family:IBM Plex Mono,monospace;font-size:12px;color:{email_clr};">{row["Email"]}</span>',1163 f'<span style="font-family:IBM Plex Mono,monospace;font-size:11px;color:{valid_clr};">{row["Email Status"]}</span>',1164 f'<span style="font-family:IBM Plex Mono,monospace;font-size:12px;color:{clr};">{row["Products"]}</span>',1165 ])1166 st.markdown(1167 html_table(["Domain", "Tier", "Email", "Status", "Products"], rows),1168 unsafe_allow_html=True,1169 )1170 1171 st.markdown("<br>", unsafe_allow_html=True)1172 st.download_button(1173 "↓ Download Full Pipeline CSV",1174 data=df_final.to_csv(index=False, encoding="utf-8-sig").encode(),1175 file_name=f"ag_pipeline_{datetime.now().strftime('%Y%m%d_%H%M%S')}.csv",1176 mime="text/csv",1177 )1178 1179 1180# ─────────────────────────────────────────────────────────────────────1181# TAB 5 — SYSTEM INFO1182# ─────────────────────────────────────────────────────────────────────1183def render_system_info() -> None:1184 st.markdown(1185 """1186 <div class="ag-card ag-card-accent">1187 <div style="font-family:'Barlow Condensed',sans-serif;font-size:22px;1188 font-weight:700;letter-spacing:2px;color:#E6EDF3;margin-bottom:3px;">1189 SYSTEM MODULES1190 </div>1191 <div style="font-family:'IBM Plex Mono',monospace;font-size:11px;color:#8B949E;">1192 Module Registry · Dependency Matrix · CLI Reference1193 </div>1194 </div>1195 """,1196 unsafe_allow_html=True,1197 )1198 1199 module_registry = [1200 {