Soulay/customs-compass
0
1"""Customs Compass - AI assistant for US sales tax, customs duties, and product compliance.2 3Single-file Streamlit application for Chinese SMEs (hardware, batteries, robotics,4electronics) exporting to the United States.5"""6 7from __future__ import annotations8 9import json10import os11import re12from html.parser import HTMLParser13from pathlib import Path14from typing import Optional15 16import pandas as pd17import requests18import streamlit as st19 20try:21 from dotenv import load_dotenv22 load_dotenv(Path(__file__).parent / ".env")23except ImportError:24 pass25 26 27# =============================================================================28# Section A — Configuration29# =============================================================================30 31st.set_page_config(32 page_title="Customs Compass — AI Trade Compliance",33 page_icon="🧭",34 layout="wide",35 initial_sidebar_state="expanded",36)37 38OLLAMA_URL = os.getenv("OLLAMA_URL", "http://localhost:11434")39OLLAMA_MODEL = os.getenv("OLLAMA_MODEL", "llama3.2:3b")40OLLAMA_TIMEOUT = int(os.getenv("OLLAMA_TIMEOUT", "180")) # 3 min — cold model load can be slow on CPU41OLLAMA_PROBE_TIMEOUT = 242 43# OrbitAI — used for premium agent features (market intel, localization,44# GTM roadmap). The main Q&A flow still uses local Ollama for privacy/cost.45ORBITAI_API_KEY = os.getenv("ORBITAI_API_KEY", "")46ORBITAI_BASE_URL = os.getenv("ORBITAI_BASE_URL", "https://api.orbitai.global/v1")47ORBITAI_MODEL = os.getenv("ORBITAI_MODEL", "gpt-5.4")48ORBITAI_TIMEOUT = 6049 50CBP_NEWSROOM_URL = "https://www.cbp.gov/newsroom"51CBP_TRADE_URL = "https://www.cbp.gov/trade"52NEWS_FETCH_TIMEOUT = 553NEWS_CACHE_TTL = 3600 # 1 hour54 55USER_AGENT = "CustomsCompass/1.0 (Educational)"56 57DATA_DIR = Path(__file__).parent58 59# US customs fees (FY2026 rates)60MPF_RATE = 0.003464 # Merchandise Processing Fee: 0.3464% ad valorem61MPF_MIN = 32.71 # USD62MPF_MAX = 634.62 # USD63HMF_RATE = 0.00125 # Harbor Maintenance Fee: 0.125% (sea freight only)64 65# Section 301 List-4A surcharge on most Chinese electronics66SECTION_301_RATE = 0.25 # 25% additional ad valorem67 68SYSTEM_PROMPT = (69 "You are Customs Compass, an AI assistant specialized in US sales tax, "70 "nexus thresholds, customs duties, and product compliance. Rules: "71 "1) Use only the provided CSV data. "72 "2) If answer not in data, say 'I don't have enough information.' "73 "3) Ask clarifying questions if product category is unclear. "74 "4) Always cite source (e.g., 'nexus_thresholds.csv'). "75 "5) Answer in [English] then [中文]. "76 "6) Provide checklist + risk flags + sources."77)78 79STATE_ABBREVIATIONS = {80 "AL": "Alabama", "AK": "Alaska", "AZ": "Arizona", "AR": "Arkansas",81 "CA": "California", "CO": "Colorado", "CT": "Connecticut", "DE": "Delaware",82 "FL": "Florida", "GA": "Georgia", "HI": "Hawaii", "ID": "Idaho",83 "IL": "Illinois", "IN": "Indiana", "IA": "Iowa", "KS": "Kansas",84 "KY": "Kentucky", "LA": "Louisiana", "ME": "Maine", "MD": "Maryland",85 "MA": "Massachusetts", "MI": "Michigan", "MN": "Minnesota", "MS": "Mississippi",86 "MO": "Missouri", "MT": "Montana", "NE": "Nebraska", "NV": "Nevada",87 "NH": "New Hampshire", "NJ": "New Jersey", "NM": "New Mexico", "NY": "New York",88 "NC": "North Carolina", "ND": "North Dakota", "OH": "Ohio", "OK": "Oklahoma",89 "OR": "Oregon", "PA": "Pennsylvania", "RI": "Rhode Island", "SC": "South Carolina",90 "SD": "South Dakota", "TN": "Tennessee", "TX": "Texas", "UT": "Utah",91 "VT": "Vermont", "VA": "Virginia", "WA": "Washington", "WV": "West Virginia",92 "WI": "Wisconsin", "WY": "Wyoming", "DC": "District of Columbia",93}94 95# Keywords mapped to HTS categories. The right side must match product_category in hts_duty_codes.csv.96PRODUCT_KEYWORDS = {97 "battery_with_charger": ["power bank", "power banks", "charger", "rechargeable battery pack"],98 "battery_only": ["battery", "batteries", "lithium", "li-ion", "lipo", "lifepo4", "cell"],99 "robotics_with_radio": ["robot", "robotics", "robotic arm", "agv", "amr"],100 "consumer_electronics": ["consumer electronics", "gadget", "electronics"],101 "medical_device": ["medical", "thermometer", "blood pressure", "pulse oximeter", "diagnostic"],102 "industrial_machinery": ["industrial machine", "cnc", "lathe", "press", "factory equipment"],103 "power_tools": ["power tool", "drill", "saw", "grinder", "impact driver"],104 "led_lighting": ["led", "lighting", "lamp", "bulb", "light fixture"],105 "drones": ["drone", "drones", "uav", "quadcopter"],106 "solar_panels": ["solar panel", "solar panels", "pv module", "photovoltaic"],107 "smart_home_devices": ["smart home", "smart plug", "smart bulb", "smart switch", "iot device"],108 "ev_charger": ["ev charger", "ev charging", "electric vehicle charger", "level 2 charger"],109 "wearables": ["smartwatch", "smart watch", "fitness tracker", "wearable"],110 "audio_equipment": ["speaker", "speakers", "headphone", "headphones", "earbuds", "audio equipment"],111}112 113NO_TAX_STATES = {"Oregon", "Delaware", "Montana", "New Hampshire", "Alaska"}114 115 116# =============================================================================117# Section B — Data Loading (cached)118# =============================================================================119 120NEXUS_COLS = ["state", "threshold_usd", "transaction_rule", "notes"]121HTS_COLS = ["product_category", "hts_code", "duty_rate", "fcc_needed", "ul_needed", "fda_needed", "notes"]122CBP_ALERTS_COLS = ["category", "title", "summary", "relevant_products", "country_focus", "severity", "action_required", "source_url"]123 124# Common English stopwords — filtered from RAG queries to avoid noise.125_STOPWORDS = frozenset({126 "a", "an", "and", "are", "as", "at", "be", "by", "for", "from", "has", "have",127 "he", "in", "is", "it", "its", "of", "on", "or", "that", "the", "to", "was",128 "were", "will", "with", "this", "these", "those", "we", "us", "our", "you",129 "your", "they", "them", "their", "i", "me", "my", "do", "does", "did", "if",130 "but", "not", "no", "any", "some", "all", "can", "could", "should", "would",131 "may", "might", "must", "shall", "what", "when", "where", "who", "how", "why",132 "which", "about", "into", "than", "then", "there", "here", "also", "more",133 "less", "very", "much", "many", "such", "so", "up", "down", "out", "over",134 "under", "again", "further", "once", "been", "being", "had", "having", "am",135 "now", "just", "only", "own", "same", "too", "ll", "re", "ve", "d", "s", "t",136})137 138 139@st.cache_data(show_spinner=False)140def load_nexus_thresholds() -> pd.DataFrame:141 path = DATA_DIR / "nexus_thresholds.csv"142 if not path.exists():143 return pd.DataFrame(columns=NEXUS_COLS)144 df = pd.read_csv(path).fillna("")145 missing = [c for c in NEXUS_COLS if c not in df.columns]146 if missing:147 raise ValueError(f"nexus_thresholds.csv missing columns: {missing}")148 df["threshold_usd"] = pd.to_numeric(df["threshold_usd"], errors="coerce").fillna(0)149 return df150 151 152@st.cache_data(show_spinner=False)153def load_hts_duty_codes() -> pd.DataFrame:154 path = DATA_DIR / "hts_duty_codes.csv"155 if not path.exists():156 return pd.DataFrame(columns=HTS_COLS)157 df = pd.read_csv(path).fillna("")158 missing = [c for c in HTS_COLS if c not in df.columns]159 if missing:160 raise ValueError(f"hts_duty_codes.csv missing columns: {missing}")161 return df162 163 164@st.cache_data(show_spinner=False)165def load_tax_rates() -> dict:166 path = DATA_DIR / "tax_rates_by_state.json"167 if not path.exists():168 return {}169 return json.loads(path.read_text(encoding="utf-8"))170 171 172@st.cache_data(show_spinner=False)173def load_cbp_alerts() -> pd.DataFrame:174 """Static CBP enforcement & tariff alerts (Section 301, UFLPA, AD/CVD, etc.)."""175 path = DATA_DIR / "cbp_alerts.csv"176 if not path.exists():177 return pd.DataFrame(columns=CBP_ALERTS_COLS)178 df = pd.read_csv(path).fillna("")179 missing = [c for c in CBP_ALERTS_COLS if c not in df.columns]180 if missing:181 raise ValueError(f"cbp_alerts.csv missing columns: {missing}")182 return df183 184 185# ---------- CBP RAG (chunks from cbp_chunks.jsonl) -----------------------186 187import math188from collections import Counter189 190_TOKEN_RE = re.compile(r"[a-z0-9]+")191 192# Page-title patterns for low-value index/listing pages we want to exclude193# from the RAG corpus. These pages are just lists of references (FR numbers,194# bulletin titles, etc.) and don't contain substantive guidance text.195_NOISE_TITLE_PATTERNS = (196 "Federal Register Notices",197 "Customs Bulletin and Decisions",198 "Notices of Action",199 "Quota Bulletins",200 "CBP Trade-Related",201)202 203 204def _is_noise_chunk(rec: dict) -> bool:205 """Heuristic: skip chunks from low-information index/listing pages.206 207 A chunk is considered noise if:208 - its page title matches one of the index-page patterns, OR209 - the text is dominated by Federal Register references (e.g. "85 FR 15714")210 — more than 4 such references signals a list page.211 """212 title = rec.get("title", "") or ""213 if any(pat in title for pat in _NOISE_TITLE_PATTERNS):214 return True215 text = rec.get("text", "") or ""216 fr_refs = re.findall(r"\b\d{2,3}\s+FR\s+\d{4,6}\b", text)217 if len(fr_refs) >= 4:218 return True219 return False220 221 222def _tokenize(text: str) -> list[str]:223 """Lowercase, extract alphanumeric tokens, drop stopwords & short tokens."""224 return [t for t in _TOKEN_RE.findall(text.lower()) if len(t) > 2 and t not in _STOPWORDS]225 226 227@st.cache_resource(show_spinner=False)228def load_cbp_chunks_index() -> dict:229 """Load cbp_chunks.jsonl + precompute TF (per chunk) and IDF (global).230 231 Returned dict has:232 - chunks: list[dict] (raw chunk records)233 - tfs: list[Counter] (per-chunk term frequencies)234 - idf: dict[str, float] (term -> inverse doc frequency)235 - N: int (total chunks)236 - avg_len: float (avg token count per chunk, for BM25 length norm)237 - lens: list[int] (per-chunk token count)238 """239 path = DATA_DIR / "cbp_chunks.jsonl"240 if not path.exists():241 return {"chunks": [], "tfs": [], "idf": {}, "N": 0, "avg_len": 0.0, "lens": []}242 243 chunks: list[dict] = []244 tfs: list[Counter] = []245 title_tokens_list: list[set[str]] = []246 lens: list[int] = []247 doc_freq: Counter = Counter()248 249 skipped_noise = 0250 with path.open(encoding="utf-8") as fh:251 for line in fh:252 line = line.strip()253 if not line:254 continue255 try:256 rec = json.loads(line)257 except json.JSONDecodeError:258 continue259 if _is_noise_chunk(rec):260 skipped_noise += 1261 continue262 text = rec.get("text", "")263 tokens = _tokenize(text)264 title_tokens = set(_tokenize(rec.get("title", "")))265 tf = Counter(tokens)266 chunks.append(rec)267 tfs.append(tf)268 title_tokens_list.append(title_tokens)269 lens.append(len(tokens))270 for term in tf.keys():271 doc_freq[term] += 1272 273 N = len(chunks)274 avg_len = (sum(lens) / N) if N else 0.0275 idf = {276 term: math.log(1 + (N - df + 0.5) / (df + 0.5))277 for term, df in doc_freq.items()278 }279 return {280 "chunks": chunks,281 "tfs": tfs,282 "title_tokens": title_tokens_list,283 "idf": idf,284 "N": N,285 "avg_len": avg_len,286 "lens": lens,287 "skipped_noise": skipped_noise,288 }289 290 291def search_cbp_chunks(292 query: str,293 index: dict,294 products: list[str],295 states: list[str],296 top_k: int = 3,297 k1: float = 1.5,298 b: float = 0.75,299) -> list[dict]:300 """BM25 retrieval over CBP chunks, deduplicated to one chunk per parent page."""301 chunks = index["chunks"]302 if not chunks:303 return []304 305 # Build query terms: tokenize question + add product category words + state names306 q_tokens = _tokenize(query)307 for p in products:308 q_tokens.extend(_tokenize(p.replace("_", " ")))309 for s in states:310 q_tokens.extend(_tokenize(s))311 if not q_tokens:312 return []313 314 q_set = set(q_tokens)315 tfs = index["tfs"]316 idf = index["idf"]317 lens = index["lens"]318 avg_len = index["avg_len"] or 1.0319 320 title_tokens_list = index.get("title_tokens", [])321 title_boost = 2.5 # multiply IDF when query term appears in the page title322 323 scored: list[tuple[float, int]] = []324 for i, tf in enumerate(tfs):325 score = 0.0326 dl = lens[i]327 norm = 1 - b + b * (dl / avg_len)328 title_set = title_tokens_list[i] if i < len(title_tokens_list) else set()329 for term in q_set:330 f = tf.get(term, 0)331 if not f:332 continue333 term_idf = idf.get(term, 0.0)334 if term in title_set:335 term_idf *= title_boost336 score += term_idf * (f * (k1 + 1)) / (f + k1 * norm)337 if score > 0:338 scored.append((score, i))339 340 scored.sort(key=lambda t: -t[0])341 342 seen_parents: set[str] = set()343 results: list[dict] = []344 for score, i in scored:345 chunk = chunks[i]346 parent = chunk.get("parent_id") or chunk.get("chunk_id")347 if parent in seen_parents:348 continue349 seen_parents.add(parent)350 # Keep the full chunk text for inline display, plus a shorter excerpt351 # for prompt injection (LLM context budget).352 full_text = (chunk.get("text") or "").strip().replace("\n", " ")353 # Collapse repeated whitespace354 full_text = re.sub(r"\s+", " ", full_text)355 if len(full_text) > 700:356 excerpt = full_text[:700].rsplit(" ", 1)[0] + "…"357 else:358 excerpt = full_text359 results.append({360 "title": chunk.get("title", ""),361 "url": chunk.get("url", ""),362 "section": chunk.get("section", ""),363 "page_type": chunk.get("page_type", ""),364 "published_date": chunk.get("published_date", ""),365 "excerpt": excerpt, # short — used in prompt366 "full_text": full_text, # full — shown in UI367 "score": round(score, 2),368 "chunk_id": chunk.get("chunk_id", ""),369 })370 if len(results) >= top_k:371 break372 373 return results374 375 376def find_relevant_alerts(377 alerts_df: pd.DataFrame,378 products: list[str],379 question: str,380) -> pd.DataFrame:381 """Return alerts matching detected products or 'all'-scoped Critical alerts."""382 if alerts_df.empty:383 return alerts_df384 question_lower = question.lower()385 mentions_china = "china" in question_lower or "chinese" in question_lower or "中国" in question386 387 def row_matches(row: pd.Series) -> bool:388 rel = str(row.get("relevant_products", "")).lower()389 # Always show Critical alerts that scope to "all"390 if row.get("severity") == "Critical" and "all" in rel:391 return True392 # Match by product overlap393 if products:394 rel_set = {p.strip() for p in rel.split(",") if p.strip()}395 if rel_set & set(products):396 return True397 if "all" in rel_set:398 return True399 # If user mentions China and alert is China-focused, include high-severity ones400 if mentions_china and str(row.get("country_focus", "")).lower() == "china":401 if row.get("severity") in ("Critical", "High"):402 return True403 return False404 405 mask = alerts_df.apply(row_matches, axis=1)406 matched = alerts_df[mask]407 # Sort: Critical → High → Medium → Info408 severity_order = {"Critical": 0, "High": 1, "Medium": 2, "Info": 3}409 matched = matched.assign(410 _sev_rank=matched["severity"].map(lambda s: severity_order.get(s, 99))411 ).sort_values("_sev_rank").drop(columns=["_sev_rank"])412 return matched.head(5)413 414 415# =============================================================================416# Section C — CBP News Fetcher417# =============================================================================418 419class _CBPNewsParser(HTMLParser):420 """Extracts anchor links and text from cbp.gov HTML.421 422 The CBP newsroom uses a Drupal site; news article links generally live under423 paths containing 'newsroom' or 'news-release'. We collect every <a> and filter424 afterwards rather than relying on a brittle CSS selector.425 """426 427 def __init__(self) -> None:428 super().__init__()429 self.links: list[dict] = []430 self._current_href: Optional[str] = None431 self._current_text: list[str] = []432 433 def handle_starttag(self, tag: str, attrs: list[tuple[str, Optional[str]]]) -> None:434 if tag == "a":435 for k, v in attrs:436 if k == "href" and v:437 self._current_href = v438 self._current_text = []439 return440 441 def handle_endtag(self, tag: str) -> None:442 if tag == "a" and self._current_href is not None:443 text = " ".join(self._current_text).strip()444 if text and len(text) > 15: # filter noise (e.g., "Home", "Login")445 self.links.append({"href": self._current_href, "text": text})446 self._current_href = None447 self._current_text = []448 449 def handle_data(self, data: str) -> None:450 if self._current_href is not None:451 self._current_text.append(data.strip())452 453 454@st.cache_data(ttl=NEWS_CACHE_TTL, show_spinner=False)455def fetch_cbp_news() -> list[dict]:456 """Fetch the latest CBP newsroom headlines. Returns [] on any failure."""457 try:458 resp = requests.get(459 CBP_NEWSROOM_URL,460 headers={"User-Agent": USER_AGENT},461 timeout=NEWS_FETCH_TIMEOUT,462 )463 if resp.status_code != 200:464 return []465 parser = _CBPNewsParser()466 parser.feed(resp.text)467 except (requests.RequestException, ValueError):468 return []469 470 items: list[dict] = []471 seen_titles: set[str] = set()472 for link in parser.links:473 href = link["href"]474 text = link["text"]475 # Filter to CBP article-like URLs476 if not any(token in href for token in ("/newsroom/", "/news-release", "/spotlights/")):477 continue478 if text in seen_titles:479 continue480 seen_titles.add(text)481 # Normalize URL482 if href.startswith("/"):483 href = "https://www.cbp.gov" + href484 items.append({"title": text, "url": href})485 if len(items) >= 20:486 break487 return items488 489 490def filter_news_by_query(491 news_items: list[dict],492 question: str,493 products: list[str],494 states: list[str],495) -> list[dict]:496 """Return news items relevant to the user query (max 5)."""497 if not news_items:498 return []499 haystack_terms = set()500 haystack_terms.update(s.lower() for s in states)501 for p in products:502 haystack_terms.update(p.replace("_", " ").lower().split())503 for word in re.findall(r"[a-zA-Z]{4,}", question.lower()):504 haystack_terms.add(word)505 506 scored: list[tuple[int, dict]] = []507 for item in news_items:508 title_lower = item["title"].lower()509 score = sum(1 for term in haystack_terms if term in title_lower)510 if score > 0:511 scored.append((score, item))512 scored.sort(key=lambda t: -t[0])513 return [item for _, item in scored[:5]]514 515 516# =============================================================================517# Section D — Retrieval Helpers518# =============================================================================519 520def extract_states(text: str) -> list[str]:521 found: set[str] = set()522 text_lower = text.lower()523 for full in STATE_ABBREVIATIONS.values():524 if full.lower() in text_lower:525 found.add(full)526 for abbr, full in STATE_ABBREVIATIONS.items():527 if re.search(rf"\b{abbr}\b", text):528 found.add(full)529 return sorted(found)530 531 532def extract_product_categories(text: str, hts_df: pd.DataFrame) -> list[str]:533 text_lower = text.lower()534 found: list[str] = []535 available = set(hts_df["product_category"].tolist()) if not hts_df.empty else set()536 for category, keywords in PRODUCT_KEYWORDS.items():537 if category not in available:538 continue539 if any(kw in text_lower for kw in keywords):540 if category not in found:541 found.append(category)542 return found543 544 545def extract_sales_amount(text: str) -> Optional[float]:546 """Parse '$200k', '200,000', '$1.5M', '500000' from text. Returns USD amount."""547 cleaned = text.replace(",", "")548 pattern = r"\$?\s*(\d+(?:\.\d+)?)\s*([kKmM])?"549 best: Optional[float] = None550 for match in re.finditer(pattern, cleaned):551 num_str, suffix = match.group(1), match.group(2)552 try:553 val = float(num_str)554 except ValueError:555 continue556 if suffix in ("k", "K"):557 val *= 1_000558 elif suffix in ("m", "M"):559 val *= 1_000_000560 # Heuristic: only accept values that look like sales figures561 if val < 1_000:562 continue563 if best is None or val > best:564 best = val565 return best566 567 568def build_context(569 question: str,570 nexus_df: pd.DataFrame,571 hts_df: pd.DataFrame,572 tax_rates: dict,573 news_items: list[dict],574 alerts_df: Optional[pd.DataFrame] = None,575 chunk_index: Optional[dict] = None,576) -> tuple[str, dict]:577 """Assemble the retrieval context block plus a structured payload."""578 states = extract_states(question)579 products = extract_product_categories(question, hts_df)580 sales = extract_sales_amount(question)581 582 relevant_news = filter_news_by_query(news_items, question, products, states)583 relevant_alerts = (584 find_relevant_alerts(alerts_df, products, question)585 if alerts_df is not None and not alerts_df.empty586 else pd.DataFrame(columns=CBP_ALERTS_COLS)587 )588 relevant_chunks = (589 search_cbp_chunks(question, chunk_index, products, states, top_k=3)590 if chunk_index and chunk_index.get("N")591 else []592 )593 594 lines: list[str] = ["## Retrieved Knowledge Base Data", ""]595 596 if states:597 lines.append("### Nexus thresholds (source: nexus_thresholds.csv)")598 nexus_subset = nexus_df[nexus_df["state"].isin(states)]599 for _, row in nexus_subset.iterrows():600 lines.append(601 f"- **{row['state']}**: threshold ${int(row['threshold_usd']):,} | "602 f"transactions: {row['transaction_rule']} | notes: {row['notes']}"603 )604 lines.append("")605 lines.append("### State sales tax rates (source: tax_rates_by_state.json)")606 for s in states:607 rate = tax_rates.get(s)608 if rate is not None:609 lines.append(f"- **{s}**: {rate}%")610 lines.append("")611 612 if products:613 lines.append("### HTS / compliance (source: hts_duty_codes.csv)")614 hts_subset = hts_df[hts_df["product_category"].isin(products)]615 for _, row in hts_subset.iterrows():616 lines.append(617 f"- **{row['product_category']}**: HTS {row['hts_code']} | "618 f"duty {row['duty_rate']} | FCC: {row['fcc_needed']} | "619 f"UL: {row['ul_needed']} | FDA: {row['fda_needed']} | {row['notes']}"620 )621 lines.append("")622 623 if sales is not None:624 lines.append(f"### Detected sales amount: ${sales:,.0f} USD")625 lines.append("")626 627 if not relevant_alerts.empty:628 lines.append("### CBP enforcement & tariff alerts (source: cbp_alerts.csv)")629 for _, row in relevant_alerts.iterrows():630 lines.append(631 f"- **[{row['severity']}] {row['category']} — {row['title']}**: "632 f"{row['summary']} _Action:_ {row['action_required']} "633 f"({row['source_url']})"634 )635 lines.append("")636 637 if relevant_chunks:638 lines.append("### CBP knowledge base excerpts (source: cbp_chunks.jsonl)")639 for c in relevant_chunks:640 lines.append(f"- **{c['title']}** ({c['url']})")641 lines.append(f" > {c['excerpt']}")642 lines.append("")643 644 if relevant_news:645 lines.append("### Recent CBP news (source: cbp.gov/newsroom)")646 for item in relevant_news:647 lines.append(f"- {item['title']} — {item['url']}")648 lines.append("")649 650 if not states and not products and not relevant_news and relevant_alerts.empty and not relevant_chunks:651 lines.append("_No matching state, product, alert, news, or CBP excerpt found._")652 653 payload = {654 "states": states,655 "products": products,656 "sales_usd": sales,657 "news": relevant_news,658 "alerts": relevant_alerts.to_dict(orient="records"),659 "chunks": relevant_chunks,660 }661 return "\n".join(lines), payload662 663 664# =============================================================================665# Section E — Risk Assessment666# =============================================================================667 668def assess_nexus_risk(sales_usd: Optional[float], threshold_usd: float) -> str:669 if sales_usd is None or threshold_usd <= 0:670 return "Unknown"671 ratio = sales_usd / threshold_usd672 if ratio >= 1.0:673 return "High"674 if ratio >= 0.7:675 return "Medium"676 return "Low"677 678 679def assess_compliance_risk(product_row: pd.Series) -> list[str]:680 flags: list[str] = []681 if str(product_row.get("fcc_needed", "")).strip().lower() == "yes":682 flags.append("FCC certification required")683 if str(product_row.get("ul_needed", "")).strip().lower() == "yes":684 flags.append("UL certification required")685 if str(product_row.get("fda_needed", "")).strip().lower() == "yes":686 flags.append("FDA clearance required")687 return flags688 689 690# =============================================================================691# Section F — LLM Integration692# =============================================================================693 694@st.cache_data(ttl=60, show_spinner=False)695def is_ollama_available() -> bool:696 try:697 resp = requests.get(f"{OLLAMA_URL}/api/tags", timeout=OLLAMA_PROBE_TIMEOUT)698 return resp.status_code == 200699 except requests.RequestException:700 return False701 702 703def call_ollama(system_prompt: str, user_prompt: str) -> str:704 payload = {705 "model": OLLAMA_MODEL,706 "messages": [707 {"role": "system", "content": system_prompt},708 {"role": "user", "content": user_prompt},709 ],710 "stream": False,711 "keep_alive": "10m", # keep model loaded in RAM between calls712 "options": {713 "temperature": 0.2,714 "num_predict": 600, # cap response length so it returns quickly715 "num_ctx": 4096, # context window716 },717 }718 resp = requests.post(719 f"{OLLAMA_URL}/api/chat",720 json=payload,721 timeout=OLLAMA_TIMEOUT,722 )723 resp.raise_for_status()724 data = resp.json()725 return data.get("message", {}).get("content", "").strip()726 727 728# --- OrbitAI client (OpenAI-compatible) ---------------------------------------729# Used for premium agent features that need a stronger model than llama3.2:3b730# (market intelligence, localization, GTM roadmap generation, document analysis).731 732def is_orbitai_configured() -> bool:733 return bool(ORBITAI_API_KEY) and ORBITAI_API_KEY.startswith("sk-")734 735 736def call_orbitai(737 system_prompt: str,738 user_prompt: str,739 model: Optional[str] = None,740 temperature: float = 0.4,741) -> str:742 """Call OrbitAI's OpenAI-compatible chat completions endpoint.743 744 Raises requests.RequestException on network errors. Callers should catch745 and fall back gracefully (typically to Ollama or a deterministic template).746 """747 if not is_orbitai_configured():748 raise RuntimeError("ORBITAI_API_KEY is not set; cannot call OrbitAI.")749 payload = {750 "model": model or ORBITAI_MODEL,751 "messages": [752 {"role": "system", "content": system_prompt},753 {"role": "user", "content": user_prompt},754 ],755 "temperature": temperature,756 "stream": False,757 }758 headers = {759 "Authorization": f"Bearer {ORBITAI_API_KEY}",760 "Content-Type": "application/json",761 "User-Agent": USER_AGENT,762 }763 resp = requests.post(764 f"{ORBITAI_BASE_URL.rstrip('/')}/chat/completions",765 json=payload,766 headers=headers,767 timeout=ORBITAI_TIMEOUT,768 )769 resp.raise_for_status()770 data = resp.json()771 choices = data.get("choices") or []772 if not choices:773 return ""774 return (choices[0].get("message") or {}).get("content", "").strip()775 776 777# --- Fallback engine -----------------------------------------------------------778 779RISK_BADGE = {780 "Low": "🟢 Low",781 "Medium": "🟡 Medium",782 "High": "🔴 High",783 "Unknown": "⚪ Unknown",784}785 786RISK_BADGE_CN = {787 "Low": "🟢 低",788 "Medium": "🟡 中",789 "High": "🔴 高",790 "Unknown": "⚪ 未知",791}792 793 794def call_fallback(795 question: str,796 payload: dict,797 nexus_df: pd.DataFrame,798 hts_df: pd.DataFrame,799 tax_rates: dict,800) -> str:801 """Build a deterministic bilingual response using only CSV lookups (no LLM)."""802 states = payload["states"]803 products = payload["products"]804 sales = payload["sales_usd"]805 806 en_lines: list[str] = ["## [English]"]807 cn_lines: list[str] = ["## [中文]"]808 sources: set[str] = set()809 overall_risk = "Low"810 811 if not states and not products:812 en_lines.append("I don't have enough information. Please mention a US state and/or a product category (e.g., 'lithium batteries to Texas').")813 cn_lines.append("信息不足。请提供一个美国州名和/或产品类别（例如：销往德克萨斯州的锂电池）。")814 return "\n".join(en_lines + [""] + cn_lines)815 816 # ---- Nexus / sales tax ----817 if states:818 en_lines.append("### Sales tax & nexus checklist")819 cn_lines.append("### 销售税与经济关联检查清单")820 for state in states:821 row = nexus_df[nexus_df["state"] == state]822 if row.empty:823 en_lines.append(f"- **{state}**: I don't have enough information.")824 cn_lines.append(f"- **{state}**：信息不足。")825 continue826 sources.add("nexus_thresholds.csv")827 threshold = float(row.iloc[0]["threshold_usd"])828 tx_rule = row.iloc[0]["transaction_rule"]829 rate = tax_rates.get(state)830 risk = assess_nexus_risk(sales, threshold)831 832 if state in NO_TAX_STATES or threshold == 0:833 en_lines.append(834 f"- **{state}**: No state sales tax obligation. Risk: {RISK_BADGE['Low']}."835 )836 cn_lines.append(837 f"- **{state}**：无州销售税义务。风险：{RISK_BADGE_CN['Low']}。"838 )839 else:840 threshold_str = f"${int(threshold):,}"841 if sales is not None:842 if risk == "High":843 en_lines.append(844 f"- **{state}**: Sales (${sales:,.0f}) exceed the {threshold_str} threshold — "845 f"you MUST register and collect sales tax (rate: {rate}%). Risk: {RISK_BADGE['High']}."846 )847 cn_lines.append(848 f"- **{state}**：销售额（${sales:,.0f}）超过 {threshold_str} 门槛 — "849 f"必须注册并征收销售税（税率：{rate}%）。风险：{RISK_BADGE_CN['High']}。"850 )851 overall_risk = "High"852 elif risk == "Medium":853 en_lines.append(854 f"- **{state}**: Sales (${sales:,.0f}) approaching the {threshold_str} threshold. "855 f"Monitor closely. Rate when triggered: {rate}%. Risk: {RISK_BADGE['Medium']}."856 )857 cn_lines.append(858 f"- **{state}**：销售额（${sales:,.0f}）接近 {threshold_str} 门槛。"859 f"请密切监控。触发后税率：{rate}%。风险：{RISK_BADGE_CN['Medium']}。"860 )861 if overall_risk != "High":862 overall_risk = "Medium"863 else:864 en_lines.append(865 f"- **{state}**: Sales (${sales:,.0f}) below the {threshold_str} threshold — "866 f"no collection required yet. Risk: {RISK_BADGE['Low']}."867 )868 cn_lines.append(869 f"- **{state}**：销售额（${sales:,.0f}）低于 {threshold_str} 门槛 — "870 f"暂无需征收。风险：{RISK_BADGE_CN['Low']}。"871 )872 else:873 en_lines.append(874 f"- **{state}**: Threshold {threshold_str}, transaction rule: {tx_rule or 'none'}, "875 f"rate: {rate}%. Provide your annual sales to assess risk."876 )877 cn_lines.append(878 f"- **{state}**：门槛 {threshold_str}，交易规则：{tx_rule or '无'}，"879 f"税率：{rate}%。请提供年销售额以评估风险。"880 )881 if overall_risk == "Low":882 overall_risk = "Unknown"883 884 # ---- Product compliance ----885 if products:886 en_lines.append("")887 cn_lines.append("")888 en_lines.append("### Customs duty & federal compliance checklist")889 cn_lines.append("### 关税与联邦合规检查清单")890 for category in products:891 row = hts_df[hts_df["product_category"] == category]892 if row.empty:893 continue894 sources.add("hts_duty_codes.csv")895 r = row.iloc[0]896 flags = assess_compliance_risk(r)897 flags_str = ", ".join(flags) if flags else "no special certification flagged"898 en_lines.append(899 f"- **{category}**: HTS code `{r['hts_code']}`, duty rate **{r['duty_rate']}**. "900 f"Required: {flags_str}. {r['notes']}"901 )902 cn_lines.append(903 f"- **{category}**：HTS 代码 `{r['hts_code']}`，关税税率 **{r['duty_rate']}**。"904 f"所需认证：{flags_str}。{r['notes']}"905 )906 if flags and overall_risk == "Low":907 overall_risk = "Medium"908 909 # ---- CBP Alerts (static enforcement / tariff data) ----910 if payload.get("alerts"):911 sources.add("cbp_alerts.csv")912 en_lines.append("")913 cn_lines.append("")914 en_lines.append("### ⚠️ CBP enforcement & tariff alerts")915 cn_lines.append("### ⚠️ CBP 执法与关税警报")916 sev_badge = {"Critical": "🔴 Critical", "High": "🟠 High", "Medium": "🟡 Medium", "Info": "🟢 Info"}917 sev_badge_cn = {"Critical": "🔴 紧急", "High": "🟠 高", "Medium": "🟡 中", "Info": "🟢 信息"}918 for alert in payload["alerts"]:919 sev = alert.get("severity", "Info")920 en_lines.append(921 f"- {sev_badge.get(sev, sev)} **{alert['category']} — {alert['title']}**: "922 f"{alert['summary']} _Action:_ {alert['action_required']} "923 f"[source]({alert['source_url']})"924 )925 cn_lines.append(926 f"- {sev_badge_cn.get(sev, sev)} **{alert['category']} — {alert['title']}**："927 f"{alert['summary']} _建议行动：_ {alert['action_required']} "928 f"[来源]({alert['source_url']})"929 )930 if sev == "Critical":931 overall_risk = "High"932 elif sev == "High" and overall_risk != "High":933 overall_risk = "Medium" if overall_risk == "Low" else overall_risk934 935 # ---- CBP knowledge base excerpts (RAG from JSONL) ----936 if payload.get("chunks"):937 sources.add("cbp_chunks.jsonl")938 en_lines.append("")939 cn_lines.append("")940 en_lines.append("### 📚 CBP knowledge base excerpts")941 cn_lines.append("### 📚 CBP 知识库摘录")942 for c in payload["chunks"]:943 en_lines.append(f"- **[{c['title']}]({c['url']})** (relevance {c['score']})")944 en_lines.append(f" > {c['excerpt']}")945 cn_lines.append(f"- **[{c['title']}]({c['url']})** （相关度 {c['score']}）")946 cn_lines.append(f" > {c['excerpt']}")947 948 # ---- News ----949 if payload.get("news"):950 sources.add("cbp.gov/newsroom")951 en_lines.append("")952 cn_lines.append("")953 en_lines.append("### Recent CBP news (informational)")954 cn_lines.append("### 近期 CBP 新闻（仅供参考）")955 for item in payload["news"]:956 en_lines.append(f"- [{item['title']}]({item['url']})")957 cn_lines.append(f"- [{item['title']}]({item['url']})")958 959 # ---- Risk summary + sources ----960 en_lines.append("")961 en_lines.append(f"**Overall risk:** {RISK_BADGE[overall_risk]}")962 en_lines.append(f"**Sources:** {', '.join(sorted(sources)) if sources else 'none'}")963 964 cn_lines.append("")965 cn_lines.append(f"**总体风险：** {RISK_BADGE_CN[overall_risk]}")966 cn_lines.append(f"**来源：** {', '.join(sorted(sources)) if sources else '无'}")967 968 return "\n".join(en_lines + [""] + cn_lines)969 970 971def get_answer(972 question: str,973 nexus_df: pd.DataFrame,974 hts_df: pd.DataFrame,975 tax_rates: dict,976 news_items: list[dict],977 alerts_df: Optional[pd.DataFrame] = None,978 chunk_index: Optional[dict] = None,979 force_fallback: bool = False,980) -> tuple[str, str, dict]:981 """Returns (answer_markdown, mode_used, payload)."""982 context, payload = build_context(983 question, nexus_df, hts_df, tax_rates, news_items, alerts_df, chunk_index984 )985 986 use_llm = (not force_fallback) and is_ollama_available()987 988 if use_llm:989 user_prompt = (990 f"User question: {question}\n\n"991 f"{context}\n\n"992 "Using ONLY the data above, produce a structured bilingual answer with:\n"993 "1. [English] section: a checklist, risk flag (Low/Medium/High), and sources.\n"994 "2. [中文] section: the same content translated to Chinese.\n"995 "If the data doesn't cover the question, say 'I don't have enough information.'"996 )997 try:998 answer = call_ollama(SYSTEM_PROMPT, user_prompt)999 if answer:1000 return answer, "🤖 Ollama (llama3.2:3b)", payload1001 except requests.RequestException:1002 pass # fall through to template fallback1003 1004 answer = call_fallback(question, payload, nexus_df, hts_df, tax_rates)1005 return answer, "📋 Rule-based fallback", payload1006 1007 1008# =============================================================================1009# Section G — UI Rendering1010# =============================================================================1011 1012EXAMPLE_QUERIES = [1013 "We sell power banks to Texas, $200k annual sales. Do we need to collect sales tax?",1014 "Our startup ships lithium batteries to California and New York. What certifications do we need?",1015 "Medical thermometer exports to Florida with $150k revenue — what are our obligations?",1016]1017 1018 1019def render_sidebar(ollama_ok: bool, news_count: int, news_enabled: bool, alerts_count: int = 0, chunks_count: int = 0) -> dict:1020 with st.sidebar:1021 st.markdown("# 🧭 Customs Compass")1022 st.caption("AI compliance assistant for US exports")1023 1024 st.markdown("### System status")1025 if ollama_ok:1026 st.success(f"🟢 Ollama online ({OLLAMA_MODEL})")1027 else:1028 st.warning("🟡 Ollama offline — fallback mode")1029 st.caption("Start Ollama and run: `ollama pull llama3.2:3b`")1030 1031 if news_enabled:1032 if news_count > 0:1033 st.success(f"🟢 CBP news ({news_count} items)")1034 else:1035 st.warning("🟡 CBP news unavailable")1036 else:1037 st.info("📴 CBP news disabled")1038 1039 news_toggle = st.checkbox("Enable live CBP news", value=news_enabled)1040 refresh_news = st.button("🔄 Refresh news now", use_container_width=True)1041 force_fallback = st.checkbox("Force fallback mode (skip LLM)", value=False)1042 1043 st.markdown("---")1044 st.markdown("### Knowledge base")1045 st.markdown("- 📄 `nexus_thresholds.csv` (51 states)")1046 st.markdown("- 📄 `hts_duty_codes.csv` (14 categories)")1047 st.markdown("- 📄 `tax_rates_by_state.json`")1048 st.markdown(f"- ⚠️ `cbp_alerts.csv` ({alerts_count} curated alerts)")1049 st.markdown(f"- 📚 `cbp_chunks.jsonl` ({chunks_count} RAG chunks)")1050 st.markdown(f"- 🌐 [CBP Newsroom]({CBP_NEWSROOM_URL})")1051 st.markdown(f"- 🌐 [CBP Trade]({CBP_TRADE_URL})")1052 1053 st.markdown("---")1054 st.markdown("### Example queries")1055 example_clicked: Optional[str] = None1056 for i, ex in enumerate(EXAMPLE_QUERIES):1057 if st.button(f"💬 Example {i + 1}", key=f"ex_{i}", use_container_width=True):1058 example_clicked = ex1059 with st.expander(f"Preview {i + 1}"):1060 st.caption(ex)1061 1062 st.markdown("---")1063 st.caption("⚠️ Educational tool. Not legal or tax advice.")1064 1065 return {1066 "news_enabled": news_toggle,1067 "refresh_news": refresh_news,1068 "force_fallback": force_fallback,1069 "example_clicked": example_clicked,1070 }1071 1072 1073def render_main_form(prefilled_question: str = "") -> dict:1074 st.markdown("## Ask Customs Compass")1075 st.caption("Describe your product and/or ask a compliance question. The assistant answers in English and 中文.")1076 1077 col1, col2 = st.columns([1, 1])1078 with col1:1079 product_desc = st.text_area(1080 "Product description (optional)",1081 placeholder="e.g., Lithium-ion power bank, 20000 mAh, with USB-C charging",1082 height=120,1083 key="product_desc",1084 )1085 with col2:1086 question = st.text_area(1087 "Your question",1088 value=prefilled_question,1089 placeholder="e.g., We sell to Texas and California, $300k sales. Do we need to collect sales tax?",1090 height=120,1091 key="question",1092 )1093 1094 submitted = st.button("🔍 Analyze (分析)", type="primary", use_container_width=True)1095 return {"product_desc": product_desc, "question": question, "submitted": submitted}1096 1097 1098def render_response(answer: str, mode: str, payload: dict, news_items: list[dict]) -> None:1099 st.markdown("---")1100 st.markdown(f"#### Response — {mode}")1101 st.markdown(answer)1102 1103 with st.expander("📊 Retrieved data used"):1104 st.json({1105 "states_detected": payload["states"],1106 "products_detected": payload["products"],1107 "sales_usd_detected": payload["sales_usd"],1108 })1109 1110 if payload.get("alerts"):1111 st.markdown(f"#### ⚠️ CBP alerts triggered ({len(payload['alerts'])})")1112 import html as _html1113 for alert in payload["alerts"]:1114 sev = alert["severity"].lower()1115 st.markdown(1116 f"""1117<div class="cc-alert cc-alert-{sev}">1118 <div class="cc-alert-head">1119 <span class="cc-badge cc-badge-{sev}">{alert['severity']}</span>1120 <span>{alert['category']} — {alert['title']}</span>1121 </div>1122 <div class="cc-alert-body">{_html.escape(alert['summary'])}</div>1123 <div class="cc-alert-action">✅ <b>Action:</b> {_html.escape(alert['action_required'])}</div>1124 <div style="margin-top:6px"><a href="{alert['source_url']}" target="_blank">🔗 Source on cbp.gov</a></div>1125</div>1126 """,1127 unsafe_allow_html=True,1128 )1129 1130 if payload.get("chunks"):1131 st.markdown(f"#### 📚 CBP knowledge base excerpts ({len(payload['chunks'])})")1132 st.caption(1133 "Full text from the most relevant CBP pages — no need to click out. The link goes to the source page."1134 )1135 import html as _html1136 for c in payload["chunks"]:1137 published = c.get("published_date", "")1138 meta = f"{c['section']} · {c['page_type']} · relevance {c['score']}"1139 if published:1140 meta += f" · {published}"1141 full = c.get("full_text") or c.get("excerpt", "")1142 safe = _html.escape(full)1143 st.markdown(1144 f"""1145<div class="cc-card">1146 <div class="cc-card-title">{_html.escape(c['title'])}</div>1147 <div class="cc-card-meta">{meta}</div>1148 <div class="cc-chunk">{safe}</div>1149 <div style="margin-top:10px"><a href="{c['url']}" target="_blank">🔗 View original page on cbp.gov</a></div>1150</div>1151 """,1152 unsafe_allow_html=True,1153 )1154 1155 if payload.get("news"):1156 with st.expander(f"📰 Relevant CBP news ({len(payload['news'])})"):1157 for item in payload["news"]:1158 st.markdown(f"- [{item['title']}]({item['url']})")1159 1160 st.caption("⚠️ This is an educational tool. Always consult a licensed tax or trade professional before making compliance decisions.")1161 1162 1163# =============================================================================1164# Section H — Premium Visual Polish (custom CSS + hero)1165# =============================================================================1166 1167_CSS = """1168<style>1169/* ----------- Fonts & base ----------- */1170@import url('https://fonts.googleapis.com/css2?family=Inter:wght@400;500;600;700;800&family=JetBrains+Mono:wght@400;500&display=swap');1171 1172html, body, [class*="css"], .stApp {1173 font-family: 'Inter', -apple-system, BlinkMacSystemFont, 'Segoe UI', sans-serif;1174}1175 1176code, pre, .stCode {1177 font-family: 'JetBrains Mono', monospace !important;1178}1179 1180/* ----------- Hide default Streamlit chrome ----------- */1181#MainMenu {visibility: hidden;}1182footer {visibility: hidden;}1183header[data-testid="stHeader"] {background: transparent;}1184 1185/* ----------- Color tokens ----------- */1186:root {1187 --cc-primary: #6366F1;1188 --cc-primary-dark: #4F46E5;1189 --cc-accent: #EC4899;1190 --cc-surface: #F8FAFC;1191 --cc-border: #E2E8F0;1192 --cc-text: #0F172A;1193 --cc-muted: #64748B;1194 --cc-success: #10B981;1195 --cc-warning: #F59E0B;1196 --cc-danger: #EF4444;1197 --cc-critical: #DC2626;1198}1199 1200/* ----------- Hero header ----------- */