SoftwareResearch/g2dm-research
0
1"""Claude-powered research engine for product/category fitment.2 3One product -> one Claude conversation that browses the official vendor site4(web_search + web_fetch) and returns a strict JSON verdict, combined with the5deterministic verbatim title/meta extracted by html_extract.py.6"""7import json8import os9import re10import threading11import time12from urllib.parse import urlparse13 14import anthropic15 16import html_extract17import prompts18 19MODEL = "claude-opus-4-8"20 21# web_search attaches citations, which are incompatible with output_config.format,22# so we ask for a trailing JSON block and parse it defensively instead.23TOOLS = [24 {"type": "web_search_20260209", "name": "web_search"},25 {"type": "web_fetch_20260209", "name": "web_fetch"},26]27 28_client = None29_client_lock = threading.Lock()30 31 32def get_client():33 global _client34 with _client_lock:35 if _client is None:36 _client = anthropic.Anthropic()37 return _client38 39 40def _registrable(host: str) -> str:41 """Crude registrable domain (last two labels), ignoring www/locale subdomains."""42 host = (host or "").lower().split(":")[0]43 parts = [p for p in host.split(".") if p]44 return ".".join(parts[-2:]) if len(parts) >= 2 else host45 46 47def _domain_changed(input_url: str, final_url: str) -> bool:48 try:49 a = _registrable(urlparse(html_extract.normalize_url(input_url)).netloc)50 b = _registrable(urlparse(final_url).netloc)51 return bool(a) and bool(b) and a != b52 except Exception:53 return False54 55 56def auth_status() -> dict:57 """Report whether credentials look available, without making a network call."""58 has_key = bool(os.environ.get("ANTHROPIC_API_KEY") or os.environ.get("ANTHROPIC_AUTH_TOKEN"))59 return {60 "has_credentials": has_key,61 "base_url": os.environ.get("ANTHROPIC_BASE_URL", ""),62 }63 64 65# ---------------------------------------------------------------------------66# JSON parsing helpers67# ---------------------------------------------------------------------------68 69_JSON_FENCE = re.compile(r"```(?:json)?\s*(\{.*?\})\s*```", re.S)70 71 72def _extract_json(text: str):73 if not text:74 return None75 m = _JSON_FENCE.search(text)76 candidate = m.group(1) if m else None77 if candidate is None:78 # Fall back to the last balanced-looking object in the text.79 start = text.rfind("{")80 end = text.rfind("}")81 if start != -1 and end != -1 and end > start:82 candidate = text[start:end + 1]83 if not candidate:84 return None85 try:86 return json.loads(candidate)87 except json.JSONDecodeError:88 return None89 90 91def _final_text(message) -> str:92 return "".join(b.text for b in message.content if getattr(b, "type", "") == "text")93 94 95def _run_conversation(messages, max_continuations: int = 6):96 """Run the server-tool agentic loop; return the final assistant text."""97 client = get_client()98 last_message = None99 for _ in range(max_continuations):100 with client.messages.stream(101 model=MODEL,102 max_tokens=8000,103 thinking={"type": "adaptive"},104 output_config={"effort": "high"},105 tools=TOOLS,106 system=prompts.RESEARCH_SYSTEM,107 messages=messages,108 ) as stream:109 last_message = stream.get_final_message()110 if last_message.stop_reason == "pause_turn":111 # Server tool loop paused; resend to continue.112 messages = messages[:1] + [113 {"role": "assistant", "content": last_message.content}114 ]115 continue116 break117 return _final_text(last_message) if last_message else ""118 119 120def _structuring_fallback(raw_text: str):121 """Ask the model to convert prior prose into the JSON object (no tools)."""122 client = get_client()123 try:124 resp = client.messages.create(125 model=MODEL,126 max_tokens=2000,127 messages=[{128 "role": "user",129 "content": (130 "Convert the following research notes into the required JSON object "131 "(same keys as specified) and output ONLY a single fenced ```json block.\n\n"132 + raw_text[-6000:]133 ),134 }],135 system=prompts.RESEARCH_SYSTEM,136 )137 return _extract_json(_final_text(resp))138 except Exception:139 return None140 141 142# ---------------------------------------------------------------------------143# Public entry point144# ---------------------------------------------------------------------------145 146def research_product(*, category, special_considerations, market,147 product_name, vendor, url, prevalidated_status="", prior_status=""):148 """Research a single product. Returns a dict of input + output columns."""149 base = {150 "vendor": vendor,151 "product_name": product_name,152 "url": url,153 "prevalidated_status": prevalidated_status,154 "prior_status": prior_status,155 "reviews_us_only": "Yes" if market.strip().upper() == "US" else "No",156 "pricing_in_usd": "",157 "website_in_english": "",158 "market_fit": "",159 "recommended": "",160 "reason": "",161 "source_url": "",162 "standalone_or_suite": "",163 "title_tag": "",164 "meta_tag": "",165 "new_url": "",166 "lifecycle_status": "",167 "approach_a": "",168 "approach_b": "",169 "confidence": None,170 "needs_human_review": False,171 "evidence": [],172 "error": "",173 }174 175 # 1) Verbatim title/meta (deterministic).176 tags = html_extract.extract_tags(url)177 base["title_tag"] = tags["title"]178 base["meta_tag"] = tags["meta_description"]179 if tags["final_url"] and _domain_changed(url, tags["final_url"]):180 base["new_url"] = tags["final_url"]181 182 site_unreachable = bool(tags["error"]) and not tags["title"]183 184 # 2) Claude research.185 user_msg = prompts.build_research_user_message(186 category=category,187 special_considerations=special_considerations,188 market=market,189 product_name=product_name,190 vendor=vendor,191 url=tags["final_url"] or url,192 prevalidated_status=prevalidated_status,193 prior_status=prior_status,194 title_tag=tags["title"],195 meta_tag=tags["meta_description"],196 )197 if site_unreachable:198 user_msg += (199 f"\n\nNOTE: a direct fetch of the URL failed ({tags['error']}). Try web_fetch "200 "yourself; if the official site is genuinely unreachable, follow the "201 "site-not-accessible rule."202 )203 204 # Retry transient failures / parse misses so a single flaky product doesn't205 # fail a large batch. Auth/permission errors won't change, so don't retry those.206 data = None207 last_err = ""208 for attempt in range(3):209 try:210 text = _run_conversation([{"role": "user", "content": user_msg}])211 data = _extract_json(text) or _structuring_fallback(text)212 if data is not None:213 break214 last_err = "Could not parse model output"215 except (anthropic.AuthenticationError, anthropic.PermissionDeniedError) as exc:216 last_err = f"{type(exc).__name__}: {exc}"217 break218 except Exception as exc: # noqa: BLE001 - surface to the UI after retries219 last_err = f"{type(exc).__name__}: {exc}"220 if attempt < 2:221 time.sleep(1.5 * (attempt + 1))222 223 if data is None:224 base["error"] = last_err or "Unknown error"225 base["needs_human_review"] = True226 return base227 228 base["pricing_in_usd"] = data.get("pricing_in_usd", "")229 base["website_in_english"] = data.get("website_in_english", "")230 base["market_fit"] = data.get("market_fit", "")231 base["recommended"] = data.get("recommended", "")232 base["reason"] = data.get("reason", "")233 base["source_url"] = data.get("source_url", "")234 base["standalone_or_suite"] = data.get("standalone_or_suite", "")235 base["lifecycle_status"] = data.get("lifecycle_status", "")236 base["approach_a"] = data.get("approach_a", "")237 base["approach_b"] = data.get("approach_b", "")238 base["confidence"] = data.get("confidence")239 base["needs_human_review"] = bool(data.get("needs_human_review", False))240 base["evidence"] = data.get("evidence", []) or []241 return base242