BonusLockSMith/web-research-agent

1#!/usr/bin/env python2"""Keyless multi-engine web search for the Web Research Agent (#13).3 4No API key required — so this runs as a free, private local tool as easily as a public demo.5Order: a local SearXNG (if GRITAI_SEARXNG / SEARXNG_URL is set — truest private) → Google News RSS6(keyless, reliable, no rate limit) → DuckDuckGo HTML → Bing HTML. Returns structured results so the7agent can fetch each URL and build numbered citations.8 9Adapted from swarm/orchestrator/tools.py (the studio's proven keyless search) into structured output.10"""11import os12import urllib.parse13 14import requests15import feedparser16from bs4 import BeautifulSoup17 18_UA = ("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "19       "(KHTML, like Gecko) Chrome/124.0 Safari/537.36")20TIMEOUT = 1521 22 23def _searxng(query, n):24    base = os.environ.get("GRITAI_SEARXNG") or os.environ.get("SEARXNG_URL")25    if not base:26        return []27    d = requests.get(f"{base}/search", params={"q": query, "format": "json"},28                     headers={"User-Agent": _UA}, timeout=TIMEOUT).json()29    return [{"title": r.get("title", ""), "url": r.get("url", ""),30             "snippet": (r.get("content", "") or "")[:300], "source": "searxng"}31            for r in d.get("results", [])[:n] if r.get("url")]32 33 34def _gnews(query, n):35    """Google News RSS — keyless, structured, fresh, no rate limit."""36    d = feedparser.parse(37        f"https://news.google.com/rss/search?q={urllib.parse.quote(query)}&hl=en-US&gl=US&ceid=US:en")38    out = []39    for e in d.entries[:n]:40        src = e.get("source", {}).get("title", "") if isinstance(e.get("source"), dict) else ""41        out.append({"title": e.get("title", ""), "url": e.get("link", ""),42                    "snippet": f"{src} · {e.get('published', '')[:16]}".strip(" ·"), "source": "google-news"})43    return [r for r in out if r["url"]]44 45 46def _ddg(query, n):47    r = requests.post("https://html.duckduckgo.com/html/", data={"q": query},48                      headers={"User-Agent": _UA}, timeout=TIMEOUT)49    soup = BeautifulSoup(r.text, "html.parser")50    out = []51    for res in soup.select(".result")[:n]:52        a = res.select_one(".result__a")53        snip = res.select_one(".result__snippet")54        if a and a.get("href"):55            out.append({"title": a.get_text(strip=True), "url": a.get("href"),56                        "snippet": (snip.get_text(strip=True) if snip else "")[:300], "source": "duckduckgo"})57    return out58 59 60def _bing(query, n):61    r = requests.get("https://www.bing.com/search", params={"q": query},62                     headers={"User-Agent": _UA}, timeout=TIMEOUT)63    soup = BeautifulSoup(r.text, "html.parser")64    out = []65    for li in soup.select("li.b_algo")[:n]:66        a = li.select_one("h2 a")67        p = li.select_one(".b_caption p") or li.select_one("p")68        if a and a.get("href"):69            out.append({"title": a.get_text(strip=True), "url": a.get("href"),70                        "snippet": (p.get_text(strip=True) if p else "")[:300], "source": "bing"})71    return out72 73 74def _dedup(results):75    seen, out = set(), []76    for r in results:77        key = (r["url"] or "").split("#")[0].rstrip("/")78        if key and key not in seen:79            seen.add(key)80            out.append(r)81    return out82 83 84def search(query: str, n: int = 6) -> list:85    """Return up to n structured results [{title,url,snippet,source}] with DIRECT, fetchable URLs.86    Order favors engines whose links can actually be read: SearXNG (local, direct) → DuckDuckGo87    (direct) → Bing (resolvable click-redirect). Google News is excluded here (encoded links that88    don't resolve to full text) — use search_news() for freshness/discovery."""89    n = int(n)90    for engine in (_searxng, _ddg, _bing):91        try:92            hits = engine(query, n)93            if hits:94                return _dedup(hits)[:n]95        except Exception:96            continue97    return []98 99 100def search_news(query: str, n: int = 6) -> list:101    """Fresh headlines via Google News RSS — keyless, no rate limit. Links are encoded redirects102    (good for discovery/snippets; not reliably full-text readable), so keep this separate from the103    readable web search above."""104    try:105        return _dedup(_gnews(query, int(n)))[:int(n)]106    except Exception:107        return []108 109 110if __name__ == "__main__":111    import sys, json112    q = " ".join(sys.argv[1:]) or "what is retrieval augmented generation"113    res = search(q, 6)114    print(f"query: {q}\n{len(res)} results:")115    print(json.dumps(res, indent=2)[:2000])116