apodex/frontier-agent-demo

1"""Bounded HTTP body reads for the web_fetch tool family."""2 3from __future__ import annotations4 5import asyncio6import os7 8import httpx9 10# Byte ceiling for one response body. Env-tunable for callers that11# legitimately need bigger text payloads.12DEFAULT_MAX_FETCH_BYTES = 5 * 1024 * 102413 14 15def max_fetch_bytes() -> int:16    raw = (os.getenv("WEB_FETCH_MAX_BYTES") or "").strip()17    if raw:18        try:19            value = int(raw)20            if value > 0:21                return value22        except ValueError:23            pass24    return DEFAULT_MAX_FETCH_BYTES25 26 27# Content-types that are data blobs, never extractable page text. PDF is28# deliberately absent: Jina converts PDFs to text, so blocking it here29# would regress the primary scrape path. Prefix match on the bare type30# (parameters like ``; charset=`` stripped by the caller helper).31_BINARY_TYPE_PREFIXES = ("image/", "audio/", "video/", "font/")32_BINARY_TYPES = frozenset({33    "application/zip",34    "application/octet-stream",35    "application/x-tar",36    "application/gzip",37    "application/x-gzip",38    "application/x-bzip2",39    "application/x-xz",40    "application/x-7z-compressed",41    "application/x-rar-compressed",42    "application/x-hdf",43    "application/x-hdf5",44    "application/vnd.ms-cab-compressed",45    "application/x-matlab-data",46    "application/x-msdownload",47    "application/wasm",48})49 50 51# URL path extensions that denote dataset / archive / binary downloads.52# Screened BEFORE any request is issued (no Jina call, no bytes on the53# wire). Text-ish data formats (.csv/.json/.txt/.xml) are deliberately54# absent: small ones are legitimate fetch targets and the byte cap55# bounds the large ones. PDF stays allowed (Jina converts it).56_BLOCKED_URL_EXTENSIONS = (57    ".zip", ".tar", ".tgz", ".tar.gz", ".tar.bz2", ".tar.xz",58    ".gz", ".bz2", ".xz", ".7z", ".rar", ".zst",59    ".mat", ".h5", ".hdf5", ".npz", ".npy", ".pkl", ".pickle",60    ".pt", ".pth", ".onnx", ".safetensors", ".parquet", ".feather",61    ".whl", ".deb", ".rpm", ".dmg", ".iso", ".exe", ".msi", ".apk",62)63 64 65def blocked_download_url(url: str) -> str | None:66    """The matched extension when ``url``'s path names a dataset/archive67    download, else ``None``. Match is on the URL *path* (query/fragment68    stripped), so ``?format=zip`` params don't false-positive."""69    try:70        from urllib.parse import unquote, urlsplit71 72        path = unquote(urlsplit(url).path).strip().lower()73    except Exception:74        return None75    for ext in _BLOCKED_URL_EXTENSIONS:76        if path.endswith(ext):77            return ext78    return None79 80 81def binary_content_type(content_type: str | None) -> str | None:82    """The normalized content-type when it denotes a non-text blob, else83    ``None``. Callers use the returned value in the error message."""84    if not content_type:85        return None86    bare = content_type.split(";", 1)[0].strip().lower()87    if bare in _BINARY_TYPES or bare.startswith(_BINARY_TYPE_PREFIXES):88        return bare89    return None90 91 92async def read_bounded(93    response: httpx.Response,94    max_bytes: int | None = None,95) -> tuple[bytes, bool]:96    """Read a streaming response up to ``max_bytes``; ``(body, truncated)``.97 98    Must be called inside the ``client.stream(...)`` context. Stopping99    early closes the connection, so a 2GB download costs at most100    ``max_bytes`` of transfer and memory.101    """102    cap = max_bytes if max_bytes is not None else max_fetch_bytes()103    chunks: list[bytes] = []104    total = 0105    async for chunk in response.aiter_bytes():106        chunks.append(chunk)107        total += len(chunk)108        if total >= cap:109            return b"".join(chunks)[:cap], True110    return b"".join(chunks), False111 112 113def decode_body(response: httpx.Response, body: bytes) -> str:114    """Decode a bounded body with the response's declared charset.115 116    ``errors="replace"`` because a truncated multi-byte sequence at the117    cap boundary must not raise. No chardet sniffing on the (possibly118    huge) body — absent/unknown charset falls back to UTF-8, matching119    the dominant real-world default.120    """121    encoding = response.charset_encoding or "utf-8"122    try:123        return body.decode(encoding, errors="replace")124    except LookupError:125        return body.decode("utf-8", errors="replace")126 127 128async def non_public_url_error(url: str) -> str:129    """Reason *url* must not be fetched, else ``""``.130 131    ``web_fetch`` is auto-approved in the terminal's risk gate, so a URL that132    arrives from page content — the prompt-injection path for a research agent133    — is requested without a human ever seeing it. Unguarded, the target may be134    a cloud metadata endpoint or any service on the deployment's private135    network, and with ``JINA_API_KEY`` set the internal URL is handed to the136    scrape provider before the fetch is even attempted.137 138    Reuses ``download_file``'s vetting so both tools share one definition of139    "public": http(s) only, no credentials in the URL, and EVERY resolved140    address global, so a split-horizon DNS answer cannot slip a private address141    through. Fail-closed — a name that cannot be resolved cannot be vetted.142    Resolution is off-loaded because it blocks.143 144    Set ``FRONTIER_AGENT_ALLOW_PRIVATE_FETCH=1`` to fetch a localhost or145    intranet service deliberately.146    """147    refusal, _addresses = await vet_public_url(url)148    return refusal149 150 151async def vet_public_url(url: str) -> tuple[str, tuple[str, ...]]:152    """``(refusal, validated_addresses)`` for *url*.153 154    Returning the addresses is what makes DNS-rebinding defence possible:155    validating a name and then letting the client resolve it a second time is a156    TOCTOU — an attacker-controlled resolver can answer with a public address157    for the check and a private one for the connection. Callers hand these158    addresses to :func:`pin_to_address` so the socket goes where the check159    looked. ``download_file`` has always pinned for this reason; the scrape160    paths now do too.161    """162    if (os.getenv("FRONTIER_AGENT_ALLOW_PRIVATE_FETCH") or "").strip() == "1":163        return "", ()164    from plugins.tools._download_runner import DownloadError, _validate_public_url165    try:166        addresses = await asyncio.to_thread(_validate_public_url, url)167    except DownloadError as exc:168        return str(exc), ()169    if isinstance(addresses, str):   # older single-address contract170        addresses = (addresses,)171    return "", tuple(addresses)172 173 174#: Headers that authenticate the caller and must not follow a redirect to a175#: different origin. httpx strips these itself when it follows redirects; a176#: hand-rolled hop loop has to do it explicitly or it leaks the credential to177#: whatever host the first origin names.178_CREDENTIAL_HEADERS = frozenset({179    "authorization", "cookie", "proxy-authorization", "www-authenticate",180})181 182 183def _origin(url: str) -> tuple[str, str, int | None]:184    parts = httpx.URL(url)185    return (parts.scheme, parts.host, parts.port)186 187 188def strip_cross_origin_credentials(189    headers: dict[str, str], from_url: str, to_url: str,190) -> dict[str, str]:191    """Drop caller credentials when a redirect hop changes origin.192 193    Same-origin hops keep them, so an authenticated fetch that redirects within194    one host still works.195    """196    try:197        if _origin(from_url) == _origin(to_url):198            return headers199    except Exception:200        pass201    return {202        name: value for name, value in headers.items()203        if name.lower() not in _CREDENTIAL_HEADERS204    }205 206 207def pin_to_address(208    url: str, addresses: tuple[str, ...], headers: dict[str, str],209) -> tuple[str, dict[str, str], dict[str, object]]:210    """Rewrite a request to dial an already-validated address.211 212    Returns ``(url, headers, extensions)``. The address replaces the URL host so213    no second DNS lookup can happen, while the original hostname is preserved214    twice over: in the ``Host`` header (virtual-host routing) and in the215    ``sni_hostname`` extension, which drives TLS SNI *and* the certificate216    hostname check — so a pinned HTTPS request still fails closed on a217    mismatched certificate.218 219    A no-op when there is nothing to pin (the private-fetch opt-in returns no220    addresses) or the URL is already literal-IP.221    """222    if not addresses:223        return url, headers, {}224    parsed = httpx.URL(url)225    hostname = parsed.host226    if not hostname or hostname == addresses[0]:227        return url, headers, {}228    pinned = str(parsed.copy_with(host=addresses[0]))229    return (230        pinned,231        {**headers, "Host": parsed.netloc.decode("ascii")},232        {"sni_hostname": hostname},233    )234 235 236#: Redirect hops a scrape may follow. Matches httpx's own default ceiling.237MAX_REDIRECT_HOPS = 20238 239 240class RedirectRefused(Exception):241    """A redirect hop pointed somewhere ``non_public_url_error`` refuses."""242 243 244async def next_hop(response: httpx.Response, current_url: str) -> str | None:245    """The vetted URL a 30x response redirects to, or ``None`` if it is final.246 247    Automatic redirect following defeats the URL guard: only the FIRST URL is248    vetted, so a public attacker-controlled page can answer 302 → localhost or249    a cloud metadata endpoint and the client follows it. Callers therefore250    disable ``follow_redirects`` and walk the chain through this, which vets251    every hop with the same rule the initial URL passed.252 253    Raises :class:`RedirectRefused` rather than returning the reason, so a254    refused hop cannot be mistaken for "no more redirects" and silently treated255    as a successful fetch.256    """257    if not response.is_redirect:258        return None259    location = response.headers.get("location", "").strip()260    if not location:261        return None262    target = str(httpx.URL(current_url).join(location))263    refusal = await non_public_url_error(target)264    if refusal:265        raise RedirectRefused(refusal)266    return target267 268 269__all__ = [270    "DEFAULT_MAX_FETCH_BYTES",271    "MAX_REDIRECT_HOPS",272    "RedirectRefused",273    "binary_content_type",274    "blocked_download_url",275    "decode_body",276    "max_fetch_bytes",277    "next_hop",278    "non_public_url_error",279    "pin_to_address",280    "read_bounded",281    "strip_cross_origin_credentials",282    "vet_public_url",283]284