TypeGPT/Webscout-API

1import re2from decimal import Decimal3from html import unescape4from math import atan2, cos, radians, sin, sqrt5from typing import Any, Dict, List, Union6from urllib.parse import unquote7import orjson8 9from .exceptions import WebscoutE10 11REGEX_STRIP_TAGS = re.compile("<.*?>")12 13 14def json_dumps(obj: Any) -> str:15    try:16        return orjson.dumps(obj).decode("utf-8")17    except Exception as ex:18        raise WebscoutE(f"{type(ex).__name__}: {ex}") from ex19 20 21def json_loads(obj: Union[str, bytes]) -> Any:22    try:23        return orjson.loads(obj)24    except Exception as ex:25        raise WebscoutE(f"{type(ex).__name__}: {ex}") from ex26 27 28def _extract_vqd(html_bytes: bytes, keywords: str) -> str:29    """Extract vqd from html bytes."""30    for c1, c1_len, c2 in (31        (b'vqd="', 5, b'"'),32        (b"vqd=", 4, b"&"),33        (b"vqd='", 5, b"'"),34    ):35        try:36            start = html_bytes.index(c1) + c1_len37            end = html_bytes.index(c2, start)38            return html_bytes[start:end].decode()39        except ValueError:40            pass41    raise WebscoutE(f"_extract_vqd() {keywords=} Could not extract vqd.")42 43 44def _text_extract_json(html_bytes: bytes, keywords: str) -> List[Dict[str, str]]:45    """text(backend="api") -> extract json from html."""46    try:47        start = html_bytes.index(b"DDG.pageLayout.load('d',") + 2448        end = html_bytes.index(b");DDG.duckbar.load(", start)49        data = html_bytes[start:end]50        result: List[Dict[str, str]] = json_loads(data)51        return result52    except Exception as ex:53        raise WebscoutE(f"_text_extract_json() {keywords=} {type(ex).__name__}: {ex}") from ex54    raise WebscoutE(f"_text_extract_json() {keywords=} return None")55 56 57def _normalize(raw_html: str) -> str:58    """Strip HTML tags from the raw_html string."""59    return unescape(REGEX_STRIP_TAGS.sub("", raw_html)) if raw_html else ""60 61 62def _normalize_url(url: str) -> str:63    """Unquote URL and replace spaces with '+'."""64    return unquote(url.replace(" ", "+")) if url else ""65 66 67def _calculate_distance(lat1: Decimal, lon1: Decimal, lat2: Decimal, lon2: Decimal) -> float:68    """Calculate distance between two points in km. Haversine formula."""69    R = 6371.0087714  # Earth's radius in km70    rlat1, rlon1, rlat2, rlon2 = map(radians, [float(lat1), float(lon1), float(lat2), float(lon2)])71    dlon, dlat = rlon2 - rlon1, rlat2 - rlat172    a = sin(dlat / 2) ** 2 + cos(rlat1) * cos(rlat2) * sin(dlon / 2) ** 273    c = 2 * atan2(sqrt(a), sqrt(1 - a))74    return R * c