ghstedpixel/app.py

1import httpx2import random3from bs4 import BeautifulSoup4from config import USER_AGENTS5 6async def extract_page_data(url: str, product_name: str, client: httpx.AsyncClient = None) -> tuple:7    """Fetches web targets dynamically swapping modern user-agent strings line-by-line to extract pricing indicators"""8    try:9        headers = {"User-Agent": random.choice(USER_AGENTS)}10        html_content = ""11        12        if client is not None:13            res = await client.get(url, headers=headers, timeout=6.0)14            html_content = res.text15        else:16            async with httpx.AsyncClient(trust_env=False) as standalone_client:17                res = await standalone_client.get(url, headers=headers, timeout=6.0)18                html_content = res.text19                20        soup = BeautifulSoup(html_content, 'html.parser')21                22        img_url = None23        og_img = soup.find("meta", property="og:image") or soup.find("meta", attrs={"name": "og:image"})24        if og_img: img_url = og_img.get("content")25                26        for s in soup(["script", "style", "nav", "footer", "header", "aside"]): s.extract()27                28        raw_lines = soup.get_text(separator='\n').split('\n')29        keywords = [k.lower() for k in product_name.split() if len(k) > 2]30        filtered_chunks = []31        for line in raw_lines:32            line_clean = line.strip()33            if not line_clean: continue34            if any(c in line_clean for c in ['$', 'USD', 'Price']) or any(k in line_clean.lower() for k in keywords):35                if len(line_clean) < 300: filtered_chunks.append(line_clean)36        dense_text = " | ".join(filtered_chunks)[:1500]37        return dense_text, img_url38    except Exception as e:39        print(f"❌ [Network Scraper Exception] Failed fetching {url}: {e}")40        return None, None