GHSsda/HotelPriceScraper
0
1import asyncio2import re3import psycopg24import random5import gc6from datetime import datetime, timedelta7from bs4 import BeautifulSoup8from playwright.async_api import async_playwright9import playwright_stealth10 11# ==========================================12# 1. CONFIGURAZIONE DB E TARGET13# ==========================================14DB_URL = "postgresql://postgres:&UWZBk?Aap@78vi@db.fsnhksmnzcvltsdnfhmn.supabase.co:5432/postgres"15GIORNI_DA_ANALIZZARE = 36516CHUNK_GIORNI = 7 17 18HOTEL_MAP = {19 "Hotel Flora": "https://www.booking.com/hotel/it/flora-milano.it.html",20 "Hotel Ibis Centro": "https://www.booking.com/hotel/it/ibismilanocentromilano.it.html",21 "Hotel Garda": "https://www.booking.com/hotel/it/garda.it.html",22 "Hotel Mennini": "https://www.booking.com/hotel/it/mennini.it.html",23 "Hotel Bernina": "https://www.booking.com/hotel/it/hotelbernina.it.html",24 "Hotel Canova": "https://www.booking.com/hotel/it/canovahotel.it.html",25 "Hotel Mythos": "https://www.booking.com/hotel/it/hotel-mythos.it.html",26 "Hotel Delle Nazioni": "https://www.booking.com/hotel/it/tidellenazioni.it.html",27 "Spice Hotel": "https://www.booking.com/hotel/it/Spice.it.html",28 "Hotel Folen": "https://www.booking.com/hotel/it/folen.it.html"29}30 31def init_db():32 conn = psycopg2.connect(DB_URL)33 c = conn.cursor()34 c.execute('''CREATE TABLE IF NOT EXISTS market_data35 (scrape_timestamp TEXT, stay_date TEXT, hotel_name TEXT, 36 room_type TEXT, price REAL, is_sold_out INTEGER)''')37 conn.commit()38 conn.close()39 40def salva_su_db(dati_estratti):41 if not dati_estratti:42 return43 44 conn = psycopg2.connect(DB_URL)45 c = conn.cursor()46 47 for dato in dati_estratti:48 c.execute('''49 INSERT INTO market_data (scrape_timestamp, stay_date, hotel_name, room_type, price, is_sold_out)50 VALUES (%s, %s, %s, %s, %s, %s)51 ''', (52 dato['scrape_timestamp'],53 dato['stay_date'],54 dato['hotel_name'],55 dato['room_type'],56 dato['price'],57 dato['is_sold_out']58 ))59 60 conn.commit()61 conn.close()62 63# Logica Bendata: T+0, salta T+1, da T+2 a T+36564def get_target_dates(total_days):65 today = datetime.now()66 dates = []67 for i in range(2, total_days + 2):68 dates.append((today + timedelta(days=i)).strftime("%Y-%m-%d"))69 return dates70 71# ==========================================72# 2. MOTORE STEALTH DINAMICO73# ==========================================74async def apply_stealth_smart(page):75 try:76 if hasattr(playwright_stealth, 'stealth_async'):77 await playwright_stealth.stealth_async(page)78 elif hasattr(playwright_stealth, 'stealth_sync'):79 playwright_stealth.stealth_sync(page)80 elif hasattr(playwright_stealth, 'stealth'):81 st_attr = getattr(playwright_stealth, 'stealth')82 if callable(st_attr):83 res = st_attr(page)84 if asyncio.iscoroutine(res):85 await res86 except Exception:87 pass88 89async def get_clean_price(page, url, checkin, checkout, adulti):90 target = f"{url.split('?')[0]}?checkin={checkin}&checkout={checkout}&group_adults={adulti}&no_rooms=1&selected_currency=EUR&lang=it&sb_price_type=total"91 92 try:93 await page.goto(target, wait_until="domcontentloaded", timeout=45000)94 await asyncio.sleep(random.uniform(2.0, 3.5)) 95 await page.mouse.wheel(0, random.randint(400, 800))96 97 content = await page.content()98 if content is None or not str(content).strip():99 return None100 101 soup = BeautifulSoup(content, 'html.parser')102 html_basso = content.lower()103 104 if 'non ci sono camere disponibili' in html_basso or 'sold out' in html_basso or 'esaurit' in html_basso or 'niente camere' in html_basso:105 return None106 107 prices = []108 for row in soup.select('.bui-price-display__value, .fc_price--actual, .prco-val-low-rate, [data-testid="price-and-discounted-price"]'):109 val = re.sub(r'[^\d]', '', row.get_text())110 if val: 111 prices.append(float(val))112 113 return min(prices) if prices else None114 except Exception:115 return None116 117# ==========================================118# 3. LOGICA DI ESTRAZIONE119# ==========================================120async def scrape_verified_logic(browser, name, url, checkin, adulti):121 context = await browser.new_context(user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36")122 page = await context.new_page()123 124 async def blocca_spazzatura(route):125 if route.request.resource_type in ["image", "stylesheet", "media", "font"]:126 await route.abort()127 else:128 await route.continue_()129 130 await page.route("**/*", blocca_spazzatura)131 await apply_stealth_smart(page) 132 133 d2_1n = (datetime.strptime(checkin, '%Y-%m-%d') + timedelta(days=1)).strftime('%Y-%m-%d')134 price = await get_clean_price(page, url, checkin, d2_1n, adulti)135 136 if price is None:137 d2_2n = (datetime.strptime(checkin, '%Y-%m-%d') + timedelta(days=2)).strftime('%Y-%m-%d')138 price_2n = await get_clean_price(page, url, checkin, d2_2n, adulti)139 if price_2n:140 price = price_2n / 2 141 142 await page.close()143 await context.close()144 gc.collect()145 146 return price147 148# ==========================================149# 4. CICLO VITALE150# ==========================================151async def esegui_ciclo():152 init_db()153 date_da_analizzare = get_target_dates(GIORNI_DA_ANALIZZARE)154 155 async with async_playwright() as p:156 browser = await p.chromium.launch(headless=True)157 ts = datetime.now().strftime('%Y-%m-%d %H:%M:%S')158 159 print(f"\n[{ts}] ⚡ Avvio HTF Engine (Playwright Stealth Mode)...")160 161 for i in range(0, len(date_da_analizzare), CHUNK_GIORNI):162 blocco_date = date_da_analizzare[i:i + CHUNK_GIORNI]163 chunk_data = []164 165 print(f"\n🚀 Scansione Blocco: {blocco_date[0]} al {blocco_date[-1]}")166 167 for checkin in blocco_date:168 for adulti in [1, 2]:169 tipo_camera = "Singola" if adulti == 1 else "Doppia"170 for nome, url in HOTEL_MAP.items():171 print(f"🕵️ {nome} | {checkin} | {tipo_camera} ... ", end="", flush=True)172 173 try:174 prezzo = await scrape_verified_logic(browser, nome, url, checkin, adulti)175 is_sold_out = 1 if prezzo is None else 0176 177 chunk_data.append({178 "scrape_timestamp": ts,179 "stay_date": checkin,180 "hotel_name": nome,181 "room_type": tipo_camera,182 "price": prezzo,183 "is_sold_out": is_sold_out184 })185 186 status = f"{prezzo}€" if not is_sold_out else "SOLD OUT"187 print(status)188 except Exception as e:189 print(f"ERRORE (Saltato in sicurezza)")190 191 await asyncio.sleep(random.uniform(1.0, 2.0))192 193 salva_su_db(chunk_data)194 print(f"💾 Blocco salvato in Supabase. Dashboard aggiornata.")195 196 await browser.close()197 print(f"✅ Ciclo annuale completo.")198 199def start_engine():200 print("🤖 Flora HTF Core Engine Online. Inizio scansione.")201 try:202 asyncio.run(esegui_ciclo())203 print("🏁 Analisi completata.")204 except Exception as e:205 print(f"🚨 Errore Fatale del Motore: {e}")206 207if __name__ == "__main__":208 start_engine()