Hprono/fbrefscraper

1import gradio as gr2import pandas as pd3import json4import time5import random6from bs4 import BeautifulSoup7from selenium import webdriver8from selenium.webdriver.chrome.options import Options9from selenium.webdriver.chrome.service import Service10from selenium.webdriver.common.by import By11from selenium.webdriver.support.ui import WebDriverWait12from selenium.webdriver.support import expected_conditions as EC13from webdriver_manager.chrome import ChromeDriverManager14 15# =========================16# DRIVER SETUP17# =========================18def setup_driver():19    chrome_options = Options()20    chrome_options.add_argument("--headless")21    chrome_options.add_argument("--no-sandbox")22    chrome_options.add_argument("--disable-dev-shm-usage")23    chrome_options.add_argument("--disable-blink-features=AutomationControlled")24    chrome_options.add_experimental_option("excludeSwitches", ["enable-automation"])25    chrome_options.add_experimental_option("useAutomationExtension", False)26    chrome_options.add_argument(27        "--user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) "28        "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"29    )30    service = Service(ChromeDriverManager().install())31    driver = webdriver.Chrome(service=service, options=chrome_options)32    driver.execute_script(33        "Object.defineProperty(navigator, 'webdriver', {get: () => undefined})"34    )35    return driver36 37# =========================38# SCRAPING FBREF39# =========================40BIG5 = ["Premier League", "La Liga", "Serie A", "Bundesliga", "Ligue 1", "Champions League"]41 42def scrape_fbref():43    driver = setup_driver()44    fixtures = []45 46    try:47        driver.get("https://fbref.com/en/matches/")48        WebDriverWait(driver, 20).until(49            EC.presence_of_element_located((By.TAG_NAME, "table"))50        )51        driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")52        time.sleep(random.uniform(2, 4))53 54        soup = BeautifulSoup(driver.page_source, "html.parser")55 56        for header in soup.find_all("h2"):57            competition = header.get_text(strip=True).replace(" Scores & Fixtures", "")58 59            # Filtrer grands championnats uniquement60            if not any(b in competition for b in BIG5):61                continue62 63            table = header.find_next("table")64            if not table:65                continue66 67            try:68                headers_row = [69                    th.get_text(strip=True)70                    for th in table.find("thead").find_all("th")71                ]72            except AttributeError:73                continue74 75            tbody = table.find("tbody")76            if not tbody:77                continue78 79            for row in tbody.find_all("tr"):80                cells = row.find_all("td")81                if not cells:82                    continue83 84                row_data = {"Competition": competition}85 86                for i, cell in enumerate(cells):87                    col_name = headers_row[i+1] if i+1 < len(headers_row) else f"Col_{i}"88                    row_data[col_name] = cell.get_text(strip=True)89                    link = cell.find("a")90                    if link and "href" in link.attrs:91                        row_data[f"{col_name}_URL"] = "https://fbref.com" + link["href"]92 93                # Garder seulement lignes avec score ou heure94                if row_data.get("Score") or row_data.get("Time"):95                    fixtures.append(row_data)96 97        return fixtures, None98 99    except Exception as e:100        return [], str(e)101 102    finally:103        driver.quit()104 105# =========================106# ANALYSE107# =========================108def analyze(fixture):109    parts = []110    score = fixture.get("Score", "")111    xg = fixture.get("xG", "")112 113    if not score or "–" not in score:114        return "Match à venir"115 116    try:117        goals = score.split("–")118        h, a = int(goals[0].strip()), int(goals[1].strip())119        total = h + a120 121        if total == 0:122            parts.append("Match fermé 0-0")123        elif total >= 5:124            parts.append(f"Match très animé ({total} buts !)")125        elif total >= 3:126            parts.append(f"Match animé ({total} buts)")127 128        if h > a + 1:129            parts.append("Victoire nette domicile")130        elif a > h + 1:131            parts.append("Victoire nette extérieur")132        elif h == a:133            parts.append("Match nul")134 135        if xg:136            try:137                xg_val = float(xg)138                if h > a and xg_val < 1.0:139                    parts.append("Hold-up domicile (xG faible)")140                elif a > h and xg_val < 1.0:141                    parts.append("Hold-up extérieur (xG faible)")142            except:143                pass144 145    except:146        return "Analyse impossible"147 148    return ". ".join(parts) if parts else "Match joué"149 150# =========================151# GET JSON (pour Scriptable)152# =========================153def get_json():154    fixtures, error = scrape_fbref()155    if error:156        return json.dumps({"error": error, "matches": []}, ensure_ascii=False)157 158    results = []159    for f in fixtures:160        results.append({161            "competition": f.get("Competition", ""),162            "home": f.get("Home", ""),163            "away": f.get("Away", ""),164            "score": f.get("Score", ""),165            "xg": f.get("xG", ""),166            "time": f.get("Time", ""),167            "venue": f.get("Venue", ""),168            "url": f.get("Score_URL", ""),169            "analysis": analyze(f)170        })171 172    return json.dumps({"matches": results}, ensure_ascii=False, indent=2)173 174# =========================175# GET CSV176# =========================177def get_csv():178    fixtures, error = scrape_fbref()179    if error:180        return f"Erreur: {error}"181    if not fixtures:182        return "Aucun match trouvé"183 184    rows = []185    for f in fixtures:186        rows.append({187            "Competition": f.get("Competition", ""),188            "Home": f.get("Home", ""),189            "Away": f.get("Away", ""),190            "Score": f.get("Score", ""),191            "xG": f.get("xG", ""),192            "Time": f.get("Time", ""),193            "Venue": f.get("Venue", ""),194            "Analyse": analyze(f)195        })196 197    return pd.DataFrame(rows).to_csv(index=False)198 199# =========================200# INTERFACE GRADIO201# =========================202with gr.Blocks(title="⚽ FBref Scraper") as demo:203    gr.Markdown("# ⚽ FBref Scraper — Big 5 Leagues")204    gr.Markdown("Scrape FBref via Selenium · Données réelles avec xG")205 206    with gr.Tab("📊 JSON — pour Scriptable"):207        gr.Markdown("Appelle ce bouton depuis Scriptable via l'API Gradio")208        btn_json = gr.Button("🔄 Scraper FBref → JSON")209        out_json = gr.Textbox(label="JSON", lines=30)210        btn_json.click(fn=get_json, outputs=out_json)211 212    with gr.Tab("📋 CSV — export"):213        btn_csv = gr.Button("🔄 Scraper FBref → CSV")214        out_csv = gr.Textbox(label="CSV", lines=30)215        btn_csv.click(fn=get_csv, outputs=out_csv)216 217demo.launch()218