Juna190825/seleniumwebscrapping

1 2import os3import time4import tempfile5import shutil6import logging7import subprocess8import socket9import random10from selenium import webdriver11from selenium.webdriver.common.by import By12from selenium.webdriver.support.ui import WebDriverWait13from selenium.webdriver.support import expected_conditions as EC14from selenium.webdriver.chrome.options import Options15from selenium.webdriver.chrome.service import Service16 17import gradio as gr18 19# Use these specific ports20WEBDRIVER_PORT = random.randint(4444, 4544)21 22# Configure logging23logging.basicConfig(24    level=logging.INFO,25    format='%(asctime)s - %(levelname)s - %(message)s',26    handlers=[27        logging.FileHandler('/tmp/webscraper.log'),28        logging.StreamHandler()29    ]30)31logger = logging.getLogger(__name__)32 33# Configure environment for matplotlib34os.environ['MPLCONFIGDIR'] = tempfile.mkdtemp()35logger.info(f"Matplotlib config directory set to: {os.environ['MPLCONFIGDIR']}")36 37def verify_chrome_installation():38    try:39        result = subprocess.run(['google-chrome-stable', '--version'],40                              capture_output=True, text=True)41        logger.info(f"Chrome version: {result.stdout}")42        return True43    except:44        logger.error("Chrome verification failed")45        return False46 47def verify_chromedriver():48    chromedriver_path = '/usr/local/bin/chromedriver'49    try:50        result = subprocess.run([chromedriver_path, '--version'],51                              capture_output=True, text=True)52        logger.info(f"ChromeDriver version: {result.stdout}")53        st = os.stat(chromedriver_path)54        if not st.st_mode & 0o111:  # Check executable bit55            logger.warning("ChromeDriver not executable, fixing permissions...")56            os.chmod(chromedriver_path, 0o755)57            logger.info("Permissions updated")58        return True59    except:60        logger.error("ChromeDriver verification failed")61        return False62 63def create_service():64    # 1. Verify chromedriver65    chromedriver_path = shutil.which('chromedriver') or '/usr/local/bin/chromedriver'66    if not os.path.exists(chromedriver_path):67        raise FileNotFoundError(f"ChromeDriver missing at {chromedriver_path}")68 69    # 2. Check port availability70    port = WEBDRIVER_PORT71    s = socket.socket()72    if s.connect_ex(('127.0.0.1', port)) == 0:73        port += 174    s.close()75 76    # 3. Ensure log directory77    log_dir = tempfile.gettempdir()78    if not os.access(log_dir, os.W_OK):79        log_dir = '/tmp'80 81    # 4. Create service82    try:83        return Service(84            executable_path=chromedriver_path,85            port=port,86            service_args=[87                '--verbose',88                '--log-path=/tmp/chromedriver.log'89            ]90        )91    except Exception as e:92        print(f"Service creation failed: {str(e)}")93        return Service()  # Fallback to simplest config94 95def verify_critical_path():96    print(f"ChromeDriver exists: {os.path.exists('/usr/local/bin/chromedriver')}")97    print(f"Chrome exists: {os.path.exists('/usr/bin/google-chrome-stable')}")98 99    print(f"ChromeDriver executable: {os.access('/usr/local/bin/chromedriver', os.X_OK)}")100    print(f"Chrome executable: {os.access('/usr/bin/google-chrome-stable', os.X_OK)}")101 102    try:103        subprocess.run(["/usr/local/bin/chromedriver", "--version"], check=True)104        subprocess.run(["/usr/bin/google-chrome-stable", "--version"], check=True)105    except subprocess.CalledProcessError as e:106        print(f"Binary test failed: {e}")107 108def setup_selenium():109    try:110        logger.info("Initializing Selenium Chrome driver...")111        options = Options()112        options.add_argument("--headless=new")113        options.add_argument("--no-sandbox")114        options.add_argument("--disable-dev-shm-usage")115        options.add_argument("--disable-gpu")116        options.add_argument("--window-size=1920,1080")117        options.add_argument("--start-maximized")118        options.add_argument("--disable-blink-features=AutomationControlled")119        options.add_argument("--disable-notifications")120        options.add_argument("--remote-debugging-port=9222")121 122        # Disable automation flag123        options.add_experimental_option("excludeSwitches", ["enable-automation"])124        options.add_experimental_option("useAutomationExtension", False)125        options.add_experimental_option("prefs", {126            "profile.default_content_setting_values.notifications": 2  # Disable notifications127        })128 129        # Mimic a real browser's user agent130        options.add_argument("user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36")131 132        logger.info("Verifying ChromeDriver...")133        verifying_chromedriver = verify_chromedriver()134        logger.info(f"ChromeDriver verification: {verifying_chromedriver}")135 136        chromedriver_path = "/usr/local/bin/chromedriver"137        # os.chmod(chromedriver_path, 0o755)  # Make sure it's executable138        logger.info(f"Setting executable permission for {chromedriver_path}")139 140        service = create_service()141        logger.info("Service created")142 143        verify_critical_path()144 145        try:146            logger.info("Attempting Chrome initialization...")147            driver = webdriver.Chrome(148                options=options,149                service=service150            )151            logger.info("Successfully initialized Chrome driver!")152            return driver153 154        except Exception as e:155            logger.error(f"Initialization failed completely: {str(e)}")156            try:157                from subprocess import run158                run([chromedriver_path, "--version"], check=True)159                run(["/usr/bin/google-chrome-stable", "--version"], check=True)160            except Exception as sub_e:161                logger.error(f"Subprocess check failed: {str(sub_e)}")162            raise163 164    except Exception as e:165        logger.error("Possible causes:")166        logger.error("- Missing or incompatible ChromeDriver")167        logger.error("- Chrome binary not found")168        logger.error("- Insufficient permissions")169        logger.error("- Missing system dependencies")170        raise RuntimeError(f"Failed to initialize WebDriver: {str(e)}")171 172def check_selenium_environment():173    try:174        driver = setup_selenium()175        driver.quit()176        return True177    except:178        return False179 180if not check_selenium_environment():181    logger.critical("Selenium environment check failed!")182    exit(1)183 184def form_input_text(word):185    logger.info(f"Generating input text for word: {word}")186    return f"use the word '{word}' based on the following sentences types, give output in a table format, no quotes around sentences, no repeating the same sentence: Simple Sentence, Compound Sentence"187 188def fetch_sentences(input_text):189    driver = None190    generated_sentences = []191    try:192        logger.info("Starting sentence fetching process...")193        logger.debug(f"Input text: {input_text[:100]}...")194 195        driver = setup_selenium()196        driver.implicitly_wait(5)197 198        logger.info("Navigating to target URL...")199        driver.get("https://copilot.microsoft.com/chat")200        print("...after: driver.get('https://copilot.microsoft.com/chat')")201 202        logger.info("Waiting for page to load...")203        WebDriverWait(driver, 50).until(EC.presence_of_element_located((By.TAG_NAME, "body")))204        time.sleep(10)205 206        textarea = WebDriverWait(driver, 20).until(207            EC.presence_of_element_located((By.ID, "userInput"))208        )209        textarea.clear()210        textarea.send_keys(input_text)211        print("...after: sending")212        time.sleep(20)213 214        button = driver.find_element(By.XPATH, "//button[@title='Submit message']")215        button.click()216        print("...after: button.click()")217        print("Text inserted and button pressed successfully!")218 219        time.sleep(20)220 221        table = WebDriverWait(driver, 10).until(222            EC.presence_of_element_located((By.XPATH, "//table"))223        )224 225        rows = table.find_elements(By.TAG_NAME, "tr")226        print(len(rows))227 228        for row in rows:229            row_cells = row.find_elements(By.TAG_NAME, "td")230            generated_sentences.append([r.text for r in row_cells])231 232        time.sleep(10)233    finally:234        if driver:235            driver.quit()236 237    generated_sentences = [item for item in generated_sentences if len(item) > 0] if len(generated_sentences) > 1 else generated_sentences238    return generated_sentences239 240def scrape_website(url):241    try:242        if not verify_chrome_installation():243            return "Error: Chrome not properly installed"244 245        current_word = 'go'246        return fetch_sentences(form_input_text(current_word))247 248    except RuntimeError as e:249        logger.error(f"Runtime error: {str(e)}")250        return f"System error: {str(e)}"251    except Exception as e:252        logger.error(f"Unexpected error: {str(e)}")253        return f"Unexpected error occurred: {str(e)}"254 255iface = gr.Interface(256    fn=scrape_website,257    inputs=[258        gr.Textbox(label="URL to scrape", placeholder="https://example.com"),259    ],260    outputs=gr.Textbox(label="Scraped Content"),261    title="Web Scraper with Selenium & Chrome",262    description="Enter a URL to scrape its content using headless Chrome browser."263)264 265iface.queue().launch(266    server_name="0.0.0.0",267    server_port=7860,268    share=False269)270