jonathanjordan21/selenium_web_scrape

1from selenium import webdriver2from selenium.common.exceptions import WebDriverException3from PIL import Image4from io import BytesIO5 6import time7 8from selenium.webdriver.common.by import By9from selenium.webdriver.support.ui import WebDriverWait10from selenium.webdriver.support import expected_conditions as EC11 12def take_webdata(url):13    options = webdriver.ChromeOptions()14    options.add_argument('--headless')15    options.add_argument('--no-sandbox')16    options.add_argument('--disable-dev-shm-usage')17 18    try:19        wd = webdriver.Chrome(options=options)20        wd.set_window_size(1080, 720)  # Adjust the window size here21        wd.get(url)22        wd.implicitly_wait(5)23        # Get the page title24        page_title = wd.title25        screenshot = wd.get_screenshot_as_png()26 27    except WebDriverException as e:28        return Image.new('RGB', (1, 1)), page_title29    finally:30        if wd:31            wd.quit()32 33    return Image.open(BytesIO(screenshot)) , page_title34 35 36def scrape_vehicle(driver):37    data_kendaraan = {}38    try:39        rows = driver.find_elements(By.CSS_SELECTOR, "table tr")40        for row in rows:41            cols = row.find_elements(By.TAG_NAME, "td")42            if len(cols) >= 3:43                key = cols[0].text.strip().lower().replace(".", "").replace(" ", "_")44                value = cols[2].text.strip()45                data_kendaraan[key] = value46    except Exception as e:47        print("Gagal parsing tabel:", e)48    49    # rincians = []50    # try:51    #     container = driver.find_element(By.ID, "det_pkb")52    #     rows = container.find_elements(By.CLASS_NAME, "row")53    #     for row in rows[1:]:  # skip header54    #         cols = row.find_elements(By.TAG_NAME, "p")55    #         if len(cols) >= 3:56    #             rincian = {57    #                 "pokok": cols[0].text.strip(),58    #                 "denda": cols[1].text.strip(),59    #                 "total": cols[2].text.strip(),60    #             }61    #             if len(cols) > 3:62    #                 rincian["jenis"] = cols[3].text.strip().upper()63    #             rincians.append(rincian)64    # except Exception as e:65    #     print("Gagal parsing det_pkb:", e)66 67    total_tagihan = []68    try:69        all_rows = driver.find_elements(By.CSS_SELECTOR, "div.row")70        for row in all_rows:71            print("[ROW TOTAL]", row.text)72            if not ("POKOK" in row.text or "DENDA" in row.text or "TOTAL" in row.text):73                cols = row.find_elements(By.TAG_NAME, "p")74                print("[COLS TOTAL]", [x.text for x in cols])75                if len(cols) >= 4:76                    total_tagihan.append({77                        "pokok": cols[0].text.strip(),78                        "denda": cols[1].text.strip(),79                        "total": cols[2].text.strip(),80                        "jenis": cols[3].text.strip()81                    })82    except Exception as e:83        print("Gagal parsing total tagihan:", e)84    85 86    rincians_pkb = []87    try:88        pkb_rows = driver.find_elements(By.CSS_SELECTOR, "#det_pkb .row")[1:]  # skip header89        for row in pkb_rows:90            print("[ROW PKB]", row.text)91            cols = row.find_elements(By.TAG_NAME, "p")92            print("[COLS PKB]", [x.text for x in cols])93            if len(cols) >= 3:94                rincians_pkb.append({95                    "pokok": cols[0].text.strip(),96                    "denda": cols[1].text.strip(),97                    "total": cols[2].text.strip()98                })99    except Exception as e:100        print("Gagal parsing det_pkb:", e)101    102 103    rincians_swd = []104    try:105        swd_rows = driver.find_elements(By.CSS_SELECTOR, "#det_swd .row")[1:]  # skip header106        for row in swd_rows:107            print("[ROW SWD]", row.text)108            cols = row.find_elements(By.TAG_NAME, "p")109            print("[COLS SWD]", [x.text for x in cols])110            if len(cols) >= 3:111                rincians_swd.append({112                    "pokok": cols[0].text.strip(),113                    "denda": cols[1].text.strip(),114                    "total": cols[2].text.strip()115                })116    except Exception as e:117        print("Gagal parsing det_swd:", e)118 119    # rincians = [total_tagihan, rincians_pkb, rincians_swd]120    return [data_kendaraan, total_tagihan, rincians_pkb, rincians_swd]121 122 123def get_vehicle_info(plate_number: str):124    # Configure headless Chrome125    options = webdriver.ChromeOptions()126    options.add_argument("--headless")127    options.add_argument("--disable-gpu")128    options.add_argument("--no-sandbox")129 130    # Path to chromedriver (adjust if needed)131    driver = webdriver.Chrome(options=options)132 133    try:    134        driver.get("https://www.jambisamsat.net/infopkb.html")135        time.sleep(1)136 137        WebDriverWait(driver, 10).until(138            EC.presence_of_element_located((By.ID, "no_polisi"))139        )140 141        input_field = driver.find_element(By.ID, "no_polisi")142        input_field.clear()143        input_field.send_keys(plate_number)144 145        submit_button = driver.find_element(By.CSS_SELECTOR, 'button.btn.btn-primary[type="submit"]')146        submit_button.click()147 148        # Wait for the new page to load149        WebDriverWait(driver, 10).until(150            EC.url_contains("infopkb.php")151        )152 153        driver.implicitly_wait(3)154        155        scroll_height = driver.execute_script("return document.body.scrollHeight")156        driver.set_window_size(1920, scroll_height + 200)  # force full-page height157 158        button = WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.ID, "show_det_pkb")))159        button.click()160        161        button = WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.ID, "show_det_swd")))162        button.click()163        164        time.sleep(1)165 166        data = scrape_vehicle(driver)167 168        print(data)169 170        page_title = driver.title171        screenshot = driver.get_screenshot_as_png()172 173        return Image.open(BytesIO(screenshot)) , page_title, data174 175    except WebDriverException as e:176        return Image.new('RGB', (1, 1)), page_title177 178    finally:179        driver.quit()