mumer119131/sephora-scraper

1import requests2from bs4 import BeautifulSoup3import re4import csv5import psycopg26import datetime7import time8 9 10class SephoraScraper:11    def __init__(self):12        self.headers = {13            'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) ',14        }15        self.categories = [16            "makeup-cosmetics", "skincare", "hair-products", "fragrance", "makeup-tools", "bath-body", "travel-size-toiletries", "gifts"17        ]18 19    def get_response(self, url):20        id_pattern = pattern = r'-(P\d+)\?'21        match = re.search(pattern, url)22        if match:23            product_id = match.group(1)24        else:25            return False26        burp0_url = f"https://www.sephora.com:443/api2/catalog/products/{product_id}?addCurrentSkuToProductChildSkus=true&includeRegionsMap=true&showContent=true&includeConfigurableSku=true&countryCode=US&removePersonalizedData=true&includeReviewFilters=true&includeReviewImages=true&sentiments=6"27        burp0_headers = {"Sec-Ch-Ua": "\"Chromium\";v=\"117\", \"Not;A=Brand\";v=\"8\"", "X-Ufe-Request": "true", "Sec-Ch-Ua-Mobile": "?0", "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.5938.132 Safari/537.36", "X-Dtpc": "5$505172501_268h23vBLLCMFBNGOLFAPFGHKUDBVTGKMEPJULD-0e0", "X-Dtreferer": "https://www.sephora.com/shop/makeup-cosmetics?currentPage=2", "Exclude_personalized_content": "true", "X-Requested-Source": "rwd", "Sec-Ch-Ua-Platform": "\"Windows\"", "Accept": "*/*", "Sec-Fetch-Site": "same-origin", "Sec-Fetch-Mode": "cors", "Sec-Fetch-Dest": "empty", "Referer": "https://www.sephora.com/product/sephora-collection-total-coverage-blending-sponge-set-60-plant-based-P482303?skuId=2497220&icid2=products%20grid:p482303:product", "Accept-Encoding": "gzip, deflate, br", "Accept-Language": "en-US,en;q=0.9"}28        try:29            response = requests.get(burp0_url, headers=burp0_headers)30            return response.json()31        except Exception as e:32            print(e)33            return False34    35    def parse_response(self, response, url):36        try:37            name = response['productDetails']['displayName']38            product_id = response['productDetails']['productId']39            try:40                ingredients = response['currentSku']['ingredientDesc'].replace('\n', ' ').replace('<b>', '').replace('</b>', '').replace('<br>', '').replace('</p>', '').replace('<p>', '').replace('<br/>', '').replace('<p>', '')41            except:42                ingredients = ''43 44            return [name, product_id, ingredients]45        except:46            return False47    def search_category_total_results(self, category):48        burp0_url = f"https://www.sephora.com:443/api/v2/catalog/categories/{category}/seo?targetSearchEngine=NLP&currentPage=2&pageSize=60&content=true&includeRegionsMap=true&headers=%5Bobject%20Object%5D&pickupRampup=true&sddRampup=true&loc=en-US&ch=rwd"49        burp0_headers = {"Sec-Ch-Ua": "\"Chromium\";v=\"117\", \"Not;A=Brand\";v=\"8\"", "Sec-Ch-Ua-Mobile": "?0", "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.5938.132 Safari/537.36", "X-Dtpc": "5$505172501_268h16vBLLCMFBNGOLFAPFGHKUDBVTGKMEPJULD-0e0", "X-Dtreferer": "https://www.sephora.com/shop/makeup-cosmetics", "X-Timestamp": "1697306065014", "Exclude_personalized_content": "true", "X-Requested-Source": "rwd", "Sec-Ch-Ua-Platform": "\"Windows\"", "Accept": "*/*", "Sec-Fetch-Site": "same-origin", "Sec-Fetch-Mode": "cors", "Sec-Fetch-Dest": "empty", "Referer": "https://www.sephora.com/shop/makeup-cosmetics?currentPage=2", "Accept-Encoding": "gzip, deflate, br", "Accept-Language": "en-US,en;q=0.9"}50        response = requests.get(burp0_url, headers=burp0_headers)51        52        data = response.json()53 54        return data['totalProducts']55    56    def get_urls_of_category_from_all_results(self, category, total_results):57 58        all_urls = []59        pages = total_results // 60 + 160        for page in range(1, pages+1):61            burp0_url = f"https://www.sephora.com:443/api/v2/catalog/categories/{category}/seo?targetSearchEngine=NLP&currentPage={page}&pageSize=60&content=true&includeRegionsMap=true&headers=%5Bobject%20Object%5D&pickupRampup=true&sddRampup=true&loc=en-US&ch=rwd"62            burp0_headers = {"Sec-Ch-Ua": "\"Chromium\";v=\"117\", \"Not;A=Brand\";v=\"8\"", "Sec-Ch-Ua-Mobile": "?0", "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.5938.132 Safari/537.36", "X-Dtpc": "5$505172501_268h16vBLLCMFBNGOLFAPFGHKUDBVTGKMEPJULD-0e0", "X-Dtreferer": "https://www.sephora.com/shop/makeup-cosmetics", "X-Timestamp": "1697306065014", "Exclude_personalized_content": "true", "X-Requested-Source": "rwd", "Sec-Ch-Ua-Platform": "\"Windows\"", "Accept": "*/*", "Sec-Fetch-Site": "same-origin", "Sec-Fetch-Mode": "cors", "Sec-Fetch-Dest": "empty", "Referer": "https://www.sephora.com/shop/makeup-cosmetics?currentPage=2", "Accept-Encoding": "gzip, deflate, br", "Accept-Language": "en-US,en;q=0.9"}63            response = requests.get(burp0_url, headers=burp0_headers)64            data = response.json()65            products = data['products']66            urls = []67            for product in products:68                urls.append(f'https://www.sephora.com{product["targetUrl"]}')69            print(urls)70            self.get_all_products_from_category_page(urls)71        # self.get_all_products_from_category_page(all_urls)72            73    def get_all_products_from_category_page(self, urls):74        for url in urls:75            response = self.get_response(url)76            product = self.parse_response(response, url=url)77            if product:78                self.save_product({79                    'title': product[0],80                    'product_id': product[1],81                    'ingredients': product[2],82                    'url': url,83                    'store_name': 'Sephora'84                })85                # save_product(product)86        return True87 88    def save_product_to_csv(self, product):89        with open('sep.csv', 'a+', encoding='utf-8', newline='') as file:90            #check if the product is already in the csv91            file.seek(0) # move the file pointer to the beginning of the file92            reader = csv.reader(file)93            product_ids = [row[1] for row in reader]94            print(product_ids)95            if product[1] in product_ids:96                return False97            98            writer = csv.writer(file)99            writer.writerow(product)100            file.close()101    def save_product(self, product):102        try:103            conn = psycopg2.connect(104                host="ep-rapid-cake-30394055.us-east-2.aws.neon.tech",105                database="ingredients-scraper",106                user="mumer113141",107                password="SFBtp4xnPeA2"108            )109            cur = conn.cursor()110            cur.execute("INSERT INTO scraper_product (title, product_id, ingredients, url, store_name, date_created) VALUES (%s, %s, %s, %s, %s, %s)", (product['title'], product['product_id'], product['ingredients'], product['url'], product['store_name'], datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")))111            conn.commit()112            cur.close()113            conn.close()114            return True115        except Exception as e:116            print(e)117            return False118    def run(self):119        for category in self.categories:120            total_results = self.search_category_total_results(category)121            self.get_urls_of_category_from_all_results(category, total_results)122            print(f'Finished {category}')123        return True124 125 126if __name__ == '__main__':127    SephoraScraper().run()