architojha/job-query

1from bs4 import BeautifulSoup2import requests3 4class ScrapeJobDetails:5    def __init__(self, urls: list):6 7        self.url_list = urls8 9    def _fetchUrlData(self, url: str):10 11        headers = {12            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36',13            'Accept-Language': 'en-US,en;q=0.9',14        }15 16        return requests.get(url, headers=headers).text17 18    def _parseHtml(self, html_content):19 20        soup = BeautifulSoup(html_content, 'lxml')21 22        tags_to_extract = ['p', 'li', 'ul']23 24        extracted_text = []25        for tag in tags_to_extract:26            for element in soup.find_all(tag):27                extracted_text.append(element.get_text(strip=True))  28 29        return "\n".join(extracted_text)30 31 32    def _run(self):33 34        result_dict = {}35 36        for url in self.url_list:37 38            html_data = self._fetchUrlData(url)39 40            result_dict[url] = self._parseHtml(html_data)41 42        return result_dict