Python-proje/presum

1from bs4 import BeautifulSoup2import requests3import re4 5def scrape_hes(url):6    HEADERS = {7        'User-Agent': 'Mozilla/5.0 (iPad; CPU OS 12_2 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148'}8    page = requests.get(url, headers=HEADERS)9 10    src = page.content  # variable to store page content11    soup = BeautifulSoup(src, "html.parser")  # beautify code12    # print(soup)13    Matches_Details = []14 15    # find all divs where exists class...16    article_content = soup.find("div", {'article-content'})17 18    all_paragraphes = article_content.find_all("p")  # get all a tags19    # matches_number = len(all_matches)20    article_text = ""21    i=022    for x in all_paragraphes:23        if i==0:24            i=125            x = x.text.strip()26            article_text = article_text+'\n'+x27            continue28        x = x.text.strip()29        article_text = article_text+'\n'+x30    #suppression espaces vides31    article_text = re.sub(r'\n[\t\n\s]+\n*',r"\n",article_text)32    return article_text.strip()33