coding-alt/AutoGPT
0
1# Generated by CodiumAI2 3import requests4 5from autogpt.commands.web_requests import scrape_text6 7"""8Code Analysis9 10Objective:11The objective of the "scrape_text" function is to scrape the text content from12a given URL and return it as a string, after removing any unwanted HTML tags and scripts.13 14Inputs:15- url: a string representing the URL of the webpage to be scraped.16 17Flow:181. Send a GET request to the given URL using the requests library and the user agent header from the config file.192. Check if the response contains an HTTP error. If it does, return an error message.203. Use BeautifulSoup to parse the HTML content of the response and extract all script and style tags.214. Get the text content of the remaining HTML using the get_text() method of BeautifulSoup.225. Split the text into lines and then into chunks, removing any extra whitespace.236. Join the chunks into a single string with newline characters between them.247. Return the cleaned text.25 26Outputs:27- A string representing the cleaned text content of the webpage.28 29Additional aspects:30- The function uses the requests library and BeautifulSoup to handle the HTTP request and HTML parsing, respectively.31- The function removes script and style tags from the HTML to avoid including unwanted content in the text output.32- The function uses a generator expression to split the text into lines and chunks, which can improve performance for large amounts of text.33"""34 35 36class TestScrapeText:37 # Tests that scrape_text() returns the expected text when given a valid URL.38 def test_scrape_text_with_valid_url(self, mocker):39 # Mock the requests.get() method to return a response with expected text40 expected_text = "This is some sample text"41 mock_response = mocker.Mock()42 mock_response.status_code = 20043 mock_response.text = f"<html><body><div><p style='color: blue;'>{expected_text}</p></div></body></html>"44 mocker.patch("requests.Session.get", return_value=mock_response)45 46 # Call the function with a valid URL and assert that it returns the expected text47 url = "http://www.example.com"48 assert scrape_text(url) == expected_text49 50 # Tests that the function returns an error message when an invalid or unreachable url is provided.51 def test_invalid_url(self, mocker):52 # Mock the requests.get() method to raise an exception53 mocker.patch(54 "requests.Session.get", side_effect=requests.exceptions.RequestException55 )56 57 # Call the function with an invalid URL and assert that it returns an error message58 url = "http://www.invalidurl.com"59 error_message = scrape_text(url)60 assert "Error:" in error_message61 62 # Tests that the function returns an empty string when the html page contains no text to be scraped.63 def test_no_text(self, mocker):64 # Mock the requests.get() method to return a response with no text65 mock_response = mocker.Mock()66 mock_response.status_code = 20067 mock_response.text = "<html><body></body></html>"68 mocker.patch("requests.Session.get", return_value=mock_response)69 70 # Call the function with a valid URL and assert that it returns an empty string71 url = "http://www.example.com"72 assert scrape_text(url) == ""73 74 # Tests that the function returns an error message when the response status code is an http error (>=400).75 def test_http_error(self, mocker):76 # Mock the requests.get() method to return a response with a 404 status code77 mocker.patch("requests.Session.get", return_value=mocker.Mock(status_code=404))78 79 # Call the function with a URL80 result = scrape_text("https://www.example.com")81 82 # Check that the function returns an error message83 assert result == "Error: HTTP 404 error"84 85 # Tests that scrape_text() properly handles HTML tags.86 def test_scrape_text_with_html_tags(self, mocker):87 # Create a mock response object with HTML containing tags88 html = "<html><body><p>This is <b>bold</b> text.</p></body></html>"89 mock_response = mocker.Mock()90 mock_response.status_code = 20091 mock_response.text = html92 mocker.patch("requests.Session.get", return_value=mock_response)93 94 # Call the function with a URL95 result = scrape_text("https://www.example.com")96 97 # Check that the function properly handles HTML tags98 assert result == "This is bold text."99