coding-alt/AutoGPT
0
1# Generated by CodiumAI2 3# Dependencies:4# pip install pytest-mock5import pytest6 7from autogpt.commands.web_requests import scrape_links8 9"""10Code Analysis11 12Objective:13The objective of the 'scrape_links' function is to scrape hyperlinks from a14given URL and return them in a formatted way.15 16Inputs:17- url: a string representing the URL to be scraped.18 19Flow:201. Send a GET request to the given URL using the requests library and the user agent header from the config file.212. Check if the response contains an HTTP error. If it does, return "error".223. Parse the HTML content of the response using the BeautifulSoup library.234. Remove any script and style tags from the parsed HTML.245. Extract all hyperlinks from the parsed HTML using the 'extract_hyperlinks' function.256. Format the extracted hyperlinks using the 'format_hyperlinks' function.267. Return the formatted hyperlinks.27 28Outputs:29- A list of formatted hyperlinks.30 31Additional aspects:32- The function uses the 'requests' and 'BeautifulSoup' libraries to send HTTP33requests and parse HTML content, respectively.34- The 'extract_hyperlinks' function is called to extract hyperlinks from the parsed HTML.35- The 'format_hyperlinks' function is called to format the extracted hyperlinks.36- The function checks for HTTP errors and returns "error" if any are found.37"""38 39 40class TestScrapeLinks:41 # Tests that the function returns a list of formatted hyperlinks when42 # provided with a valid url that returns a webpage with hyperlinks.43 def test_valid_url_with_hyperlinks(self):44 url = "https://www.google.com"45 result = scrape_links(url)46 assert len(result) > 047 assert isinstance(result, list)48 assert isinstance(result[0], str)49 50 # Tests that the function returns correctly formatted hyperlinks when given a valid url.51 def test_valid_url(self, mocker):52 # Mock the requests.get() function to return a response with sample HTML containing hyperlinks53 mock_response = mocker.Mock()54 mock_response.status_code = 20055 mock_response.text = (56 "<html><body><a href='https://www.google.com'>Google</a></body></html>"57 )58 mocker.patch("requests.Session.get", return_value=mock_response)59 60 # Call the function with a valid URL61 result = scrape_links("https://www.example.com")62 63 # Assert that the function returns correctly formatted hyperlinks64 assert result == ["Google (https://www.google.com)"]65 66 # Tests that the function returns "error" when given an invalid url.67 def test_invalid_url(self, mocker):68 # Mock the requests.get() function to return an HTTP error response69 mock_response = mocker.Mock()70 mock_response.status_code = 40471 mocker.patch("requests.Session.get", return_value=mock_response)72 73 # Call the function with an invalid URL74 result = scrape_links("https://www.invalidurl.com")75 76 # Assert that the function returns "error"77 assert "Error:" in result78 79 # Tests that the function returns an empty list when the html contains no hyperlinks.80 def test_no_hyperlinks(self, mocker):81 # Mock the requests.get() function to return a response with sample HTML containing no hyperlinks82 mock_response = mocker.Mock()83 mock_response.status_code = 20084 mock_response.text = "<html><body><p>No hyperlinks here</p></body></html>"85 mocker.patch("requests.Session.get", return_value=mock_response)86 87 # Call the function with a URL containing no hyperlinks88 result = scrape_links("https://www.example.com")89 90 # Assert that the function returns an empty list91 assert result == []92 93 # Tests that scrape_links() correctly extracts and formats hyperlinks from94 # a sample HTML containing a few hyperlinks.95 def test_scrape_links_with_few_hyperlinks(self, mocker):96 # Mock the requests.get() function to return a response with a sample HTML containing hyperlinks97 mock_response = mocker.Mock()98 mock_response.status_code = 20099 mock_response.text = """100 <html>101 <body>102 <div id="google-link"><a href="https://www.google.com">Google</a></div>103 <div id="github"><a href="https://github.com">GitHub</a></div>104 <div id="CodiumAI"><a href="https://www.codium.ai">CodiumAI</a></div>105 </body>106 </html>107 """108 mocker.patch("requests.Session.get", return_value=mock_response)109 110 # Call the function being tested111 result = scrape_links("https://www.example.com")112 113 # Assert that the function returns a list of formatted hyperlinks114 assert isinstance(result, list)115 assert len(result) == 3116 assert result[0] == "Google (https://www.google.com)"117 assert result[1] == "GitHub (https://github.com)"118 assert result[2] == "CodiumAI (https://www.codium.ai)"119