TGChandu/Web-Research-Agent

1 2import openai3import os4import requests5from bs4 import BeautifulSoup6 7def search_query_serpapi(query, num_results=5):8    try:9        api_key = os.environ['SERPAPI_KEY']10        params = {11            "engine": "google",12            "q": query,13            "num": num_results,14            "api_key": api_key15        }16        response = requests.get("https://serpapi.com/search", params=params)17        results = response.json().get("organic_results", [])18        links = [result['link'] for result in results]19        return links if links else mock_search_results(query)20    except Exception as e:21        print(f"Error using SerpAPI: {e}")22        return mock_search_results(query)23 24def mock_search_results(query):25    print(f"Returning mock search results for query: {query}")26    return [27        f"https://example.com/{query.replace(' ', '_')}/result1",28        f"https://example.com/{query.replace(' ', '_')}/result2",29        f"https://example.com/{query.replace(' ', '_')}/result3"30    ]31 32def scrape_website(url):33    try:34        headers = {'User-Agent': 'Mozilla/5.0'}35        response = requests.get(url, headers=headers, timeout=5)36        soup = BeautifulSoup(response.content, 'html.parser')37        paragraphs = soup.find_all(['p', 'h1', 'h2', 'h3'])38        text = '\n'.join([para.get_text(strip=True) for para in paragraphs])39        return text if text else "No text found on this page."40    except Exception as e:41        print(f"Failed to scrape {url}: {e}")42        return "Error scraping this site."43 44def summarize_content(content, query):45    try:46        openai.api_key = os.environ['OPENAI_API_KEY']47        prompt = (48            f"The following is some text extracted from websites based on the query: '{query}'. "49            "Please summarize the key findings, latest research, and important points from this content. "50            "Make the summary clear and concise:\n\n"51            f"{content}\n\n"52            "Summary:"53        )54        response = openai.ChatCompletion.create(55            model="gpt-3.5-turbo",56            messages=[{"role": "user", "content": prompt}],57            max_tokens=800,58            temperature=0.559        )60        summary = response['choices'][0]['message']['content'].strip()61        return summary62    except Exception as e:63        print(f"Error during summarization: {e}")64        return "Could not summarize the content."65