Sahil1694/Atlan

1import os
2import json
3import time
4import requests
5from bs4 import BeautifulSoup
6from urllib.parse import urljoin, urlparse
7
8# --- Configuration ---
9
10# The starting points for our crawl.
11START_URLS = [
12    "https://docs.atlan.com/",
13    "https://developer.atlan.com/"
14]
15
16# The path where the final scraped data will be saved.
17SAVE_PATH = "data/knowledge_base.json"
18
19# It's a good practice to identify your scraper with a User-Agent.
20HEADERS = {
21    'User-Agent': 'AtlanSupportCopilotBot/1.0'
22}
23
24# --- Core Logic ---
25
26def get_all_links(base_url: str) -> set:
27    """
28    Crawls a starting URL to find all unique, same-domain links.
29    """
30    found_links = set()
31    try:
32        response = requests.get(base_url, headers=HEADERS)
33        response.raise_for_status() # Raise an exception for bad status codes
34        
35        soup = BeautifulSoup(response.content, 'html.parser')
36        base_domain = urlparse(base_url).netloc
37        
38        for a_tag in soup.find_all('a', href=True):
39            href = a_tag['href']
40            # Create a full, absolute URL from a relative or absolute path
41            full_url = urljoin(base_url, href)
42            
43            # Keep the link only if it belongs to the same domain
44            if urlparse(full_url).netloc == base_domain:
45                # Clean off any URL fragments (#section-links)
46                found_links.add(full_url.split('#')[0])
47                
48    except requests.RequestException as e:
49        pass
50
51    return found_links
52
53def scrape_page_content(url: str) -> dict | None:
54    """
55    Scrapes the main textual content from a single URL.
56    Returns a dictionary with the URL and its content, or None on failure.
57    """
58    try:
59        response = requests.get(url, headers=HEADERS)
60        response.raise_for_status()
61        
62        soup = BeautifulSoup(response.content, 'html.parser')
63        
64        # **This is the most important part to customize.**
65        # We need to find the main content container. After inspecting the sites,
66        # 'main' and 'article' tags seem to hold the relevant content.
67        main_content = soup.find('main') or soup.find('article')
68        
69        if main_content:
70            # Use .get_text() to extract all text from the container.
71            # `separator=' '` joins text from different tags with a space.
72            # `strip=True` removes leading/trailing whitespace.
73            text = main_content.get_text(separator=' ', strip=True)
74            return {"url": url, "content": text}
75        else:
76            # If no main content is found, we can skip this page.
77            return None
78            
79    except requests.RequestException as e:
80        return None
81
82# --- Main Execution ---
83
84if __name__ == "__main__":
85    all_site_links = set()
86    
87    # 1. Discover all links from the starting URLs
88    for url in START_URLS:
89        links = get_all_links(url)
90        all_site_links.update(links)
91    
92    scraped_data = []
93    
94    # 2. Scrape the content from each unique link
95    total_links = len(all_site_links)
96    for i, link in enumerate(list(all_site_links), 1):
97        content_dict = scrape_page_content(link)
98        if content_dict and content_dict['content']:
99            scraped_data.append(content_dict)
100        
101        # Be a good web citizen! Wait 1 second between requests.
102        time.sleep(1)
103        
104    # 3. Save the final data to a JSON file
105    try:
106        # Ensure the 'data' directory exists
107        os.makedirs(os.path.dirname(SAVE_PATH), exist_ok=True)
108        
109        with open(SAVE_PATH, 'w', encoding='utf-8') as f:
110            json.dump(scraped_data, f, indent=4, ensure_ascii=False)
111    except Exception as e:
112        pass
113