TechNeur/course_search

1import requests
2from bs4 import BeautifulSoup
3import pandas as pd
4import time
5
6base_url = "https://courses.analyticsvidhya.com/collections?page="
7course_url_base = "https://courses.analyticsvidhya.com"
8
9course_data = []
10
11for page in range(1,9):
12    print(f"Scraping page {page}...")
13    response = requests.get(base_url + str(page))
14    soup = BeautifulSoup(response.text, 'html.parser')
15    
16    course_section = soup.find_all('div', class_="collections__product-cards collections__product-cards___0b9ab")
17    if not course_section:
18        print("No course section found, skipping this page.")
19        continue
20    
21    courses = course_section[0].find_all('li')
22    
23    for course in courses:
24        link_tag = course.find('a', href=True)
25        if not link_tag:
26            continue
27        course_relative_link = link_tag['href']
28        course_link = course_url_base + course_relative_link
29        
30        course_response = requests.get(course_link)
31        course_soup = BeautifulSoup(course_response.text, 'html.parser')
32        
33        title_tag = course_soup.find('h1', class_="section__heading")
34        if title_tag:
35            course_title = title_tag.get_text(strip=True)
36        else:
37            course_title = "N/A"
38        
39        description_tag = course_soup.find_all('div', class_="rich-text__container")
40        course_description = " ".join([p.get_text(strip=True) for tag in description_tag for p in tag.find_all('p')]) if description_tag else "N/A"
41        
42        curriculum_section = course_soup.find('div', class_="course-curriculum__container")
43        if curriculum_section:
44            curriculum_content = []
45            
46            chapters = curriculum_section.find_all('li', class_="course-curriculum__chapter")
47            for chapter in chapters:
48                title = chapter.find('h5', class_="course-curriculum__chapter-title")
49                if title:
50                    curriculum_content.append(title.get_text(strip=True))
51                    
52                    chapter_content = chapter.find('ul', class_="course-curriculum__chapter-content")
53                    if chapter_content:
54                        curriculum_content.extend(
55                            [f"  - {item.get_text(strip=True)}" for item in chapter_content.find_all('li')]
56                        )
57            course_curriculum = "\n".join(curriculum_content) if curriculum_content else "N/A"
58        else:
59            course_curriculum = "N/A"
60        
61        course_data.append({
62            "Course Title": course_title,
63            "Course Description": course_description,
64            "Course Curriculum": course_curriculum,
65            "Link": course_link
66        })
67        
68        time.sleep(1)
69
70df = pd.DataFrame(course_data)
71file_path = r"C:\Users\rachi\OneDrive\Desktop\Analytics VIdya - Gen AI\analytics_vidhya_courses.xlsx"
72df.to_excel(file_path, index=False)
73print(f"Data saved to {file_path}")
74