RituJoshi/RecommendationSystem

1{2 "cells": [3  {4   "cell_type": "code",5   "execution_count": 2,6   "id": "4f772539",7   "metadata": {},8   "outputs": [],9   "source": [10    "# data_collection\n",11    "import requests\n",12    "from bs4 import BeautifulSoup\n",13    "import pandas as pd\n",14    "\n",15    "url = \"https://www.shl.com/solutions/products/product-catalog/\"\n",16    "headers = {\"User-Agent\": \"Mozilla/5.0\"}  # Avoid 403 Forbidden errors\n",17    "response = requests.get(url, headers=headers)\n",18    "soup = BeautifulSoup(response.content, \"html.parser\")\n",19    "\n",20    "assessments = []\n",21    "for assessment in soup.find_all(\"div\", class_=\"assessment-card\"):  # Update class name!\n",22    "    # Extract fields (adjust tags/classes to match SHL's HTML)\n",23    "    name = assessment.find(\"h2\").text.strip()\n",24    "    description = assessment.find(\"p\", class_=\"description\").text.strip()  # Update class!\n",25    "    url = assessment.find(\"a\")[\"href\"]\n",26    "    \n",27    "    # Example: Extract duration (adjust logic)\n",28    "    duration_text = assessment.find(\"span\", class_=\"duration\").text.strip()  # Update class!\n",29    "    duration = int(duration_text.replace(\" mins\", \"\"))  # Convert to integer\n",30    "    \n",31    "    # Repeat for remote_support, adaptive_support, test_type\n",32    "    assessments.append({\n",33    "        \"description\": description,\n",34    "        \"url\": url,\n",35    "        \"duration\": duration,  # Actual extracted value\n",36    "        \"remote_support\": \"Yes\",  # Replace with extracted value\n",37    "        \"adaptive_support\": \"No\",  # Replace with extracted value\n",38    "        \"test_type\": \"Technical\"  # Replace with extracted value\n",39    "    })\n",40    "\n",41    "# Save to CSV\n",42    "df = pd.DataFrame(assessments)\n",43    "df.to_csv(\"data/shl_assessments.csv\", index=False)\n",44    "\n",45    "\n",46    "\n",47    "\n",48    "\n",49    "\n",50    "\n",51    "\n",52    "\n",53    "\n",54    "\n",55    "\n",56    "\n",57    "\n",58    "\n",59    "\n",60    "\n",61    "\n",62    "\n",63    "\n",64    "\n",65    "\n",66    "\n",67    "\n",68    "\n"69   ]70  }71 ],72 "metadata": {73  "kernelspec": {74   "display_name": "Python 3 (ipykernel)",75   "language": "python",76   "name": "python3"77  },78  "language_info": {79   "codemirror_mode": {80    "name": "ipython",81    "version": 382   },83   "file_extension": ".py",84   "mimetype": "text/x-python",85   "name": "python",86   "nbconvert_exporter": "python",87   "pygments_lexer": "ipython3",88   "version": "3.10.0"89  }90 },91 "nbformat": 4,92 "nbformat_minor": 593}94