celise88/Pathfinder
3
1import requests2from bs4 import BeautifulSoup3from cleantext import clean4import pandas as pd5import numpy as np6 7onet = pd.read_csv('static/ONET_JobTitles.csv')8headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.1.2 Safari/605.1.15'}9 10def remove_new_line(value):11 return ''.join(value.splitlines())12 13def get_onet_code(jobtitle):14 onetCode = onet.loc[onet['JobTitle'] == jobtitle, 'onetCode']15 onetCode = onetCode.reindex().tolist()[0]16 return onetCode17 18def get_onet_description(onetCode):19 url = "https://www.onetonline.org/link/summary/" + onetCode20 response = requests.get(url, headers=headers, verify=False)21 soup = BeautifulSoup(response.text, 'html.parser')22 jobdescription = soup.p.get_text()23 return jobdescription24 25def get_onet_tasks(onetCode):26 headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.1.2 Safari/605.1.15'}27 url = "https://www.onetonline.org/link/result/" + onetCode + "?c=tk&n_tk=0&s_tk=IM&c_tk=0"28 response = requests.get(url, headers=headers, verify=False)29 soup = BeautifulSoup(response.text, 'html.parser')30 tasks = str(soup.get_text('reportsubdesc')).replace("reportsubdesc", " ").replace("ImportanceCategoryTask ", "")31 tasks = clean(tasks)32 if len(tasks.split('show all show top 10')) > 1:33 tasks = tasks.split('show all show top 10')[1]34 tasks = tasks.split('occupations related to multiple tasks')[0]35 tasks = remove_new_line(tasks).replace("related occupations", " ").replace("core", " - ").replace("supplemental", "").replace("not available", "").replace(" )importance category task", "").replace(" find ", "")36 tasks = tasks.split(". ")37 tasks = [''.join(map(lambda c: '' if c in '0123456789-' else c, task)) for task in tasks]38 return tasks39 else: 40 return pd.DataFrame([("We're sorry."), ("This occupation is currently undergoing updates."), ("Please try again later.")])41 42def get_onet_activities(onetCode):43 headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.1.2 Safari/605.1.15'}44 45 activities_url = "https://www.onetonline.org/link/result/" + onetCode + "?c=wa&n_wa=0&s_wa=IM&c_wa=0"46 47 response = requests.get(activities_url, headers=headers, verify=False)48 soup = BeautifulSoup(response.text, 'html.parser')49 tasks = str(soup.get_text('reportsubdesc')).replace("reportsubdesc", " ").replace("ImportanceCategoryTask ", "")50 tasks = clean(tasks)51 if len(tasks.split('show all show top 10')) > 1:52 tasks = tasks.split('show all show top 10')[1]53 tasks = tasks.split('back to top')[0]54 tasks = remove_new_line(tasks).replace("related occupations", " ").replace("importance work activity", " ")55 tasks = tasks.split(". ")56 split_data = [item.split(" -- ")[0] for item in tasks]57 num_desc = []58 for i in range(len(tasks)):59 temp = [','.join(item) for item in split_data][i].split(',')60 num_desc.append([''.join([c for c in temp if c in '0123456789']), ''.join([c for c in temp if c not in '0123456789']).replace(' ) ', '')])61 df = pd.DataFrame(num_desc, columns = ['Importance', 'Activity'])62 df = df[df['Importance'] != '']63 activities = df64 return activities65 else: 66 return pd.DataFrame([("We're sorry."), ("This occupation is currently undergoing updates."), ("Please try again later.")])67 68 69def get_onet_context(onetCode):70 headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.1.2 Safari/605.1.15'}71 72 context_url = "https://www.onetonline.org/link/result/" + onetCode + "?c=cx&n_cx=0&c_cx=0&s_cx=n"73 74 response = requests.get(context_url, headers=headers, verify=False)75 soup = BeautifulSoup(response.text, 'html.parser')76 tasks = str(soup.get_text('reportsubdesc')).replace("reportsubdesc", " ").replace("ImportanceCategoryTask ", "")77 tasks = clean(tasks)78 if len(tasks.split('show all show top 10')) > 1:79 tasks = tasks.split('show all show top 10')[1]80 tasks = tasks.split('back to top')[0]81 tasks = remove_new_line(tasks).replace("related occupations", " ").replace("importance work activity", " ")82 tasks = tasks.split("? ")83 split_data = [item.split(" -- ")[0] for item in tasks]84 num_desc = []85 for i in range(len(tasks)):86 temp = [','.join(item) for item in split_data][i].split(',')87 num_desc.append([''.join([c for c in temp if c in '0123456789']), ''.join([c for c in temp if c not in '0123456789']).replace(')context work context', '')])88 df2 = pd.DataFrame(num_desc, columns = ['Importance', 'Condition'])89 df2 = df2[df2['Importance'] != '']90 context = df291 if len(context.index) < 5:92 context_url = "https://www.onetonline.org/link/result/" + onetCode + "?c=wc&n_wc=0&c_wc=0"93 response = requests.get(context_url, headers=headers, verify=False)94 soup = BeautifulSoup(response.text, 'html.parser')95 tasks = str(soup.get_text('reportsubdesc')).replace("reportsubdesc", " ").replace("ImportanceCategoryTask ", "")96 tasks = clean(tasks)97 if len(tasks.split('show all show top 10')) > 1:98 tasks = tasks.split('show all show top 10')[1]99 tasks = tasks.split('back to top')[0]100 tasks = remove_new_line(tasks).replace("related occupations", " ").replace("importance work activity", " ")101 tasks = tasks.split("? ")102 split_data = [item.split(" -- ")[0] for item in tasks]103 num_desc = []104 for i in range(len(tasks)):105 temp = [','.join(item) for item in split_data][i].split(',')106 num_desc.append([''.join([c for c in temp if c in '0123456789']), ''.join([c for c in temp if c not in '0123456789']).replace(')context work context', '')])107 df2 = pd.DataFrame(num_desc, columns = ['Importance', 'Condition'])108 df2 = df2[df2['Importance'] != '']109 context = df2110 return context111 else: 112 return pd.DataFrame([("We're sorry."), ("This occupation is currently undergoing updates."), ("Please try again later.")])113 114 115def get_onet_skills(onetCode):116 headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.1.2 Safari/605.1.15'}117 118 skills_url = "https://www.onetonline.org/link/result/" + onetCode + "?c=sk&n_sk=0&s_sk=IM&c_sk=0"119 120 response = requests.get(skills_url, headers=headers, verify=False)121 soup = BeautifulSoup(response.text, 'html.parser')122 tasks = str(soup.get_text('reportsubdesc')).replace("reportsubdesc", " ").replace("ImportanceCategoryTask ", "")123 tasks = clean(tasks)124 if len(tasks.split('show all show top 10')) > 1:125 tasks = tasks.split('show all show top 10')[1]126 tasks = tasks.split('back to top')[0]127 tasks = remove_new_line(tasks).replace("related occupations", " ").replace(")importance skill", " ")128 tasks = tasks.split(". ")129 split_data = [item.split(" -- ")[0] for item in tasks]130 num_desc = []131 for i in range(len(tasks)):132 temp = [','.join(item) for item in split_data][i].split(',')133 num_desc.append([''.join([c for c in temp if c in '0123456789']), ''.join([c for c in temp if c not in '0123456789']).replace(')context work context', '')])134 df3 = pd.DataFrame(num_desc, columns = ['Importance', 'Skill'])135 df3 = df3[df3['Importance'] != '']136 skills = df3137 return skills138 else: 139 return pd.DataFrame([("We're sorry."), ("This occupation is currently undergoing updates."), ("Please try again later.")])140 141 142def get_onet_knowledge(onetCode):143 headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.1.2 Safari/605.1.15'}144 145 knowledge_url = "https://www.onetonline.org/link/result/" + onetCode + "?c=kn&n_kn=0&s_kn=IM&c_kn=0"146 147 response = requests.get(knowledge_url, headers=headers, verify=False)148 soup = BeautifulSoup(response.text, 'html.parser')149 tasks = str(soup.get_text('reportsubdesc')).replace("reportsubdesc", " ").replace("ImportanceCategoryTask ", "")150 tasks = clean(tasks)151 if len(tasks.split('show all show top 10')) > 1:152 tasks = tasks.split('show all show top 10')[1]153 tasks = tasks.split('back to top')[0]154 tasks = remove_new_line(tasks).replace("related occupations", " ").replace(")importance knowledge", " ")155 tasks = tasks.split(". ")156 split_data = [item.split(" -- ")[0] for item in tasks]157 num_desc = []158 for i in range(len(tasks)):159 temp = [','.join(item) for item in split_data][i].split(',')160 num_desc.append([''.join([c for c in temp if c in '0123456789']), ''.join([c for c in temp if c not in '0123456789']).replace(')context work context', '')])161 df4 = pd.DataFrame(num_desc, columns = ['Importance', 'Knowledge'])162 df4 = df4[df4['Importance'] != '']163 knowledge = df4164 return knowledge165 else: 166 return pd.DataFrame([("We're sorry."), ("This occupation is currently undergoing updates."), ("Please try again later.")])167 168 169def get_onet_abilities(onetCode):170 headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.1.2 Safari/605.1.15'}171 172 abilities_url = "https://www.onetonline.org/link/result/" + onetCode + "?c=ab&n_ab=0&s_ab=IM&c_ab=0"173 174 response = requests.get(abilities_url, headers=headers, verify=False)175 soup = BeautifulSoup(response.text, 'html.parser')176 tasks = str(soup.get_text('reportsubdesc')).replace("reportsubdesc", " ").replace("ImportanceCategoryTask ", "")177 tasks = clean(tasks)178 if len(tasks.split('show all show top 10')) > 1:179 tasks = tasks.split('show all show top 10')[1]180 tasks = tasks.split('back to top')[0]181 tasks = remove_new_line(tasks).replace("related occupations", " ").replace(")importance ability", " ")182 tasks = tasks.split(". ")183 split_data = [item.split(" -- ")[0] for item in tasks]184 num_desc = []185 for i in range(len(tasks)):186 temp = [','.join(item) for item in split_data][i].split(',')187 num_desc.append([''.join([c for c in temp if c in '0123456789']), ''.join([c for c in temp if c not in '0123456789']).replace(')context work context', '')])188 df5 = pd.DataFrame(num_desc, columns = ['Importance', 'Ability'])189 df5 = df5[df5['Importance'] != '']190 abilities = df5191 return abilities192 else:193 return pd.DataFrame([("We're sorry."), ("This occupation is currently undergoing updates."), ("Please try again later.")])194 195 196 197def get_onet_interests(onetCode):198 headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.1.2 Safari/605.1.15'}199 200 interests_url = "https://www.onetonline.org/link/result/" + onetCode + "?c=in&c_in=0"201 202 response = requests.get(interests_url, headers=headers, verify=False)203 soup = BeautifulSoup(response.text, 'html.parser')204 tasks = str(soup.get_text('reportsubdesc')).replace("reportsubdesc", " ").replace("ImportanceCategoryTask ", "")205 tasks = clean(tasks)206 tasks = tasks.split("occupational interest interest")[1]#.replace('bright outlook', '').replace('updated 2023', '')207 if len(tasks.split('back to top')) > 1:208 tasks = tasks.split('back to top')[0]209 tasks = remove_new_line(tasks).replace("related occupations", " ").replace(")importance interest", " ")210 tasks = tasks.split(". ")211 split_data = [item.split(" -- ")[0] for item in tasks]212 num_desc = []213 for i in range(len(tasks)):214 temp = [','.join(item) for item in split_data][i].split(',')215 num_desc.append([''.join([c for c in temp if c in '0123456789']), ''.join([c for c in temp if c not in '0123456789']).replace(')context work context', '')])216 df6 = pd.DataFrame(num_desc, columns = ['Importance', 'Interest'])217 df6 = df6[df6['Importance'] != '']218 interests = df6219 return interests220 else: 221 return pd.DataFrame([("We're sorry."), ("This occupation is currently undergoing updates."), ("Please try again later.")])222 223 224def get_onet_values(onetCode):225 headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.1.2 Safari/605.1.15'}226 227 values_url = "https://www.onetonline.org/link/result/" + onetCode + "?c=wv&c_wv=0"228 229 response = requests.get(values_url, headers=headers, verify=False)230 soup = BeautifulSoup(response.text, 'html.parser')231 tasks = str(soup.get_text('reportsubdesc')).replace("reportsubdesc", " ").replace("ImportanceCategoryTask ", "")232 tasks = clean(tasks)233 tasks = tasks.split('extent work value')[1]234 tasks = tasks.split('back to top')[0]235 tasks = remove_new_line(tasks).replace("related occupations", " ").replace(")importance value", " ")236 tasks = tasks.split(". ")237 split_data = [item.split(" -- ")[0] for item in tasks]238 num_desc = []239 for i in range(len(tasks)):240 temp = [','.join(item) for item in split_data][i].split(',')241 num_desc.append([''.join([c for c in temp if c in '0123456789']), ''.join([c for c in temp if c not in '0123456789']).replace(')context work context', '')])242 df7 = pd.DataFrame(num_desc, columns = ['Importance', 'Value'])243 df7 = df7[df7['Importance'] != '']244 values = df7245 return values246 247def get_onet_styles(onetCode):248 headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.1.2 Safari/605.1.15'}249 250 style_url = "https://www.onetonline.org/link/result/" + onetCode + "?c=ws&n_ws=0&c_ws=0"251 252 response = requests.get(style_url, headers=headers, verify=False)253 soup = BeautifulSoup(response.text, 'html.parser')254 tasks = str(soup.get_text('reportsubdesc')).replace("reportsubdesc", " ").replace("ImportanceCategoryTask ", "")255 tasks = clean(tasks)256 tasks = tasks.split('show all show top 10')[1]257 tasks = tasks.split('back to top')[0]258 tasks = remove_new_line(tasks).replace("related occupations", " ").replace(")importance work style", "").replace(")importance style", " ")259 tasks = tasks.split(". ")260 split_data = [item.split(" -- ")[0] for item in tasks]261 num_desc = []262 for i in range(len(tasks)):263 temp = [','.join(item) for item in split_data][i].split(',')264 num_desc.append([''.join([c for c in temp if c in '0123456789']), ''.join([c for c in temp if c not in '0123456789']).replace(')context work context', '')])265 df8 = pd.DataFrame(num_desc, columns = ['Importance', 'Style'])266 df8 = df8[df8['Importance'] != '']267 styles = df8268 return styles269 270def get_job_postings(onetCode, state):271 headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.1.2 Safari/605.1.15'}272 url = "https://www.onetonline.org/link/localjobs/" + onetCode + "?st=" + state273 response = requests.get(url, headers=headers, verify=False)274 soup = BeautifulSoup(response.text, 'html.parser')275 jobs = str(soup.get_text("tbody")).split('PostedtbodyTitle and CompanytbodyLocation')[1].split('Sources:')[0].split("tbody")276 jobs = jobs[5:45]277 starts = np.linspace(start=0, stop=len(jobs)-4,num= 10)278 stops = np.linspace(start=3, stop=len(jobs)-1, num= 10)279 jobpostings = []280 for i in range(0,10):281 jobpostings.append(str([' '.join(jobs[int(starts[i]):int(stops[i])])]).replace("['", '').replace("']", ''))282 links = str(soup.find_all('a', href=True)).split("</small>")[1].split(', <a href="https://www.careeronestop.org/"')[0].split(' data-bs-toggle="modal" ')283 linklist = []284 for i in range(1, len(links)):285 links[i] = "https://www.onetonline.org" + str(links[i]).split(' role="button">')[0].replace("href=", "")286 linklist.append(links[i].replace('"', ''))287 return jobpostings, linklist