chelscelis/resume-screening-classification
3
1import altair as alt2# import datetime3import joblib4import nltk5import numpy as np6import pandas as pd7import re8import streamlit as st 9import time10 11from gensim.corpora import Dictionary12from gensim.models import KeyedVectors, TfidfModel13from gensim.similarities import SoftCosineSimilarity, SparseTermSimilarityMatrix, WordEmbeddingSimilarityIndex14from gensim.similarities.annoy import AnnoyIndexer15from io import BytesIO16from nltk import pos_tag, word_tokenize17from nltk.corpus import stopwords, wordnet18from nltk.stem import PorterStemmer, WordNetLemmatizer19from pandas.api.types import is_categorical_dtype, is_numeric_dtype20from PIL import Image21from scipy.sparse import csr_matrix, hstack22 23nltk.download('averaged_perceptron_tagger')24nltk.download('punkt')25nltk.download('stopwords')26nltk.download('wordnet')27 28stop_words = set(stopwords.words('english'))29lemmatizer = WordNetLemmatizer()30stemmer = PorterStemmer()31 32def addZeroFeatures(matrix):33 maxFeatures = 1803834 numDocs, numTerms = matrix.shape35 missingFeatures = maxFeatures - numTerms36 if missingFeatures > 0:37 zeroFeatures = csr_matrix((numDocs, missingFeatures), dtype=np.float64)38 matrix = hstack([matrix, zeroFeatures])39 return matrix40 41@st.cache_data(max_entries = 1, show_spinner = False)42def classifyResumes(df):43 progressBar = st.progress(0)44 progressBar.progress(0, text = "Preprocessing data ...")45 startTime = time.time()46 df['cleanedResume'] = df.Resume.apply(lambda x: performStemming(x))47 resumeText = df['cleanedResume'].values48 progressBar.progress(20, text = "Extracting features ...")49 vectorizer = loadTfidfVectorizer()50 wordFeatures = vectorizer.transform(resumeText)51 wordFeaturesWithZeros = addZeroFeatures(wordFeatures)52 progressBar.progress(40, text = "Reducing dimensionality ...")53 finalFeatures = dimensionalityReduction(wordFeaturesWithZeros)54 progressBar.progress(60, text = "Predicting categories ...")55 knn = loadKnnModel()56 predictedCategories = knn.predict(finalFeatures)57 progressBar.progress(80, text = "Finishing touches ...")58 le = loadLabelEncoder()59 df['Industry Category'] = le.inverse_transform(predictedCategories)60 df['Industry Category'] = pd.Categorical(df['Industry Category'])61 df.drop(columns = ['cleanedResume'], inplace = True)62 endTime = time.time()63 elapsedSeconds = endTime - startTime64 hours, remainder = divmod(int(elapsedSeconds), 3600)65 minutes, _ = divmod(remainder, 60)66 secondsWithDecimals = '{:.2f}'.format(elapsedSeconds % 60)67 elapsedTimeStr = f'{hours} h : {minutes} m : {secondsWithDecimals} s'68 progressBar.progress(100, text = f'Classification Complete!')69 time.sleep(1)70 progressBar.empty()71 st.info(f'Finished classifying {len(resumeText)} resumes - {elapsedTimeStr}')72 return df 73 74def clickClassify():75 st.session_state.processClf = True76 77def clickRank():78 st.session_state.processRank = True79 80def convertDfToXlsx(df):81 output = BytesIO()82 writer = pd.ExcelWriter(output, engine = 'xlsxwriter')83 df.to_excel(writer, index = False, sheet_name = 'Sheet1')84 workbook = writer.book85 worksheet = writer.sheets['Sheet1']86 format1 = workbook.add_format({'num_format': '0.00'}) 87 worksheet.set_column('A:A', None, format1) 88 writer.close()89 processedData = output.getvalue()90 return processedData91 92def createBarChart(df):93 valueCounts = df['Industry Category'].value_counts().reset_index()94 valueCounts.columns = ['Industry Category', 'Count']95 newDataframe = pd.DataFrame(valueCounts)96 barChart = alt.Chart(newDataframe,97 ).mark_bar(98 color = '#56B6C2',99 size = 13 100 ).encode(101 x = alt.X('Count:Q', axis = alt.Axis(format = 'd'), title = 'Number of Resumes'),102 y = alt.Y('Industry Category:N', title = 'Category'),103 tooltip = ['Industry Category', 'Count']104 ).properties(105 title = 'Number of Resumes per Category',106 )107 return barChart108 109def dimensionalityReduction(features):110 nca = joblib.load('nca_model.joblib')111 features = nca.transform(features.toarray())112 return features 113 114def filterDataframeClf(df: pd.DataFrame) -> pd.DataFrame:115 modify = st.toggle("Add filters", key = 'filter-clf-1')116 if not modify:117 return df118 df = df.copy()119 modificationContainer = st.container()120 with modificationContainer:121 toFilterColumns = st.multiselect("Filter table on", df.columns, key = 'filter-clf-2')122 for column in toFilterColumns:123 left, right = st.columns((1, 20))124 left.write("↳")125 widgetKey = f'filter-clf-{toFilterColumns.index(column)}-{column}'126 if is_categorical_dtype(df[column]):127 userCatInput = right.multiselect(128 f'Values for {column}',129 df[column].unique(),130 default = list(df[column].unique()),131 key = widgetKey 132 )133 df = df[df[column].isin(userCatInput)]134 elif is_numeric_dtype(df[column]):135 _min = float(df[column].min())136 _max = float(df[column].max())137 step = (_max - _min) / 100138 userNumInput = right.slider(139 f'Values for {column}',140 min_value = _min,141 max_value = _max,142 value = (_min, _max),143 step = step,144 key = widgetKey 145 )146 df = df[df[column].between(*userNumInput)]147 else:148 userTextInput = right.text_input(149 f'Substring or regex in {column}',150 key = widgetKey 151 )152 if userTextInput:153 userTextInput = userTextInput.lower()154 df = df[df[column].astype(str).str.lower().str.contains(userTextInput)]155 return df156 157def filterDataframeRnk(df: pd.DataFrame) -> pd.DataFrame:158 modify = st.toggle("Add filters", key = 'filter-rnk-1')159 if not modify:160 return df161 df = df.copy()162 modificationContainer = st.container()163 with modificationContainer:164 toFilterColumns = st.multiselect("Filter table on", df.columns, key = 'filter-rnk-2')165 for column in toFilterColumns:166 left, right = st.columns((1, 20))167 left.write("↳")168 widgetKey = f'filter-rnk-{toFilterColumns.index(column)}-{column}'169 if is_categorical_dtype(df[column]):170 userCatInput = right.multiselect(171 f'Values for {column}',172 df[column].unique(),173 default = list(df[column].unique()),174 key = widgetKey175 )176 df = df[df[column].isin(userCatInput)]177 elif is_numeric_dtype(df[column]):178 _min = float(df[column].min())179 _max = float(df[column].max())180 step = (_max - _min) / 100181 userNumInput = right.slider(182 f'Values for {column}',183 min_value = _min,184 max_value = _max,185 value = (_min, _max),186 step = step,187 key = widgetKey188 )189 df = df[df[column].between(*userNumInput)]190 else:191 userTextInput = right.text_input(192 f'Substring or regex in {column}',193 key = widgetKey194 )195 if userTextInput:196 userTextInput = userTextInput.lower()197 df = df[df[column].astype(str).str.lower().str.contains(userTextInput)]198 return df199 200def getWordnetPos(tag):201 if tag.startswith('J'):202 return wordnet.ADJ203 elif tag.startswith('V'):204 return wordnet.VERB205 elif tag.startswith('N'):206 return wordnet.NOUN207 elif tag.startswith('R'):208 return wordnet.ADV209 else:210 return wordnet.NOUN211 212def loadKnnModel():213 knnModelFileName = f'knn_model.joblib'214 return joblib.load(knnModelFileName)215 216def loadLabelEncoder():217 labelEncoderFileName = f'label_encoder.joblib'218 return joblib.load(labelEncoderFileName)219 220def loadTfidfVectorizer():221 tfidfVectorizerFileName = f'tfidf_vectorizer.joblib' 222 return joblib.load(tfidfVectorizerFileName)223 224def performLemmatization(text):225 text = re.sub('http\S+\s*', ' ', text)226 text = re.sub('RT|cc', ' ', text)227 text = re.sub('#\S+', '', text)228 text = re.sub('@\S+', ' ', text)229 text = re.sub('[%s]' % re.escape("""!"#$%&'()*+,-./:;<=>?@[\]^_`{|}~"""), ' ', text)230 text = re.sub(r'[^\x00-\x7f]',r' ', text)231 text = re.sub('\s+', ' ', text)232 words = word_tokenize(text)233 words = [234 lemmatizer.lemmatize(word.lower(), pos = getWordnetPos(pos)) 235 for word, pos in pos_tag(words) if word.lower() not in stop_words236 ]237 return words238 239def performStemming(text):240 text = re.sub('http\S+\s*', ' ', text)241 text = re.sub('RT|cc', ' ', text)242 text = re.sub('#\S+', '', text)243 text = re.sub('@\S+', ' ', text)244 text = re.sub('[%s]' % re.escape("""!"#$%&'()*+,-./:;<=>?@[\]^_`{|}~"""), ' ', text)245 text = re.sub(r'[^\x00-\x7f]',r' ', text)246 text = re.sub('\s+', ' ', text)247 words = word_tokenize(text)248 words = [stemmer.stem(word.lower()) for word in words if word.lower() not in stop_words]249 text = ' '.join(words)250 return text 251 252@st.cache_data253def loadModel():254 model_path = 'wiki-news-300d-1M-subword.vec'255 model = KeyedVectors.load_word2vec_format(model_path)256 return model257 258model = loadModel()259 260@st.cache_data(max_entries = 1, show_spinner = False)261def rankResumes(text, df):262 progressBar = st.progress(0)263 progressBar.progress(0, text = "Preprocessing data ...")264 startTime = time.time()265 jobDescriptionText = performLemmatization(text)266 df['cleanedResume'] = df['Resume'].apply(lambda x: performLemmatization(x))267 documents = [jobDescriptionText] + df['cleanedResume'].tolist()268 progressBar.progress(13, text = "Creating a dictionary ...")269 dictionary = Dictionary(documents)270 progressBar.progress(25, text = "Creating a TF-IDF model ...")271 tfidf = TfidfModel(dictionary = dictionary)272 progressBar.progress(38, text = "Creating a Similarity Index...")273 words = [word for word, count in dictionary.most_common()]274 wordVectors = model.vectors_for_all(words, allow_inference = False)275 indexer = AnnoyIndexer(wordVectors, num_trees = 300)276 similarityIndex = WordEmbeddingSimilarityIndex(wordVectors, kwargs = {'indexer': indexer})277 progressBar.progress(50, text = "Creating a Similarity Matrix...")278 similarityMatrix = SparseTermSimilarityMatrix(similarityIndex, dictionary, tfidf)279 progressBar.progress(63, text = "Setting up job description as the query ...")280 query = tfidf[dictionary.doc2bow(jobDescriptionText)]281 progressBar.progress(75, text = "Calculating semantic similarities ...")282 index = SoftCosineSimilarity(283 tfidf[[dictionary.doc2bow(resume) for resume in df['cleanedResume']]],284 similarityMatrix 285 )286 similarities = index[query]287 progressBar.progress(88, text = "Finishing touches ...")288 df['Similarity Score (-1 to 1)'] = similarities289 df['Rank'] = df['Similarity Score (-1 to 1)'].rank(ascending=False, method='dense').astype(int)290 df.sort_values(by='Rank', inplace=True)291 df.drop(columns = ['cleanedResume'], inplace = True)292 endTime = time.time()293 elapsedSeconds = endTime - startTime294 hours, remainder = divmod(int(elapsedSeconds), 3600)295 minutes, _ = divmod(remainder, 60)296 secondsWithDecimals = '{:.2f}'.format(elapsedSeconds % 60)297 elapsedTimeStr = f'{hours} h : {minutes} m : {secondsWithDecimals} s'298 progressBar.progress(100, text = f'Ranking Complete!')299 time.sleep(1)300 progressBar.empty()301 st.info(f'Finished ranking {len(df)} resumes - {elapsedTimeStr}')302 return df 303 304def writeGettingStarted():305 st.write("""306 ## Hello, Welcome! 307 In today's competitive job market, the process of manually screening resumes has become a daunting task for recruiters and hiring managers. 308 The sheer volume of applications received for a single job posting can make it extremely time-consuming to identify the most suitable candidates efficiently. 309 This often leads to missed opportunities and the potential loss of top-tier talent.310 311 The ***Resume Screening & Classification*** website application aims to help alleviate the challenges posed by manual resume screening. 312 The main objectives are:313 - To classify the resumes into their most suitable job industry category314 - To compare the resumes to the job description and rank them by similarity315 """)316 st.divider()317 st.write("""318 ## Input Guide 319 #### For the Job Description: 320 Ensure the job description is saved in a text (.txt) file. 321 Kindly outline the responsibilities, qualifications, and skills associated with the position.322 323 #### For the Resumes: 324 Resumes must be compiled in an excel (.xlsx) file. 325 The organization of columns is up to you but ensure that the "Resume" column is present.326 The values under this column should include all the relevant details for each resume.327 """)328 st.info("""329 ##### NOTE:330 - If the "Resume" column is not present, the classification/ranking process will not be executed.331 - If there are multiple "Resume" columns, the first occurrence will be taken into account while the remaining duplicates are given a different column name.332 """)333 st.divider()334 st.write("""335 ## Demo Walkthrough336 #### Classify Tab:337 The web app will classify the resumes into their most suitable job industry category.338 Currently the Category Scope consists of the following:339 """)340 column1, column2 = st.columns(2)341 with column1:342 st.write("""343 - Aviation344 - Business development345 - Culinary346 - Education347 - Engineering348 - Finance349 """)350 with column2:351 st.write("""352 - Fitness353 - Healthcare354 - HR355 - Information Technology356 - Public relations357 """)358 with st.expander('Classification Steps'):359 st.write("""360 ##### Upload Resumes & Start Processing:361 - Navigate to the "Classify" tab.362 - Upload the Excel file (.xlsx) containing the resumes you want to classify. Ensure that your Excel file has the "Resume" column containing the resume texts.363 - Click the "Start Processing" button.364 - The app will analyze the resumes and categorize them into job industry categories.365 ######366 """)367 imgClf1 = Image.open('clf-1.png')368 st.image(imgClf1, use_column_width = True, output_format = "PNG")369 st.write("""370 ##### View Bar Chart:371 - A bar chart will appear, showing the number of resumes per category, helping you visualize the distribution.372 ######373 """)374 imgClf2 = Image.open('clf-2.png')375 st.image(imgClf2, use_column_width = True, output_format = "PNG")376 st.write("""377 ##### Add Filters:378 - You can apply filters to the dataframe to narrow down your results.379 ######380 """)381 imgClf3 = Image.open('clf-3.png')382 st.image(imgClf3, use_column_width = True, output_format = "PNG")383 st.write("""384 ##### Donwload Results:385 - Once you've applied filters or are satisfied with the results, you can download the current dataframe as an Excel file by clicking the "Save Current Output as XLSX" button.386 ####387 """)388 imgClf4 = Image.open('clf-4.png')389 st.image(imgClf4, use_column_width = True, output_format = "PNG")390 st.write("""391 #### Rank Tab:392 The web app will rank the resumes based on their semantic similarity to the job description. 393 The similarity score ranges from -1 to 1.394 A score of 1 is achieved when Document A and Document B are identical.395 396 ##### **Kindly take note:**397 398 It's important to note that these scores are not absolute and may change when more resumes are added in the comparison.399 The ranking algorithm dynamically adjusts its results based on the entire set of uploaded resumes.400 We recommend considering the scores as a relative measure rather than an absolute determination.401 """)402 with st.expander('Ranking Steps'):403 st.write("""404 ##### Upload Files & Start Processing:405 - Navigate to the "Rank" tab.406 - Upload the job description as a text file. This file should contain the description of the job you want to compare resumes against.407 - Upload the Excel file that contains the resumes you want to rank.408 - Click the "Start Processing" button.409 - The app will analyze the job description and rank the resumes based on their semantic similarity to the job description.410 ######411 """)412 imgRnk1 = Image.open('rnk-1.png')413 st.image(imgRnk1, use_column_width = True, output_format = "PNG")414 st.write("""415 ##### View Job Description:416 - The output will display the contents of the job description for reference.417 ######418 """)419 imgRnk2 = Image.open('rnk-2.png')420 st.image(imgRnk2, use_column_width = True, output_format = "PNG")421 st.write("""422 ##### Add Filters:423 - You can apply filters to the dataframe to narrow down your results.424 ######425 """)426 imgRnk3 = Image.open('rnk-3.png')427 st.image(imgRnk3, use_column_width = True, output_format = "PNG")428 st.write("""429 ##### Donwload Results:430 - Once you've applied filters or are satisfied with the results, you can download the current dataframe as an Excel file by clicking the "Save Current Output as XLSX" button.431 ####432 """)433 imgRnk4 = Image.open('rnk-4.png')434 st.image(imgRnk4, use_column_width = True, output_format = "PNG")435 436