FadouaFGM/Stackoverflow_Questions_Categorisation
3
1# # <center> <font color='blue'> **Projet 5: Catégorisez automatiquement des questions**2# # <center> <font color='goldenrode'> **Notebook: API**3# 4# ## link to the created HuggingFace Space:5# 6# # https://huggingface.co/spaces/FadouaFGM/Stackoverflow_Questions_Categorisation7# 8# 9# <center><img src="stackoverflow2.png" align="center"/> </center>10 11# 12# 13# **On commence tout d'abord par la définition des fonctions globales utilisées pour le nettoyage du texte, ensuite la vectorisation et le modèle ML choisi lors de l'étude menée dans le notebook précédent.**14 15 16 17 18from sklearn.pipeline import Pipeline19from sklearn.feature_extraction.text import CountVectorizer20from sklearn.multiclass import OneVsRestClassifier21from sklearn.linear_model import LogisticRegression22from sklearn.decomposition import LatentDirichletAllocation23import pandas as pd24import numpy as np25import matplotlib.pyplot as plt26import seaborn as sns27import time28import warnings29import re30import nltk31import spacy32import re33 34from nltk.tokenize import WordPunctTokenizer35from nltk.corpus import stopwords36 37nlp = spacy.load("en_core_web_sm")38nlp.Defaults.stop_words.add("`,")39nlp.Defaults.stop_words.add("``")40 41 42# ### **Définition des fonctions et modèle**43 44 45# Define functions46 47#lemmatize text without stop or punctuation words48def lemmatize(text):49 doc = nlp(text)50 tokens = [token.lemma_ for token in doc if not (token.is_stop or token.is_digit or token.is_punct)]51 return ' '.join(tokens)52 53def tokenization(text):54 tokens = WordPunctTokenizer().tokenize(text)55 return tokens56 57 58# Function to preprocess text59def clean(text):60 # Lower case61 text = text.lower()62 # Removing paragraph numbers63 text = re.sub(r'[0-9]+.\t', '', text)64 # Change the pattern C# to csharp65 pattern = r'c\#' 66 text = re.sub(pattern, 'csharp', text)67 # Removing web and HTML links68 text = re.sub(r'http\S+', '', text)69 # Removing special characters70 text = re.sub("</p>", '', str(text))71 text = re.sub("<p>", '', str(text))72 text = re.sub("</pre>", '', str(text))73 text = re.sub("<pre>", '', str(text))74 text = re.sub("&", '', str(text))75 text = re.sub(";", '', str(text))76 text = re.sub("gt", ' ', str(text))77 text = re.sub("pre", '', str(text))78 # Removing any reference to outside text79 text = re.sub("[\(\[].*?[\)\]]", "", str(text))80 # Removing numbers81 text = re.sub('[0-9]', '', str(text))82 # Removing new line characters83 text = re.sub('\n ', '', str(text))84 text = re.sub('\n', ' ', str(text))85 # Removing apostrophes86 text = re.sub("'s", '', str(text))87 # Removing hyphens88 text = re.sub("-", ' ', str(text))89 text = re.sub("—", '', str(text))90 # Removing > or < or = signs91 text = re.sub("<", ' ', str(text))92 text = re.sub(">", '', str(text))93 text = re.sub("=", '', str(text))94 # Removing quotation marks95 text = re.sub('\"', '', str(text))96 # Removing quotation marks97 text = re.sub('/', '', str(text))98 # Use regex to delete all what's inside < >99 CLEANR = re.compile('<.*?>')100 text = re.sub(CLEANR, '', text)101 102 return text103 104def remove_code(text):105 106 #first position of the code in code107 codepointer=text.find('<code>')108 result=''109 110 while codepointer!=-1:111 #last position of /code112 codeender=text.find(u'</code>',codepointer)113 #the code between pointer and ender114 result=result+text[codepointer:codeender+7]115 codepointer=text.find('<code>',codeender)116 117 listOfWords2remove = ([i for i in result.split()])118 119 for i in listOfWords2remove:120 text = text.replace(i, '') 121 122 return text123 124def text_processing(dfoftext):125 126 cleaneddftext = dfoftext.apply(lambda txt : remove_code(txt))127 cleaneddftext = cleaneddftext.apply(lambda txt : clean(txt))128 cleaneddftext = cleaneddftext.apply(lambda txt : lemmatize(txt))129 130 return cleaneddftext131 132# Define function to predict with the new list of thresholds with attributing a threshold per label133def predict_with_thresholds(y_prob, thresholds):134 y_pred = np.zeros_like(y_prob)135 for i in range(y_prob.shape[1]):136 y_pred[:, i] = (y_prob[:, i] >= thresholds[i]).astype(int)137 return y_pred138 139 140import joblib141 142def makeprediction(text):143 # load the pre-trained TfidfVectorizer from disk144 tfidfvectorizer = joblib.load('tfidf_vectorizer_100523.joblib')145 146 # load the pre-trained Linear_SGD classifier from disk147 ovr = joblib.load('linear_regression_classifier_100523.joblib')148 149 # Processing the text150 cleanedtext = text_processing(text)151 #print(cleanedtext)152 #print(type(cleanedtext))153 154 # applying the model and reconstruction predicted targets155 texttfidf = tfidfvectorizer.transform(cleanedtext)156 157 # make prediction with pretrained classifier158 ypred = ovr.predict_proba(texttfidf)159 #print(ypred)160 161 # recontructing tags from predicted y162 thresholds = joblib.load('thresholds_100523.joblib')163 labels = joblib.load('labels_100523.joblib')164 165 y_pred_thr = predict_with_thresholds(ypred,thresholds)166 #print(y_pred_thr)167 168 tags_pred = [[labels[i] for i in range(len(yp)) if yp[i] == 1] for yp in y_pred_thr]169 #tags_pred = tags_pred.apply(lambda x: x if x else ['no predicted labels'])170 171 # Predict with unsupervised model the most important topics and key words related to the document172 # Load unsupervised pretrained model and dictionary173 lda_model = joblib.load('best_lda_model.joblib')174 lda_dictionary = joblib.load('lda_dictionary.joblib')175 176 # process the document to be a corpus as needed for LDA algorithm177 cleanedtext = pd.Series(cleanedtext)178 text_lda = ' '.join((cleanedtext.tolist())) 179 corpus = lda_dictionary.doc2bow(text_lda.split())180 181 # get the topic distribution for the document182 doc_topics = lda_model.get_document_topics(corpus)183 184 # filter topics with probability > 0.2 and sort them by probability185 important_topics = sorted([(topic, prob) for topic, prob in doc_topics if prob > 0.2], key=lambda x: x[1], reverse=True)186 187 # print the most important topics related to the document and their probability188 print('The most important topics related to the document, and there probabilities, are:')189 for topic, prob in important_topics:190 print(f'Topic {topic}: {prob:.2f}')191 192 # get the main keywords for each important topic193 important_topic_ids = [topic for topic, _ in important_topics]194 topic_keywords = lda_model.show_topics(num_topics=-1, formatted=False)195 main_keywords = []196 main_keywords = []197 for topic_id, topic_prob in topic_keywords:198 if topic_id in important_topic_ids:199 keywords = [(word, prob) for word, prob in topic_prob if prob > 0.05]200 main_keywords.append(keywords)201 202 return tags_pred, important_topics, main_keywords203 204# ### **Implémentation de l'API: GradioAPI**205import gradio as gra206from typing import List207 208def predict(text: List[str]):209 data = [[text]]210 data = pd.DataFrame(data, columns = ['Text'])211 tags, topics, keywords = makeprediction(data['Text'])212 #return {"tags!😎": tags, "Related Topics": topics, "Most important key words": keywords } 213 return tags, topics, keywords214 215inputs = gra.inputs.Textbox(label="Question to predict tags, topics, and keywords", lines=10)216outputs = [217 gra.outputs.Textbox(label="Tags"),218 gra.outputs.Textbox(label="Related Topics"),219 gra.outputs.Textbox(label="Related Keywords")220]221title = "Prediction: Tags, Topics, and Keywords for StackOverflow Questions"222 223app = gra.Interface(fn=predict, inputs=inputs, outputs=outputs, title=title)224 225app.launch(debug=True,enable_queue=True)226 227#link to the created HuggingFace Space228# https://huggingface.co/spaces/FadouaFGM/Stackoverflow_Questions_Categorisation229# Process to update parameters of the model or any other changes on notebooks230# copy all files to the cloned repository231# cp ../Formation_ML/P5/*.joblib .232# cp ../Formation_ML/P5/app.py .233# add all files and commit234# git add .235# git commit -m "Update model parameters"236# git push237 238#pip freeze > requirements.txt239 240 241 242 243