hongaik/service_text_classification

1import re2from gensim.models.keyedvectors import KeyedVectors3from transformers import pipeline4import pickle5import numpy as np6import pandas as pd7 8w2v = KeyedVectors.load('models/word2vec')9w2v_vocab = set(sorted(w2v.index_to_key))10model = pickle.load(open('models/w2v_ovr_svc.sav', 'rb'))11classifier = pipeline("zero-shot-classification",12                      model="facebook/bart-large-mnli", framework='pt'13                     )14 15labels = [16    'communication', 'waiting time',17       'information', 'user interface',18       'facilities', 'location', 'price'19]20 21sample_file = pd.read_csv('sample.csv').to_csv(index=False).encode('utf-8')22 23print('utils imported!')24 25def get_sentiment_label_facebook(list_of_sent_dicts):26    if list_of_sent_dicts['labels'][0] == 'negative':27        return 'negative'28    else:29        return 'positive'30 31def get_single_prediction(text):32    33    # manipulate data into a format that we pass to our model34    text = text.lower() #lower case35    text = re.sub('[^0-9a-zA-Z\s]', '', text) #remove special char, punctuation36 37    # Remove OOV words38    text = ' '.join([i for i in text.split() if i in w2v_vocab])39    40    # Vectorise text and store in new dataframe. Sentence vector = average of word vectors41    text_vectors = np.mean([w2v[i] for i in text.split()], axis=0)42 43    # Make predictions44    results = model.predict_proba(text_vectors.reshape(1,300)).squeeze().round(2)45    pred_prob = pd.DataFrame({'topic': labels, 'probability': results}).sort_values('probability', ascending=True)46    47    # Get sentiment48    sentiment_results = classifier(text, 49                                candidate_labels=['positive', 'negative'], 50                                hypothesis_template='The sentiment of this is {}')51    sentiment_prob = pd.DataFrame({'sentiment': sentiment_results['labels'], 'probability': sentiment_results['scores']})52    53    return (pred_prob, sentiment_prob)54 55def get_multiple_predictions(csv):56    57    df = pd.read_csv(csv)58    df.columns = ['sequence']59 60    df['sequence_clean'] = df['sequence'].str.lower() #lower case61    df['sequence_clean'] = df['sequence_clean'].str.strip()62    df['sequence_clean'] = df['sequence_clean'].str.replace('[^0-9a-zA-Z\s]','') #remove special char, punctuation63 64    # Remove OOV words65    df['sequence_clean'] = df['sequence_clean'].apply(lambda x: ' '.join([i for i in x.split() if i in w2v_vocab]))66 67    # Remove rows with blank string68    invalid = df[(pd.isna(df['sequence_clean'])) | (df['sequence_clean'] == '')]69    invalid.drop(columns=['sequence_clean'], inplace=True)70    71    # Drop rows with blank string72    df.dropna(inplace=True)73    df = df[df['sequence_clean'] != ''].reset_index(drop=True)74    75    # Vectorise text and store in new dataframe. Sentence vector = average of word vectors76    series_text_vectors = pd.DataFrame(df['sequence_clean'].apply(lambda x: np.mean([w2v[i] for i in x.split()], axis=0)).values.tolist())77    78    # Get predictions79    pred_results = pd.DataFrame(model.predict(series_text_vectors), columns = labels)80    81    # Join back to original sequence82    final_results = df.join(pred_results)83    final_results['others'] = final_results[labels].max(axis=1)84    final_results['others'] = final_results['others'].apply(lambda x: 1 if x == 0 else 0)85    86    # Get sentiment labels87    final_results['sentiment'] = final_results['sequence_clean'].apply(lambda x: get_sentiment_label_facebook(classifier(x, 88                                                            candidate_labels=['positive', 'negative'], 89                                                            hypothesis_template='The sentiment of this is {}'))90                                                                )91    92    final_results.drop(columns=['sequence_clean'], inplace=True)93    94    # Append invalid rows95    if len(invalid) == 0:96        return final_results.to_csv(index=False).encode('utf-8')97    else:98        return pd.concat([final_results, invalid]).reset_index(drop=True).to_csv(index=False).encode('utf-8')