pikaduck/setfit-twitter-data-sentiment-analysis

1"""2    @author : Sakshi Tantak3"""4 5# Imports6import re7import string8import pickle9from time import time10 11import nltk12from nltk.tokenize import word_tokenize13from nltk.stem import WordNetLemmatizer14from nltk.corpus import stopwords15 16import emoji17 18from paths import COUNT_VECTORIZER_PATH, TFIDF_VECTORIZER_PATH, NB_MODEL_PATH as MODEL_PATH19 20nltk.download('punkt')21nltk.download('omw-1.4')22nltk.download('stopwords')23nltk.download('wordnet')24 25stops = stopwords.words('english')26negatives = ['no','nor','not','ain','aren',"aren't",'couldn',"couldn't",'didn',"didn't",'doesn',"doesn't",'hadn',"hadn't",'hasn',27  "hasn't",'haven',"haven't",'isn',"isn't",'mightn',"mightn't",'mustn',"mustn't",'needn',"needn't",'shan',"shan't",'shouldn',"shouldn't",28  'wasn',"wasn't",'weren',"weren't","won't",'wouldn',"wouldn't",'don',"don't"]29stops = set([stop for stop in stops if stop not in negatives])30 31lemmatizer = WordNetLemmatizer()32MODEL, COUNT_VECTORIZER, TFIDF = None, None, None33 34def clean_text(text):35    text = re.sub(r'[\.]+', '.', text)36    # print(text)37    text = re.sub(r'[\!]+', '!', text)38    # print(text)39    text = re.sub(r'[\?]+', '!', text)40    # print(text)41    text = re.sub(r'\s+', ' ', text).strip().lower()42    # print(text)43    text = re.sub(r'@\w+', '', text).strip().lower()44    # print(text)45    text = re.sub(r'\s[n]+[o]+', ' no', text)46    # print(text)47    text = re.sub(r'n\'t', 'n not', text)48    # print(text)49    text = re.sub(r'\'nt', 'n not', text)50    # print(text)51    text = re.sub(r'\'re', ' are', text)52    # print(text)53    text = re.sub(r'\'s', ' is', text)54    # print(text)55    text = re.sub(r'\'d', ' would', text)56    # print(text)57    text = re.sub(r'\'ll', ' will', text)58    # print(text)59    text = re.sub(r'\'ve', ' have', text)60    # print(text)61    text = re.sub(r'\'m', ' am', text)62    # print(text)63    # map variations of nope to no64    text = re.sub(r'\s[n]+[o]+[p]+[e]+', ' no', text)65    # print(text)66    # clean websites mentioned in text67    text = re.sub(r'(https|http)?:\/\/(\w|\.|\/|\?|\=|\&|\%|\~)*\b', '', text, flags=re.MULTILINE).strip()68    # print(text)69    text = re.sub(r'(www.)(\w|\.|\/|\?|\=|\&|\%)*\b', '', text, flags=re.MULTILINE).strip()70    # print(text)71    text = re.sub(r'\w+.com', '', text).strip()72    # print(text)73    text = emoji.demojize(text)74    return text75 76def remove_punctuation(text):77    translator = str.maketrans(string.punctuation, ' '*len(string.punctuation))78    text = text.translate(translator)79    return re.sub(r'\s+', ' ', text).strip()80 81def remove_numbers(text):82  return re.sub(r'[0-9]+', '', text)83 84def remove_stopwords_and_lemmatize(text):85    tokens = word_tokenize(text)86    tokens = [token.strip() for token in tokens if token.strip() not in stops]87    tokens = [lemmatizer.lemmatize(token) for token in tokens]88    return ' '.join(tokens)89 90def load_model():91    global MODEL, COUNT_VECTORIZER, TFIDF92 93    if MODEL is None:94        with open(MODEL_PATH, 'rb') as f:95            print('Loading classifier ...')96            start = time()97            MODEL = pickle.load(f)98            print(f'Time taken to load model = {time() - start}')99        f.close()100 101    if COUNT_VECTORIZER is None:102        with open(COUNT_VECTORIZER_PATH, 'rb') as f:103            print('Loading count vectorizer ...')104            start = time()105            COUNT_VECTORIZER = pickle.load(f)106            print(f'Time taken to load count vectorizer = {time() - start}')107        f.close()108 109    if TFIDF is None:110        with open(TFIDF_VECTORIZER_PATH, 'rb') as f:111            print('Loading tfidf vectorizer ...')112            start = time()113            TFIDF = pickle.load(f)114            print(f'Time taken to load tfidf vectorizer = {time() - start}')115        f.close()116 117def predict(text):118    if MODEL is None:119        load_model()120 121    text = clean_text(text)122    text = remove_numbers(text)123    text = remove_punctuation(text)124    text = remove_stopwords_and_lemmatize(text)125 126    vector = COUNT_VECTORIZER.transform([text]).toarray()127    vector = TFIDF.transform(vector).toarray()128    start = time()129    prediction = MODEL.predict(vector)130    print(prediction)131    prediction = MODEL.predict(vector).item()132    print(f'Inference time = {time() - start}')133    return ('positive', 1) if prediction == 1 else ('negative', 1)134 135if __name__ == '__main__':136    text = input('Enter tweet : ')137    # text = "i am so bored!!!"138    prediction = predict(text)139    print(text, ' : ', prediction)