prithush/Disaster_Tweet_Prediction

1# importing Libraries2 3import streamlit as st4import PIL5from PIL import Image6import tensorflow as tf7from nltk.stem import WordNetLemmatizer8from nltk.tokenize import RegexpTokenizer9import re10import string11import numpy as np12import pandas as pd13import nltk14 15try:                                                                         # Check if wordnet is installed16    nltk.find("corpora/wordnet.zip")          17except LookupError:18    nltk.download('wordnet')19 20# ----------------------------------------------------------------------------------21# read files22try:23    acronyms_dict, contractions_dict, stops24except NameError:25    acronyms_dict = pd.read_json("acronym.json", typ = "series")26    contractions_dict = pd.read_json("contractions.json", typ = "series")27    stops = list(pd.read_csv('stopwords.csv').values.flatten())28 29# ----------------------------------------------------------------------------------30# Defining tokenizer31regexp = RegexpTokenizer("[\w']+")32 33# preprocess Function34def preprocess(text):35    36    text = text.lower()                                                                                        # lowercase37    text = text.strip()                                                                                        # whitespaces38    39    # Removing html tags40    html = re.compile(r'<.*?>')41    text = html.sub(r'', text)                                                                                 # html tags42    43    # Removing emoji patterns44    emoji_pattern = re.compile("["45                           u"\U0001F600-\U0001F64F"  # emoticons46                           u"\U0001F300-\U0001F5FF"  # symbols & pictographs47                           u"\U0001F680-\U0001F6FF"  # transport & map symbols48                           u"\U0001F1E0-\U0001F1FF"  # flags (iOS)49                           u"\U00002702-\U000027B0"50                           u"\U000024C2-\U0001F251"51                           "]+", flags = re.UNICODE)52    text = emoji_pattern.sub(r'', text)                                                                         # unicode char53    54    # Removing urls55    http = "https?://\S+|www\.\S+" # matching strings beginning with http (but not just "http")56    pattern = r"({})".format(http) # creating pattern57    text = re.sub(pattern, "", text)                                                                            # remove urls58    59    # Removing twitter usernames60    pattern = r'@[\w_]+'61    text = re.sub(pattern, "", text)                                                                            # remove @twitter usernames62    63    # Removing punctuations and numbers64    punct_str = string.punctuation + string.digits65    punct_str = punct_str.replace("'", "")66    punct_str = punct_str.replace("-", "")67    text = text.translate(str.maketrans('', '', punct_str))                                                     # punctuation and numbers68    69    # Replacing "-" in text with empty space70    text = text.replace("-", " ")                                                                               # "-"71    72    # Substituting acronyms73    words = []74    for word in regexp.tokenize(text):75        if word in acronyms_dict.index:76            words = words + acronyms_dict[word].split()77        else:78            words = words + word.split()79    text = ' '.join(words)                                                                                       # acronyms80    81    # Substituting Contractions82    words = []83    for word in regexp.tokenize(text):84        if word in contractions_dict.index:85            words = words + contractions_dict[word].split()86        else:87            words = words + word.split()88    text = " ".join(words)                                                                                       # contractions89    90    punct_str = string.punctuation91    text = text.translate(str.maketrans('', '', punct_str))                                                     # punctuation again to remove "'"92    93    # lemmatization94    lemmatizer = WordNetLemmatizer()95    text = " ".join([lemmatizer.lemmatize(word) for word in regexp.tokenize(text)])                             # lemmatize96    97    # Stopwords Removal98    text = ' '.join([word for word in regexp.tokenize(text) if word not in stops])                              # stopwords99    100    # Removing all characters except alphabets and " " (space)101    filter = string.ascii_letters + " "102    text = "".join([chr for chr in text if chr in filter])                                                      # remove all characters except alphabets and " " (space)103    104    # Removing words with one alphabet occuring more than 3 times continuously105    pattern = r'\b\w*?(.)\1{2,}\w*\b'106    text = re.sub(pattern, "", text).strip()                                                                    # remove words with one alphabet occuring more than 3 times continuously107    108    # Removing words with less than 3 characters109    short_words = r'\b\w{1,2}\b'110    text = re.sub(short_words, "", text)                                                                     # remove words with less than 3 characters111    112    # return final output113    return text114 115# ===============================================================================================================116                                       # STREAMLIT117 118# App Devolopment Starts119st.set_page_config(layout="wide")120st.write("# Disaster Tweet Predictor")121 122img = Image.open("dis_image.png")123st.image(img)124 125tweet = st.text_input(label = "Enter or paste your tweet here", value = "")126 127# Defining a function to store the model in streamlit cache memory128@st.cache_resource129def cache_model(model_name):130    model = tf.keras.models.load_model(model_name)131    return model132 133model = cache_model("transfer_tweet")134 135# if user gives any input136if len(tweet) > 0:137    clean_tweet = preprocess(tweet)                   # cleans tweet138    y_pred = model.predict([clean_tweet])             # gives probability of class = 1139    y_pred_num = int(np.round(y_pred)[0][0])          # get final prediction of output class140    141    if y_pred_num == 0:142        st.write(f"#### Non-Disaster tweet with disaster probability {round(y_pred[0][0]*100, 4)}%")143    else:144        st.write(f"#### Disaster tweet with disaster probability {round(y_pred[0][0]*100, 4)}%")145 146# ==============================================================================================================147