Tihsrah-CD/Hinglish-Text-Normalizer
1
1import streamlit as st2import pandas as pd3import pickle4from tqdm import tqdm5from Levenshtein import distance as lev6import joblib7from googletrans import Translator8from indictrans import Transliterator9from pyphonetics import RefinedSoundex10from bs4 import BeautifulSoup11import re12import torch13from transformers import AutoTokenizer, AutoModelForSequenceClassification14 15# Load sentiment analysis model and tokenizer16tokenizer = AutoTokenizer.from_pretrained("Seethal/sentiment_analysis_generic_dataset")17model = AutoModelForSequenceClassification.from_pretrained("Seethal/sentiment_analysis_generic_dataset")18 19# Define a function to get the sentiment from the model20def get_sentiment(text):21 inputs = tokenizer(text, return_tensors='pt', truncation=True, padding=True)22 outputs = model(**inputs)23 sentiment = torch.argmax(outputs.logits, dim=1).item()24 return 'Positive' if sentiment == 1 else 'Negative'25 26 27def closest_match(word, vocabulary):28 best_match = None29 best_distance = float('inf')30 for vocab_word in vocabulary:31 dist = lev(word, vocab_word)32 if dist < best_distance:33 best_distance = dist34 best_match = vocab_word35 return best_match36 37def main():38 st.title('Text Processing App')39 rs = RefinedSoundex()40 normalized_string_final=[]41 translator = Translator()42 trn = Transliterator(source='eng', target='hin')43 44 with open(r'./english_vocab.pkl', "rb") as fp:45 english = pickle.load(fp)46 english_vocab=english 47 with open(r'./hinglish_vocab.pkl', "rb") as fp:48 hinglish = pickle.load(fp)49 hinglish_vocab=hinglish 50 51 english_vocab['and'] = ['and']52 english_vocab['is'] = ['is']53 54 def clean_tweet(tweet):55 text=re.sub(r'@ [A-Za-z0-9\']+','',tweet)56 text=BeautifulSoup(text,'lxml').get_text()57 text=re.sub(r'https (//)[A-Za-z0-9. ]*(/) [A-Za-z0-9]+','',text)58 text=re.sub(r'https[A-Za-z0-9/. ]*','',text)59 text=re.sub("[^a-zA-Z]"," ",text)60 text=re.sub(r'\bRT\b',' ',text)61 text=re.sub(r'\bnan\b',' ',text)62 return text63 64 input_text = st.text_area("Enter the text:")65 total_translated = []66 if st.button('Process'):67 data = {'Text': [input_text]}68 df1 = pd.DataFrame(data)69 df1['Text'] = df1['Text'].apply(clean_tweet)70 cleaned_text = df1['Text'].tolist()[0]71 total_text = [cleaned_text]72 st.write("Input Text:", total_text)73 74 for i in tqdm(total_text):75 test_text=i.split()76 not_changed_idx=[]77 for i in range(len(test_text)):78 not_changed_idx.append(0)79 changed_text=[]80 changed_idx=[]81 82 for i in range(len(test_text)):83 for key in english_vocab:84 done=085 for val in english_vocab[key]:86 if(test_text[i]==val):87 changed_text.append(key)88 changed_idx.append(i)89 not_changed_idx[i]=190 done=191 break92 if done==1:93 break94 95 96 normalized_string=[]97 res = dict(zip(changed_idx, changed_text))98 for i in range(len(test_text)):99 try:100 normalized_string.append(res[i])101 except:102 normalized_string.append(test_text[i])103 print("English Normalized String:", normalized_string)104 105 # hinglish word change106 test_list = [i for i in range(len(test_text))]107 changed_hing_idx = [i for i in test_list if i not in changed_idx]108 hinglish_text_part = [test_text[i] for i in changed_hing_idx]109 changed_text2 = []110 changed_idx2 = []111 112 for i in range(len(hinglish_text_part)):113 for key in hinglish_vocab:114 done = 0115 for val in hinglish_vocab[key]:116 if hinglish_text_part[i] == val:117 changed_text2.append(key)118 changed_idx2.append(i)119 done = 1120 break121 if done == 1:122 break123 124 normalized_string2 = []125 res2 = dict(zip(changed_idx2, changed_text2))126 for i in range(len(hinglish_text_part)):127 try:128 normalized_string2.append(res2[i])129 except:130 normalized_string2.append(hinglish_text_part[i])131 132 for i in changed_idx:133 normalized_string2.append(res[i])134 135 print("Hinglish Normalized String:", normalized_string)136 137 # finding phoneme and leventise distance for unchanged word138 for i in range(len(not_changed_idx)):139 try:140 if not_changed_idx[i] == 0:141 eng_phoneme_correction = []142 for j in english_vocab:143 try:144 phoneme = rs.distance(normalized_string2[i], j)145 except:146 pass147 if phoneme <= 1:148 eng_phoneme_correction.append(j)149 eng_lev_correction = []150 for k in eng_phoneme_correction:151 dist = lev(normalized_string2[i], k)152 if dist <= 2:153 eng_lev_correction.append(k)154 155 eng_lev_correction.extend(hing_lev_correction)156 new_correction = eng_lev_correction157 eng_lev_correction = []158 for l in new_correction:159 dist = lev(normalized_string2[i], l)160 eng_lev_correction.append(dist)161 min_val = min(eng_lev_correction)162 min_idx = eng_lev_correction.index(min_val)163 164 suggestion = closest_match(new_correction[min_idx], english_vocab.keys())165 normalized_string2[i] = suggestion166 except:167 pass168 169 normalized_string_final = normalized_string2170 print("Phoneme levenshtein Distionary suggestion Normalized String:", normalized_string_final)171 172 # sentence tagging173 classifier = joblib.load(r"./classifer.joblib")174 classify = []175 for i in normalized_string:176 test_classify = classifier(i)177 classify.append(test_classify[0].get("label"))178 179 for i in range(len(classify)):180 if classify[i] == 'en':181 try:182 normalized_string[i] = translator.translate(normalized_string[i], src='en', dest='hi').text183 except:184 normalized_string[i] = "delete"185 print("English -> Hindi Translated String:", normalized_string)186 187 conversion_list = [trn.transform(i) for i in normalized_string]188 print("Hinglish -> Hindi Transliterated String:", conversion_list)189 190 sentence = [" ".join(conversion_list)]191 translated = []192 for i in sentence:193 try:194 translated_text = translator.translate(i, src='hi', dest='en')195 translated.append(translated_text.text)196 except:197 translated.append("delete")198 print("Hindi -> English Translated String:", translated)199 total_translated.append(translated[0])200 201 st.write("English Normalized String:", normalized_string)202 st.write("Hinglish Normalized String:", normalized_string)203 st.write("Phoneme Levenshtein Dictionary Suggestion Normalized String:", normalized_string_final)204 st.write("English -> Hindi Translated String:", normalized_string)205 st.write("Hinglish -> Hindi Transliterated String:", conversion_list)206 st.write("Hindi -> English Translated String:", translated)207 208 # Get the sentiment of the translated text209 sentiment = get_sentiment(translated[0])210 st.write("Sentiment of Translated Text:", sentiment)211 212if __name__ == '__main__':213 main()