harshithakr/mapping_bert_topic_copy

1import pandas as pd2import re3from nltk import ngrams4from nltk.corpus import wordnet5from nltk.corpus import stopwords6from nltk.tokenize import word_tokenize7from nltk.stem import WordNetLemmatizer8import nltk9nltk.download('wordnet')10nltk.download('stopwords')11nltk.download('punkt')12nltk.download('stopwords')13stop_words = set(stopwords.words('english'))14 15 16stop_words_2 = ('show','international','exhibition','trade','fair','global','conference','world',17 'expo','event','wellknown','popular','new', 'together',18'latest','offer','trend','sector','exhibitor','th','one','like','also','held','well','etc','u','bb',19  'provide', 'provides', 'provide','day','attendee','year', 'best','top','management',20  'brings','bring','event','topic','visitor','buyer','brand','take','u','national','great','come')21 22stop_words = stop_words.union(stop_words_2)23 24list_location = []25for col in ['name','capital','region','subregion']:#countries26  list_location.extend(list(set(pd.read_csv('countries.csv')[col])))27list_location.extend(list(set(pd.read_csv('states.csv')['name'])))28list_location.extend(list(set(pd.read_csv('cities.csv')['name'])))29list_location.extend(list(set(pd.read_csv('zones.csv')['Zone'])))30 31locations_removal = set([x.lower() for x in list_location if not pd.isna(x)])32 33locations_removal.discard('nan')34 35stop_words_bert   = stop_words.union(locations_removal).union(stop_words_2)36 37def preprocess_text(keyword):38  keyword = ' '.join([w for w in word_tokenize(keyword) if not w.lower() in stop_words])39  keyword = keyword.replace('/', ' ')40  keyword = re.sub(r"^[^a-zA-Z0-9]+|[^a-zA-Z0-9\)]+$", " ", keyword).strip()41  keyword = keyword.replace('_', ' ')42  keyword = keyword.replace('&', ' ').strip()43  keyword = keyword.encode('ascii', 'ignore').decode('utf-8').strip().lower()44  keyword = re.sub(r'[^a-zA-Z\s]', '', keyword)45  words = word_tokenize(keyword)46  words = [word for word in words if word not in stop_words]47  lemmatizer = WordNetLemmatizer()48  words = list(set([lemmatizer.lemmatize(word) for word in words]))49  words = [word for word in words if word not in stop_words]50  processed_text = ' '.join(words)51  processed_text = re.sub(r'\b\w*([a-zA-Z])\1{10,}\w*\b', '', processed_text)52  return processed_text53 54def bert_preprocess(keyword):55 56    # Remove abbreviations57    keyword = re.sub(r"\b[A-Z\.]{2,}\b", ' ', keyword)58 59    # Convert to lowercase60    keyword = keyword.lower()61 62    # Tokenize and remove stop words63    keyword = ' '.join([w for w in word_tokenize(keyword) if re.sub(r'[^\w\s]', '', w.lower()) not in stop_words_bert])64 65    # Remove special characters, unwanted patterns, and symbols66    keyword = re.sub(r"^[^a-zA-Z0-9]+|[^a-zA-Z0-9\)]+$", " ", keyword)67    keyword = re.sub(r'[^a-zA-Z\s]', ' ', keyword)68 69    # Clean up and lemmatize words70    lemmatizer = WordNetLemmatizer()71    words  = [w for w in word_tokenize(keyword)]72    words = [lemmatizer.lemmatize(word) for word in words]73 74    # Remove repeated characters75    processed_text = re.sub(r'\b\w*([a-zA-Z])\1{10,}\w*\b', '', ' '.join(words))76 77    # Join words and remove unnecessary spaces78    processed_text = ' '.join(processed_text.split())79 80    return processed_text81 82def lam_list(list_words):83    list_words = [x.strip().lower() for x in list_words]84    lemmatizer = WordNetLemmatizer()85    list_words_v = [lemmatizer.lemmatize(word,pos='v') for word in list_words]86    list_words_n = [lemmatizer.lemmatize(word,pos='n') for word in list_words]87    return list_words_v, list_words_n