harshithakr/mapping_bert_topic_copy

1import pickle2import pandas as pd3from sentence_transformers import SentenceTransformer, util4from preprocess_function import preprocess_text5from topics_extraction import classify6 7model_sent = SentenceTransformer("all-mpnet-base-v2")8 9sector_model = pickle.load(open('sector_knn.sav', 'rb'))10indus_model = pickle.load(open('indus_knn.sav', 'rb'))11 12def get_mapping(prep_text):13    14  tags_list = classify(prep_text)15  tags_list = tags_list['tags']16 17  if tags_list!=[]:18    19      event_discr_embeddings = model_sent.encode([' '.join(tags_list)],20                                          batch_size=250,21                                          show_progress_bar=True)22        23      event_embedd = event_discr_embeddings[0]24      25      sectors = pd.read_excel('sect_other.xlsx', sheet_name = 'sectors')26      sectors['name_clean'] = sectors['name'].str.replace('&','').str.strip()27      sectors['name_clean'] = sectors['name_clean'].str.replace('IT','information technology').str.replace(',','').str.lower()28    29      industries = pd.read_excel('sect_other.xlsx', sheet_name = 'other_indus')30      industries['industries_name_clean'] = industries['name'].str.replace('&','').str.strip()31      industries['industries_name_clean'] = industries['industries_name_clean'].str.replace('IT','information technology').str.replace(',','').str.lower()32 33      n_neighbors = 134      threshold = 0.4035    36      #sectors37      distances, indices = sector_model.kneighbors([event_embedd], n_neighbors=2)38      name_index = indices[0]39      distance_name = str(distances[0])40      topic_name = []41      for index_i in name_index:42        topic_name.append(sectors['name_clean'].tolist()[index_i])43      #topic_name = str(topic_name)44    45    46      #industries47      distances_indus, indices_indus = indus_model.kneighbors([event_embedd], n_neighbors=n_neighbors)48      name_index_indus = indices_indus[0][0]49      distance_name_indus = distances_indus[0][0]50      topic_name_indus = industries['industries_name_clean'].tolist()[name_index_indus]51    52    53      return topic_name, distance_name, topic_name_indus, distance_name_indus,tags_list54 55  else:56      return 'no tags identified', None, None, None,tags_list