taskswithcode/semantic_clustering
2
1import time2import sys3import streamlit as st4import string5from io import StringIO 6import pdb7import json8from twc_embeddings import HFModel,SimCSEModel,SGPTModel,CausalLMModel,SGPTQnAModel9from twc_openai_embeddings import OpenAIModel10from twc_clustering import TWCClustering11import torch12import requests13import socket14 15 16MAX_INPUT = 500017 18SEM_SIMILARITY="1"19DOC_RETRIEVAL="2"20CLUSTERING="3"21 22 23use_case = {"1":"Finding similar phrases/sentences","2":"Retrieving semantically matching information to a query. It may not be a factual match","3":"Clustering"}24use_case_url = {"1":"https://huggingface.co/spaces/taskswithcode/semantic_similarity","2":"https://huggingface.co/spaces/taskswithcode/semantic_search","3":""}25 26 27 28from transformers import BertTokenizer, BertForMaskedLM29 30 31APP_NAME = "hf/semantic_clustering"32INFO_URL = "https://www.taskswithcode.com/stats/"33 34 35 36 37 38def get_views(action):39 ret_val = 040 hostname = socket.gethostname()41 ip_address = socket.gethostbyname(hostname)42 if ("view_count" not in st.session_state):43 try:44 app_info = {'name': APP_NAME,"action":action,"host":hostname,"ip":ip_address}45 res = requests.post(INFO_URL, json = app_info).json()46 print(res)47 data = res["count"]48 except:49 data = 050 ret_val = data51 st.session_state["view_count"] = data52 else:53 ret_val = st.session_state["view_count"]54 if (action != "init"):55 app_info = {'name': APP_NAME,"action":action,"host":hostname,"ip":ip_address}56 res = requests.post(INFO_URL, json = app_info).json()57 return "{:,}".format(ret_val)58 59 60 61 62def construct_model_info_for_display(model_names):63 options_arr = []64 markdown_str = f"<div style=\"font-size:16px; color: #2f2f2f; text-align: left\"><br/><b>Models evaluated ({len(model_names)})</b><br/><i>The selected models satisfy one or more of the following (1) state-of-the-art (2) the most downloaded models on Hugging Face (3) Large Language Models (e.g. GPT-3)</i></div>"65 markdown_str += f"<div style=\"font-size:2px; color: #2f2f2f; text-align: left\"><br/></div>"66 for node in model_names:67 options_arr .append(node["name"])68 if (node["mark"] == "True"):69 markdown_str += f"<div style=\"font-size:16px; color: #5f5f5f; text-align: left\"> • Model: <a href=\'{node['paper_url']}\' target='_blank'>{node['name']}</a><br/> Code released by: <a href=\'{node['orig_author_url']}\' target='_blank'>{node['orig_author']}</a><br/> Model info: <a href=\'{node['sota_info']['sota_link']}\' target='_blank'>{node['sota_info']['task']}</a></div>"70 if ("Note" in node):71 markdown_str += f"<div style=\"font-size:16px; color: #a91212; text-align: left\"> {node['Note']}<a href=\'{node['alt_url']}\' target='_blank'>link</a></div>"72 markdown_str += "<div style=\"font-size:16px; color: #5f5f5f; text-align: left\"><br/></div>"73 74 markdown_str += "<div style=\"font-size:12px; color: #9f9f9f; text-align: left\"><b>Note:</b><br/>• Uploaded files are loaded into non-persistent memory for the duration of the computation. They are not cached</div>"75 limit = "{:,}".format(MAX_INPUT)76 markdown_str += f"<div style=\"font-size:12px; color: #9f9f9f; text-align: left\">• User uploaded file has a maximum limit of {limit} sentences.</div>"77 return options_arr,markdown_str78 79 80st.set_page_config(page_title='TWC - Compare popular/state-of-the-art models for semantic clustering using sentence embeddings', page_icon="logo.jpg", layout='centered', initial_sidebar_state='auto',81 menu_items={82 'About': 'This app was created by taskswithcode. http://taskswithcode.com'83 84 })85col,pad = st.columns([85,15])86 87with col:88 st.image("long_form_logo_with_icon.png")89 90 91@st.experimental_memo92def load_model(model_name,model_class,load_model_name):93 try:94 ret_model = None95 obj_class = globals()[model_class]96 ret_model = obj_class()97 ret_model.init_model(load_model_name)98 assert(ret_model is not None)99 except Exception as e:100 st.error(f"Unable to load model class:{model_class} model_name: {model_name} load_model_name: {load_model_name} {str(e)}")101 pass102 return ret_model103 104 105 106@st.experimental_memo107def cached_compute_similarity(input_file_name,sentences,_model,model_name,threshold,_cluster,clustering_type):108 texts,embeddings = _model.compute_embeddings(input_file_name,sentences,is_file=False)109 results = _cluster.cluster(None,texts,embeddings,threshold,clustering_type)110 return results111 112 113def uncached_compute_similarity(input_file_name,sentences,_model,model_name,threshold,cluster,clustering_type):114 with st.spinner('Computing vectors for sentences'):115 texts,embeddings = _model.compute_embeddings(input_file_name,sentences,is_file=False)116 results = cluster.cluster(None,texts,embeddings,threshold,clustering_type)117 #st.success("Similarity computation complete")118 return results119 120DEFAULT_HF_MODEL = "sentence-transformers/paraphrase-MiniLM-L6-v2"121def get_model_info(model_names,model_name):122 for node in model_names:123 if (model_name == node["name"]):124 return node,model_name125 return get_model_info(model_names,DEFAULT_HF_MODEL)126 127 128def run_test(model_names,model_name,input_file_name,sentences,display_area,threshold,user_uploaded,custom_model,clustering_type):129 display_area.text("Loading model:" + model_name)130 #Note. model_name may get mapped to new name in the call below for custom models131 orig_model_name = model_name132 model_info,model_name = get_model_info(model_names,model_name)133 if (model_name != orig_model_name):134 load_model_name = orig_model_name135 else:136 load_model_name = model_info["model"]137 if ("Note" in model_info):138 fail_link = f"{model_info['Note']} [link]({model_info['alt_url']})"139 display_area.write(fail_link)140 if (user_uploaded and "custom_load" in model_info and model_info["custom_load"] == "False"):141 fail_link = f"{model_info['Note']} [link]({model_info['alt_url']})"142 display_area.write(fail_link)143 return {"error":fail_link}144 model = load_model(model_name,model_info["class"],load_model_name)145 display_area.text("Model " + model_name + " load complete")146 try:147 if (user_uploaded):148 results = uncached_compute_similarity(input_file_name,sentences,model,model_name,threshold,st.session_state["cluster"],clustering_type)149 else:150 display_area.text("Computing vectors for sentences")151 results = cached_compute_similarity(input_file_name,sentences,model,model_name,threshold,st.session_state["cluster"],clustering_type)152 display_area.text("Similarity computation complete")153 return results154 155 except Exception as e:156 st.error("Some error occurred during prediction" + str(e))157 st.stop()158 return {}159 160 161 162 163 164def display_results(orig_sentences,results,response_info,app_mode,model_name):165 main_sent = f"<div style=\"font-size:14px; color: #2f2f2f; text-align: left\">{response_info}<br/><br/></div>"166 main_sent += f"<div style=\"font-size:14px; color: #2f2f2f; text-align: left\">Showing results for model: <b>{model_name}</b></div>"167 score_text = "cosine distance"168 main_sent += f"<div style=\"font-size:14px; color: #6f6f6f; text-align: left\">Clustering by {score_text}. <b>{len(results['clusters'])} clusters</b>. mean:{results['info']['mean']:.2f}; std:{results['info']['std']:.2f}; current threshold:{results['info']['current_threshold']}<br/>Threshold hints:{str(results['info']['zscores'])}<br/>Overlap stats(overlap,freq):{str(results['info']['overlap'])}</div>"169 body_sent = []170 download_data = {}171 for i in range(len(results["clusters"])):172 pivot_index = results["clusters"][i]["pivot_index"]173 pivot_sent = orig_sentences[pivot_index]174 pivot_index += 1175 d_cluster = {}176 download_data[i + 1] = d_cluster177 d_cluster["pivot"] = {"pivot_index":pivot_index,"sent":pivot_sent,"children":{}}178 body_sent.append(f"<div style=\"font-size:16px; color: #2f2f2f; text-align: left\">{pivot_index}] {pivot_sent} <b><i>(Cluster {i+1})</i></b> </div>")179 neighs_dict = results["clusters"][i]["neighs"]180 for key in neighs_dict:181 cosine_dist = neighs_dict[key]182 child_index = key183 sentence = orig_sentences[child_index]184 child_index += 1185 body_sent.append(f"<div style=\"font-size:16px; color: #2f2f2f; text-align: left\">{child_index}] {sentence} <b>{cosine_dist:.2f}</b></div>")186 d_cluster["pivot"]["children"][sentence] = f"{cosine_dist:.2f}" 187 body_sent.append(f"<div style=\"font-size:16px; color: #2f2f2f; text-align: left\"> </div>")188 main_sent = main_sent + "\n" + '\n'.join(body_sent)189 st.markdown(main_sent,unsafe_allow_html=True)190 st.session_state["download_ready"] = json.dumps(download_data,indent=4)191 get_views("submit")192 193 194def init_session():195 if ("model_name" not in st.session_state):196 st.session_state["model_name"] = "ss_test"197 st.session_state["download_ready"] = None 198 st.session_state["model_name"] = "ss_test"199 st.session_state["threshold"] = 1.5200 st.session_state["file_name"] = "default"201 st.session_state["overlapped"] = "overlapped"202 st.session_state["cluster"] = TWCClustering()203 else:204 print("Skipping init session")205 206def app_main(app_mode,example_files,model_name_files,clus_types):207 init_session()208 with open(example_files) as fp:209 example_file_names = json.load(fp) 210 with open(model_name_files) as fp:211 model_names = json.load(fp)212 with open(clus_types) as fp:213 cluster_types = json.load(fp)214 curr_use_case = use_case[app_mode].split(".")[0]215 st.markdown("<h5 style='text-align: center;'>Compare popular/state-of-the-art models for semantic clustering using sentence embeddings</h5>", unsafe_allow_html=True)216 st.markdown(f"<p style='font-size:14px; color: #4f4f4f; text-align: center'><i>Or compare your own model with state-of-the-art/popular models</p>", unsafe_allow_html=True)217 st.markdown(f"<div style='color: #4f4f4f; text-align: left'>Use cases for sentence embeddings<br/> • <a href=\'{use_case_url['1']}\' target='_blank'>{use_case['1']}</a><br/> • <a href=\'{use_case_url['2']}\' target='_blank'>{use_case['2']}</a><br/> • {use_case['3']}<br/><i>This app illustrates <b>'{curr_use_case}'</b> use case</i></div>", unsafe_allow_html=True)218 st.markdown(f"<div style='color: #9f9f9f; text-align: right'>views: {get_views('init')}</div>", unsafe_allow_html=True)219 220 221 try:222 223 224 with st.form('twc_form'):225 226 step1_line = "Upload text file(one sentence in a line) or choose an example text file below"227 if (app_mode == DOC_RETRIEVAL):228 step1_line += ". The first line is treated as the query"229 uploaded_file = st.file_uploader(step1_line, type=".txt")230 231 selected_file_index = st.selectbox(label=f'Example files ({len(example_file_names)})', 232 options = list(dict.keys(example_file_names)), index=0, key = "twc_file")233 st.write("")234 options_arr,markdown_str = construct_model_info_for_display(model_names)235 selection_label = 'Select Model'236 selected_model = st.selectbox(label=selection_label, 237 options = options_arr, index=0, key = "twc_model")238 st.write("")239 custom_model_selection = st.text_input("Model not listed above? Type any Hugging Face sentence embedding model name ", "",key="custom_model")240 hf_link_str = "<div style=\"font-size:12px; color: #9f9f9f; text-align: left\"><a href='https://huggingface.co/models?pipeline_tag=sentence-similarity' target = '_blank'>List of Hugging Face sentence embedding models</a><br/><br/><br/></div>"241 st.markdown(hf_link_str, unsafe_allow_html=True)242 threshold = st.number_input('Choose a zscore threshold (number of std devs from mean)',value=st.session_state["threshold"],min_value = 0.0,step=.01)243 st.write("")244 clustering_type = st.selectbox(label=f'Select type of clustering', 245 options = list(dict.keys(cluster_types)), index=0, key = "twc_cluster_types")246 st.write("")247 submit_button = st.form_submit_button('Run')248 249 250 input_status_area = st.empty()251 display_area = st.empty()252 if submit_button:253 start = time.time()254 if uploaded_file is not None:255 st.session_state["file_name"] = uploaded_file.name256 sentences = StringIO(uploaded_file.getvalue().decode("utf-8")).read()257 else:258 st.session_state["file_name"] = example_file_names[selected_file_index]["name"]259 sentences = open(example_file_names[selected_file_index]["name"]).read()260 sentences = sentences.split("\n")[:-1]261 if (len(sentences) > MAX_INPUT):262 st.info(f"Input sentence count exceeds maximum sentence limit. First {MAX_INPUT} out of {len(sentences)} sentences chosen")263 sentences = sentences[:MAX_INPUT]264 if (len(custom_model_selection) != 0):265 run_model = custom_model_selection266 else:267 run_model = selected_model268 st.session_state["model_name"] = selected_model269 st.session_state["threshold"] = threshold270 st.session_state["overlapped"] = cluster_types[clustering_type]["type"]271 results = run_test(model_names,run_model,st.session_state["file_name"],sentences,display_area,threshold,(uploaded_file is not None),(len(custom_model_selection) != 0),cluster_types[clustering_type]["type"])272 display_area.empty()273 with display_area.container():274 if ("error" in results):275 st.error(results["error"])276 else:277 device = 'GPU' if torch.cuda.is_available() else 'CPU'278 response_info = f"Computation time on {device}: {time.time() - start:.2f} secs for {len(sentences)} sentences"279 if (len(custom_model_selection) != 0):280 st.info("Custom model overrides model selection in step 2 above. So please clear the custom model text box to choose models from step 2")281 display_results(sentences,results,response_info,app_mode,run_model)282 #st.json(results)283 st.download_button(284 label="Download results as json",285 data= st.session_state["download_ready"] if st.session_state["download_ready"] != None else "",286 disabled = False if st.session_state["download_ready"] != None else True,287 file_name= (st.session_state["model_name"] + "_" + str(st.session_state["threshold"]) + "_" + st.session_state["overlapped"] + "_" + '_'.join(st.session_state["file_name"].split(".")[:-1]) + ".json").replace("/","_"),288 mime='text/json',289 key ="download" 290 )291 292 293 294 except Exception as e:295 st.error("Some error occurred during loading" + str(e))296 st.stop() 297 298 st.markdown(markdown_str, unsafe_allow_html=True)299 300 301 302if __name__ == "__main__":303 #print("comand line input:",len(sys.argv),str(sys.argv))304 #app_main(sys.argv[1],sys.argv[2],sys.argv[3])305 #app_main("1","sim_app_examples.json","sim_app_models.json")306 app_main("3","clus_app_examples.json","clus_app_models.json","clus_app_clustypes.json")307 308