sanjayw/SparkNLP_NER
0
1import streamlit as st2 3 4st.set_page_config(5 layout="centered", # Can be "centered" or "wide". In the future also "dashboard", etc.6 initial_sidebar_state="auto", # Can be "auto", "expanded", "collapsed"7 page_title='Extractive Summarization', # String or None. Strings get appended with "• Streamlit".8 page_icon='./favicon.png', # String, anything supported by st.image, or None.9)10 11 12import pandas as pd13import numpy as np14import json 15import os16import sys17sys.path.append(os.path.abspath('./'))18import streamlit_apps_config as config19from streamlit_ner_output import show_html2, jsl_display_annotations, get_color20 21import sparknlp22from sparknlp.base import *23from sparknlp.annotator import *24from pyspark.sql import functions as F25from sparknlp_display import NerVisualizer26from pyspark.ml import Pipeline27from pyspark.sql.types import StringType28 29spark= sparknlp.start()30 31## Marking down NER Style32st.markdown(config.STYLE_CONFIG, unsafe_allow_html=True)33 34root_path = config.project_path35 36########## To Remove the Main Menu Hamburger ########37 38hide_menu_style = """39 <style>40 #MainMenu {visibility: hidden;}41 </style>42 """43st.markdown(hide_menu_style, unsafe_allow_html=True)44 45########## Side Bar ########46 47## loading logo(newer version with href)48import base6449@st.cache(allow_output_mutation=True)50def get_base64_of_bin_file(bin_file):51 with open(bin_file, 'rb') as f:52 data = f.read()53 return base64.b64encode(data).decode()54 55@st.cache(allow_output_mutation=True)56def get_img_with_href(local_img_path, target_url):57 img_format = os.path.splitext(local_img_path)[-1].replace('.', '')58 bin_str = get_base64_of_bin_file(local_img_path)59 html_code = f'''60 <a href="{target_url}">61 <img height="90%" width="90%" src="data:image/{img_format};base64,{bin_str}" />62 </a>'''63 return html_code64 65logo_html = get_img_with_href('./jsl-logo.png', 'https://www.johnsnowlabs.com/')66st.sidebar.markdown(logo_html, unsafe_allow_html=True)67 68 69 70#sidebar info71model_name= ["nerdl_fewnerd_100d", "ner_conll_elmo", "ner_mit_movie_complex_distilbert_base_cased", "ner_conll_albert_large_uncased", "onto_100"]72st.sidebar.title("Pretrained model to test")73selected_model = st.sidebar.selectbox("", model_name)74 75######## Main Page #########76 77if selected_model == "nerdl_fewnerd_100d":78 app_title= "Detect up to 8 entity types in general domain texts"79 app_description= "Named Entity Recognition model aimed to detect up to 8 entity types from general domain texts. This model was trained on the Few-NERD/inter public dataset using Spark NLP, and it is available in Spark NLP Models hub. "80 st.title(app_title)81 st.markdown("<h2>"+app_description+"</h2>" , unsafe_allow_html=True)82 st.markdown("**`PERSON`** **,** **`ORGANIZATION`** **,** **`LOCATION`** **,** **`ART`** **,** **`BUILDING`** **,** **`PRODUCT`** **,** **`EVENT`** **,** **`OTHER`**", unsafe_allow_html=True)83 84elif selected_model== "ner_conll_elmo":85 app_title= "Detect up to 4 entity types in general domain texts"86 app_description= "Named Entity Recognition model aimed to detect up to 4 entity types from general domain texts. This model was trained on the CoNLL 2003 text corpus using Spark NLP, and it is available in Spark NLP Models hub. "87 st.title(app_title)88 st.markdown("<h2>"+app_description+"</h2>" , unsafe_allow_html=True)89 st.markdown("**`PER`** **,** **`LOC`** **,** **`ORG`** **,** **`MISC` **", unsafe_allow_html=True)90 91elif selected_model== "ner_mit_movie_complex_distilbert_base_cased":92 app_title= "Detect up to 12 entity types in movie domain texts"93 app_description= "Named Entity Recognition model aimed to detect up to 12 entity types from movie domain texts. This model was trained on the MIT Movie Corpus complex queries dataset to detect movie trivia using Spark NLP, and it is available in Spark NLP Models hub. "94 st.title(app_title)95 st.markdown("<h2>"+app_description+"</h2>" , unsafe_allow_html=True)96 st.markdown("""**`ACTOR`** **,** **`AWARD`** **,** **`CHARACTER_NAME`** **,** **`DIRECTOR`** **,** **`GENRE`** **,** **`OPINION`** **,** **`ORIGIN`** **,** **`PLOT`**,97 **`QUOTE`** **,** **`RELATIONSHIP`** **,** **`SOUNDTRACK`** **,** **`YEAR` **""", unsafe_allow_html=True)98 99 100elif selected_model=="ner_conll_albert_large_uncased":101 app_title= "Detect up to 4 entity types in general domain texts"102 app_description= "Named Entity Recognition model aimed to detect up to 4 entity types from general domain texts. This model was trained on the CoNLL 2003 text corpus using Spark NLP, and it is available in Spark NLP Models hub. "103 st.title(app_title)104 st.markdown("<h2>"+app_description+"</h2>" , unsafe_allow_html=True)105 st.markdown("**`PER`** **,** **`LOC`** **,** **`ORG`** **,** **`MISC` **", unsafe_allow_html=True)106 107elif selected_model=="onto_100":108 app_title= "Detect up to 18 entity types in general domain texts"109 app_description= "Named Entity Recognition model aimed to detect up to 18 entity types from general domain texts. This model was trained with GloVe 100d word embeddings using Spark NLP, so be sure to use same embeddings in the pipeline. It is available in Spark NLP Models hub. "110 st.title(app_title)111 st.markdown("<h2>"+app_description+"</h2>" , unsafe_allow_html=True)112 st.markdown("""**`CARDINAL`** **,** **`EVENT`** **,** **`WORK_OF_ART`** **,** **`ORG`** **,** **`DATE`** **,** **`GPE`** **,** **`PERSON`** **,** **`PRODUCT`**,113 **`NORP`** **,** **`ORDINAL`** **,** **`MONEY`** **,** **`LOC` **, **`FAC`** **,** **`LAW`** **,** **`TIME`** **,** **`PERCENT`** **,** **`QUANTITY`** **,** **`LANGUAGE` **""", unsafe_allow_html=True)114 115 116st.subheader("")117 118 119 120 121 122#caching the models in the dictionary123@st.cache(allow_output_mutation=True, show_spinner=False)124def load_sparknlp_models():125 ner_models_list= ["nerdl_fewnerd_100d", "ner_conll_elmo", "ner_mit_movie_complex_distilbert_base_cased", 126 "ner_conll_albert_large_uncased", "onto_100"]127 embeddings_list= ["glove_100d", "elmo", "distilbert_base_cased", "albert_large_uncased", "glove_100d_for_onto"]128 129 130 documentAssembler = DocumentAssembler()\131 .setInputCol("text")\132 .setOutputCol("document")133 134 sentenceDetector= SentenceDetector()\135 .setInputCols(["document"])\136 .setOutputCol("sentence")137 138 tokenizer = Tokenizer()\139 .setInputCols(["sentence"])\140 .setOutputCol("token")141 142 ner_converter= NerConverter()\143 .setInputCols(["document", "token", "ner"])\144 .setOutputCol("ner_chunk")145 146 model_dict= {147 'documentAssembler': documentAssembler,148 'sentenceDetector': sentenceDetector,149 'tokenizer': tokenizer,150 'ner_converter': ner_converter151 }152 153 for embeddings_name, ner_model_name in zip(embeddings_list, ner_models_list):154 155 try:156 if embeddings_name=="glove_100d":157 model_dict[embeddings_name]= WordEmbeddingsModel.pretrained(embeddings_name, "en")\158 .setInputCols(["sentence", "token"])\159 .setOutputCol("embeddings")160 161 elif embeddings_name=="elmo":162 model_dict[embeddings_name]= ElmoEmbeddings.pretrained(embeddings_name, "en")\163 .setInputCols(["token", "document"])\164 .setOutputCol("embeddings")\165 .setPoolingLayer("elmo")166 167 elif embeddings_name=="distilbert_base_cased":168 model_dict[embeddings_name]= DistilBertEmbeddings\169 .pretrained(embeddings_name, 'en')\170 .setInputCols(["token", "document"])\171 .setOutputCol("embeddings")172 173 elif embeddings_name=="albert_large_uncased":174 model_dict[embeddings_name]= AlbertEmbeddings\175 .pretrained(embeddings_name, 'en')\176 .setInputCols(["document", "token"])\177 .setOutputCol("embeddings")178 179 elif embeddings_name=="glove_100d_for_onto":180 model_dict[embeddings_name]= WordEmbeddingsModel.pretrained("glove_100d", "en")\181 .setInputCols(["sentence", "token"])\182 .setOutputCol("embeddings")183 184 185 model_dict[ner_model_name]= NerDLModel.pretrained(ner_model_name, "en")\186 .setInputCols(["document", "token", "embeddings"])\187 .setOutputCol("ner")188 189 190 except:191 pass192 return model_dict193 194 195 196placeholder_= st.empty()197placeholder_.info("If you are launching the app for the first time, it may take some time (approximately 1 minute) for SparkNLP models to load...")198nlp_dict= load_sparknlp_models()199placeholder_.empty()200 201 202 203 204if selected_model=="ner_conll_albert_large_uncased":205 text= st.text_input("Type here your text and press enter to run:", value="Mark Knopfler was born in Glasgow, Scotland. He is a British singer-songwriter, guitarist, and record producer. He became known as the lead guitarist, singer and songwriter of the rock band Dire Straits.")206 207elif selected_model=="ner_mit_movie_complex_distilbert_base_cased":208 text= st.text_input("Type here your text and press enter to run:", value="It's only appropriate that Solaris, Russian filmmaker Andrei Tarkovsky's psychological sci-fi classic from 1972, contains an equally original and mind-bending score. Solaris explores the inadequacies of time and memory on an enigmatic planet below a derelict space station. To reinforce the film's chilling setting, Tarkovsky commissioned composer Eduard Artemiev to construct an electronic soundscape reflecting planet Solaris' amorphous and mysterious surface")209 210elif selected_model=="ner_conll_elmo":211 text= st.text_input("Type here your text and press enter to run: ", value="Tottenham Hotspur Football Club, commonly referred to as Tottenham or Spurs, is an English professional football club based in Tottenham, London, that competes in the Premier League, the top flight of English football.")212 213elif selected_model=="onto_100":214 text= st.text_input("Type here your text and press enter to run: ", value="William Henry Gates III (born October 28, 1955) is an American business magnate, software developer, investor, and philanthropist. He is best known as the co-founder of Microsoft Corporation. During his career at Microsoft, Gates held the positions of chairman, chief executive officer (CEO), president and chief software architect, while also being the largest individual shareholder until May 2014. He is one of the best-known entrepreneurs and pioneers of the microcomputer revolution of the 1970s and 1980s. Born and raised in Seattle, Washington, Gates co-founded Microsoft with childhood friend Paul Allen in 1975, in Albuquerque, New Mexico; it went on to become the world's largest personal computer software company. Gates led the company as chairman and CEO until stepping down as CEO in January 2000, but he remained chairman and became chief software architect.")215 216else:217 text= st.text_input("Type here your text and press enter to run:", value="12 Corazones ('12 Hearts') is Spanish-language dating game show produced in the United States for the television network Telemundo since January 2005, based on its namesake Argentine TV show format. The show is filmed in Los Angeles and revolves around the twelve Zodiac signs that identify each contestant. In 2008, Ho filmed a cameo in the Steven Spielberg feature film The Cloverfield Paradox, as a news pundit.")218 219 220 221def build_pipeline(text, model_name=selected_model):222 223 base_pipeline= Pipeline(stages=[224 nlp_dict["documentAssembler"],225 nlp_dict["sentenceDetector"],226 nlp_dict["tokenizer"]227 ])228 229 fewnerd_pipeline= Pipeline(stages=[230 base_pipeline,231 nlp_dict["glove_100d"],232 nlp_dict[model_name],233 nlp_dict["ner_converter"]234 ])235 236 elmo_pipeline= Pipeline(stages=[237 base_pipeline,238 nlp_dict["elmo"],239 nlp_dict[model_name],240 nlp_dict["ner_converter"]241 ])242 243 movie_pipeline= Pipeline(stages=[244 base_pipeline,245 nlp_dict["distilbert_base_cased"],246 nlp_dict[model_name],247 nlp_dict["ner_converter"]248 ])249 250 albert_pipeline= Pipeline(stages=[251 base_pipeline,252 nlp_dict["albert_large_uncased"],253 nlp_dict[model_name],254 nlp_dict["ner_converter"]255 ])256 257 onto_pipeline= Pipeline(stages=[258 base_pipeline,259 nlp_dict["glove_100d_for_onto"],260 nlp_dict[model_name],261 nlp_dict["ner_converter"]262 ])263 264 265 text_df = spark.createDataFrame([[text]]).toDF("text")266 267 if model_name=="nerdl_fewnerd_100d":268 pipeline_model= fewnerd_pipeline.fit(text_df)269 270 elif model_name=="ner_conll_elmo":271 pipeline_model= elmo_pipeline.fit(text_df)272 273 elif model_name=="ner_mit_movie_complex_distilbert_base_cased":274 pipeline_model= movie_pipeline.fit(text_df)275 276 elif model_name=="ner_conll_albert_large_uncased":277 pipeline_model= albert_pipeline.fit(text_df)278 279 elif model_name=="onto_100":280 pipeline_model= onto_pipeline.fit(text_df)281 282 result = pipeline_model.transform(text_df).toPandas()283 284 return result285 286#placeholder for warning287placeholder= st.empty()288placeholder.info("Processing...")289 290result= build_pipeline(text)291placeholder.empty()292 293df= pd.DataFrame({"ner_chunk": result["ner_chunk"].iloc[0]})294 295labels_set = set()296for i in df['ner_chunk'].values:297 labels_set.add(i[4]['entity'])298labels_set = list(labels_set)299 300labels = st.sidebar.multiselect(301 "NER Labels", options=labels_set, default=list(labels_set)302 )303 304show_html2(text, df, labels, "Text annotated with identified Named Entities")305 306try_link="""<a href="https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Public/3.SparkNLP_Pretrained_Models.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" style="zoom: 1.3" alt="Open In Colab"/></a>"""307st.sidebar.title('')308st.sidebar.markdown("<h1> Try it yourself: </h1>" , unsafe_allow_html=True)309st.sidebar.markdown(try_link, unsafe_allow_html=True)310 311st.sidebar.info("""Want to see more? 312- Check Spark NLP in action, including our Spark NLP for Healthcare & Spark OCR demos at [here](https://nlp.johnsnowlabs.com/demos)313- Check our 4.4K+ models available in Spark NLP Models Hub [here](https://nlp.johnsnowlabs.com/models)""")314