Update app.py

app.py

@@ -1,21 +1,19 @@
+import pandas as pd
+import numpy as np
+import json 
+import os
+import sys
 import streamlit as st
 
 
+import streamlit_apps_config as config
 st.set_page_config(
     layout="centered",  # Can be "centered" or "wide". In the future also "dashboard", etc.
     initial_sidebar_state="auto",  # Can be "auto", "expanded", "collapsed"
     page_title='Extractive Summarization',  # String or None. Strings get appended with "• Streamlit".
     page_icon='./favicon.png',  # String, anything supported by st.image, or None.
 )
-
-
-import pandas as pd
-import numpy as np
-import json 
-import os
-import sys
 sys.path.append(os.path.abspath('./'))
-import streamlit_apps_config as config
 from streamlit_ner_output import show_html2, jsl_display_annotations, get_color
 
 import sparknlp
@@ -117,7 +115,195 @@ st.subheader("")
 
 
 
+#caching the models in the dictionary
+@st.cache(allow_output_mutation=True, show_spinner=False)
+def load_sparknlp_models():
+    ner_models_list= ["nerdl_fewnerd_100d", "ner_conll_elmo", "ner_mit_movie_complex_distilbert_base_cased", "ner_conll_albert_large_uncased", "onto_100"]
+    embeddings_list= ["glove_100d", "elmo", "distilbert_base_cased", "albert_large_uncased", "glove_100d_for_onto"]
+
+
+    documentAssembler = DocumentAssembler()\
+        .setInputCol("text")\
+        .setOutputCol("document")
+
+    sentenceDetector= SentenceDetector()\
+        .setInputCols(["document"])\
+        .setOutputCol("sentence")
+
+    tokenizer = Tokenizer()\
+        .setInputCols(["sentence"])\
+        .setOutputCol("token")
+
+    ner_converter= NerConverter()\
+        .setInputCols(["document", "token", "ner"])\
+        .setOutputCol("ner_chunk")
+
+    model_dict= {
+        'documentAssembler': documentAssembler,
+        'sentenceDetector': sentenceDetector,
+        'tokenizer': tokenizer,
+        'ner_converter': ner_converter
+    }
+
+    for embeddings_name, ner_model_name in zip(embeddings_list, ner_models_list):
+
+        try:
+          if embeddings_name=="glove_100d":
+            model_dict[embeddings_name]= WordEmbeddingsModel.pretrained(embeddings_name, "en")\
+                                                  .setInputCols(["sentence", "token"])\
+                                                  .setOutputCol("embeddings")
+
+          elif embeddings_name=="elmo":
+            model_dict[embeddings_name]= ElmoEmbeddings.pretrained(embeddings_name, "en")\
+                                                  .setInputCols(["token", "document"])\
+                                                  .setOutputCol("embeddings")\
+                                                  .setPoolingLayer("elmo")
+
+          elif embeddings_name=="distilbert_base_cased":
+              model_dict[embeddings_name]= DistilBertEmbeddings\
+                                                   .pretrained(embeddings_name, 'en')\
+                                                   .setInputCols(["token", "document"])\
+                                                   .setOutputCol("embeddings")
+
+          elif embeddings_name=="albert_large_uncased":
+              model_dict[embeddings_name]= AlbertEmbeddings\
+                                                    .pretrained(embeddings_name, 'en')\
+                                                    .setInputCols(["document", "token"])\
+                                                    .setOutputCol("embeddings")
+
+          elif embeddings_name=="glove_100d_for_onto":
+              model_dict[embeddings_name]= WordEmbeddingsModel.pretrained("glove_100d", "en")\
+                                                    .setInputCols(["sentence", "token"])\
+                                                    .setOutputCol("embeddings")
+
+
+          model_dict[ner_model_name]= NerDLModel.pretrained(ner_model_name, "en")\
+                                              .setInputCols(["document", "token", "embeddings"])\
+                                              .setOutputCol("ner")
+
+          
+        except:
+            pass
+    return model_dict
+
+
+
+placeholder_= st.empty()
+placeholder_.info("If you are launching the app for the first time, it may take some time (approximately 1 minute) for SparkNLP models to load...")
+nlp_dict= load_sparknlp_models()
+placeholder_.empty()
+
+
+
+
+if selected_model=="ner_conll_albert_large_uncased":
+    text= st.text_input("Type here your text and press enter to run:", value="Mark Knopfler was born in Glasgow, Scotland. He is a British singer-songwriter, guitarist, and record producer. He became known as the lead guitarist, singer and songwriter of the rock band Dire Straits.")
+
+elif selected_model=="ner_mit_movie_complex_distilbert_base_cased":
+    text= st.text_input("Type here your text and press enter to run:", value="It's only appropriate that Solaris, Russian filmmaker Andrei Tarkovsky's psychological sci-fi classic from 1972, contains an equally original and mind-bending score. Solaris explores the inadequacies of time and memory on an enigmatic planet below a derelict space station. To reinforce the film's chilling setting, Tarkovsky commissioned composer Eduard Artemiev to construct an electronic soundscape reflecting planet Solaris' amorphous and mysterious surface")
+
+elif selected_model=="ner_conll_elmo":
+    text= st.text_input("Type here your text and press enter to run: ", value="Tottenham Hotspur Football Club, commonly referred to as Tottenham or Spurs, is an English professional football club based in Tottenham, London, that competes in the Premier League, the top flight of English football.")
+
+elif selected_model=="onto_100":
+    text= st.text_input("Type here your text and press enter to run: ", value="William Henry Gates III (born October 28, 1955) is an American business magnate, software developer, investor, and philanthropist. He is best known as the co-founder of Microsoft Corporation. During his career at Microsoft, Gates held the positions of chairman, chief executive officer (CEO), president and chief software architect, while also being the largest individual shareholder until May 2014. He is one of the best-known entrepreneurs and pioneers of the microcomputer revolution of the 1970s and 1980s. Born and raised in Seattle, Washington, Gates co-founded Microsoft with childhood friend Paul Allen in 1975, in Albuquerque, New Mexico; it went on to become the world's largest personal computer software company. Gates led the company as chairman and CEO until stepping down as CEO in January 2000, but he remained chairman and became chief software architect.")
+
+else:
+    text= st.text_input("Type here your text and press enter to run:", value="12 Corazones ('12 Hearts') is Spanish-language dating game show produced in the United States for the television network Telemundo since January 2005, based on its namesake Argentine TV show format. The show is filmed in Los Angeles and revolves around the twelve Zodiac signs that identify each contestant. In 2008, Ho filmed a cameo in the Steven Spielberg feature film The Cloverfield Paradox, as a news pundit.")
+
+
+
+def build_pipeline(text, model_name=selected_model):
+
+    base_pipeline= Pipeline(stages=[
+                           nlp_dict["documentAssembler"],
+                           nlp_dict["sentenceDetector"],
+                           nlp_dict["tokenizer"]
+                            ])
+
+    fewnerd_pipeline= Pipeline(stages=[
+                           base_pipeline,
+                           nlp_dict["glove_100d"],
+                           nlp_dict[model_name],
+                           nlp_dict["ner_converter"]
+                            ])
+
+    elmo_pipeline= Pipeline(stages=[
+                           base_pipeline,
+                           nlp_dict["elmo"],
+                           nlp_dict[model_name],
+                           nlp_dict["ner_converter"]
+                            ])
+                        
+    movie_pipeline= Pipeline(stages=[
+                           base_pipeline,
+                           nlp_dict["distilbert_base_cased"],
+                           nlp_dict[model_name],
+                           nlp_dict["ner_converter"]
+                            ])
+
+    albert_pipeline= Pipeline(stages=[
+                           base_pipeline,
+                           nlp_dict["albert_large_uncased"],
+                           nlp_dict[model_name],
+                           nlp_dict["ner_converter"]
+                            ])
+
+    onto_pipeline= Pipeline(stages=[
+                           base_pipeline,
+                           nlp_dict["glove_100d_for_onto"],
+                           nlp_dict[model_name],
+                           nlp_dict["ner_converter"]
+                            ])
+
+    
+    empty_df = spark.createDataFrame([[""]]).toDF("text")
+
+    if model_name=="nerdl_fewnerd_100d":
+        pipeline_model= fewnerd_pipeline.fit(empty_df)
+
+    elif model_name=="ner_conll_elmo":
+        pipeline_model=  elmo_pipeline.fit(empty_df)
+
+    elif model_name=="ner_mit_movie_complex_distilbert_base_cased":
+        pipeline_model= movie_pipeline.fit(empty_df)
+
+    elif model_name=="ner_conll_albert_large_uncased":
+        pipeline_model= albert_pipeline.fit(empty_df)
+
+    elif model_name=="onto_100":
+        pipeline_model= onto_pipeline.fit(empty_df)
+
+
+    text_df= spark.createDataFrame(pd.DataFrame({"text": [text]}))
+    result= pipeline_model.transform(text_df).toPandas()
+
+    return result
+
+#placeholder for warning
+placeholder= st.empty()
+placeholder.info("Processing...")
+
+result= build_pipeline(text)
+placeholder.empty()
+
+df= pd.DataFrame({"ner_chunk": result["ner_chunk"].iloc[0]})
+
+labels_set = set()
+for i in df['ner_chunk'].values:
+    labels_set.add(i[4]['entity'])
+labels_set = list(labels_set)
+
+labels = st.sidebar.multiselect(
+        "NER Labels", options=labels_set, default=list(labels_set)
+    )
+
+show_html2(text, df, labels, "Text annotated with identified Named Entities")
 
+try_link="""<a href="https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Public/3.SparkNLP_Pretrained_Models.ipynb"><img src="https://colab.research.google.com/assets/colab-badge.svg" style="zoom: 1.3" alt="Open In Colab"/></a>"""
+st.sidebar.title('')
+st.sidebar.markdown('Try it yourself:')
+st.sidebar.markdown(try_link, unsafe_allow_html=True)