Upload 6 files

.streamlit/config.toml

@@ -0,0 +1,3 @@
+[theme]
+base="light"
+primaryColor="#29B4E8"

Demo.py

@@ -0,0 +1,158 @@
+import streamlit as st
+import sparknlp
+import os
+import pandas as pd
+
+from sparknlp.base import *
+from sparknlp.annotator import *
+from pyspark.ml import Pipeline
+from sparknlp.pretrained import PretrainedPipeline
+from annotated_text import annotated_text
+
+# Page configuration
+st.set_page_config(
+    layout="wide", 
+    page_title="Spark NLP Demos App", 
+    initial_sidebar_state="auto"
+)
+
+# CSS for styling
+st.markdown("""
+    <style>
+        .main-title {
+            font-size: 36px;
+            color: #4A90E2;
+            font-weight: bold;
+            text-align: center;
+        }
+        .section p, .section ul {
+            color: #666666;
+        }
+    </style>
+""", unsafe_allow_html=True)
+
+@st.cache_resource
+def init_spark():
+    return sparknlp.start()
+
+@st.cache_resource
+def create_pipeline(model):
+
+    documentAssembler = DocumentAssembler() \
+    .setInputCol('text') \
+    .setOutputCol('document')
+
+    tokenizer = Tokenizer() \
+        .setInputCols(['document']) \
+        .setOutputCol('token')
+
+    if model == "ner_dl":
+        embeddings = WordEmbeddingsModel.pretrained('glove_100d') \
+            .setInputCols(["document", 'token']) \
+            .setOutputCol("embeddings")
+    elif model == "ner_dl_bert":
+        embeddings = BertEmbeddings.pretrained('bert_base_cased','en') \
+            .setInputCols(['document', 'token']) \
+            .setOutputCol('embeddings')
+
+    ner_model = NerDLModel.pretrained(model, 'en') \
+        .setInputCols(['document', 'token', 'embeddings']) \
+        .setOutputCol('ner')
+
+    ner_converter = NerConverter() \
+        .setInputCols(['document', 'token', 'ner']) \
+        .setOutputCol('ner_chunk')
+
+    nlp_pipeline = Pipeline(
+        stages=[
+            documentAssembler, 
+            tokenizer,
+            embeddings,
+            ner_model,
+            ner_converter])
+
+    return nlp_pipeline
+
+def fit_data(pipeline, data):
+  empty_df = spark.createDataFrame([['']]).toDF('text')
+  pipeline_model = pipeline.fit(empty_df)
+  model = LightPipeline(pipeline_model)
+  result = model.fullAnnotate(data)
+  return result
+
+def annotate(data):
+    document, chunks, labels = data["Document"], data["NER Chunk"], data["NER Label"]
+    annotated_words = []
+    for chunk, label in zip(chunks, labels):
+        parts = document.split(chunk, 1)
+        if parts[0]:
+            annotated_words.append(parts[0])
+        annotated_words.append((chunk, label))
+        document = parts[1]
+    if document:
+        annotated_words.append(document)
+    annotated_text(*annotated_words)
+
+# Set up the page layout
+st.markdown('<div class="main-title">State-of-the-Art Named Entity Recognition with Spark NLP</div>', unsafe_allow_html=True)
+
+# Sidebar content
+model = st.sidebar.selectbox(
+    "Choose the pretrained model",
+    ["ner_dl", "ner_dl_bert"],
+    help="For more info about the models visit: https://sparknlp.org/models"
+)
+
+# Reference notebook link in sidebar
+link = """
+<a href="https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/NER_EN.ipynb">
+    <img src="https://colab.research.google.com/assets/colab-badge.svg" style="zoom: 1.3" alt="Open In Colab"/>
+</a>
+"""
+st.sidebar.markdown('Reference notebook:')
+st.sidebar.markdown(link, unsafe_allow_html=True)
+
+# Load examples
+examples = [
+        "William Henry Gates III (born October 28, 1955) is an American business magnate, software developer, investor, and philanthropist. He is best known as the co-founder of Microsoft Corporation. During his career at Microsoft, Gates held the positions of chairman, chief executive officer (CEO), president and chief software architect, while also being the largest individual shareholder until May 2014. He is one of the best-known entrepreneurs and pioneers of the microcomputer revolution of the 1970s and 1980s. Born and raised in Seattle, Washington, Gates co-founded Microsoft with childhood friend Paul Allen in 1975, in Albuquerque, New Mexico; it went on to become the world's largest personal computer software company. Gates led the company as chairman and CEO until stepping down as CEO in January 2000, but he remained chairman and became chief software architect. During the late 1990s, Gates had been criticized for his business tactics, which have been considered anti-competitive. This opinion has been upheld by numerous court rulings. In June 2006, Gates announced that he would be transitioning to a part-time role at Microsoft and full-time work at the Bill & Melinda Gates Foundation, the private charitable foundation that he and his wife, Melinda Gates, established in 2000.[9] He gradually transferred his duties to Ray Ozzie and Craig Mundie. He stepped down as chairman of Microsoft in February 2014 and assumed a new post as technology adviser to support the newly appointed CEO Satya Nadella.",
+        "The Mona Lisa is a 16th century oil painting created by Leonardo. It's held at the Louvre in Paris.",
+        "When Sebastian Thrun started working on self-driving cars at Google in 2007, few people outside of the company took him seriously. “I can tell you very senior CEOs of major American car companies would shake my hand and turn away because I wasn’t worth talking to,” said Thrun, now the co-founder and CEO of online higher education startup Udacity, in an interview with Recode earlier this week.",
+        "Facebook is a social networking service launched as TheFacebook on February 4, 2004. It was founded by Mark Zuckerberg with his college roommates and fellow Harvard University students Eduardo Saverin, Andrew McCollum, Dustin Moskovitz and Chris Hughes. The website's membership was initially limited by the founders to Harvard students, but was expanded to other colleges in the Boston area, the Ivy League, and gradually most universities in the United States and Canada.",
+        "The history of natural language processing generally started in the 1950s, although work can be found from earlier periods. In 1950, Alan Turing published an article titled 'Computing Machinery and Intelligence' which proposed what is now called the Turing test as a criterion of intelligence",
+        "Geoffrey Everest Hinton is an English Canadian cognitive psychologist and computer scientist, most noted for his work on artificial neural networks. Since 2013 he divides his time working for Google and the University of Toronto. In 2017, he cofounded and became the Chief Scientific Advisor of the Vector Institute in Toronto.",
+        "When I told John that I wanted to move to Alaska, he warned me that I'd have trouble finding a Starbucks there.",
+        "Steven Paul Jobs was an American business magnate, industrial designer, investor, and media proprietor. He was the chairman, chief executive officer (CEO), and co-founder of Apple Inc., the chairman and majority shareholder of Pixar, a member of The Walt Disney Company's board of directors following its acquisition of Pixar, and the founder, chairman, and CEO of NeXT. Jobs is widely recognized as a pioneer of the personal computer revolution of the 1970s and 1980s, along with Apple co-founder Steve Wozniak. Jobs was born in San Francisco, California, and put up for adoption. He was raised in the San Francisco Bay Area. He attended Reed College in 1972 before dropping out that same year, and traveled through India in 1974 seeking enlightenment and studying Zen Buddhism.",
+        "Titanic is a 1997 American epic romance and disaster film directed, written, co-produced, and co-edited by James Cameron. Incorporating both historical and fictionalized aspects, it is based on accounts of the sinking of the RMS Titanic, and stars Leonardo DiCaprio and Kate Winslet as members of different social classes who fall in love aboard the ship during its ill-fated maiden voyage.",
+        "Other than being the king of the north, John Snow is a an english physician and a leader in the development of anaesthesia and medical hygiene. He is considered for being the first one using data to cure cholera outbreak in 1834."
+]
+
+# st.subheader("Automatically detect phrases expressing dates and normalize them with respect to a reference date.")
+selected_text = st.selectbox("Select an example", examples)
+custom_input = st.text_input("Try it with your own Sentence!")
+
+text_to_analyze = custom_input if custom_input else selected_text
+
+st.subheader('Full example text')
+HTML_WRAPPER = """<div class="scroll entities" style="overflow-x: auto; border: 1px solid #e6e9ef; border-radius: 0.25rem; padding: 1rem; margin-bottom: 2.5rem; white-space:pre-wrap">{}</div>"""
+st.markdown(HTML_WRAPPER.format(text_to_analyze), unsafe_allow_html=True)
+
+# Initialize Spark and create pipeline
+spark = init_spark()
+pipeline = create_pipeline(model)
+output = fit_data(pipeline, text_to_analyze)
+
+# Display matched sentence
+st.subheader("Processed output:")
+
+results = {
+    'Document': output[0]['document'][0].result,
+    'NER Chunk': [n.result for n in output[0]['ner_chunk']],
+    "NER Label": [n.metadata['entity'] for n in output[0]['ner_chunk']]
+}
+
+annotate(results)
+
+with st.expander("View DataFrame"):
+    df = pd.DataFrame({'NER Chunk': results['NER Chunk'], 'NER Label': results['NER Label']})
+    df.index += 1
+    st.dataframe(df)

Dockerfile

@@ -0,0 +1,70 @@
+# Download base image ubuntu 18.04
+FROM ubuntu:18.04
+
+# Set environment variables
+ENV NB_USER jovyan
+ENV NB_UID 1000
+ENV HOME /home/${NB_USER}
+
+# Install required packages
+RUN apt-get update && apt-get install -y \
+    tar \
+    wget \
+    bash \
+    rsync \
+    gcc \
+    libfreetype6-dev \
+    libhdf5-serial-dev \
+    libpng-dev \
+    libzmq3-dev \
+    python3 \
+    python3-dev \
+    python3-pip \
+    unzip \
+    pkg-config \
+    software-properties-common \
+    graphviz \
+    openjdk-8-jdk \
+    ant \
+    ca-certificates-java \
+    && apt-get clean \
+    && update-ca-certificates -f;
+
+# Install Python 3.8 and pip
+RUN add-apt-repository ppa:deadsnakes/ppa \
+    && apt-get update \
+    && apt-get install -y python3.8 python3-pip \
+    && apt-get clean;
+
+# Set up JAVA_HOME
+ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64/
+RUN mkdir -p ${HOME} \
+    && echo "export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/" >> ${HOME}/.bashrc \
+    && chown -R ${NB_UID}:${NB_UID} ${HOME}
+
+# Create a new user named "jovyan" with user ID 1000
+RUN useradd -m -u ${NB_UID} ${NB_USER}
+
+# Switch to the "jovyan" user
+USER ${NB_USER}
+
+# Set home and path variables for the user
+ENV HOME=/home/${NB_USER} \
+    PATH=/home/${NB_USER}/.local/bin:$PATH
+
+# Set the working directory to the user's home directory
+WORKDIR ${HOME}
+
+# Upgrade pip and install Python dependencies
+RUN python3.8 -m pip install --upgrade pip
+COPY requirements.txt /tmp/requirements.txt
+RUN python3.8 -m pip install -r /tmp/requirements.txt
+
+# Copy the application code into the container at /home/jovyan
+COPY --chown=${NB_USER}:${NB_USER} . ${HOME}
+
+# Expose port for Streamlit
+EXPOSE 7860
+
+# Define the entry point for the container
+ENTRYPOINT ["streamlit", "run", "Demo.py", "--server.port=7860", "--server.address=0.0.0.0"]

pages/Workflow & Model Overview.py

@@ -0,0 +1,355 @@
+import streamlit as st
+
+# Custom CSS for better styling
+st.markdown("""
+    <style>
+        .main-title {
+            font-size: 36px;
+            color: #4A90E2;
+            font-weight: bold;
+            text-align: center;
+        }
+        .sub-title {
+            font-size: 24px;
+            color: #4A90E2;
+            margin-top: 20px;
+        }
+        .section {
+            background-color: #f9f9f9;
+            padding: 15px;
+            border-radius: 10px;
+            margin-top: 20px;
+        }
+        .section h2 {
+            font-size: 22px;
+            color: #4A90E2;
+        }
+        .section p, .section ul {
+            color: #666666;
+        }
+        .link {
+            color: #4A90E2;
+            text-decoration: none;
+        }
+    </style>
+""", unsafe_allow_html=True)
+
+# Main Title
+st.markdown('<div class="main-title">The Ultimate Guide to Named Entity Recognition with Spark NLP</div>', unsafe_allow_html=True)
+
+# Introduction
+st.markdown("""
+<div class="section">
+    <p>Named Entity Recognition (NER) is the task of identifying important words in a text and associating them with a category. For example, we may be interested in finding all the personal names in documents, or company names in news articles. Other examples include domain-specific uses such as identifying all disease names in a clinical text, or company trading codes in financial ones.</p>
+    <p>NER can be implemented with many approaches. In this post, we introduce two methods: using a manually crafted list of entities (gazetteer) or regular expressions, and using deep learning with the NerDL model. Both approaches leverage the scalability of Spark NLP with Python.</p>
+</div>
+""", unsafe_allow_html=True)
+
+st.image("images/ner.png")
+
+# Introduction to Spark NLP
+st.markdown('<div class="sub-title">Introduction to Spark NLP</div>', unsafe_allow_html=True)
+st.markdown("""
+<div class="section">
+    <p>Spark NLP is an open-source library maintained by John Snow Labs. It is built on top of Apache Spark and Spark ML and provides simple, performant & accurate NLP annotations for machine learning pipelines that can scale easily in a distributed environment.</p>
+    <p>To install Spark NLP, you can simply use any package manager like conda or pip. For example, using pip you can simply run <code>pip install spark-nlp</code>. For different installation options, check the official <a href="https://nlp.johnsnowlabs.com/docs/en/install" target="_blank" class="link">documentation</a>.</p>
+</div>
+""", unsafe_allow_html=True)
+
+# Using NerDL Model
+st.markdown('<div class="sub-title">Using NerDL Model</div>', unsafe_allow_html=True)
+st.markdown("""
+<div class="section">
+    <p>The NerDL model in Spark NLP is a deep learning-based approach for NER tasks. It uses a Char CNNs - BiLSTM - CRF architecture that achieves state-of-the-art results in most datasets. The training data should be a labeled Spark DataFrame in the format of CoNLL 2003 IOB with annotation type columns.</p>
+</div>
+""", unsafe_allow_html=True)
+
+# Setup Instructions
+st.markdown('<div class="sub-title">Setup</div>', unsafe_allow_html=True)
+st.markdown('<p>To install Spark NLP in Python, use your favorite package manager (conda, pip, etc.). For example:</p>', unsafe_allow_html=True)
+st.code("""
+pip install spark-nlp
+pip install pyspark
+""", language="bash")
+
+st.markdown("<p>Then, import Spark NLP and start a Spark session:</p>", unsafe_allow_html=True)
+st.code("""
+import sparknlp
+
+# Start Spark Session
+spark = sparknlp.start()
+""", language='python')
+
+# Example Usage with NerDL Model
+st.markdown('<div class="sub-title">Example Usage with NerDL Model</div>', unsafe_allow_html=True)
+st.markdown("""
+<div class="section">
+    <p>Below is an example of how to set up and use the NerDL model for named entity recognition:</p>
+</div>
+""", unsafe_allow_html=True)
+st.code('''
+from sparknlp.base import *
+from sparknlp.annotator import *
+from pyspark.ml import Pipeline
+
+# Document Assembler
+document_assembler = DocumentAssembler() \\
+    .setInputCol("text") \\
+    .setOutputCol("document")
+
+# Sentence Detector
+sentence_detector = SentenceDetector() \\
+    .setInputCols(["document"]) \\
+    .setOutputCol("sentence")
+
+# Tokenizer
+tokenizer = Tokenizer() \\
+    .setInputCols(["sentence"]) \\
+    .setOutputCol("token")
+
+# Word Embeddings
+embeddings = WordEmbeddingsModel.pretrained() \\
+    .setInputCols(["sentence", "token"]) \\
+    .setOutputCol("bert")
+
+# NerDL Model
+ner_tagger = NerDLModel.pretrained() \\
+    .setInputCols(["sentence", "token", "bert"]) \\
+    .setOutputCol("ner")
+
+# Pipeline
+pipeline = Pipeline().setStages([
+    document_assembler,
+    sentence_detector,
+    tokenizer,
+    embeddings,
+    ner_tagger
+])
+
+# Example sentence
+example = """
+William Henry Gates III (born October 28, 1955) is an American business magnate, software developer, investor, and philanthropist. 
+He is best known as the co-founder of Microsoft Corporation. Throughout his career at Microsoft, Gates held various positions, 
+including chairman, chief executive officer (CEO), president, and chief software architect. He was also the largest individual 
+shareholder until May 2014.
+
+Gates is recognized as one of the foremost entrepreneurs and pioneers of the microcomputer revolution of the 1970s and 1980s. 
+Born and raised in Seattle, Washington, he co-founded Microsoft with childhood friend Paul Allen in 1975. Initially established 
+in Albuquerque, New Mexico, Microsoft grew to become the world’s largest personal computer software company.
+
+Gates led Microsoft as chairman and CEO until January 2000, when he stepped down as CEO but continued as chairman and chief 
+software architect. During the late 1990s, Gates faced criticism for business practices considered anti-competitive, an opinion 
+upheld by numerous court rulings.
+
+In June 2006, Gates announced his transition to a part-time role at Microsoft while dedicating full time to the Bill & Melinda Gates 
+Foundation, a private charitable organization he established with his wife, Melinda Gates, in 2000. Gates gradually transferred 
+his responsibilities to Ray Ozzie and Craig Mundie and stepped down as chairman of Microsoft in February 2014. He then assumed 
+the role of technology adviser to support the newly appointed CEO, Satya Nadella.
+"""
+
+data = spark.createDataFrame([[example]]).toDF("text")
+
+# Transforming data
+result = pipeline.fit(data).transform(data)
+result.select("ner.result").show(truncate=False)
+''', language="python")
+
+st.text("""
++-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+|result                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                         |
++-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+|[O, B-PER, I-PER, I-PER, I-PER, O, O, O, O, O, O, O, O, O, B-MISC, O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, B-ORG, I-ORG, O, O, O, O, O, B-ORG, O, B-PER, O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, B-LOC, O, O, O, B-LOC, O, B-LOC, O, B-PER, O, B-ORG, O, O, O, B-PER, I-PER, O, O, O, O, B-LOC, O, B-LOC, I-LOC, O, O, O, O, O, O, O, O, O, O, O, O, O, O, B-PER, O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, B-PER, O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, B-PER, O, O, O, O, O, O, O, O, O, O, O, B-ORG, O, O, O, O, O, B-ORG, I-ORG, I-ORG, I-ORG, I-ORG, O, O, O, O, O, O, O, O, O, O, O, B-PER, I-PER, O, O, O, O, O, O, O, O, O, O, B-PER, I-PER, O, B-PER, I-PER, O, O, O, O, O, O, O, B-ORG, O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, O, B-PER, I-PER, O]|
++-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+""")
+
+# Using EntityRuler Annotator
+st.markdown('<div class="sub-title">Using EntityRuler Annotator</div>', unsafe_allow_html=True)
+st.markdown("""
+<div class="section">
+    <p>In addition to the deep learning-based approach, Spark NLP also supports a rule-based method for NER using the EntityRuler annotator. This method involves using a gazetteer or regular expressions to identify entities in the text.</p>
+</div>
+""", unsafe_allow_html=True)
+
+# Example Usage with EntityRuler
+st.markdown('<div class="sub-title">Example Usage with EntityRuler</div>', unsafe_allow_html=True)
+st.markdown("""
+<div class="section">
+    <p>For the NER tasks based on gazetteer list, we will use the EntityRuler annotator, which has both Approach and Model versions.</p>
+    <p>As this annotator consists in finding the entities based in a list of desired names, the EntityRulerApproach annotator will store the given list in the EntityRulerModel parameters. All we need is a JSON or CSV file with the list of names or regex rules. For example, we may use the following entities.json file:</p>
+</div>
+""", unsafe_allow_html=True)
+st.code("""
+[
+    {
+        "label": "PERSON",
+        "patterns": [
+           "John",
+           "John Snow"
+       ]
+    },
+    {
+        "label": "PERSON",
+        "patterns": [
+            "Eddard",
+            "Eddard Stark"
+        ]
+    },
+    {
+        "label": "LOCATION",
+        "patterns": [
+            "Winterfell"
+        ]
+    },
+    {
+        "label": "DATE",
+        "patterns": [
+            "[0-9]{4}-[0-9]{1,2}-[0-9]{1,2}"
+        ],
+        "regex": true
+    }
+]
+""", language="json")
+
+# Pipeline Setup
+st.markdown('<div class="sub-title">Pipeline Setup</div>', unsafe_allow_html=True)
+st.code("""
+from sparknlp.base import DocumentAssembler, Pipeline
+from sparknlp.annotator import EntityRulerApproach, Tokenizer
+
+document_assembler = DocumentAssembler() \\
+    .setInputCol("text") \\
+    .setOutputCol("document")
+
+tokenizer = Tokenizer() \\
+    .setInputCols(["document"]) \\
+    .setOutputCol("token")
+
+entity_ruler = EntityRulerApproach() \\
+    .setInputCols(["document", "token"]) \\
+    .setOutputCol("entity") \\
+    .setPatternsResource("entities.json")
+
+pipeline = Pipeline(stages=[document_assembler, tokenizer, entity_ruler])
+""", language="python")
+
+# Example Sentences
+st.markdown('<div class="sub-title">Example Sentences</div>', unsafe_allow_html=True)
+st.code('''
+example = """Game of Thrones was released in 2011-04-17.
+Lord Eddard Stark was the head of House Stark.
+John Snow lives in Winterfell."""
+
+data = spark.createDataFrame([[example]]).toDF("text")
+pipeline_model = pipeline.fit(data)
+''', language="python")
+
+# Save and Load Model
+st.markdown('<div class="sub-title">We can Save and Load Model for future use (optional)</div>', unsafe_allow_html=True)
+st.code("""
+pipeline_model.stages[-1].write().overwrite().save('my_entityruler')
+
+entity_ruler = EntityRulerModel.load("my_entityruler") \\
+    .setInputCols(["document", "token"]) \\
+    .setOutputCol("entity")
+""", language="python")
+
+# Result Visualization
+st.markdown('<div class="sub-title">Result Visualization</div>', unsafe_allow_html=True)
+st.code("""
+result = pipeline_model.transform(data)
+import pyspark.sql.functions as F
+
+result.select(
+    F.explode(F.col("entity")).alias("entity")
+).select(
+    F.col("entity.result").alias("keyword"),
+    F.col("entity.metadata").alias("metadata")
+).select(
+    F.col("keyword"),
+    F.expr("metadata['entity']").alias("label")
+).show()
+""", language="python")
+
+st.text("""
++------------+--------+
+|     keyword|   label|
++------------+--------+
+|  2011-04-17|    DATE|
+|Eddard Stark|  PERSON|
+|   John Snow|  PERSON|
+|  Winterfell|LOCATION|
++------------+--------+
+""")
+
+# Non-English Languages
+st.markdown('<div class="sub-title">Non-English Languages</div>', unsafe_allow_html=True)
+
+st.markdown("""
+<p>The EntityRuler annotator utilizes the Aho-Corasick algorithm, which may not handle languages with unique characters or alphabets effectively. For example:</p>
+<div class="section">
+<ul>
+    <li>Spanish includes the ñ character.</li>
+    <li>Portuguese uses ç.</li>
+    <li>Many languages have accented characters (á, ú, ê, etc.).</li>
+</ul>
+</div>
+<p>To accommodate these characters, use the <code>.setAlphabetResource</code> parameter.</p>
+<p>When a character is missing from the alphabet, you might encounter errors like this:</p>
+<pre><code>Py4JJavaError: An error occurred while calling o69.fit.
+: java.lang.UnsupportedOperationException: Char ú not found on alphabet. Please check alphabet</code></pre>
+<p>To define a custom alphabet, create a text file (e.g., <code>custom_alphabet.txt</code>) with all required characters:</p>
+""", unsafe_allow_html=True)
+st.code("""
+abcdefghijklmnopqrstuvwxyz
+ABCDEFGHIJKLMNOPQRSTUVWXYZ
+áúéêçñ
+ÁÚÉÊÇÑ
+""")
+
+st.markdown("""
+<p>Alternatively, you can use predefined alphabets for common languages. For instance, for Spanish:</p>
+""", unsafe_allow_html=True)
+st.code("""
+entity_ruler = (
+    EntityRulerApproach()
+    .setInputCols(["sentence"])
+    .setOutputCol("entity")
+    .setPatternsResource("locations.json")
+    .setAlphabetResource("Spanish")
+)
+""")
+
+# Summary
+st.markdown('<div class="sub-title">Summary</div>', unsafe_allow_html=True)
+st.markdown("""
+<div class="section">
+    <p>In this article, we talked about named entity recognition using both deep learning-based and rule-based methods. We introduced how to perform the task using the open-source Spark NLP library with Python, which can be used at scale in the Spark ecosystem. These methods can be used for natural language processing applications in various fields, including finance and healthcare.</p>
+</div>
+""", unsafe_allow_html=True)
+
+# References
+st.markdown('<div class="sub-title">References</div>', unsafe_allow_html=True)
+st.markdown("""
+<div class="section">
+    <ul>
+        <li><a class="link" href="https://nlp.johnsnowlabs.com/docs/en/annotators#entityruler" target="_blank" rel="noopener">EntityRuler</a> annotator documentation</li>
+        <li>Python Docs: <a class="link" href="https://nlp.johnsnowlabs.com/api/python/reference/autosummary/sparknlp/annotator/er/entity_ruler/index.html#sparknlp.annotator.er.entity_ruler.EntityRulerApproach" target="_blank" rel="noopener">EntityRulerApproach</a>, <a class="link" href="https://nlp.johnsnowlabs.com/api/python/reference/autosummary/sparknlp/annotator/er/entity_ruler/index.html#sparknlp.annotator.er.entity_ruler.EntityRulerModel">EntityRulerModel</a></li>
+        <li>Scala Docs: <a class="link" href="https://nlp.johnsnowlabs.com/api/com/johnsnowlabs/nlp/annotators/er/EntityRulerApproach.html" target="_blank" rel="noopener">EntityRulerApproach</a>, <a class="link" href="https://nlp.johnsnowlabs.com/api/com/johnsnowlabs/nlp/annotators/er/EntityRulerModel.html">EntityRulerModel</a></li>
+        <li><a class="link" href="https://nlp.johnsnowlabs.com/recognize_entitie" target="_blank" rel="noopener">Visualization demos for NER in Spark NLP</a></li>
+        <li><a class="link" href="https://www.johnsnowlabs.com/named-entity-recognition-ner-with-bert-in-spark-nlp/">Named Entity Recognition (NER) with BERT in Spark NLP</a></li>
+    </ul>
+</div>
+""", unsafe_allow_html=True)
+
+st.markdown('<div class="sub-title">Community & Support</div>', unsafe_allow_html=True)
+st.markdown("""
+<div class="section">
+    <ul>
+        <li><a class="link" href="https://sparknlp.org/" target="_blank">Official Website</a>: Documentation and examples</li>
+        <li><a class="link" href="https://join.slack.com/t/spark-nlp/shared_invite/zt-198dipu77-L3UWNe_AJ8xqDk0ivmih5Q" target="_blank">Slack</a>: Live discussion with the community and team</li>
+        <li><a class="link" href="https://github.com/JohnSnowLabs/spark-nlp" target="_blank">GitHub</a>: Bug reports, feature requests, and contributions</li>
+        <li><a class="link" href="https://medium.com/spark-nlp" target="_blank">Medium</a>: Spark NLP articles</li>
+        <li><a class="link" href="https://www.youtube.com/channel/UCmFOjlpYEhxf_wJUDuz6xxQ/videos" target="_blank">YouTube</a>: Video tutorials</li>
+    </ul>
+</div>
+""", unsafe_allow_html=True)

requirements.txt

@@ -0,0 +1,6 @@
+streamlit
+st-annotated-text
+pandas
+numpy
+spark-nlp
+pyspark