Upload 5 files

.streamlit/config.toml

@@ -0,0 +1,3 @@
+[theme]
+base="light"
+primaryColor="#29B4E8"

Demo.py

@@ -0,0 +1,161 @@
+import streamlit as st
+import sparknlp
+import os
+import pandas as pd
+
+from sparknlp.base import *
+from sparknlp.annotator import *
+from pyspark.ml import Pipeline
+from sparknlp.pretrained import PretrainedPipeline
+from annotated_text import annotated_text
+
+# Page configuration
+st.set_page_config(
+    layout="wide", 
+    initial_sidebar_state="auto"
+)
+
+# CSS for styling
+st.markdown("""
+    <style>
+        .main-title {
+            font-size: 36px;
+            color: #4A90E2;
+            font-weight: bold;
+            text-align: center;
+        }
+        .section {
+            background-color: #f9f9f9;
+            padding: 10px;
+            border-radius: 10px;
+            margin-top: 10px;
+        }
+        .section p, .section ul {
+            color: #666666;
+        }
+    </style>
+""", unsafe_allow_html=True)
+
+@st.cache_resource
+def init_spark():
+    return sparknlp.start()
+
+@st.cache_resource
+def create_pipeline(model):
+    document_assembler = DocumentAssembler() \
+        .setInputCol("text") \
+        .setOutputCol("document")
+
+    sentence_detector = SentenceDetector() \
+        .setInputCols(["document"]) \
+        .setOutputCol("sentence")
+
+    word_segmenter = WordSegmenterModel.pretrained("wordseg_large", "zh") \
+        .setInputCols(["sentence"]) \
+        .setOutputCol("token")
+
+    embeddings = BertEmbeddings.pretrained(name='bert_base_chinese', lang='zh') \
+        .setInputCols(["document", "token"]) \
+        .setOutputCol("embeddings")
+
+    ner = NerDLModel.pretrained(model, "zh") \
+        .setInputCols(["document", "token", "embeddings"]) \
+        .setOutputCol("ner")
+
+    ner_converter = NerConverter() \
+        .setInputCols(["sentence", "token", "ner"]) \
+        .setOutputCol("entities")
+
+    pipeline = Pipeline(stages=[document_assembler, sentence_detector, word_segmenter, embeddings, ner, ner_converter])
+    return pipeline
+
+def fit_data(pipeline, data):
+  empty_df = spark.createDataFrame([['']]).toDF('text')
+  pipeline_model = pipeline.fit(empty_df)
+  model = LightPipeline(pipeline_model)
+  result = model.fullAnnotate(data)
+  return result
+
+def annotate(data):
+    document, chunks, labels = data["Document"], data["NER Chunk"], data["NER Label"]
+    annotated_words = []
+    for chunk, label in zip(chunks, labels):
+        parts = document.split(chunk, 1)
+        if parts[0]:
+            annotated_words.append(parts[0])
+        annotated_words.append((chunk, label))
+        document = parts[1]
+    if document:
+        annotated_words.append(document)
+    annotated_text(*annotated_words)
+
+# Set up the page layout
+st.markdown('<div class="main-title">Recognize entities in Urdu text</div>', unsafe_allow_html=True)
+st.markdown("""
+<div class="section">
+    <p>This demo utilizes embeddings-based NER model for Urdu texts, using the urduvec_140M_300d word embeddings</p>
+</div>
+""", unsafe_allow_html=True)
+
+# Sidebar content
+model = st.sidebar.selectbox(
+    "Choose the pretrained model",
+    ["ner_msra_bert_768d", "ner_weibo_bert_768d"],
+    help="For more info about the models visit: https://sparknlp.org/models"
+)
+
+# Reference notebook link in sidebar
+link = """
+<a href="https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/NER.ipynb">
+    <img src="https://colab.research.google.com/assets/colab-badge.svg" style="zoom: 1.3" alt="Open In Colab"/>
+</a>
+"""
+st.sidebar.markdown('Reference notebook:')
+st.sidebar.markdown(link, unsafe_allow_html=True)
+
+# Load examples
+examples = [
+    "当 前 ， 在 中共十五大 精 神 的 指 引 下 ， 在 以 江泽民 同 志 为 核 心 的 中共中央 领 导 下 ， 全 党 和 全 国 各 族 人 民 正 高 举 邓小平 理 论 伟 大 旗 帜 ， 同 心 同 德 ， 团 结 奋 斗 ， 沿 着 建 设 有 中国 特 色 的 社 会 主 义 道 路 阔 步 前 进 。",
+    "中共中央 致 中国致公党十一大 的 贺 词 各 位 代 表 、 各 位 同 志 ： 在 中国致公党第十一次全国代表大会 隆 重 召 开 之 际 ， 中国共产党中央委员会 谨 向 大 会 表 示 热 烈 的 祝 贺 ， 向 致公党 的 同 志 们 致 以 亲 切 的 问 候 ！",
+    "数 百 名 华 人 、 华 侨 、 留 学 人 员 、 我 国 驻 纽约 总 领 事 馆 代 表 在 机 场 挥 舞 中 美 两 国 国 旗 ， 热 烈 欢 迎 江 主 席 访 问 波士顿 。",
+    "到 机 场 迎 接 江 主 席 的 美 方 人 员 有 马萨诸塞州 州 长 和 波士顿 市 长 等 。",
+    "又 讯 中国 国 家 主 席 江泽民 １ 日 上 午 应 邀 在 美国 著 名 学 府 哈佛大学 发 表 重 要 演 讲 。",
+    "江 主 席 来 到 哈佛大学 时 ， 受 到 哈佛大学 校 长 陆登庭 及 哈佛 各 学 院 院 长 的 热 烈 欢 迎 。",
+    "本 报 纽约 １ ０ 月 ３ １ 日 电 记 者 陈特�� 、 周德武 报 道 ： 今 天 晚 上 ， 美中贸易全国委员会 和 美国中国商会 在 纽约 举 行 盛 大 宴 会 欢 迎 江泽民 主 席 。",
+    "哈佛大学 校 长 陆登庭 对 江 主 席 访 问 哈佛 并 发 表 演 讲 表 示 欢 迎 。",
+    "美中贸易全国委员会 主 席 费希尔 和 美国中国商会 会 长 沈被章 先 后 致 词 。"
+]
+
+selected_text = st.selectbox("Select an example", examples)
+custom_input = st.text_input("Try it with your own Sentence!")
+
+text_to_analyze = custom_input if custom_input else selected_text
+
+st.subheader('Full example text')
+HTML_WRAPPER = """<div class="scroll entities" style="overflow-x: auto; border: 1px solid #e6e9ef; border-radius: 0.25rem; padding: 1rem; margin-bottom: 2.5rem; white-space:pre-wrap">{}</div>"""
+st.markdown(HTML_WRAPPER.format(text_to_analyze), unsafe_allow_html=True)
+
+# Initialize Spark and create pipeline
+spark = init_spark()
+pipeline = create_pipeline(model)
+output = fit_data(pipeline, text_to_analyze)
+
+# Display matched sentence
+st.subheader("Processed output:")
+
+results = {
+    'Document': output[0]['document'][0].result,
+    'NER Chunk': [n.result for n in output[0]['ner_chunk']],
+    "NER Label": [n.metadata['entity'] for n in output[0]['ner_chunk']]
+}
+
+annotate(results)
+
+with st.expander("View DataFrame"):
+    df = pd.DataFrame({'NER Chunk': results['NER Chunk'], 'NER Label': results['NER Label']})
+    df.index += 1
+    st.dataframe(df)
+
+
+
+

Dockerfile

@@ -0,0 +1,70 @@
+# Download base image ubuntu 18.04
+FROM ubuntu:18.04
+
+# Set environment variables
+ENV NB_USER jovyan
+ENV NB_UID 1000
+ENV HOME /home/${NB_USER}
+
+# Install required packages
+RUN apt-get update && apt-get install -y \
+    tar \
+    wget \
+    bash \
+    rsync \
+    gcc \
+    libfreetype6-dev \
+    libhdf5-serial-dev \
+    libpng-dev \
+    libzmq3-dev \
+    python3 \
+    python3-dev \
+    python3-pip \
+    unzip \
+    pkg-config \
+    software-properties-common \
+    graphviz \
+    openjdk-8-jdk \
+    ant \
+    ca-certificates-java \
+    && apt-get clean \
+    && update-ca-certificates -f;
+
+# Install Python 3.8 and pip
+RUN add-apt-repository ppa:deadsnakes/ppa \
+    && apt-get update \
+    && apt-get install -y python3.8 python3-pip \
+    && apt-get clean;
+
+# Set up JAVA_HOME
+ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64/
+RUN mkdir -p ${HOME} \
+    && echo "export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/" >> ${HOME}/.bashrc \
+    && chown -R ${NB_UID}:${NB_UID} ${HOME}
+
+# Create a new user named "jovyan" with user ID 1000
+RUN useradd -m -u ${NB_UID} ${NB_USER}
+
+# Switch to the "jovyan" user
+USER ${NB_USER}
+
+# Set home and path variables for the user
+ENV HOME=/home/${NB_USER} \
+    PATH=/home/${NB_USER}/.local/bin:$PATH
+
+# Set the working directory to the user's home directory
+WORKDIR ${HOME}
+
+# Upgrade pip and install Python dependencies
+RUN python3.8 -m pip install --upgrade pip
+COPY requirements.txt /tmp/requirements.txt
+RUN python3.8 -m pip install -r /tmp/requirements.txt
+
+# Copy the application code into the container at /home/jovyan
+COPY --chown=${NB_USER}:${NB_USER} . ${HOME}
+
+# Expose port for Streamlit
+EXPOSE 7860
+
+# Define the entry point for the container
+ENTRYPOINT ["streamlit", "run", "Demo.py", "--server.port=7860", "--server.address=0.0.0.0"]

pages/Workflow & Model Overview.py

@@ -0,0 +1,215 @@
+import streamlit as st
+
+# Custom CSS for better styling
+st.markdown("""
+    <style>
+        .main-title {
+            font-size: 36px;
+            color: #4A90E2;
+            font-weight: bold;
+            text-align: center;
+        }
+        .sub-title {
+            font-size: 24px;
+            color: #4A90E2;
+            margin-top: 20px;
+        }
+        .section {
+            background-color: #f9f9f9;
+            padding: 15px;
+            border-radius: 10px;
+            margin-top: 20px;
+        }
+        .section h2 {
+            font-size: 22px;
+            color: #4A90E2;
+        }
+        .section p, .section ul {
+            color: #666666;
+        }
+        .link {
+            color: #4A90E2;
+            text-decoration: none;
+        }
+    </style>
+""", unsafe_allow_html=True)
+
+# Main Title
+st.markdown('<div class="main-title">Named Entity Recognition for Chinese (BERT-MSRA Dataset)</div>', unsafe_allow_html=True)
+
+# Description
+st.markdown('<div class="sub-title">Description</div>', unsafe_allow_html=True)
+st.markdown("""
+<div class="section">
+    <p>This model annotates named entities in a text, which can be used to find features such as names of people, places, and organizations. The model does not read words directly but instead reads word embeddings, which represent words as points such that more semantically similar words are closer together.</p>
+    <p>This model uses the pre-trained <code>bert_base_chinese</code> embeddings model from BertEmbeddings annotator as an input, so be sure to use the same embeddings in the pipeline.</p>
+</div>
+""", unsafe_allow_html=True)
+
+# Predicted Entities
+st.markdown('<div class="sub-title">Predicted Entities</div>', unsafe_allow_html=True)
+st.markdown("""
+<div class="section">
+    <ul>
+        <li>Persons-PER</li>
+        <li>Locations-LOC</li>
+        <li>Organizations-ORG</li>
+    </ul>
+</div>
+""", unsafe_allow_html=True)
+
+# How to use
+st.markdown('<div class="sub-title">How to use</div>', unsafe_allow_html=True)
+st.markdown("""
+<div class="section">
+    <p>To use this model, follow these steps in Python:</p>
+</div>
+""", unsafe_allow_html=True)
+st.code("""
+from sparknlp.base import *
+from sparknlp.annotator import *
+from pyspark.ml import Pipeline
+
+# Define the components of the pipeline
+document_assembler = DocumentAssembler() \\
+    .setInputCol("text") \\
+    .setOutputCol("document")
+
+sentence_detector = SentenceDetector() \\
+    .setInputCols(["document"]) \\
+    .setOutputCol("sentence")
+
+word_segmenter = WordSegmenterModel.pretrained("wordseg_large", "zh") \\
+    .setInputCols(["sentence"]) \\
+    .setOutputCol("token")
+
+embeddings = BertEmbeddings.pretrained(name='bert_base_chinese', lang='zh') \\
+    .setInputCols(["document", "token"]) \\
+    .setOutputCol("embeddings")
+
+ner = NerDLModel.pretrained("ner_msra_bert_768d", "zh") \\
+    .setInputCols(["document", "token", "embeddings"]) \\
+    .setOutputCol("ner")
+
+ner_converter = NerConverter() \\
+    .setInputCols(["sentence", "token", "ner"]) \\
+    .setOutputCol("entities")
+
+# Create the pipeline
+pipeline = Pipeline(stages=[document_assembler, sentence_detector, word_segmenter, embeddings, ner, ner_converter])
+
+# Create sample data
+example = spark.createDataFrame([['马云在浙江省杭州市出生，是阿里巴巴集团的主要创始人。']], ["text"])
+
+# Fit and transform data with the pipeline
+result = pipeline.fit(example).transform(example)
+
+# Select the result, entity
+result.select(
+    expr("explode(entities) as entity")
+).select(
+    col("entity.result").alias("chunk"),
+    col("entity.metadata").getItem("entity").alias("ner_label")
+).show(truncate=False)
+""", language="python")
+
+# Results
+import pandas as pd
+
+# Create the data for the DataFrame
+data = {
+    "token": ["马云", "浙江省", "杭州市", "出生", "阿里巴巴集团", "创始人"],
+    "ner": ["PER", "LOC", "LOC", "ORG", "ORG", "PER"]
+}
+
+# Creating the DataFrame
+df = pd.DataFrame(data)
+df.index += 1
+st.dataframe(df)
+
+# Model Information
+st.markdown('<div class="sub-title">Model Information</div>', unsafe_allow_html=True)
+st.markdown("""
+<div class="section">
+    <p>The <code>ner_msra_bert_768d</code> model details are as follows:</p>
+    <ul>
+        <li><strong>Model Name:</strong> ner_msra_bert_768d</li>
+        <li><strong>Type:</strong> ner</li>
+        <li><strong>Compatibility:</strong> Spark NLP 2.7.0+</li>
+        <li><strong>License:</strong> Open Source</li>
+        <li><strong>Edition:</strong> Official</li>
+        <li><strong>Input Labels:</strong> [sentence, token, embeddings]</li>
+        <li><strong>Output Labels:</strong> [ner]</li>
+        <li><strong>Language:</strong> zh</li>
+    </ul>
+</div>
+""", unsafe_allow_html=True)
+
+# Data Source
+st.markdown('<div class="sub-title">Data Source</div>', unsafe_allow_html=True)
+st.markdown("""
+<div class="section">
+    <p>The model was trained on the MSRA (Levow, 2006) data set created by “Microsoft Research Asia”.</p>
+</div>
+""", unsafe_allow_html=True)
+
+# Benchmarking
+st.markdown('<div class="sub-title">Benchmarking</div>', unsafe_allow_html=True)
+st.markdown("""
+<div class="section">
+    <p>Evaluating the performance of NER models is crucial to understanding their effectiveness in real-world applications. Below are the benchmark results for the <code>ner_msra_bert_768d</code> model, focusing on various named entity categories. The metrics used include precision, recall, and F1-score, which are standard for evaluating classification models.</p>
+</div>
+""", unsafe_allow_html=True)
+st.markdown("""
+---
+|      ner_tag        | precision | recall | f1-score | support |
+|--------------|-----------|--------|----------|---------|
+| LOC          | 0.97      | 0.97   | 0.97     | 2777    |
+| O            | 1.00      | 1.00   | 1.00     | 146826  |
+| ORG          | 0.88      | 0.99   | 0.93     | 1292    |
+| PER          | 0.97      | 0.97   | 0.97     | 1430    |
+| accuracy     | 1.00      | 152325 |          |         |
+| macro avg    | 0.95      | 0.98   | 0.97     | 152325  |
+| weighted avg | 1.00      | 1.00   | 1.00     | 152325  |
+---
+""", unsafe_allow_html=True)
+
+st.markdown("""
+<div class="section">
+    <p>These results demonstrate the model's ability to accurately identify and classify named entities in Chinese text. Precision measures the accuracy of the positive predictions, recall measures the model's ability to find all relevant instances, and F1-score provides a balance between precision and recall.</p>
+</div>
+""", unsafe_allow_html=True)
+
+# Conclusion/Summary
+st.markdown('<div class="sub-title">Conclusion</div>', unsafe_allow_html=True)
+st.markdown("""
+<div class="section">
+    <p>The <code>ner_msra_bert_768d</code> model demonstrates effective named entity recognition in Chinese texts, with high performance metrics across different entity types. This model leverages <code>bert_base_chinese</code> embeddings to enhance its understanding and accuracy in identifying entities like persons, locations, and organizations. Its integration into Spark NLP allows for efficient and scalable processing of Chinese text data, making it a valuable tool for researchers and developers working with Chinese language applications.</p>
+</div>
+""", unsafe_allow_html=True)
+
+# References
+st.markdown('<div class="sub-title">References</div>', unsafe_allow_html=True)
+st.markdown("""
+<div class="section">
+    <ul>
+        <li><a class="link" href="https://sparknlp.org/api/python/reference/autosummary/sparknlp/annotator/ner/ner_dl/index.html" target="_blank" rel="noopener">NerDLModel</a> annotator documentation</li>
+        <li>Model Used: <a class="link" href="https://sparknlp.org/2021/01/03/ner_msra_bert_768d_zh.html" rel="noopener">ner_msra_bert_768d_zh</a></li>
+        <li>The model was trained on the <a class="link" href="https://www.aclweb.org/anthology/W06-0115/">MSRA (Levow, 2006)</a> data set created by “Microsoft Research Asia”.</li>
+        <li><a class="link" href="https://nlp.johnsnowlabs.com/recognize_entitie" target="_blank" rel="noopener">Visualization demos for NER in Spark NLP</a></li>
+        <li><a class="link" href="https://www.johnsnowlabs.com/named-entity-recognition-ner-with-bert-in-spark-nlp/">Named Entity Recognition (NER) with BERT in Spark NLP</a></li>
+    </ul>
+</div>
+""", unsafe_allow_html=True)
+
+# Community & Support
+st.markdown('<div class="sub-title">Community & Support</div>', unsafe_allow_html=True)
+st.markdown("""
+<div class="section">
+    <ul>
+        <li><a class="link" href="https://sparknlp.org/" target="_blank">Official Website</a>: Documentation and examples</li>
+        <li><a class="link" href="https://github.com/JohnSnowLabs/spark-nlp" target="_blank">GitHub Repository</a>: Report issues or contribute</li>
+        <li><a class="link" href="https://forum.johnsnowlabs.com/" target="_blank">Community Forum</a>: Ask questions, share ideas, and get support</li>
+    </ul>
+</div>
+""", unsafe_allow_html=True)

requirements.txt

@@ -0,0 +1,6 @@
+streamlit
+st-annotated-text
+pandas
+numpy
+spark-nlp
+pyspark