Upload 5 files

.streamlit/config.toml

@@ -0,0 +1,3 @@
+[theme]
+base="light"
+primaryColor="#29B4E8"

Demo.py

@@ -0,0 +1,158 @@
+import streamlit as st
+import sparknlp
+import os
+import pandas as pd
+
+from sparknlp.base import *
+from sparknlp.annotator import *
+from pyspark.ml import Pipeline
+from sparknlp.pretrained import PretrainedPipeline
+from annotated_text import annotated_text
+
+# Page configuration
+st.set_page_config(
+    layout="wide", 
+    initial_sidebar_state="auto"
+)
+
+# CSS for styling
+st.markdown("""
+    <style>
+        .main-title {
+            font-size: 36px;
+            color: #4A90E2;
+            font-weight: bold;
+            text-align: center;
+        }
+        .section {
+            background-color: #f9f9f9;
+            padding: 10px;
+            border-radius: 10px;
+            margin-top: 10px;
+        }
+        .section p, .section ul {
+            color: #666666;
+        }
+    </style>
+""", unsafe_allow_html=True)
+
+@st.cache_resource
+def init_spark():
+    return sparknlp.start()
+
+@st.cache_resource
+def create_pipeline(model):
+    document_assembler = DocumentAssembler() \
+        .setInputCol("text") \
+        .setOutputCol("document")
+
+    sentence_detector = SentenceDetector() \
+        .setInputCols(["document"]) \
+        .setOutputCol("sentence")
+
+    word_segmenter = WordSegmenterModel.pretrained("wordseg_kaist_ud", "ko") \
+        .setInputCols(["sentence"]) \
+        .setOutputCol("token")
+
+    embeddings = WordEmbeddingsModel.pretrained("glove_840B_300", "xx") \
+        .setInputCols(["document", "token"]) \
+        .setOutputCol("embeddings")
+
+    ner = NerDLModel.pretrained("ner_kmou_glove_840B_300d", "ko") \
+        .setInputCols(["document", "token", "embeddings"]) \
+        .setOutputCol("ner")
+
+    ner_converter = NerConverter().setInputCols(["document", "token", "ner"]).setOutputCol("ner_chunk")
+
+    pipeline = Pipeline(stages=[document_assembler, sentence_detector, word_segmenter, embeddings, ner, ner_converter])
+    return nlpPipeline
+
+def fit_data(pipeline, data):
+  empty_df = spark.createDataFrame([['']]).toDF('text')
+  pipeline_model = pipeline.fit(empty_df)
+  model = LightPipeline(pipeline_model)
+  result = model.fullAnnotate(data)
+  return result
+
+def annotate(data):
+    document, chunks, labels = data["Document"], data["NER Chunk"], data["NER Label"]
+    annotated_words = []
+    for chunk, label in zip(chunks, labels):
+        parts = document.split(chunk, 1)
+        if parts[0]:
+            annotated_words.append(parts[0])
+        annotated_words.append((chunk, label))
+        document = parts[1]
+    if document:
+        annotated_words.append(document)
+    annotated_text(*annotated_words)
+
+# Set up the page layout
+st.markdown('<div class="main-title">Recognize entities in Urdu text</div>', unsafe_allow_html=True)
+st.markdown("""
+<div class="section">
+    <p>This model uses the pre-trained <code>glove_840B_300</code> embeddings model from WordEmbeddings annotator as an input</p>
+</div>
+""", unsafe_allow_html=True)
+
+# Sidebar content
+model = st.sidebar.selectbox(
+    "Choose the pretrained model",
+    ["ner_kmou_glove_840B_300d"],
+    help="For more info about the models visit: https://sparknlp.org/models"
+)
+
+# Reference notebook link in sidebar
+link = """
+<a href="https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/public/NER_KO.ipynb">
+    <img src="https://colab.research.google.com/assets/colab-badge.svg" style="zoom: 1.3" alt="Open In Colab"/>
+</a>
+"""
+st.sidebar.markdown('Reference notebook:')
+st.sidebar.markdown(link, unsafe_allow_html=True)
+
+# Load examples
+examples = [
+    """ARD , ZDF 등 공영 TV 와 바이에른주 방송 , 북부 독일 방송 등 은 이 날 한국 의 총선 소식 과 관련 , 여당 의 과반수 의석 확보 와 신당 의 득표 율 이 이번 선거 의 최대 관심사 이 라고 보도 하 ㄴ 데 잇 어 저녁 시간 부터 는 수 차례 에 걸치 어 개표 상황 과 정당 별 의석 전망 을 속보 로 전하 았 다 .""",
+    """두 나라 관계 는 중국 의 인권 문제 와 핵확산 방지 문제 , 통상 문제 및 최근 의 F 16 전투기 대 대만 판매 등 을 놓 고 이미 위험선 상 에 오 아 있 는데 클린턴 행정부 의 등장 으로 양국 관계 가 더욱 경색 되 ㄹ 것 을 걱정 하 는 분위기 .""",
+    """서울대 건축공학 과 를 졸업 하 ㄴ 이 씨 는 한국건축가협회""",
+    """나 는 다시 순자 를 양동 에서 빼내 기 위하 아서 창신 팔동""",
+    """헤라신전 서 채화 되 ㄴ 지 보름 , 지구 의 반바퀴 를 돌 아 제주공항 에 첫발 을 내디디 ㄴ 이래 로 열이틀""",
+    """다음 은 홍콩 의 권위지 명보 와 일본 도쿄 ( 동경 ) 신문 이 24일""",
+    """최 영사 가 우리 외교관 이 며 그 신변보호 책임 이 주재국 이 ㄴ 러시아 에 있 다는 점 에서 러시아 는 이 같 은 우리 정부 요구 에 응하 아야 하 ㄹ 의무 가 있 다 .""",
+    """판 에 박 은 듯 하 ㄴ 깨끗 하 ㄴ 글씨 로 , 처음 단군 님 이 니 신라 , 백제 , 고구려 이 니 띄엄띄엄 어른 들 한테 서 귀결 로 들어오 던 얘기 들 이 참말 로 씌 어 있 었 다 ."""
+]
+
+selected_text = st.selectbox("Select an example", examples)
+custom_input = st.text_input("Try it with your own Sentence!")
+
+text_to_analyze = custom_input if custom_input else selected_text
+
+st.subheader('Full example text')
+HTML_WRAPPER = """<div class="scroll entities" style="overflow-x: auto; border: 1px solid #e6e9ef; border-radius: 0.25rem; padding: 1rem; margin-bottom: 2.5rem; white-space:pre-wrap">{}</div>"""
+st.markdown(HTML_WRAPPER.format(text_to_analyze), unsafe_allow_html=True)
+
+# Initialize Spark and create pipeline
+spark = init_spark()
+pipeline = create_pipeline(model)
+output = fit_data(pipeline, text_to_analyze)
+
+# Display matched sentence
+st.subheader("Processed output:")
+
+results = {
+    'Document': output[0]['document'][0].result,
+    'NER Chunk': [n.result for n in output[0]['ner_chunk']],
+    "NER Label": [n.metadata['entity'] for n in output[0]['ner_chunk']]
+}
+
+annotate(results)
+
+with st.expander("View DataFrame"):
+    df = pd.DataFrame({'NER Chunk': results['NER Chunk'], 'NER Label': results['NER Label']})
+    df.index += 1
+    st.dataframe(df)
+
+
+
+

Dockerfile

@@ -0,0 +1,70 @@
+# Download base image ubuntu 18.04
+FROM ubuntu:18.04
+
+# Set environment variables
+ENV NB_USER jovyan
+ENV NB_UID 1000
+ENV HOME /home/${NB_USER}
+
+# Install required packages
+RUN apt-get update && apt-get install -y \
+    tar \
+    wget \
+    bash \
+    rsync \
+    gcc \
+    libfreetype6-dev \
+    libhdf5-serial-dev \
+    libpng-dev \
+    libzmq3-dev \
+    python3 \
+    python3-dev \
+    python3-pip \
+    unzip \
+    pkg-config \
+    software-properties-common \
+    graphviz \
+    openjdk-8-jdk \
+    ant \
+    ca-certificates-java \
+    && apt-get clean \
+    && update-ca-certificates -f;
+
+# Install Python 3.8 and pip
+RUN add-apt-repository ppa:deadsnakes/ppa \
+    && apt-get update \
+    && apt-get install -y python3.8 python3-pip \
+    && apt-get clean;
+
+# Set up JAVA_HOME
+ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64/
+RUN mkdir -p ${HOME} \
+    && echo "export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/" >> ${HOME}/.bashrc \
+    && chown -R ${NB_UID}:${NB_UID} ${HOME}
+
+# Create a new user named "jovyan" with user ID 1000
+RUN useradd -m -u ${NB_UID} ${NB_USER}
+
+# Switch to the "jovyan" user
+USER ${NB_USER}
+
+# Set home and path variables for the user
+ENV HOME=/home/${NB_USER} \
+    PATH=/home/${NB_USER}/.local/bin:$PATH
+
+# Set the working directory to the user's home directory
+WORKDIR ${HOME}
+
+# Upgrade pip and install Python dependencies
+RUN python3.8 -m pip install --upgrade pip
+COPY requirements.txt /tmp/requirements.txt
+RUN python3.8 -m pip install -r /tmp/requirements.txt
+
+# Copy the application code into the container at /home/jovyan
+COPY --chown=${NB_USER}:${NB_USER} . ${HOME}
+
+# Expose port for Streamlit
+EXPOSE 7860
+
+# Define the entry point for the container
+ENTRYPOINT ["streamlit", "run", "Demo.py", "--server.port=7860", "--server.address=0.0.0.0"]

pages/Workflow & Model Overview.py

@@ -0,0 +1,220 @@
+import streamlit as st
+
+# Custom CSS for better styling
+st.markdown("""
+    <style>
+        .main-title {
+            font-size: 36px;
+            color: #4A90E2;
+            font-weight: bold;
+            text-align: center;
+        }
+        .sub-title {
+            font-size: 24px;
+            color: #4A90E2;
+            margin-top: 20px;
+        }
+        .section {
+            background-color: #f9f9f9;
+            padding: 15px;
+            border-radius: 10px;
+            margin-top: 20px;
+        }
+        .section h2 {
+            font-size: 22px;
+            color: #4A90E2;
+        }
+        .section p, .section ul {
+            color: #666666;
+        }
+        .link {
+            color: #4A90E2;
+            text-decoration: none;
+        }
+    </style>
+""", unsafe_allow_html=True)
+
+# Main Title
+st.markdown('<div class="main-title">Named Entity Recognition for Korean (GloVe 840B 300d)</div>', unsafe_allow_html=True)
+
+# Description
+st.markdown('<div class="sub-title">Description</div>', unsafe_allow_html=True)
+st.markdown("""
+<div class="section">
+    <p>This model annotates named entities in a text, which can be used to find features such as names of people, places, and organizations in the BIO format. The model does not read words directly but instead reads word embeddings, which represent words as points such that more semantically similar words are closer together.</p>
+    <p>This model uses the pre-trained <code>glove_840B_300</code> embeddings model from WordEmbeddings annotator as an input, so be sure to use the same embeddings in the pipeline.</p>
+</div>
+""", unsafe_allow_html=True)
+
+# Predicted Entities
+st.markdown('<div class="sub-title">Predicted Entities</div>', unsafe_allow_html=True)
+st.markdown("""
+<div class="section">
+    <ul>
+        <li>Dates-DT</li>
+        <li>Locations-LC</li>
+        <li>Organizations-OG</li>
+        <li>Persons-PS</li>
+        <li>Time-TI</li>
+    </ul>
+</div>
+""", unsafe_allow_html=True)
+
+# How to use
+st.markdown('<div class="sub-title">How to use</div>', unsafe_allow_html=True)
+st.markdown("""
+<div class="section">
+    <p>To use this model, follow these steps in Python:</p>
+</div>
+""", unsafe_allow_html=True)
+st.code("""
+from sparknlp.base import *
+from sparknlp.annotator import *
+from pyspark.ml import Pipeline
+
+# Define the components of the pipeline
+document_assembler = DocumentAssembler() \\
+    .setInputCol("text") \\
+    .setOutputCol("document")
+
+sentence_detector = SentenceDetector() \\
+    .setInputCols(["document"]) \\
+    .setOutputCol("sentence")
+
+word_segmenter = WordSegmenterModel.pretrained("wordseg_kaist_ud", "ko") \\
+    .setInputCols(["sentence"]) \\
+    .setOutputCol("token")
+
+embeddings = WordEmbeddingsModel.pretrained("glove_840B_300", "xx") \\
+    .setInputCols(["document", "token"]) \\
+    .setOutputCol("embeddings")
+
+ner = NerDLModel.pretrained("ner_kmou_glove_840B_300d", "ko") \\
+    .setInputCols(["document", "token", "embeddings"]) \\
+    .setOutputCol("ner")
+
+ner_converter = NerConverter().setInputCols(["document", "token", "ner"]).setOutputCol("ner_chunk")
+
+# Create the pipeline
+pipeline = Pipeline(stages=[document_assembler, sentence_detector, word_segmenter, embeddings, ner, ner_converter])
+
+# Create sample data
+example = spark.createDataFrame([['라이프니츠 의 주도 로 베를린 에 세우 어 지 ㄴ 베를린 과학아카데미']], ["text"])
+
+# Fit and transform data with the pipeline
+result = pipeline.fit(example).transform(example)
+
+# Select the result, entity
+result.select(
+    expr("explode(ner_chunk) as ner_chunk")
+).select(
+    col("ner_chunk.result").alias("chunk"),
+    col("ner_chunk.metadata").getItem("entity").alias("ner_label")
+).show(truncate=False)
+""", language="python")
+
+# Results
+import pandas as pd
+
+# Create the data for the DataFrame
+data = {
+    "token": ["라이프니츠", "베를린", "과학아카데미"],
+    "ner": ["B-PS", "B-OG", "I-OG"]
+}
+
+# Creating the DataFrame
+df = pd.DataFrame(data)
+df.index += 1
+st.dataframe(df)
+
+# Model Information
+st.markdown('<div class="sub-title">Model Information</div>', unsafe_allow_html=True)
+st.markdown("""
+<div class="section">
+    <p>The <code>ner_kmou_glove_840B_300d</code> model details are as follows:</p>
+    <ul>
+        <li><strong>Model Name:</strong> ner_kmou_glove_840B_300d</li>
+        <li><strong>Type:</strong> ner</li>
+        <li><strong>Compatibility:</strong> Spark NLP 2.7.0+</li>
+        <li><strong>License:</strong> Open Source</li>
+        <li><strong>Edition:</strong> Official</li>
+        <li><strong>Input Labels:</strong> [sentence, token, embeddings]</li>
+        <li><strong>Output Labels:</strong> [ner]</li>
+        <li><strong>Language:</strong> ko</li>
+    </ul>
+</div>
+""", unsafe_allow_html=True)
+
+# Data Source
+st.markdown('<div class="sub-title">Data Source</div>', unsafe_allow_html=True)
+st.markdown("""
+<div class="section">
+    <p>The model was trained by the Korea Maritime and Ocean University NLP data set.</p>
+</div>
+""", unsafe_allow_html=True)
+
+# Benchmarking
+st.markdown('<div class="sub-title">Benchmarking</div>', unsafe_allow_html=True)
+st.markdown("""
+<div class="section">
+    <p>Evaluating the performance of NER models is crucial to understanding their effectiveness in real-world applications. Below are the benchmark results for the <code>ner_kmou_glove_840B_300d</code> model, focusing on various named entity categories. The metrics used include precision, recall, and F1-score, which are standard for evaluating classification models.</p>
+</div>
+""", unsafe_allow_html=True)
+st.markdown("""
+---
+|    ner_tag   | precision | recall | f1-score | support |
+|:------------:|:---------:|:------:|:--------:|:-------:|
+|     B-DT     |    0.95   |  0.29  |   0.44   |   132   |
+|     B-LC     |    0.00   |  0.00  |   0.00   |   166   |
+|     B-OG     |    1.00   |  0.06  |   0.11   |   149   |
+|     B-PS     |    0.86   |  0.13  |   0.23   |   287   |
+|     B-TI     |    0.50   |  0.05  |   0.09   |    20   |
+|     I-DT     |    0.94   |  0.36  |   0.52   |   164   |
+|     I-LC     |    0.00   |  0.00  |   0.00   |    4    |
+|     I-OG     |    1.00   |  0.08  |   0.15   |    25   |
+|     I-PS     |    1.00   |  0.08  |   0.15   |    12   |
+|     I-TI     |    0.50   |  0.10  |   0.17   |    10   |
+|       O      |    0.94   |  1.00  |   0.97   |  12830  |
+|   accuracy   |    0.94   |  13799 |          |         |
+|   macro avg  |    0.70   |  0.20  |   0.26   |  13799  |
+| weighted avg |    0.93   |  0.94  |   0.92   |  13799  |
+""", unsafe_allow_html=True)
+
+st.markdown("""
+<div class="section">
+    <p>These results demonstrate the model's ability to accurately identify and classify named entities in Korean text. Precision measures the accuracy of the positive predictions, recall measures the model's ability to find all relevant instances, and F1-score provides a balance between precision and recall.</p>
+</div>
+""", unsafe_allow_html=True)
+
+# Conclusion/Summary
+st.markdown('<div class="sub-title">Conclusion</div>', unsafe_allow_html=True)
+st.markdown("""
+<div class="section">
+    <p>The <code>ner_kmou_glove_840B_300d</code> model demonstrates effective named entity recognition in Korean texts, with varied performance metrics across different entity types. This model leverages <code>glove_840B_300</code> embeddings to enhance its understanding and accuracy in identifying entities like persons, locations, organizations, and more. Its integration into Spark NLP allows for efficient and scalable processing of Korean text data, making it a valuable tool for researchers and developers working with Korean language applications.</p>
+</div>
+""", unsafe_allow_html=True)
+
+# References
+st.markdown('<div class="sub-title">References</div>', unsafe_allow_html=True)
+st.markdown("""
+<div class="section">
+    <ul>
+        <li><a class="link" href="https://sparknlp.org/api/python/reference/autosummary/sparknlp/annotator/ner/ner_dl/index.html" target="_blank" rel="noopener">NerDLModel</a> annotator documentation</li>
+        <li>Model Used: <a class="link" href="https://sparknlp.org/2021/01/03/ner_ud_kaist_glove_840B_300d_ko.html" rel="noopener">ner_kmou_glove_840B_300d</a></li>
+        <li><a class="link" href="https://nlp.johnsnowlabs.com/recognize_entitie" target="_blank" rel="noopener">Visualization demos for NER in Spark NLP</a></li>
+        <li><a class="link" href="https://www.johnsnowlabs.com/named-entity-recognition-ner-with-bert-in-spark-nlp/">Named Entity Recognition (NER) with BERT in Spark NLP</a></li>
+    </ul>
+</div>
+""", unsafe_allow_html=True)
+
+# Community & Support
+st.markdown('<div class="sub-title">Community & Support</div>', unsafe_allow_html=True)
+st.markdown("""
+<div class="section">
+    <ul>
+        <li><a class="link" href="https://sparknlp.org/" target="_blank">Official Website</a>: Documentation and examples</li>
+        <li><a class="link" href="https://github.com/JohnSnowLabs/spark-nlp" target="_blank">GitHub Repository</a>: Report issues or contribute</li>
+        <li><a class="link" href="https://forum.johnsnowlabs.com/" target="_blank">Community Forum</a>: Ask questions, share ideas, and get support</li>
+    </ul>
+</div>
+""", unsafe_allow_html=True)

requirements.txt

@@ -0,0 +1,6 @@
+streamlit
+st-annotated-text
+pandas
+numpy
+spark-nlp
+pyspark