pages/Workflow & Model Overview.py

import streamlit as st
import pandas as pd

# Custom CSS for better styling
st.markdown("""
    <style>
        .main-title {
            font-size: 36px;
            color: #4A90E2;
            font-weight: bold;
            text-align: center;
        }
        .sub-title {
            font-size: 24px;
            color: #4A90E2;
            margin-top: 20px;
        }
        .section {
            background-color: #f9f9f9;
            padding: 15px;
            border-radius: 10px;
            margin-top: 20px;
        }
        .section h2 {
            font-size: 22px;
            color: #4A90E2;
        }
        .section p, .section ul {
            color: #666666;
        }
        .link {
            color: #4A90E2;
            text-decoration: none;
        }
    </style>
""", unsafe_allow_html=True)

# Main Title
st.markdown('<div class="main-title">Detect Entities in Urdu (urduvec_140M_300d embeddings)</div>', unsafe_allow_html=True)

# Introduction
st.markdown("""
<div class="section">
    <p>Named Entity Recognition (NER) models identify and categorize important entities in a text. This page details a word embeddings-based NER model for Urdu texts, using the <code>urduvec_140M_300d</code> word embeddings. The model is pretrained and available for use with Spark NLP.</p>
</div>
""", unsafe_allow_html=True)

# Model Description
st.markdown('<div class="sub-title">Description</div>', unsafe_allow_html=True)
st.markdown("""
<div class="section">
    <p>This model uses Urdu word embeddings to find 7 different types of entities in Urdu text. It is trained using <code>urduvec_140M_300d</code> word embeddings, so please use the same embeddings in the pipeline. It can identify the following types of entities:</p>
    <ul>
        <li>PER (Persons)</li>
        <li>LOC (Locations)</li>
        <li>ORG (Organizations)</li>
        <li>DATE (Dates)</li>
        <li>TIME (Times)</li>
        <li>DESIGNATION (Designations)</li>
        <li>NUMBER (Numbers)</li>
    </ul>
</div>
""", unsafe_allow_html=True)

# Setup Instructions
st.markdown('<div class="sub-title">Setup</div>', unsafe_allow_html=True)
st.markdown('<p>To use the model, you need Spark NLP installed. You can install it using pip:</p>', unsafe_allow_html=True)
st.code("""
pip install spark-nlp
pip install pyspark
""", language="bash")

st.markdown("<p>Then, import Spark NLP and start a Spark session:</p>", unsafe_allow_html=True)
st.code("""
import sparknlp

# Start Spark Session
spark = sparknlp.start()
""", language='python')

# Example Usage
st.markdown('<div class="sub-title">Example Usage with Urdu NER Model</div>', unsafe_allow_html=True)
st.markdown("""
<div class="section">
    <p>Below is an example of how to set up and use the <code>uner_mk_140M_300d</code> model for named entity recognition in Urdu:</p>
</div>
""", unsafe_allow_html=True)
st.code('''
from sparknlp.base import *
from sparknlp.annotator import *
from pyspark.ml import Pipeline

# Define the components of the pipeline
documentAssembler = DocumentAssembler() \\
    .setInputCol("text") \\
    .setOutputCol("document")

sentence_detector = SentenceDetector() \\
    .setInputCols(["document"]) \\
    .setOutputCol("sentence")

tokenizer = Tokenizer() \\
    .setInputCols(["sentence"]) \\
    .setOutputCol("token")

word_embeddings = WordEmbeddingsModel.pretrained("urduvec_140M_300d", "ur") \\
    .setInputCols(["sentence", "token"]) \\
    .setOutputCol("embeddings")

ner = NerDLModel.pretrained("uner_mk_140M_300d", "ur") \\
    .setInputCols(["sentence", "token", "embeddings"]) \\
    .setOutputCol("ner")

ner_converter = NerConverter().setInputCols(["sentence", "token", "ner"]).setOutputCol("ner_chunk")

# Create the pipeline
pipeline = Pipeline(stages=[documentAssembler, sentence_detector, tokenizer, word_embeddings, ner, ner_converter])

# Create sample data
example = """
بریگیڈیئر ایڈ بٹلر سنہ دوہزارچھ میں ہلمند کے فوجی کمانڈر تھے۔
"""
data = spark.createDataFrame([[example]]).toDF("text")

# Fit and transform data with the pipeline
result = pipeline.fit(data).transform(data)

# Select the result, entity
result.select(
    expr("explode(ner_chunk) as ner_chunk")
).select(
    col("ner_chunk.result").alias("chunk"),
    col("ner_chunk.metadata").getItem("entity").alias("ner_label")
).show(truncate=False)
''', language="python")

import pandas as pd

# Create the data for the DataFrame
data = {
    "chunk": [
        "بریگیڈیئر", 
        "ایڈ بٹلر", 
        "سنہ دوہزارچھ", 
        "ہلمند"
    ],
    "ner_label": [
        "DESIGNATION", 
        "PERSON", 
        "DATE", 
        "LOCATION"
    ]
}

# Creating the DataFrame
df = pd.DataFrame(data)
df.index += 1
st.dataframe(df)

# Model Information
st.markdown('<div class="sub-title">Model Information</div>', unsafe_allow_html=True)
st.markdown("""
<div class="section">
    <p>The <code>uner_mk_140M_300d</code> model details are as follows:</p>
    <ul>
        <li><strong>Model Name:</strong> uner_mk_140M_300d</li>
        <li><strong>Type:</strong> ner</li>
        <li><strong>Compatibility:</strong> Spark NLP 4.0.2+</li>
        <li><strong>License:</strong> Open Source</li>
        <li><strong>Edition:</strong> Official</li>
        <li><strong>Input Labels:</strong> [document, token, word_embeddings]</li>
        <li><strong>Output Labels:</strong> [ner]</li>
        <li><strong>Language:</strong> ur</li>
        <li><strong>Size:</strong> 14.8 MB</li>
    </ul>
</div>
""", unsafe_allow_html=True)

# Benchmark Section
st.markdown('<div class="sub-title">Benchmark</div>', unsafe_allow_html=True)
st.markdown("""
<div class="section">
    <p>Evaluating the performance of NER models is crucial to understanding their effectiveness in real-world applications. Below are the benchmark results for the <code>uner_mk_140M_300d</code> model, focusing on various named entity categories. The metrics used include precision, recall, and F1-score, which are standard for evaluating classification models.</p>
</div>
""", unsafe_allow_html=True)
st.markdown("""
---
| Label            | TP    | FP  | FN  | Precision | Recall  | F1-Score |
|------------------|-------|-----|-----|-----------|---------|----------|
| I-TIME           | 12    | 10  | 1   | 0.545455  | 0.923077| 0.685714 |
| B-PERSON         | 2808  | 846 | 535 | 0.768473  | 0.839964| 0.802630 |
| B-DATE           | 34    | 6   | 6   | 0.850000  | 0.850000| 0.850000 |
| I-DATE           | 45    | 1   | 2   | 0.978261  | 0.957447| 0.967742 |
| B-DESIGNATION    | 49    | 30  | 16  | 0.620253  | 0.753846| 0.680556 |
| I-LOCATION       | 2110  | 750 | 701 | 0.737762  | 0.750623| 0.744137 |
| B-TIME           | 11    | 9   | 3   | 0.550000  | 0.785714| 0.647059 |
| I-ORGANIZATION   | 2006  | 772 | 760 | 0.722102  | 0.725235| 0.723665 |
| I-NUMBER         | 18    | 6   | 2   | 0.750000  | 0.900000| 0.818182 |
| B-LOCATION       | 5428  | 1255| 582 | 0.812210  | 0.903161| 0.855275 |
| B-NUMBER         | 194   | 36  | 27  | 0.843478  | 0.877828| 0.860298 |
| B-ORGANIZATION   | 4364  | 1092| 990 | 0.799926  | 0.815058| 0.807421 |
| I-DESIGNATION    | 57    | 15  | 10  | 0.791667  | 0.850746| 0.820896 |
| B-MISC           | 18    | 19  | 13  | 0.486486  | 0.580645| 0.529412 |
| I-MISC           | 10    | 11  | 10  | 0.476190  | 0.500000| 0.487805 |
| I-PERSON         | 1891  | 689 | 622 | 0.732723  | 0.752499| 0.742486 |
---
""", unsafe_allow_html=True)

st.markdown("""
<div class="section">
    <p>These results demonstrate the model's ability to accurately identify and classify named entities in Urdu text. Precision measures the accuracy of the positive predictions, recall measures the model's ability to find all relevant instances, and F1-score provides a balance between precision and recall.</p>
</div>
""", unsafe_allow_html=True)

# Try the Model
st.markdown('<div class="sub-title">Try the Model</div>', unsafe_allow_html=True)
st.markdown("""
<div class="section">
    <p>You can use the <code>LightPipeline</code> to quickly test the model on small texts. Here is an example:</p>
</div>
""", unsafe_allow_html=True)
st.code('''
from sparknlp.base import LightPipeline

# Create a LightPipeline
light_pipeline = LightPipeline(pipeline.fit(data))

# Annotate a simple text
example_text = "بریگیڈیئر ایڈ بٹلر سنہ دوہزارچھ میں ہلمند کے فوجی کمانڈر تھے۔"
annotations = light_pipeline.fullAnnotate(example_text)

# Display the annotations
for annotation in annotations[0]['ner_chunk']:
    print(annotation.result, "->", annotation.metadata['entity'])
''', language="python")

# Conclusion/Summary
st.markdown('<div class="sub-title">Conclusion</div>', unsafe_allow_html=True)
st.markdown("""
<div class="section">
    <p>The <code>uner_mk_140M_300d</code> model demonstrates effective named entity recognition in Urdu texts, with strong performance metrics across various entity types. This model leverages <code>urduvec_140M_300d</code> embeddings to enhance its understanding and accuracy in identifying entities like persons, locations, organizations, and more. Its integration into Spark NLP allows for efficient and scalable processing of Urdu text data, making it a valuable tool for researchers and developers working with Urdu language applications.</p>
</div>
""", unsafe_allow_html=True)

# References
st.markdown('<div class="sub-title">References</div>', unsafe_allow_html=True)
st.markdown("""
<div class="section">
    <ul>
        <li><a class="link" href="https://sparknlp.org/api/python/reference/autosummary/sparknlp/annotator/ner/ner_dl/index.html" target="_blank" rel="noopener">NerDLModel</a> annotator documentation</li>
        <li>Model Used: <a class="link" href="https://sparknlp.org/2022/08/09/uner_mk_140M_300d_ur_3_0.html" rel="noopener">uner_mk_140M_300d_ur_3_0</a></li>
        <li><a class="link" href="https://www.cs.bgu.ac.il/~elhadad/nlpproj/naama/" target="_blank" rel="noopener">Data Source</a></li>
        <li><a class="link" href="https://nlp.johnsnowlabs.com/recognize_entitie" target="_blank" rel="noopener">Visualization demos for NER in Spark NLP</a></li>
        <li><a class="link" href="https://www.johnsnowlabs.com/named-entity-recognition-ner-with-bert-in-spark-nlp/">Named Entity Recognition (NER) with BERT in Spark NLP</a></li>
    </ul>
</div>
""", unsafe_allow_html=True)

# Community & Support
st.markdown('<div class="sub-title">Community & Support</div>', unsafe_allow_html=True)
st.markdown("""
<div class="section">
    <ul>
        <li><a class="link" href="https://sparknlp.org/" target="_blank">Official Website</a>: Documentation and examples</li>
        <li><a class="link" href="https://github.com/JohnSnowLabs/spark-nlp" target="_blank">GitHub Repository</a>: Report issues or contribute</li>
        <li><a class="link" href="https://forum.johnsnowlabs.com/" target="_blank">Community Forum</a>: Ask questions, share ideas, and get support</li>
    </ul>
</div>
""", unsafe_allow_html=True)