Upload 10 files

.streamlit/config.toml

@@ -0,0 +1,3 @@
+[theme]
+base="light"
+primaryColor="#29B4E8"

Demo.py

@@ -0,0 +1,156 @@
+import streamlit as st
+import sparknlp
+import os
+import pandas as pd
+
+from sparknlp.base import *
+from sparknlp.annotator import *
+from pyspark.ml import Pipeline
+from sparknlp.pretrained import PretrainedPipeline
+from annotated_text import annotated_text
+
+# Page configuration
+st.set_page_config(
+    layout="wide", 
+    initial_sidebar_state="auto"
+)
+
+# CSS for styling
+st.markdown("""
+    <style>
+        .main-title {
+            font-size: 36px;
+            color: #4A90E2;
+            font-weight: bold;
+            text-align: center;
+        }
+        .section {
+            background-color: #f9f9f9;
+            padding: 10px;
+            border-radius: 10px;
+            margin-top: 10px;
+        }
+        .section p, .section ul {
+            color: #666666;
+        }
+    </style>
+""", unsafe_allow_html=True)
+
+@st.cache_resource
+def init_spark():
+    return sparknlp.start()
+
+@st.cache_resource
+def create_pipeline(model):
+    document_assembler = DocumentAssembler() \
+        .setInputCol("text") \
+        .setOutputCol("document")
+
+    sentence_detector = SentenceDetectorDLModel.pretrained("sentence_detector_dl", "en") \
+        .setInputCols(["document"]) \
+        .setOutputCol("sentence")
+
+    tokenizer = Tokenizer() \
+        .setInputCols(["sentence"]) \
+        .setOutputCol("token")
+
+    token_classifier = RoBertaForTokenClassification.pretrained("roberta_token_classifier_timex_semeval", "en") \
+        .setInputCols(["sentence", "token"]) \
+        .setOutputCol("ner")
+
+    ner_converter = NerConverter() \
+        .setInputCols(["sentence", "token", "ner"]) \
+        .setOutputCol("ner_chunk")
+
+    pipeline = Pipeline(stages=[
+        document_assembler,
+        sentence_detector,
+        tokenizer,
+        token_classifier,
+        ner_converter
+    ])
+    return pipeline
+
+def fit_data(pipeline, data):
+  empty_df = spark.createDataFrame([['']]).toDF('text')
+  pipeline_model = pipeline.fit(empty_df)
+  model = LightPipeline(pipeline_model)
+  result = model.fullAnnotate(data)
+  return result
+
+def annotate(data):
+    document, chunks, labels = data["Document"], data["NER Chunk"], data["NER Label"]
+    annotated_words = []
+    for chunk, label in zip(chunks, labels):
+        parts = document.split(chunk, 1)
+        if parts[0]:
+            annotated_words.append(parts[0])
+        annotated_words.append((chunk, label))
+        document = parts[1]
+    if document:
+        annotated_words.append(document)
+    annotated_text(*annotated_words)
+
+# Set up the page layout
+st.markdown('<div class="main-title">Detect Time-related Terminology</div>', unsafe_allow_html=True)
+st.markdown("""
+<div class="section">
+    <p>Identify and classify time-related entities in text to provide a structured representation of temporal information. This model detects various time expressions, such as dates, times, intervals, and more, enabling automated systems to process and respond to time-sensitive queries accurately and efficiently</p>
+</div>
+""", unsafe_allow_html=True)
+
+# Sidebar content
+model = st.sidebar.selectbox(
+    "Choose the pretrained model",
+    ["roberta_token_classifier_timex_semeval"],
+    help="For more info about the models visit: https://sparknlp.org/models"
+)
+
+# Reference notebook link in sidebar
+link = """
+<a href="https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/NER.ipynb">
+    <img src="https://colab.research.google.com/assets/colab-badge.svg" style="zoom: 1.3" alt="Open In Colab"/>
+</a>
+"""
+st.sidebar.markdown('Reference notebook:')
+st.sidebar.markdown(link, unsafe_allow_html=True)
+
+# Load examples
+folder_path = f"inputs/{model}"
+examples = [
+    lines[1].strip() 
+    for filename in os.listdir(folder_path) 
+    if filename.endswith('.txt') 
+    for lines in [open(os.path.join(folder_path, filename), 'r', encoding='utf-8').readlines()] 
+    if len(lines) >= 2
+]
+
+selected_text = st.selectbox("Select an example", examples)
+custom_input = st.text_input("Try it with your own Sentence!")
+
+text_to_analyze = custom_input if custom_input else selected_text
+
+st.subheader('Full example text')
+HTML_WRAPPER = """<div class="scroll entities" style="overflow-x: auto; border: 1px solid #e6e9ef; border-radius: 0.25rem; padding: 1rem; margin-bottom: 2.5rem; white-space:pre-wrap">{}</div>"""
+st.markdown(HTML_WRAPPER.format(text_to_analyze), unsafe_allow_html=True)
+
+# Initialize Spark and create pipeline
+spark = init_spark()
+pipeline = create_pipeline(model)
+output = fit_data(pipeline, text_to_analyze)
+
+# Display matched sentence
+st.subheader("Processed output:")
+
+results = {
+    'Document': output[0]['document'][0].result,
+    'NER Chunk': [n.result for n in output[0]['ner_chunk']],
+    "NER Label": [n.metadata['entity'] for n in output[0]['ner_chunk']]
+}
+
+annotate(results)
+
+with st.expander("View DataFrame"):
+    df = pd.DataFrame({'NER Chunk': results['NER Chunk'], 'NER Label': results['NER Label']})
+    df.index += 1
+    st.dataframe(df)

Dockerfile

@@ -0,0 +1,70 @@
+# Download base image ubuntu 18.04
+FROM ubuntu:18.04
+
+# Set environment variables
+ENV NB_USER jovyan
+ENV NB_UID 1000
+ENV HOME /home/${NB_USER}
+
+# Install required packages
+RUN apt-get update && apt-get install -y \
+    tar \
+    wget \
+    bash \
+    rsync \
+    gcc \
+    libfreetype6-dev \
+    libhdf5-serial-dev \
+    libpng-dev \
+    libzmq3-dev \
+    python3 \
+    python3-dev \
+    python3-pip \
+    unzip \
+    pkg-config \
+    software-properties-common \
+    graphviz \
+    openjdk-8-jdk \
+    ant \
+    ca-certificates-java \
+    && apt-get clean \
+    && update-ca-certificates -f;
+
+# Install Python 3.8 and pip
+RUN add-apt-repository ppa:deadsnakes/ppa \
+    && apt-get update \
+    && apt-get install -y python3.8 python3-pip \
+    && apt-get clean;
+
+# Set up JAVA_HOME
+ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64/
+RUN mkdir -p ${HOME} \
+    && echo "export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/" >> ${HOME}/.bashrc \
+    && chown -R ${NB_UID}:${NB_UID} ${HOME}
+
+# Create a new user named "jovyan" with user ID 1000
+RUN useradd -m -u ${NB_UID} ${NB_USER}
+
+# Switch to the "jovyan" user
+USER ${NB_USER}
+
+# Set home and path variables for the user
+ENV HOME=/home/${NB_USER} \
+    PATH=/home/${NB_USER}/.local/bin:$PATH
+
+# Set the working directory to the user's home directory
+WORKDIR ${HOME}
+
+# Upgrade pip and install Python dependencies
+RUN python3.8 -m pip install --upgrade pip
+COPY requirements.txt /tmp/requirements.txt
+RUN python3.8 -m pip install -r /tmp/requirements.txt
+
+# Copy the application code into the container at /home/jovyan
+COPY --chown=${NB_USER}:${NB_USER} . ${HOME}
+
+# Expose port for Streamlit
+EXPOSE 7860
+
+# Define the entry point for the container
+ENTRYPOINT ["streamlit", "run", "Demo.py", "--server.port=7860", "--server.address=0.0.0.0"]

inputs/roberta_token_classifier_timex_semeval/Example1.txt

@@ -0,0 +1,2 @@
+Model training was started at 22:12C and it took 3 days from Tuesday ... 
+Model training was started at 22:12C and it took 3 days from Tuesday to Friday.

inputs/roberta_token_classifier_timex_semeval/Example2.txt

@@ -0,0 +1,2 @@
+My college certificate program will be between January and June and classes will be from Monday t... 
+My college certificate program will be between January and June and classes will be from Monday to Thursday

inputs/roberta_token_classifier_timex_semeval/Example3.txt

@@ -0,0 +1,2 @@
+Compared to the previous year in Canada, the skiing season star... 
+Compared to the previous year in Canada, the skiing season started early.

inputs/roberta_token_classifier_timex_semeval/Example4.txt

@@ -0,0 +1,2 @@
+Spring walks, which have been made since 2011, started on the first Sunday of April this year, the w... 
+Spring walks, which have been made since 2011, started on the first Sunday of April this year, the walking event, which attracted a lot of attention, started at 2 PM and lasted for 3 hours and 20 minutes.

inputs/roberta_token_classifier_timex_semeval/Example5.txt

@@ -0,0 +1,2 @@
+if a New Yorker plans to meet someone in Los Angeles at 9 AM, and makes a calendar entry at 9 AM (wh... 
+if a New Yorker plans to meet someone in Los Angeles at 9 AM, and makes a calendar entry at 9 AM (which the computer assumes is New York time), the calendar entry will be at 6 AM if taking the computer's time zone.

pages/Workflow & Model Overview.py

@@ -0,0 +1,275 @@
+import streamlit as st
+
+# Custom CSS for better styling
+st.markdown("""
+    <style>
+        .main-title {
+            font-size: 36px;
+            color: #4A90E2;
+            font-weight: bold;
+            text-align: center;
+        }
+        .sub-title {
+            font-size: 24px;
+            color: #4A90E2;
+            margin-top: 20px;
+        }
+        .section {
+            background-color: #f9f9f9;
+            padding: 15px;
+            border-radius: 10px;
+            margin-top: 20px;
+        }
+        .section h2 {
+            font-size: 22px;
+            color: #4A90E2;
+        }
+        .section p, .section ul {
+            color: #666666;
+        }
+        .link {
+            color: #4A90E2;
+            text-decoration: none;
+        }
+        .benchmark-table {
+            width: 100%;
+            border-collapse: collapse;
+            margin-top: 20px;
+        }
+        .benchmark-table th, .benchmark-table td {
+            border: 1px solid #ddd;
+            padding: 8px;
+            text-align: left;
+        }
+        .benchmark-table th {
+            background-color: #4A90E2;
+            color: white;
+        }
+    </style>
+""", unsafe_allow_html=True)
+
+# Main Title
+st.markdown('<div class="main-title">Detect Time-related Terminology</div>', unsafe_allow_html=True)
+
+# Description
+st.markdown("""
+<div class="section">
+    <p><strong>Detect Time-related Terminology</strong> is a crucial NLP task that involves identifying and classifying key temporal entities in text. This app leverages the <strong>roberta_token_classifier_timex_semeval</strong> model, which has been imported from Hugging Face and trained to detect time-related terminology using RoBERTa embeddings and RobertaForTokenClassification for NER purposes.</p>
+</div>
+""", unsafe_allow_html=True)
+
+# What is NER
+st.markdown('<div class="sub-title">What is Named Entity Recognition (NER)?</div>', unsafe_allow_html=True)
+st.markdown("""
+<div class="section">
+    <p><strong>Named Entity Recognition (NER)</strong> is a process in Natural Language Processing (NLP) that locates and classifies named entities into predefined categories such as dates, times, periods, and other temporal expressions. For example, in the sentence "Model training was started at 22:12C and it took 3 days from Tuesday to Friday," NER helps identify '22:12C' as a time period, '3 days' as a calendar interval, and 'Tuesday' and 'Friday' as days of the week.</p>
+    <p>NER models are trained to understand the context and semantics of entities within text, enabling automated systems to recognize and categorize these entities accurately. This capability is essential for developing intelligent systems capable of processing and responding to user queries efficiently.</p>
+</div>
+""", unsafe_allow_html=True)
+
+# Predicted Entities
+st.markdown('<div class="sub-title">Predicted Entities</div>', unsafe_allow_html=True)
+st.markdown("""
+<div class="section">
+    <ul>
+        <li><strong>Period:</strong> Specific times such as "22:12C".</li>
+        <li><strong>Year:</strong> Years like "2023".</li>
+        <li><strong>Calendar-Interval:</strong> Intervals such as "3 days".</li>
+        <li><strong>Month-Of-Year:</strong> Months like "January".</li>
+        <li><strong>Day-Of-Month:</strong> Specific days like "15th".</li>
+        <li><strong>Day-Of-Week:</strong> Days like "Tuesday".</li>
+        <li><strong>Hour-Of-Day:</strong> Hours such as "10 AM".</li>
+        <li><strong>Minute-Of-Hour:</strong> Minutes like "45".</li>
+        <li><strong>Number:</strong> Numerical values like "3".</li>
+        <li><strong>Second-Of-Minute:</strong> Seconds like "30".</li>
+        <li><strong>Time-Zone:</strong> Time zones such as "PST".</li>
+        <li><strong>Part-Of-Day:</strong> Parts of the day like "morning".</li>
+        <li><strong>Season-Of-Year:</strong> Seasons like "summer".</li>
+        <li><strong>AMPM-Of-Day:</strong> "AM" or "PM".</li>
+        <li><strong>Part-Of-Week:</strong> Parts of the week like "weekend".</li>
+        <li><strong>Week-Of-Year:</strong> Weeks like "week 42".</li>
+        <li><strong>Two-Digit-Year:</strong> Years represented in two digits like "'99".</li>
+        <li><strong>Sum:</strong> Total values of time periods, e.g., "3 days and 2 hours".</li>
+        <li><strong>Difference:</strong> Subtracted time periods, e.g., "5 days ago".</li>
+        <li><strong>Union:</strong> Combination of multiple time-related entities.</li>
+        <li><strong>Intersection:</strong> Overlapping time periods.</li>
+        <li><strong>Every-Nth:</strong> Repeated intervals, e.g., "every 3rd day".</li>
+        <li><strong>This:</strong> Referring to the current period, e.g., "this week".</li>
+        <li><strong>Last:</strong> Referring to the previous period, e.g., "last year".</li>
+        <li><strong>Next:</strong> Referring to the following period, e.g., "next month".</li>
+        <li><strong>Before:</strong> Time before a specific point, e.g., "before noon".</li>
+        <li><strong>After:</strong> Time after a specific point, e.g., "after 5 PM".</li>
+        <li><strong>Between:</strong> Time between two points, e.g., "between Monday and Friday".</li>
+        <li><strong>NthFromStart:</strong> Nth position from the start.</li>
+        <li><strong>NthFromEnd:</strong> Nth position from the end.</li>
+        <li><strong>Frequency:</strong> How often something occurs, e.g., "weekly".</li>
+        <li><strong>Modifier:</strong> Modifiers for time-related entities.</li>
+    </ul>
+</div>
+""", unsafe_allow_html=True)
+
+# How to Use the Model
+st.markdown('<div class="sub-title">How to Use the Model</div>', unsafe_allow_html=True)
+st.markdown("""
+<div class="section">
+    <p>To use this model, follow these steps in Python:</p>
+</div>
+""", unsafe_allow_html=True)
+st.code('''
+from sparknlp.base import *
+from sparknlp.annotator import *
+from pyspark.ml import Pipeline
+from pyspark.sql.functions import col, expr, round, concat, lit
+
+# Define the components of the pipeline
+document_assembler = DocumentAssembler() \\
+    .setInputCol("text") \\
+    .setOutputCol("document")
+
+sentence_detector = SentenceDetectorDLModel.pretrained("sentence_detector_dl", "en") \\
+    .setInputCols(["document"]) \\
+    .setOutputCol("sentence")
+
+tokenizer = Tokenizer() \\
+    .setInputCols(["sentence"]) \\
+    .setOutputCol("token")
+
+token_classifier = RoBertaForTokenClassification.pretrained("roberta_token_classifier_timex_semeval", "en") \\
+    .setInputCols(["sentence", "token"]) \\
+    .setOutputCol("ner")
+
+ner_converter = NerConverter() \\
+    .setInputCols(["sentence", "token", "ner"]) \\
+    .setOutputCol("ner_chunk")
+
+# Create the pipeline
+pipeline = Pipeline(stages=[
+    document_assembler,
+    sentence_detector,
+    tokenizer,
+    token_classifier,
+    ner_converter
+])
+
+# Create some example data
+text = "Model training was started at 22:12C and it took 3 days from Tuesday to Friday."
+data = spark.createDataFrame([[text]]).toDF("text")
+
+# Apply the pipeline to the data
+model = pipeline.fit(data)
+result = model.transform(data)
+
+# Select the result, entity
+result.select(
+    expr("explode(ner_chunk) as ner_chunk")
+).select(
+    col("ner_chunk.result").alias("chunk"),
+    col("ner_chunk.metadata.entity").alias("entity")
+).show(truncate=False)
+''', language='python')
+
+# Results
+
+st.text("""
++-------+-----------------+
+|chunk  |entity           |
++-------+-----------------+
+|took   |Frequency        |
+|3      |Number           |
+|days   |Calendar-Interval|
+|Tuesday|Day-Of-Week      |
+|to     |Between          |
+|Friday |Day-Of-Week      |
++-------+-----------------+
+""")
+
+# Model Information
+st.markdown('<div class="sub-title">Model Information</div>', unsafe_allow_html=True)
+st.markdown("""
+<div class="section">
+    <table class="benchmark-table">
+        <tr>
+            <th>Model Name</th>
+            <td>roberta_token_classifier_timex_semeval</td>
+        </tr>
+        <tr>
+            <th>Compatibility</th>
+            <td>Spark NLP 3.3.4+</td>
+        </tr>
+        <tr>
+            <th>License</th>
+            <td>Open Source</td>
+        </tr>
+        <tr>
+            <th>Edition</th>
+            <td>Official</td>
+        </tr>
+        <tr>
+            <th>Input Labels</th>
+            <td>[sentence, token]</td>
+        </tr>
+        <tr>
+            <th>Output Labels</th>
+            <td>[ner]</td>
+        </tr>
+        <tr>
+            <th>Language</th>
+            <td>en</td>
+        </tr>
+        <tr>
+            <th>Size</th>
+            <td>439.5 MB</td>
+        </tr>
+        <tr>
+            <th>Case sensitive</th>
+            <td>true</td>
+        </tr>
+        <tr>
+            <th>Max sentence length</th>
+            <td>256</td>
+        </tr>
+    </table>
+</div>
+""", unsafe_allow_html=True)
+
+# Data Source
+st.markdown('<div class="sub-title">Data Source</div>', unsafe_allow_html=True)
+st.markdown("""
+<div class="section">
+    <p>For more information about the dataset used to train this model, visit the <a class="link" href="https://huggingface.co/clulab/roberta-timex-semeval" target="_blank">Hugging Face page</a>.</p>
+</div>
+""", unsafe_allow_html=True)
+
+# Conclusion
+st.markdown('<div class="sub-title">Conclusion</div>', unsafe_allow_html=True)
+st.markdown("""
+<div class="section">
+    <p>Detecting time-related terminology is essential for a wide range of applications. This model, leveraging RoBERTa embeddings and RobertaForTokenClassification, provides robust capabilities for identifying and classifying temporal entities within text.</p>
+    <p>By integrating this model into your systems, you can enhance scheduling, event tracking, historical data analysis, and more. The high accuracy and comprehensive coverage of time-related entities make this model a valuable tool for many applications.</p>
+</div>
+""", unsafe_allow_html=True)
+
+# References
+st.markdown('<div class="sub-title">References</div>', unsafe_allow_html=True)
+st.markdown("""
+<div class="section">
+    <ul>
+        <li><a class="link" href="https://sparknlp.org/api/com/johnsnowlabs/nlp/annotators/classifier/dl/RoBertaForTokenClassification.html#!=(x$1:Any):Boolean" target="_blank" rel="noopener">RoBertaForTokenClassification</a> annotator documentation</li>
+        <li>Model Used: <a class="link" href="https://sparknlp.org/2021/12/28/roberta_token_classifier_timex_semeval_en.html" rel="noopener">roberta_token_classifier_timex_semeval_en</a></li>
+        <li><a class="link" href="https://nlp.johnsnowlabs.com/recognize_entitie" target="_blank" rel="noopener">Visualization demos for NER in Spark NLP</a></li>
+        <li><a class="link" href="https://www.johnsnowlabs.com/named-entity-recognition-ner-with-bert-in-spark-nlp/">Named Entity Recognition (NER) with BERT in Spark NLP</a></li>
+    </ul>
+</div>
+""", unsafe_allow_html=True)
+
+# Community & Support
+st.markdown('<div class="sub-title">Community & Support</div>', unsafe_allow_html=True)
+st.markdown("""
+<div class="section">
+    <ul>
+        <li><a class="link" href="https://sparknlp.org/" target="_blank">Official Website</a>: Documentation and examples</li>
+        <li><a class="link" href="https://github.com/JohnSnowLabs/spark-nlp" target="_blank">GitHub Repository</a>: Report issues or contribute</li>
+        <li><a class="link" href="https://forum.johnsnowlabs.com/" target="_blank">Community Forum</a>: Ask questions, share ideas, and connect with other users</li>
+    </ul>
+</div>
+""", unsafe_allow_html=True)

requirements.txt

@@ -0,0 +1,6 @@
+streamlit
+st-annotated-text
+pandas
+numpy
+spark-nlp
+pyspark