Upload 12 files

.streamlit/config.toml

@@ -0,0 +1,3 @@
+[theme]
+base="light"
+primaryColor="#29B4E8"

Demo.py

@@ -0,0 +1,121 @@
+import streamlit as st
+import sparknlp
+import os
+import pandas as pd
+
+from sparknlp.base import *
+from sparknlp.annotator import *
+from pyspark.ml import Pipeline
+from sparknlp.pretrained import PretrainedPipeline
+
+# Page configuration
+st.set_page_config(
+    layout="wide", 
+    page_title="Spark NLP Demos App", 
+    initial_sidebar_state="auto"
+)
+
+# CSS for styling
+st.markdown("""
+    <style>
+        .main-title {
+            font-size: 36px;
+            color: #4A90E2;
+            font-weight: bold;
+            text-align: center;
+        }
+        .section p, .section ul {
+            color: #666666;
+        }
+    </style>
+""", unsafe_allow_html=True)
+
+@st.cache_resource
+def init_spark():
+    return sparknlp.start()
+
+@st.cache_resource
+def create_pipeline():
+    document = DocumentAssembler() \
+            .setInputCol("text") \
+            .setOutputCol("document")
+
+    # Step 2: Sentence Detection
+    sentenceDetector = SentenceDetector() \
+                .setInputCols("document") \
+                .setOutputCol("sentences")
+
+    # Step 3: Tokenization
+    token = Tokenizer() \
+                .setInputCols("sentences") \
+                .setOutputCol("tokens") \
+                .setContextChars(["(", ")", "?", "!", ".", ","])
+
+    # Step 4: Coreference Resolution
+    corefResolution = SpanBertCorefModel().pretrained("spanbert_base_coref") \
+                .setInputCols(["sentences", "tokens"]) \
+                .setOutputCol("corefs") \
+                .setCaseSensitive(False)
+
+    # Define the pipeline
+    pipeline = Pipeline(stages=[document, sentenceDetector, token, corefResolution])
+
+def fit_data(pipeline, data):
+    empty_df = spark.createDataFrame([['']]).toDF('text')
+    pipeline_model = pipeline.fit(empty_df)
+    model = LightPipeline(pipeline_model)
+    results = model.fullAnnotate(data)
+    return results
+
+# Set up the page layout
+st.markdown('<div class="main-title">State-of-the-Art Coreference Resolution in Spark NLP</div>', unsafe_allow_html=True)
+
+# Sidebar content
+model_name = st.sidebar.selectbox(
+    "Choose the pretrained model",
+    ['spanbert_base_coref'],
+    help="For more info about the models visit: https://sparknlp.org/models"
+)
+
+# Reference notebook link in sidebar
+link = """
+<a href="https://github.com/JohnSnowLabs/spark-nlp/blob/master/examples/python/annotation/text/english/coreference-resolution/Coreference_Resolution_SpanBertCorefModel.ipynb#L117">
+    <img src="https://colab.research.google.com/assets/colab-badge.svg" style="zoom: 1.3" alt="Open In Colab"/>
+</a>
+"""
+st.sidebar.markdown('Reference notebook:')
+st.sidebar.markdown(link, unsafe_allow_html=True)
+
+# Load examples
+examples = [
+    "Alice went to the market. She bought some fresh vegetables there. The tomatoes she purchased were particularly ripe.",
+    "Dr. Smith is a renowned surgeon. He has performed over a thousand successful operations. His colleagues respect him a lot.",
+    "The company announced a new product launch. It is expected to revolutionize the industry. The CEO was very excited about it.",
+    "Jennifer enjoys hiking. She goes to the mountains every weekend. Her favorite spot is the Blue Ridge Mountains.",
+    "The team won the championship. They celebrated their victory with a huge party. Their coach praised their hard work and dedication.",
+    "Michael is studying computer science. He finds artificial intelligence fascinating. His dream is to work at a leading tech company.",
+    "The book was well-received by critics. It was praised for its intricate plot and well-developed characters. The author felt proud of his work.",
+    "Sarah adopted a kitten. She named it Whiskers. Whiskers loves to play with her and often follows her around the house.",
+    "The project was completed ahead of schedule. It was a collaborative effort. The team members were rewarded for their contribution.",
+    "Tom is a skilled guitarist. He plays in a local band. His performances are always energetic and captivating."
+]
+
+# st.subheader("Automatically detect phrases expressing dates and normalize them with respect to a reference date.")
+selected_text = st.selectbox("Select an example", examples)
+custom_input = st.text_input("Try it with your own Sentence!")
+
+text_to_analyze = custom_input if custom_input else selected_text
+
+st.subheader('Full example text')
+st.write(text_to_analyze)
+
+# Initialize Spark and create pipeline
+spark = init_spark()
+pipeline = create_pipeline()
+output = fit_data(pipeline, text_to_analyze)
+
+# Display matched sentence
+st.subheader("Processed output:")
+df = extract_to_dataframe(output)
+df.index += 1
+st.dataframe(df)

Dockerfile

@@ -0,0 +1,70 @@
+# Download base image ubuntu 18.04
+FROM ubuntu:18.04
+
+# Set environment variables
+ENV NB_USER jovyan
+ENV NB_UID 1000
+ENV HOME /home/${NB_USER}
+
+# Install required packages
+RUN apt-get update && apt-get install -y \
+    tar \
+    wget \
+    bash \
+    rsync \
+    gcc \
+    libfreetype6-dev \
+    libhdf5-serial-dev \
+    libpng-dev \
+    libzmq3-dev \
+    python3 \
+    python3-dev \
+    python3-pip \
+    unzip \
+    pkg-config \
+    software-properties-common \
+    graphviz \
+    openjdk-8-jdk \
+    ant \
+    ca-certificates-java \
+    && apt-get clean \
+    && update-ca-certificates -f;
+
+# Install Python 3.8 and pip
+RUN add-apt-repository ppa:deadsnakes/ppa \
+    && apt-get update \
+    && apt-get install -y python3.8 python3-pip \
+    && apt-get clean;
+
+# Set up JAVA_HOME
+ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64/
+RUN mkdir -p ${HOME} \
+    && echo "export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/" >> ${HOME}/.bashrc \
+    && chown -R ${NB_UID}:${NB_UID} ${HOME}
+
+# Create a new user named "jovyan" with user ID 1000
+RUN useradd -m -u ${NB_UID} ${NB_USER}
+
+# Switch to the "jovyan" user
+USER ${NB_USER}
+
+# Set home and path variables for the user
+ENV HOME=/home/${NB_USER} \
+    PATH=/home/${NB_USER}/.local/bin:$PATH
+
+# Set the working directory to the user's home directory
+WORKDIR ${HOME}
+
+# Upgrade pip and install Python dependencies
+RUN python3.8 -m pip install --upgrade pip
+COPY requirements.txt /tmp/requirements.txt
+RUN python3.8 -m pip install -r /tmp/requirements.txt
+
+# Copy the application code into the container at /home/jovyan
+COPY --chown=${NB_USER}:${NB_USER} . ${HOME}
+
+# Expose port for Streamlit
+EXPOSE 7860
+
+# Define the entry point for the container
+ENTRYPOINT ["streamlit", "run", "Demo.py", "--server.port=7860", "--server.address=0.0.0.0"]

inputs/date_matcher/Example1.txt

@@ -0,0 +1,5 @@
+David visited the restaurant yesterday with his family.  He also visited and the day before, but at ... 
+David visited the restaurant yesterday with his family. 
+He also visited and the day before, but at that time he was alone.
+David again visited today with his colleagues.
+He and his friends really liked the food and hoped to visit again tomorrow.

inputs/date_matcher/Example2.txt

@@ -0,0 +1,5 @@
+In March 2003 she was seen in the office and appeared to be extremely disturbed emotionally. On 2003... 
+In March 2003 she was seen in the office and appeared to be extremely disturbed emotionally.
+On 2003-04-04 she again visited and talked about the effects of the medication she has been taking, and seemed positive and in much better shape.
+She again visited on Fri, 22/4/2003 and looked better.
+She has been working out and taking her medicines since April 1st 2003.

inputs/date_matcher/Example3.txt

@@ -0,0 +1,6 @@
+I have a very busy schedule these days. I have meetings from 7pm. till 11pm. I have 3 meetings the d... 
+I have a very busy schedule these days. I have meetings from 7pm. till 11pm.
+I have 3 meetings the day after, and have submission deadlines approaching as well.
+By next mon I have to finalise the architecture, for which i'll have to hold multiple meetings with ARM.
+Then i'll have to discuss dev plans by next tuesday and develop a thorough plan.
+The plan should be ready by Nov 30th.

inputs/date_matcher/Example4.txt

@@ -0,0 +1,4 @@
+When Tom visited the Bahamas last year, it was his first time travelling. Since then he was travelle... 
+When Tom visited the Bahamas last year, it was his first time travelling.
+Since then he was travelled a lot. For example, he visited Hawaii last week.
+The last time we talked, he was planning to travel to Alaska next month.

inputs/date_matcher/Example5.txt

@@ -0,0 +1,3 @@
+Isn't it weird that all my family members have the same birth day and month? All of us were born on ... 
+Isn't it weird that all my family members have the same birth day and month? All of us were born on 1st Jan
+    Dad was born on 01/01/1900. Mom has a birth date of 1st Jan 1902. And I was born on 2000/01/01

pages/Workflow & Model Overview.py

@@ -0,0 +1,216 @@
+import streamlit as st
+
+# Custom CSS for better styling
+st.markdown("""
+    <style>
+        .main-title {
+            font-size: 36px;
+            color: #4A90E2;
+            font-weight: bold;
+            text-align: center;
+        }
+        .sub-title {
+            font-size: 24px;
+            color: #4A90E2;
+            margin-top: 20px;
+        }
+        .section {
+            background-color: #f9f9f9;
+            padding: 15px;
+            border-radius: 10px;
+            margin-top: 20px;
+        }
+        .section h2 {
+            font-size: 22px;
+            color: #4A90E2;
+        }
+        .section p, .section ul {
+            color: #666666;
+        }
+        .link {
+            color: #4A90E2;
+            text-decoration: none;
+        }
+    </style>
+""", unsafe_allow_html=True)
+
+# Introduction
+st.markdown('<div class="main-title">Coreference Resolution with BERT-based Models in Spark NLP</div>', unsafe_allow_html=True)
+
+st.markdown("""
+<div class="section">
+    <p>Welcome to the Spark NLP Coreference Resolution Demo App! Coreference resolution is a crucial task in Natural Language Processing (NLP) that involves identifying and linking all expressions within a text that refer to the same real-world entity. This can be useful for a wide range of applications, such as text understanding, information extraction, and question answering.</p>
+    <p>Using Spark NLP, it is possible to perform coreference resolution with high accuracy using BERT-based models. This app demonstrates how to use the SpanBertCoref annotator to resolve coreferences in text data.</p>
+</div>
+""", unsafe_allow_html=True)
+
+st.image('images/Coreference-Resolution.png', use_column_width='auto')
+
+# About Coreference Resolution
+st.markdown('<div class="sub-title">About Coreference Resolution</div>', unsafe_allow_html=True)
+st.markdown("""
+<div class="section">
+    <p>Coreference resolution is the task of identifying and linking all expressions within a text that refer to the same real-world entity, such as a person, object, or concept. This technique involves analyzing a text and identifying all expressions that refer to a specific entity, such as “he,” “she,” “it,” or “they.” These expressions are then linked together to form a “coreference chain,” representing all the different ways that entity is referred to in the text.</p>
+    <p>For example, given the sentence, “John went to the store. He bought some groceries,” a coreference resolution model would identify that “John” and “He” both refer to the same entity and produce a cluster of coreferent mentions.</p>
+</div>
+""", unsafe_allow_html=True)
+
+# Using SpanBertCoref in Spark NLP
+st.markdown('<div class="sub-title">Using SpanBertCoref in Spark NLP</div>', unsafe_allow_html=True)
+st.markdown("""
+<div class="section">
+    <p>The SpanBertCoref annotator in Spark NLP allows users to perform coreference resolution with high accuracy using BERT-based models. This annotator can identify and link expressions that refer to the same entity in text data, providing valuable insights from unstructured text data.</p>
+    <p>The SpanBertCoref annotator in Spark NLP offers:</p>
+    <ul>
+        <li>Accurate coreference resolution using BERT-based models</li>
+        <li>Identification and linking of multiple coreferent expressions</li>
+        <li>Efficient processing of large text datasets</li>
+        <li>Integration with other Spark NLP components for comprehensive NLP pipelines</li>
+    </ul>
+</div>
+""", unsafe_allow_html=True)
+
+st.markdown('<h2 class="sub-title">Example Usage in Python</h2>', unsafe_allow_html=True)
+st.markdown('<p>Here’s how you can implement coreference resolution using the SpanBertCoref annotator in Spark NLP:</p>', unsafe_allow_html=True)
+
+# Setup Instructions
+st.markdown('<div class="sub-title">Setup</div>', unsafe_allow_html=True)
+st.markdown('<p>To install Spark NLP in Python, use your favorite package manager (conda, pip, etc.). For example:</p>', unsafe_allow_html=True)
+st.code("""
+pip install spark-nlp
+pip install pyspark
+""", language="bash")
+
+st.markdown("<p>Then, import Spark NLP and start a Spark session:</p>", unsafe_allow_html=True)
+st.code("""
+import sparknlp
+
+# Start Spark Session
+spark = sparknlp.start()
+""", language='python')
+
+# Coreference Resolution Example
+st.markdown('<div class="sub-title">Example Usage: Coreference Resolution with SpanBertCoref</div>', unsafe_allow_html=True)
+st.code('''
+from sparknlp.base import DocumentAssembler, Pipeline
+from sparknlp.annotator import (
+    SentenceDetector,
+    Tokenizer,
+    SpanBertCorefModel
+)
+import pyspark.sql.functions as F
+
+# Step 1: Transforms raw texts to document annotation
+document = DocumentAssembler() \\
+            .setInputCol("text") \\
+            .setOutputCol("document")
+
+# Step 2: Sentence Detection
+sentenceDetector = SentenceDetector() \\
+            .setInputCols("document") \\
+            .setOutputCol("sentences")
+
+# Step 3: Tokenization
+token = Tokenizer() \\
+            .setInputCols("sentences") \\
+            .setOutputCol("tokens") \\
+            .setContextChars(["(", ")", "?", "!", ".", ","])
+
+# Step 4: Coreference Resolution
+corefResolution = SpanBertCorefModel().pretrained("spanbert_base_coref") \\
+            .setInputCols(["sentences", "tokens"]) \\
+            .setOutputCol("corefs") \\
+            .setCaseSensitive(False)
+
+# Define the pipeline
+pipeline = Pipeline(stages=[document, sentenceDetector, token, corefResolution])
+
+# Create the dataframe
+data = spark.createDataFrame([["Ana is a Graduate Student at UT Dallas. She loves working in Natural Language Processing at the Institute. Her hobbies include blogging, dancing, and singing."]]).toDF("text")
+
+# Fit the dataframe to the pipeline to get the model
+model = pipeline.fit(data)
+
+# Transform the data to get predictions
+result = model.transform(data)
+
+# Display the extracted coreferences
+result.selectExpr("explode(corefs) AS coref").selectExpr("coref.result as token", "coref.metadata").show(truncate=False)
+''', language='python')
+
+st.text("""
++-------------+----------------------------------------------------------------------------------------+
+|token        |metadata                                                                                |
++-------------+----------------------------------------------------------------------------------------+
+|ana          |{head.sentence -> -1, head -> ROOT, head.begin -> -1, head.end -> -1, sentence -> 0}    |
+|she          |{head.sentence -> 0, head -> ana, head.begin -> 0, head.end -> 2, sentence -> 1}        |
+|her          |{head.sentence -> 0, head -> ana, head.begin -> 0, head.end -> 2, sentence -> 2}        |
+|ut dallas    |{head.sentence -> -1, head -> ROOT, head.begin -> -1, head.end -> -1, sentence -> 0}    |
+|the institute|{head.sentence -> 0, head -> ut dallas, head.begin -> 29, head.end -> 37, sentence -> 1}|
++-------------+----------------------------------------------------------------------------------------+
+""")
+
+st.markdown("""
+<p>The code snippet demonstrates how to set up a pipeline in Spark NLP to resolve coreferences in text data using the SpanBertCoref annotator. The resulting DataFrame contains the coreferent mentions and their metadata.</p>
+""", unsafe_allow_html=True)
+
+# One-liner Alternative
+st.markdown('<div class="sub-title">One-liner Alternative</div>', unsafe_allow_html=True)
+st.markdown("""
+<div class="section">
+    <p>In October 2022, John Snow Labs released the open-source <code>johnsnowlabs</code> library that contains all the company products, open-source and licensed, under one common library. This simplified the workflow, especially for users working with more than one of the libraries (e.g., Spark NLP + Healthcare NLP). This new library is a wrapper on all of John Snow Lab’s libraries and can be installed with pip:</p>
+    <p><code>pip install johnsnowlabs</code></p>
+    <p>To run coreference resolution with one line of code, we can simply:</p>
+</div>
+""", unsafe_allow_html=True)
+st.code("""
+# Import the NLP module which contains Spark NLP and NLU libraries
+from johnsnowlabs import nlp
+
+sample_text = "Ana is a Graduate Student at UT Dallas. She loves working in Natural Language Processing at the Institute. Her hobbies include blogging, dancing, and singing."
+
+# Returns a pandas DataFrame, we select the desired columns
+nlp.load('en.coreference.spanbert').predict(sample_text, output_level='sentence')
+""", language='python')
+
+st.image('images/johnsnowlabs-output.png', use_column_width='auto')
+
+st.markdown("""
+<p>This approach demonstrates how to use the <code>johnsnowlabs</code> library to perform coreference resolution with a single line of code. The resulting DataFrame contains the coreferent mentions and their metadata.</p>
+""", unsafe_allow_html=True)
+
+# Conclusion
+st.markdown("""
+<div class="section">
+    <h2>Conclusion</h2>
+    <p>In this app, we demonstrated how to use Spark NLP's SpanBertCoref annotator to resolve coreferences in text data. These powerful tools enable users to efficiently process large datasets and identify coreferent mentions, providing deeper insights for various applications. By integrating these annotators into your NLP pipelines, you can enhance the extraction of valuable entity relationships from unstructured text, improving text understanding, information extraction, and question answering.</p>
+</div>
+""", unsafe_allow_html=True)
+
+# References and Additional Information
+st.markdown('<div class="sub-title">For additional information, please check the following references.</div>', unsafe_allow_html=True)
+
+st.markdown("""
+<div class="section">
+        <ul>
+            <li>Documentation :&nbsp;<a href="https://nlp.johnsnowlabs.com/docs/en/transformers#spanbertcoref" target="_blank" rel="noopener">SpanBertCoref</a></li>
+            <li>Python Docs :&nbsp;<a href="https://nlp.johnsnowlabs.com/api/python/reference/autosummary/sparknlp/annotator/coref/spanbert_coref/index.html#sparknlp.annotator.coref.spanbert_coref.SpanBertCorefModel" target="_blank" rel="noopener">SpanBertCoref</a></li>
+            <li>Scala Docs :&nbsp;<a href="https://nlp.johnsnowlabs.com/api/com/johnsnowlabs/nlp/annotators/coref/SpanBertCorefModel.html" target="_blank" rel="noopener">SpanBertCoref</a></li>
+            <li>Academic Reference Paper:&nbsp;SpanBERT: <a href="https://arxiv.org/abs/1907.10529" target="_blank" rel="noopener nofollow">Improving Pre-training by Representing and Predicting Spans</a></li>
+            <li>John Snow Labs&nbsp;<a href="https://nlp.johnsnowlabs.com/2022/06/14/spanbert_base_coref_en_3_0.html" target="_blank" rel="noopener">SpanBertCoref Model</a></li>
+        </ul>
+</div>
+""", unsafe_allow_html=True)
+
+st.markdown('<div class="sub-title">Community & Support</div>', unsafe_allow_html=True)
+st.markdown("""
+<div class="section">
+    <ul>
+        <li><a class="link" href="https://sparknlp.org/" target="_blank">Official Website</a>: Documentation and examples</li>
+        <li><a class="link" href="https://join.slack.com/t/spark-nlp/shared_invite/zt-198dipu77-L3UWNe_AJ8xqDk0ivmih5Q" target="_blank">Slack</a>: Live discussion with the community and team</li>
+        <li><a class="link" href="https://github.com/JohnSnowLabs/spark-nlp" target="_blank">GitHub</a>: Bug reports, feature requests, and contributions</li>
+        <li><a class="link" href="https://medium.com/spark-nlp" target="_blank">Medium</a>: Spark NLP articles</li>
+        <li><a class="link" href="https://www.youtube.com/channel/UCmFOjlpYEhxf_wJUDuz6xxQ/videos" target="_blank">YouTube</a>: Video tutorials</li>
+    </ul>
+</div>
+""", unsafe_allow_html=True)

requirements.txt

@@ -0,0 +1,5 @@
+streamlit
+pandas
+numpy
+spark-nlp
+pyspark