Upload 2 files

Demo.py

@@ -0,0 +1,126 @@
+import streamlit as st
+import sparknlp
+import os
+import pandas as pd
+
+from sparknlp.base import *
+from sparknlp.annotator import *
+from pyspark.ml import Pipeline
+from sparknlp.pretrained import PretrainedPipeline
+
+# Page configuration
+st.set_page_config(
+    layout="wide", 
+    page_title="Spark NLP Demos App", 
+    initial_sidebar_state="auto"
+)
+
+# CSS for styling
+st.markdown("""
+    <style>
+        .main-title {
+            font-size: 36px;
+            color: #4A90E2;
+            font-weight: bold;
+            text-align: center;
+        }
+        .section p, .section ul {
+            color: #666666;
+        }
+    </style>
+""", unsafe_allow_html=True)
+
+@st.cache_resource
+def init_spark():
+    return sparknlp.start()
+
+@st.cache_resource
+def create_pipeline():
+    document = DocumentAssembler() \
+            .setInputCol("text") \
+            .setOutputCol("document")
+
+    # Step 2: Sentence Detection
+    sentenceDetector = SentenceDetector() \
+                .setInputCols("document") \
+                .setOutputCol("sentences")
+
+    # Step 3: Tokenization
+    token = Tokenizer() \
+                .setInputCols("sentences") \
+                .setOutputCol("tokens") \
+                .setContextChars(["(", ")", "?", "!", ".", ","])
+
+    # Step 4: Coreference Resolution
+    corefResolution = SpanBertCorefModel().pretrained("spanbert_base_coref") \
+                .setInputCols(["sentences", "tokens"]) \
+                .setOutputCol("corefs") \
+                .setCaseSensitive(False)
+
+    # Define the pipeline
+    pipeline = Pipeline(stages=[document, sentenceDetector, token, corefResolution])
+    
+    return pipeline
+
+def fit_data(pipeline, data):
+    empty_df = spark.createDataFrame([['']]).toDF('text')
+    pipeline_model = pipeline.fit(empty_df)
+    model = LightPipeline(pipeline_model)
+    results = model.fullAnnotate(data)
+    return results
+
+# Set up the page layout
+st.markdown('<div class="main-title">State-of-the-Art Coreference Resolution in Spark NLP</div>', unsafe_allow_html=True)
+
+# Sidebar content
+model_name = st.sidebar.selectbox(
+    "Choose the pretrained model",
+    ['spanbert_base_coref'],
+    help="For more info about the models visit: https://sparknlp.org/models"
+)
+
+# Reference notebook link in sidebar
+link = """
+<a href="https://github.com/JohnSnowLabs/spark-nlp/blob/master/examples/python/annotation/text/english/coreference-resolution/Coreference_Resolution_SpanBertCorefModel.ipynb#L117">
+    <img src="https://colab.research.google.com/assets/colab-badge.svg" style="zoom: 1.3" alt="Open In Colab"/>
+</a>
+"""
+st.sidebar.markdown('Reference notebook:')
+st.sidebar.markdown(link, unsafe_allow_html=True)
+
+# Load examples
+examples = [
+    "Alice went to the market. She bought some fresh vegetables there. The tomatoes she purchased were particularly ripe.",
+    "Dr. Smith is a renowned surgeon. He has performed over a thousand successful operations. His colleagues respect him a lot.",
+    "The company announced a new product launch. It is expected to revolutionize the industry. The CEO was very excited about it.",
+    "Jennifer enjoys hiking. She goes to the mountains every weekend. Her favorite spot is the Blue Ridge Mountains.",
+    "The team won the championship. They celebrated their victory with a huge party. Their coach praised their hard work and dedication.",
+    "Michael is studying computer science. He finds artificial intelligence fascinating. His dream is to work at a leading tech company.",
+    "Tom is a skilled guitarist. He plays in a local band. His performances are always energetic and captivating."
+]
+
+# st.subheader("Automatically detect phrases expressing dates and normalize them with respect to a reference date.")
+selected_text = st.selectbox("Select an example", examples)
+custom_input = st.text_input("Try it with your own Sentence!")
+
+text_to_analyze = custom_input if custom_input else selected_text
+
+st.subheader('Full example text')
+st.write(text_to_analyze)
+
+# Initialize Spark and create pipeline
+spark = init_spark()
+pipeline = create_pipeline()
+output = fit_data(pipeline, text_to_analyze)
+
+# Display matched sentence
+st.subheader("Processed output:")
+
+results = {
+    'tokens': [coref.result for coref in output[0]['corefs']],
+    'corefs': [coref.metadata for coref in output[0]['corefs']]
+}
+
+df = pd.DataFrame(results)
+df.index += 1
+st.dataframe(df)

pages/Workflow & Model Overview.py

@@ -0,0 +1,216 @@
+import streamlit as st
+
+# Custom CSS for better styling
+st.markdown("""
+    <style>
+        .main-title {
+            font-size: 36px;
+            color: #4A90E2;
+            font-weight: bold;
+            text-align: center;
+        }
+        .sub-title {
+            font-size: 24px;
+            color: #4A90E2;
+            margin-top: 20px;
+        }
+        .section {
+            background-color: #f9f9f9;
+            padding: 15px;
+            border-radius: 10px;
+            margin-top: 20px;
+        }
+        .section h2 {
+            font-size: 22px;
+            color: #4A90E2;
+        }
+        .section p, .section ul {
+            color: #666666;
+        }
+        .link {
+            color: #4A90E2;
+            text-decoration: none;
+        }
+    </style>
+""", unsafe_allow_html=True)
+
+# Introduction
+st.markdown('<div class="main-title">Coreference Resolution with BERT-based Models in Spark NLP</div>', unsafe_allow_html=True)
+
+st.markdown("""
+<div class="section">
+    <p>Welcome to the Spark NLP Coreference Resolution Demo App! Coreference resolution is a crucial task in Natural Language Processing (NLP) that involves identifying and linking all expressions within a text that refer to the same real-world entity. This can be useful for a wide range of applications, such as text understanding, information extraction, and question answering.</p>
+    <p>Using Spark NLP, it is possible to perform coreference resolution with high accuracy using BERT-based models. This app demonstrates how to use the SpanBertCoref annotator to resolve coreferences in text data.</p>
+</div>
+""", unsafe_allow_html=True)
+
+st.image('images/Coreference-Resolution.png', use_column_width='auto')
+
+# About Coreference Resolution
+st.markdown('<div class="sub-title">About Coreference Resolution</div>', unsafe_allow_html=True)
+st.markdown("""
+<div class="section">
+    <p>Coreference resolution is the task of identifying and linking all expressions within a text that refer to the same real-world entity, such as a person, object, or concept. This technique involves analyzing a text and identifying all expressions that refer to a specific entity, such as “he,” “she,” “it,” or “they.” These expressions are then linked together to form a “coreference chain,” representing all the different ways that entity is referred to in the text.</p>
+    <p>For example, given the sentence, “John went to the store. He bought some groceries,” a coreference resolution model would identify that “John” and “He” both refer to the same entity and produce a cluster of coreferent mentions.</p>
+</div>
+""", unsafe_allow_html=True)
+
+# Using SpanBertCoref in Spark NLP
+st.markdown('<div class="sub-title">Using SpanBertCoref in Spark NLP</div>', unsafe_allow_html=True)
+st.markdown("""
+<div class="section">
+    <p>The SpanBertCoref annotator in Spark NLP allows users to perform coreference resolution with high accuracy using BERT-based models. This annotator can identify and link expressions that refer to the same entity in text data, providing valuable insights from unstructured text data.</p>
+    <p>The SpanBertCoref annotator in Spark NLP offers:</p>
+    <ul>
+        <li>Accurate coreference resolution using BERT-based models</li>
+        <li>Identification and linking of multiple coreferent expressions</li>
+        <li>Efficient processing of large text datasets</li>
+        <li>Integration with other Spark NLP components for comprehensive NLP pipelines</li>
+    </ul>
+</div>
+""", unsafe_allow_html=True)
+
+st.markdown('<h2 class="sub-title">Example Usage in Python</h2>', unsafe_allow_html=True)
+st.markdown('<p>Here’s how you can implement coreference resolution using the SpanBertCoref annotator in Spark NLP:</p>', unsafe_allow_html=True)
+
+# Setup Instructions
+st.markdown('<div class="sub-title">Setup</div>', unsafe_allow_html=True)
+st.markdown('<p>To install Spark NLP in Python, use your favorite package manager (conda, pip, etc.). For example:</p>', unsafe_allow_html=True)
+st.code("""
+pip install spark-nlp
+pip install pyspark
+""", language="bash")
+
+st.markdown("<p>Then, import Spark NLP and start a Spark session:</p>", unsafe_allow_html=True)
+st.code("""
+import sparknlp
+
+# Start Spark Session
+spark = sparknlp.start()
+""", language='python')
+
+# Coreference Resolution Example
+st.markdown('<div class="sub-title">Example Usage: Coreference Resolution with SpanBertCoref</div>', unsafe_allow_html=True)
+st.code('''
+from sparknlp.base import DocumentAssembler, Pipeline
+from sparknlp.annotator import (
+    SentenceDetector,
+    Tokenizer,
+    SpanBertCorefModel
+)
+import pyspark.sql.functions as F
+
+# Step 1: Transforms raw texts to document annotation
+document = DocumentAssembler() \\
+            .setInputCol("text") \\
+            .setOutputCol("document")
+
+# Step 2: Sentence Detection
+sentenceDetector = SentenceDetector() \\
+            .setInputCols("document") \\
+            .setOutputCol("sentences")
+
+# Step 3: Tokenization
+token = Tokenizer() \\
+            .setInputCols("sentences") \\
+            .setOutputCol("tokens") \\
+            .setContextChars(["(", ")", "?", "!", ".", ","])
+
+# Step 4: Coreference Resolution
+corefResolution = SpanBertCorefModel().pretrained("spanbert_base_coref") \\
+            .setInputCols(["sentences", "tokens"]) \\
+            .setOutputCol("corefs") \\
+            .setCaseSensitive(False)
+
+# Define the pipeline
+pipeline = Pipeline(stages=[document, sentenceDetector, token, corefResolution])
+
+# Create the dataframe
+data = spark.createDataFrame([["Ana is a Graduate Student at UT Dallas. She loves working in Natural Language Processing at the Institute. Her hobbies include blogging, dancing, and singing."]]).toDF("text")
+
+# Fit the dataframe to the pipeline to get the model
+model = pipeline.fit(data)
+
+# Transform the data to get predictions
+result = model.transform(data)
+
+# Display the extracted coreferences
+result.selectExpr("explode(corefs) AS coref").selectExpr("coref.result as token", "coref.metadata").show(truncate=False)
+''', language='python')
+
+st.text("""
++-------------+----------------------------------------------------------------------------------------+
+|token        |metadata                                                                                |
++-------------+----------------------------------------------------------------------------------------+
+|ana          |{head.sentence -> -1, head -> ROOT, head.begin -> -1, head.end -> -1, sentence -> 0}    |
+|she          |{head.sentence -> 0, head -> ana, head.begin -> 0, head.end -> 2, sentence -> 1}        |
+|her          |{head.sentence -> 0, head -> ana, head.begin -> 0, head.end -> 2, sentence -> 2}        |
+|ut dallas    |{head.sentence -> -1, head -> ROOT, head.begin -> -1, head.end -> -1, sentence -> 0}    |
+|the institute|{head.sentence -> 0, head -> ut dallas, head.begin -> 29, head.end -> 37, sentence -> 1}|
++-------------+----------------------------------------------------------------------------------------+
+""")
+
+st.markdown("""
+<p>The code snippet demonstrates how to set up a pipeline in Spark NLP to resolve coreferences in text data using the SpanBertCoref annotator. The resulting DataFrame contains the coreferent mentions and their metadata.</p>
+""", unsafe_allow_html=True)
+
+# One-liner Alternative
+st.markdown('<div class="sub-title">One-liner Alternative</div>', unsafe_allow_html=True)
+st.markdown("""
+<div class="section">
+    <p>In October 2022, John Snow Labs released the open-source <code>johnsnowlabs</code> library that contains all the company products, open-source and licensed, under one common library. This simplified the workflow, especially for users working with more than one of the libraries (e.g., Spark NLP + Healthcare NLP). This new library is a wrapper on all of John Snow Lab’s libraries and can be installed with pip:</p>
+    <p><code>pip install johnsnowlabs</code></p>
+    <p>To run coreference resolution with one line of code, we can simply:</p>
+</div>
+""", unsafe_allow_html=True)
+st.code("""
+# Import the NLP module which contains Spark NLP and NLU libraries
+from johnsnowlabs import nlp
+
+sample_text = "Ana is a Graduate Student at UT Dallas. She loves working in Natural Language Processing at the Institute. Her hobbies include blogging, dancing, and singing."
+
+# Returns a pandas DataFrame, we select the desired columns
+nlp.load('en.coreference.spanbert').predict(sample_text, output_level='sentence')
+""", language='python')
+
+st.image('images/johnsnowlabs-output.png', use_column_width='auto')
+
+st.markdown("""
+<p>This approach demonstrates how to use the <code>johnsnowlabs</code> library to perform coreference resolution with a single line of code. The resulting DataFrame contains the coreferent mentions and their metadata.</p>
+""", unsafe_allow_html=True)
+
+# Conclusion
+st.markdown("""
+<div class="section">
+    <h2>Conclusion</h2>
+    <p>In this app, we demonstrated how to use Spark NLP's SpanBertCoref annotator to resolve coreferences in text data. These powerful tools enable users to efficiently process large datasets and identify coreferent mentions, providing deeper insights for various applications. By integrating these annotators into your NLP pipelines, you can enhance the extraction of valuable entity relationships from unstructured text, improving text understanding, information extraction, and question answering.</p>
+</div>
+""", unsafe_allow_html=True)
+
+# References and Additional Information
+st.markdown('<div class="sub-title">For additional information, please check the following references.</div>', unsafe_allow_html=True)
+
+st.markdown("""
+<div class="section">
+        <ul>
+            <li>Documentation :&nbsp;<a href="https://nlp.johnsnowlabs.com/docs/en/transformers#spanbertcoref" target="_blank" rel="noopener">SpanBertCoref</a></li>
+            <li>Python Docs :&nbsp;<a href="https://nlp.johnsnowlabs.com/api/python/reference/autosummary/sparknlp/annotator/coref/spanbert_coref/index.html#sparknlp.annotator.coref.spanbert_coref.SpanBertCorefModel" target="_blank" rel="noopener">SpanBertCoref</a></li>
+            <li>Scala Docs :&nbsp;<a href="https://nlp.johnsnowlabs.com/api/com/johnsnowlabs/nlp/annotators/coref/SpanBertCorefModel.html" target="_blank" rel="noopener">SpanBertCoref</a></li>
+            <li>Academic Reference Paper:&nbsp;SpanBERT: <a href="https://arxiv.org/abs/1907.10529" target="_blank" rel="noopener nofollow">Improving Pre-training by Representing and Predicting Spans</a></li>
+            <li>John Snow Labs&nbsp;<a href="https://nlp.johnsnowlabs.com/2022/06/14/spanbert_base_coref_en_3_0.html" target="_blank" rel="noopener">SpanBertCoref Model</a></li>
+        </ul>
+</div>
+""", unsafe_allow_html=True)
+
+st.markdown('<div class="sub-title">Community & Support</div>', unsafe_allow_html=True)
+st.markdown("""
+<div class="section">
+    <ul>
+        <li><a class="link" href="https://sparknlp.org/" target="_blank">Official Website</a>: Documentation and examples</li>
+        <li><a class="link" href="https://join.slack.com/t/spark-nlp/shared_invite/zt-198dipu77-L3UWNe_AJ8xqDk0ivmih5Q" target="_blank">Slack</a>: Live discussion with the community and team</li>
+        <li><a class="link" href="https://github.com/JohnSnowLabs/spark-nlp" target="_blank">GitHub</a>: Bug reports, feature requests, and contributions</li>
+        <li><a class="link" href="https://medium.com/spark-nlp" target="_blank">Medium</a>: Spark NLP articles</li>
+        <li><a class="link" href="https://www.youtube.com/channel/UCmFOjlpYEhxf_wJUDuz6xxQ/videos" target="_blank">YouTube</a>: Video tutorials</li>
+    </ul>
+</div>
+""", unsafe_allow_html=True)