Upload 10 files

.streamlit/config.toml

@@ -0,0 +1,3 @@
+[theme]
+base="light"
+primaryColor="#29B4E8"

Demo.py

@@ -0,0 +1,156 @@
+import streamlit as st
+import sparknlp
+import os
+import pandas as pd
+
+from sparknlp.base import *
+from sparknlp.annotator import *
+from pyspark.ml import Pipeline
+from sparknlp.pretrained import PretrainedPipeline
+from annotated_text import annotated_text
+
+# Page configuration
+st.set_page_config(
+    layout="wide", 
+    page_title="Spark NLP Demos App", 
+    initial_sidebar_state="auto"
+)
+
+# CSS for styling
+st.markdown("""
+    <style>
+        .main-title {
+            font-size: 36px;
+            color: #4A90E2;
+            font-weight: bold;
+            text-align: center;
+        }
+        .section p, .section ul {
+            color: #666666;
+        }
+        .stTable {
+            margin-left: auto;
+            margin-right: auto;
+        }
+    </style>
+""", unsafe_allow_html=True)
+
+@st.cache_resource
+def init_spark():
+    return sparknlp.start()
+
+@st.cache_resource
+def create_pipeline(model):
+    document_assembler = DocumentAssembler() \
+    .setInputCol('text') \
+    .setOutputCol('document')
+
+    sentence_detector = SentenceDetector() \
+        .setInputCols(['document']) \
+        .setOutputCol('sentences')
+
+    tokenizer = Tokenizer() \
+        .setInputCols(['sentences']) \
+        .setOutputCol('tokens') \
+        .setContextChars(['(', ')', '?', '!', '.', ','])
+
+    keywords = YakeKeywordExtraction() \
+        .setInputCols('tokens') \
+        .setOutputCol('keywords') \
+        .setMinNGrams(2) \
+        .setMaxNGrams(5) \
+        .setNKeywords(100) \
+        .setStopWords(StopWordsCleaner().getStopWords())
+
+    pipeline = Pipeline(stages=[
+        document_assembler, 
+        sentence_detector,
+        tokenizer,
+        keywords
+    ])
+
+    return pipeline
+
+def fit_data(pipeline, data):
+    empty_df = spark.createDataFrame([['']]).toDF('text')
+    pipeline_model = pipeline.fit(empty_df)
+    model = LightPipeline(pipeline_model)
+    results = model.fullAnnotate(data)[0]
+
+    return results
+    
+def highlight_keywords(data):
+    document_text = data["document"][0].result
+    keywords = data["keywords"]
+    annotations = []
+    last_index = 0
+
+    for keyword in keywords:
+        keyword_text = keyword.result
+        start_index = document_text.find(keyword_text, last_index)
+        if start_index != -1:
+            if start_index > last_index:
+                annotations.append(document_text[last_index:start_index])
+            annotations.append((keyword_text, 'Key Word'))
+            last_index = start_index + len(keyword_text)
+    
+    if last_index < len(document_text):
+        annotations.append(document_text[last_index:])
+
+    annotated_text(*annotations)
+
+# Set up the page layout
+st.markdown('<div class="main-title">Detect Key Phrases With Spark NLP</div>', unsafe_allow_html=True)
+
+# Sidebar content
+model = st.sidebar.selectbox(
+    "Choose the pretrained model",
+    ["yake_model"],
+    help="For more info about the models visit: https://sparknlp.org/models"
+)
+
+# Reference notebook link in sidebar
+link = """
+<a href="https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/streamlit_notebooks/KEYPHRASE_EXTRACTION.ipynb">
+    <img src="https://colab.research.google.com/assets/colab-badge.svg" style="zoom: 1.3" alt="Open In Colab"/>
+</a>
+"""
+st.sidebar.markdown('Reference notebook:')
+st.sidebar.markdown(link, unsafe_allow_html=True)
+
+# Load examples
+folder_path = f"inputs/{model}"
+examples = [
+    lines[1].strip() 
+    for filename in os.listdir(folder_path) 
+    if filename.endswith('.txt') 
+    for lines in [open(os.path.join(folder_path, filename), 'r', encoding='utf-8').readlines()] 
+    if len(lines) >= 2
+]
+
+selected_text = st.selectbox("Select a sample text", examples)
+custom_input = st.text_input("Try it for yourself!")
+
+if custom_input:
+    selected_text = custom_input
+elif selected_text:
+    selected_text = selected_text
+
+st.subheader('Selected Text')
+st.write(selected_text)
+
+# Initialize Spark and create pipeline
+spark = init_spark()
+pipeline = create_pipeline(model)
+output = fit_data(pipeline, selected_text)
+
+# Display output
+st.subheader("Annotated Document:")
+highlight_keywords(output)
+
+keys_df = pd.DataFrame([(k.result, k.begin, k.end, k.metadata['score'], k.metadata['sentence']) for k in output['keywords']],
+                       columns=['keywords', 'begin', 'end', 'score', 'sentence'])
+keys_df['score'] = keys_df['score'].astype(float)
+# ordered by relevance
+with st.expander("View Data Table"):
+    st.table(keys_df.sort_values(['sentence', 'score']))

Dockerfile

@@ -0,0 +1,70 @@
+# Download base image ubuntu 18.04
+FROM ubuntu:18.04
+
+# Set environment variables
+ENV NB_USER jovyan
+ENV NB_UID 1000
+ENV HOME /home/${NB_USER}
+
+# Install required packages
+RUN apt-get update && apt-get install -y \
+    tar \
+    wget \
+    bash \
+    rsync \
+    gcc \
+    libfreetype6-dev \
+    libhdf5-serial-dev \
+    libpng-dev \
+    libzmq3-dev \
+    python3 \
+    python3-dev \
+    python3-pip \
+    unzip \
+    pkg-config \
+    software-properties-common \
+    graphviz \
+    openjdk-8-jdk \
+    ant \
+    ca-certificates-java \
+    && apt-get clean \
+    && update-ca-certificates -f;
+
+# Install Python 3.8 and pip
+RUN add-apt-repository ppa:deadsnakes/ppa \
+    && apt-get update \
+    && apt-get install -y python3.8 python3-pip \
+    && apt-get clean;
+
+# Set up JAVA_HOME
+ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64/
+RUN mkdir -p ${HOME} \
+    && echo "export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/" >> ${HOME}/.bashrc \
+    && chown -R ${NB_UID}:${NB_UID} ${HOME}
+
+# Create a new user named "jovyan" with user ID 1000
+RUN useradd -m -u ${NB_UID} ${NB_USER}
+
+# Switch to the "jovyan" user
+USER ${NB_USER}
+
+# Set home and path variables for the user
+ENV HOME=/home/${NB_USER} \
+    PATH=/home/${NB_USER}/.local/bin:$PATH
+
+# Set the working directory to the user's home directory
+WORKDIR ${HOME}
+
+# Upgrade pip and install Python dependencies
+RUN python3.8 -m pip install --upgrade pip
+COPY requirements.txt /tmp/requirements.txt
+RUN python3.8 -m pip install -r /tmp/requirements.txt
+
+# Copy the application code into the container at /home/jovyan
+COPY --chown=${NB_USER}:${NB_USER} . ${HOME}
+
+# Expose port for Streamlit
+EXPOSE 7860
+
+# Define the entry point for the container
+ENTRYPOINT ["streamlit", "run", "Demo.py", "--server.port=7860", "--server.address=0.0.0.0"]

inputs/yake_model/Example1.txt

@@ -0,0 +1,2 @@
+Extracting keywords from texts has become a challenge for individuals and organizations as the infor... 
+Extracting keywords from texts has become a challenge for individuals and organizations as the information grows in complexity and size. The need to automate this task so that text can be processed in a timely and adequate manner has led to the emergence of automatic keyword extraction tools. Yake is a novel feature-based system for multi-lingual keyword extraction, which supports texts of different sizes, domain or languages. Unlike other approaches, Yake does not rely on dictionaries nor thesauri, neither is trained against any corpora. Instead, it follows an unsupervised approach which builds upon features extracted from the text, making it thus applicable to documents written in different languages without the need for further knowledge. This can be beneficial for a large number of tasks and a plethora of situations where access to training corpora is either limited or restricted.

inputs/yake_model/Example2.txt

@@ -0,0 +1,6 @@
+Iodine deficiency is a lack of the trace element iodine, an essential nutrient in the diet. It may r... 
+Iodine deficiency is a lack of the trace element iodine, an essential nutrient in the diet. It may result in metabolic problems such as goiter, sometimes as an endemic goiter as well as cretinism due to untreated congenital hypothyroidism, which results in developmental delays and other health problems. Iodine deficiency is an important global health issue, especially for fertile and pregnant women. It is also a preventable cause of intellectual disability.
+
+Iodine is an essential dietary mineral for neurodevelopment among offsprings and toddlers. The thyroid hormones thyroxine and triiodothyronine contain iodine. In areas where there is little iodine in the diet, typically remote inland areas where no marine foods are eaten, iodine deficiency is common. It is also common in mountainous regions of the world where food is grown in iodine-poor soil.
+
+Prevention includes adding small amounts of iodine to table salt, a product known as iodized salt. Iodine compounds have also been added to other foodstuffs, such as flour, water and milk, in areas of deficiency. Seafood is also a well known source of iodine.

inputs/yake_model/Example3.txt

@@ -0,0 +1,4 @@
+The Prague Quadrennial of Performance Design and Space was established in 1967 to bring the best of ... 
+The Prague Quadrennial of Performance Design and Space was established in 1967 to bring the best of design for performance, scenography, and theatre architecture to the front line of cultural activities to be experienced by professional and emerging artists as well as the general public. The quadrennial exhibitions, festivals, and educational programs act as a global catalyst of creative progress by encouraging experimentation, networking, innovation, and future collaborations. PQ aims to honor, empower and celebrate the work of designers, artists and architects while inspiring and educating audiences, who are the most essential element of any live performance. The Prague Quadrennial strives to present performance design as an art form concerned with creation of active performance environments, that are far beyond merely decorative or beautiful, but emotionally charged, where design can become a quest, a question, an argument, a threat, a resolution, an agent of change, or a provocation. Performance design is a collaborative field where designers mix, fuse and blur the lines between multiple artistic disciplines to search for new approaches and new visions.
+
+The Prague Quadrennial organizes an expansive program of international projects and activities between the main quadrennial events – performances, exhibitions, symposia, workshops, residencies, and educational initiatives serve as an international platform for exploring the practice, theory and education of contemporary performance design in the most encompassing terms.

inputs/yake_model/Example4.txt

@@ -0,0 +1,8 @@
+Author Nathan Wiseman-Trowse explained that the "approach to the sheer physicality of sound" integra... 
+Author Nathan Wiseman-Trowse explained that the "approach to the sheer physicality of sound" integral to dream pop was "arguably pioneered in popular music by figures such as Phil Spector and Brian Wilson". The music of the Velvet Underground in the 1960s and 1970s, which experimented with repetition, tone, and texture over conventional song structure, was also an important touchstone in the genre's development George Harrison's 1970 album All Things Must Pass, with its Spector-produced Wall of Sound and fluid arrangements, led music journalist John Bergstrom to credit it as a progenitor of the genre.
+
+Reynolds described dream pop bands as "a wave of hazy neo-psychedelic groups", noting the influence of the "ethereal soundscapes" of bands such as Cocteau Twins. Rolling Stone's Kory Grow described "modern dream pop" as originating with the early 1980s work of Cocteau Twins and their contemporaries, while PopMatters' AJ Ramirez noted an evolutionary line from gothic rock to dream pop. Grow considered Julee Cruise's 1989 album Floating into the Night, written and produced by David Lynch and Angelo Badalamenti, as a significant development of the dream pop sound which "gave the genre its synthy sheen." The influence of Cocteau Twins extended to the expansion of the genre's influence into Cantopop and Mandopop through the music of Faye Wong, who covered multiple Cocteau Twins songs, including tracks featured in Chungking Express, in which she also acted. Cocteau Twins would go on to collaborate with Wong on original songs of hers, and Wong contributed vocals to a limited release of a late Cocteau Twins single.
+
+In the early 1990s, some dream pop acts influenced by My Bloody Valentine, such as Seefeel, were drawn to techno and began utilizing elements such as samples and sequenced rhythms. Ambient pop music was described by AllMusic as "essentially an extension of the dream pop that emerged in the wake of the shoegazer movement", distinct for its incorporation of electronic textures.
+
+Much of the music associated with the 2009-coined term "chillwave" could be considered dream pop. In the opinion of Grantland's David Schilling, when "chillwave" was popularized, the discussion that followed among music journalists and bloggers revealed that labels such as "shoegaze" and "dream pop" were ultimately "arbitrary and meaningless".

inputs/yake_model/Example5.txt

@@ -0,0 +1,6 @@
+North Ingria was located in the Karelian Isthmus, between Finland and Soviet Russia. It was establis... 
+North Ingria was located in the Karelian Isthmus, between Finland and Soviet Russia. It was established 23 January 1919. The republic was first served by a post office at the Rautu railway station on the Finnish side of the border. As the access across the border was mainly restricted, the North Ingrian postal service was finally launched in the early 1920. The man behind the idea was the lieutenant colonel Georg Elfvengren, head of the governing council of North Ingria. He was also known as an enthusiastic stamp collector. The post office was opened at the capital village of Kirjasalo.
+
+The first series of North Ingrian stamps were issued in 21 March 1920. They were based on the 1917 Finnish "Model Saarinen" series, a stamp designed by the Finnish architect Eliel Saarinen. The first series were soon sold to collectors, as the postage stamps became the major financial source of the North Ingrian government. The second series was designed for the North Ingrian postal service and issued 2 August 1920. The value of both series was in Finnish marks and similar to the postal fees of Finland. The number of letters sent from North Ingria was about 50 per day, most of them were carried to Finland. They were mainly sent by the personnel of the Finnish occupying forces. Large number of letters were also sent in pure philatelic purposes.
+
+With the Treaty of Tartu, the area was re-integrated into Soviet Russia and the use of the North Ingrian postage stamps ended in 4 December 1920. Stamps were still sold in Finland in 1921 with an overprinting "Inkerin hyväksi" (For the Ingria), but they were no longer valid. Funds of the sale went for the North Ingrian refugees.

pages/Workflow & Model Overview.py

@@ -0,0 +1,235 @@
+import streamlit as st
+import pandas as pd
+
+# Custom CSS for better styling
+st.markdown("""
+    <style>
+        .main-title {
+            font-size: 36px;
+            color: #4A90E2;
+            font-weight: bold;
+            text-align: center;
+        }
+        .sub-title {
+            font-size: 24px;
+            color: #4A90E2;
+            margin-top: 20px;
+        }
+        .section {
+            background-color: #f9f9f9;
+            padding: 15px;
+            border-radius: 10px;
+            margin-top: 20px;
+        }
+        .section h2 {
+            font-size: 22px;
+            color: #4A90E2;
+        }
+        .section p, .section ul {
+            color: #666666;
+        }
+        .link {
+            color: #4A90E2;
+            text-decoration: none;
+        }
+    </style>
+""", unsafe_allow_html=True)
+
+# Introduction
+st.markdown('<div class="main-title">Keyword Extraction from Texts with Python and Spark NLP</div>', unsafe_allow_html=True)
+
+st.markdown("""
+<div class="section">
+    <p>Welcome to the Spark NLP Keyword Extraction Demo App! Keyword extraction is a technique in natural language processing (NLP) that involves automatically identifying the most important words or phrases in a document or corpus. Keywords extracted from a text can be used in a variety of ways, including:</p>
+    <ul>
+        <li>Document indexing</li>
+        <li>Document summarization</li>
+        <li>Content categorization</li>
+        <li>Content tagging</li>
+        <li>Search engine optimization</li>
+    </ul>
+    <p>This app demonstrates how to use Spark NLP's YakeKeywordExtraction annotator to perform keyword extraction using Python.</p>
+</div>
+""", unsafe_allow_html=True)
+
+# About Keyword Extraction
+st.markdown('<div class="sub-title">About Keyword Extraction</div>', unsafe_allow_html=True)
+st.markdown("""
+<div class="section">
+    <p>Extracting keywords from texts has become difficult for individuals and organizations as the complexity and volume of information have grown. The need to automate this task so that text can be processed promptly and adequately has led to the emergence of automatic keyword extraction tools. NLP and Python libraries help in the process.</p>
+</div>
+""", unsafe_allow_html=True)
+
+# Using YakeKeywordExtraction in Spark NLP
+st.markdown('<div class="sub-title">Using YakeKeywordExtraction in Spark NLP</div>', unsafe_allow_html=True)
+st.markdown("""
+<div class="section">
+    <p>Yake! is a novel feature-based system for multi-lingual keyword extraction, which supports texts of different sizes, domains, or languages. Unlike other approaches, Yake! does not rely on dictionaries or thesauri, nor is it trained against any corpora. Instead, it follows an unsupervised approach which builds upon features extracted from the text.</p>
+</div>
+""", unsafe_allow_html=True)
+
+st.markdown('<h2 class="sub-title">Example Usage in Python</h2>', unsafe_allow_html=True)
+st.markdown('<p>Here’s how you can implement keyword extraction using the YakeKeywordExtraction annotator in Spark NLP:</p>', unsafe_allow_html=True)
+
+# Setup Instructions
+st.markdown('<div class="sub-title">Setup</div>', unsafe_allow_html=True)
+st.markdown('<p>To install Spark NLP and extract keywords in Python, simply use your favorite package manager (conda, pip, etc.). For example:</p>', unsafe_allow_html=True)
+st.code("""
+pip install spark-nlp
+pip install pyspark
+""", language="bash")
+
+st.markdown("<p>Then, import Spark NLP and start a Spark session:</p>", unsafe_allow_html=True)
+st.code("""
+import sparknlp
+
+# Start Spark Session
+spark = sparknlp.start()
+""", language='python')
+
+# Keyword Extraction Example
+st.markdown('<div class="sub-title">Example Usage: Keyword Extraction with YakeKeywordExtraction</div>', unsafe_allow_html=True)
+st.code('''
+from sparknlp.base import DocumentAssembler, Pipeline
+from sparknlp.annotator import SentenceDetector, Tokenizer, YakeKeywordExtraction
+import pyspark.sql.functions as F
+
+# Step 1: Transforms raw texts to document annotation
+document = DocumentAssembler() \\
+    .setInputCol("text") \\
+    .setOutputCol("document")
+
+# Step 2: Sentence Detection
+sentenceDetector = SentenceDetector() \\
+    .setInputCols(["document"]) \\
+    .setOutputCol("sentence")
+
+# Step 3: Tokenization
+token = Tokenizer() \\
+    .setInputCols(["sentence"]) \\
+    .setOutputCol("token") \\
+    .setContextChars(["(", ")", "?", "!", ".", ","])
+
+# Step 4: Keyword Extraction
+keywords = YakeKeywordExtraction() \\
+    .setInputCols(["token"]) \\
+    .setOutputCol("keywords")
+
+# Define the pipeline
+yake_pipeline = Pipeline(stages=[document, sentenceDetector, token, keywords])
+
+# Create an empty dataframe
+empty_df = spark.createDataFrame([['']]).toDF("text")
+
+# Fit the dataframe to get the model
+yake_model = yake_pipeline.fit(empty_df)
+
+# Using LightPipeline
+from sparknlp.base import LightPipeline
+
+light_model = LightPipeline(yake_model)
+text = """
+google is acquiring data science community kaggle. Sources tell us that google is acquiring kaggle, a platform that hosts data science and machine learning competitions. Details about the transaction remain somewhat vague , but given that google is hosting its Cloud Next conference in san francisco this week, the official announcement could come as early as tomorrow. Reached by phone, kaggle co-founder ceo anthony goldbloom declined to deny that the acquisition is happening. google itself declined 'to comment on rumors'. kaggle, which has about half a million data scientists on its platform, was founded by Goldbloom and Ben Hamner in 2010. The service got an early start and even though it has a few competitors like DrivenData, TopCoder and HackerRank, it has managed to stay well ahead of them by focusing on its specific niche. The service is basically the de facto home for running data science and machine learning competitions. With kaggle, google is buying one of the largest and most active communities for data scientists - and with that, it will get increased mindshare in this community, too (though it already has plenty of that thanks to Tensorflow and other projects). kaggle has a bit of a history with google, too, but that's pretty recent. Earlier this month, google and kaggle teamed up to host a $100,000 machine learning competition around classifying YouTube videos. That competition had some deep integrations with the google Cloud platform, too. Our understanding is that google will keep the service running - likely under its current name. While the acquisition is probably more about Kaggle's community than technology, kaggle did build some interesting tools for hosting its competition and 'kernels', too. On kaggle, kernels are basically the source code for analyzing data sets and developers can share this code on the platform (the company previously called them 'scripts'). Like similar competition-centric sites, kaggle also runs a job board, too. It's unclear what google will do with that part of the service. According to Crunchbase, kaggle raised $12.5 million (though PitchBook says it's $12.75) since its launch in 2010. Investors in kaggle include Index Ventures, SV Angel, Max Levchin, Naval Ravikant, google chief economist Hal Varian, Khosla Ventures and Yuri Milner
+"""
+
+light_result = light_model.fullAnnotate(text)[0]
+
+import pandas as pd
+
+keys_df = pd.DataFrame([(k.result, k.begin, k.end, k.metadata['score'], k.metadata['sentence']) for k in light_result['keywords']],
+                       columns=['keywords', 'begin', 'end', 'score', 'sentence'])
+keys_df['score'] = keys_df['score'].astype(float)
+# ordered by relevance
+keys_df.sort_values(['sentence', 'score']).head(100)
+''', language='python')
+
+data = {
+    "Keyword": ["data science", "acquiring data", "google is acquiring", "community kaggle", "science community", "acquiring data science", "data science", "machine learning", "learning competitions", "acquiring kaggle", "google is acquiring", "hosts data", "science and machine", "google cloud", "cloud platform", "google cloud platform", "index ventures", "khosla ventures", "yuri milner", "sv angel", "max levchin", "naval ravikant", "hal varian", "cloud next", "next conference", "cloud next conference", "goldbloom declined", "anthony goldbloom", "data scientists", "ben hamner", "million data", "data science", "machine learning", "learning competitions", "running data", "science and machine", "data scientists", "machine learning"],
+    "Begin": [21, 11, 1, 34, 26, 11, 123, 140, 148, 83, 73, 117, 128, 1450, 1457, 1450, 2197, 2287, 2307, 2213, 2223, 2236, 2275, 262, 268, 262, 419, 411, 567, 629, 559, 895, 912, 920, 887, 900, 1024, 1333],
+    "End": [32, 24, 19, 49, 42, 32, 134, 155, 168, 98, 91, 126, 146, 1461, 1470, 1470, 2210, 2301, 2317, 2220, 2233, 2249, 2284, 271, 282, 282, 436, 427, 581, 638, 570, 906, 927, 940, 898, 918, 1038, 1348],
+    "Score": [0.255856, 0.844244, 1.039254, 1.040628, 1.152803, 1.263860, 0.255856, 0.466911, 0.762934, 0.849239, 1.039254, 1.203691, 1.257900, 0.611960, 0.796338, 1.070615, 0.904638, 0.904638, 1.008957, 1.045587, 1.045587, 1.045587, 1.045587, 0.514866, 0.994605, 1.242611, 1.078377, 1.323419, 0.562581, 0.623279, 1.210565, 0.255856, 0.466911, 0.762934, 1.183755, 1.257900, 0.562581, 0.466911],
+    "Sentence": [0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 10, 10, 10, 17, 17, 17, 17, 17, 17, 17, 2, 2, 2, 3, 3, 4, 4, 4, 6, 6, 6, 6, 6, 7, 9]
+}
+
+df = pd.DataFrame(data)
+
+st.markdown(
+    """
+    <style>
+    .stTable {
+        margin-left: auto;
+        margin-right: auto;
+    }
+    </style>
+    """,
+    unsafe_allow_html=True
+)
+
+with st.expander("View Data Table"):
+    st.table(df)
+
+st.markdown("""
+<p>The code snippet demonstrates how to set up a pipeline in Spark NLP to perform keyword extraction on text data using the YakeKeywordExtraction annotator. The resulting DataFrame contains the keywords and their corresponding scores.</p>
+""", unsafe_allow_html=True)
+
+# Highlighting Keywords in a Text
+st.markdown('<div class="sub-title">Highlighting Keywords in a Text</div>', unsafe_allow_html=True)
+st.markdown("""
+<div class="section">
+    <p>In addition to getting the keywords as a dataframe, it is also possible to highlight the extracted keywords in the text.</p>
+    <p>In this example, a dataset of 7537 texts were used — samples from the PubMed, which is a free resource supporting the search and retrieval of biomedical and life sciences literature.</p>
+</div>
+""", unsafe_allow_html=True)
+
+st.code("""
+import re
+from pyspark.sql.functions import udf
+from pyspark.sql.types import StringType
+
+def highlight_keywords(text, keywords):
+    for keyword in keywords:
+        text = re.sub(fr'\\b{keyword}\\b', f'**{keyword}**', text, flags=re.IGNORECASE)
+    return text
+
+highlight_udf = udf(highlight_keywords, StringType())
+
+df_with_highlights = df.withColumn("highlighted_text", highlight_udf("text", "keywords"))
+df_with_highlights.select("highlighted_text").show(truncate=False)
+""", language='python')
+
+# Conclusion
+st.markdown('<div class="sub-title">Conclusion</div>', unsafe_allow_html=True)
+st.markdown("""
+<div class="section">
+    <p>In this demo, we demonstrated how to extract keywords from texts using the YakeKeywordExtraction annotator in Spark NLP. We provided step-by-step instructions on setting up the environment, creating a pipeline, and running the keyword extraction. Additionally, we explored how to highlight extracted keywords in the text.</p>
+</div>
+""", unsafe_allow_html=True)
+
+# References and Additional Information
+st.markdown('<div class="sub-title">For additional information, please check the following references.</div>', unsafe_allow_html=True)
+
+st.markdown("""
+<div class="section">
+    <ul>
+        <li>Documentation <a class="link" href="https://nlp.johnsnowlabs.com/docs/en/annotators#yakekeywordextraction" target="_blank" rel="noopener">YakeKeywordExtraction</a></li>
+        <li>Python keyword extraction: Docs about are <a class="link" href="https://nlp.johnsnowlabs.com/api/python/reference/autosummary/sparknlp/annotator/keyword_extraction/yake_keyword_extraction/index.html" target="_blank" rel="noopener">here</a></li>
+        <li>Scala Docs: <a class="link" href="https://nlp.johnsnowlabs.com/api/com/johnsnowlabs/nlp/annotators/keyword/yake/YakeKeywordExtraction.html">YakeKeywordExtraction</a></li>
+        <li>For extended examples of usage, see the <a class="link" href="https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Public/8.Keyword_Extraction_YAKE.ipynb" target="_blank" rel="noopener nofollow">Spark NLP Workshop repository</a>.</li>
+        <li>Reference Paper: <a class="link" href="https://www.sciencedirect.com/science/article/abs/pii/S0020025519308588" target="_blank" rel="noopener nofollow">YAKE! Keyword extraction from single documents using multiple local features</a></li>
+        </ul>
+</div>
+""", unsafe_allow_html=True)
+
+st.markdown('<div class="sub-title">Community & Support</div>', unsafe_allow_html=True)
+st.markdown("""
+<div class="section">
+    <ul>
+        <li><a class="link" href="https://sparknlp.org/" target="_blank">Official Website</a>: Documentation and examples</li>
+        <li><a class="link" href="https://join.slack.com/t/spark-nlp/shared_invite/zt-198dipu77-L3UWNe_AJ8xqDk0ivmih5Q" target="_blank">Slack</a>: Live discussion with the community and team</li>
+        <li><a class="link" href="https://github.com/JohnSnowLabs/spark-nlp" target="_blank">GitHub</a>: Bug reports, feature requests, and contributions</li>
+        <li><a class="link" href="https://medium.com/spark-nlp" target="_blank">Medium</a>: Spark NLP articles</li>
+        <li><a class="link" href="https://www.youtube.com/channel/UCmFOjlpYEhxf_wJUDuz6xxQ/videos" target="_blank">YouTube</a>: Video tutorials</li>
+    </ul>
+</div>
+""", unsafe_allow_html=True)
+

requirements.txt

@@ -0,0 +1,5 @@
+streamlit
+pandas
+numpy
+spark-nlp
+pyspark