import streamlit as st import sparknlp import os import pandas as pd from sparknlp.base import * from sparknlp.annotator import * from pyspark.ml import Pipeline from sparknlp.pretrained import PretrainedPipeline # Page configuration st.set_page_config( layout="wide", initial_sidebar_state="auto" ) # CSS for styling st.markdown(""" """, unsafe_allow_html=True) @st.cache_resource def init_spark(): return sparknlp.start() @st.cache_resource def create_pipeline(model): document_assembler = DocumentAssembler()\ .setInputCol("text")\ .setOutputCol("document") sentence_detector = SentenceDetector() \ .setInputCols(["document"]) \ .setOutputCol("sentence") tokenizer = Tokenizer() \ .setInputCols(["sentence"]) \ .setOutputCol("token") word_embeddings = WordEmbeddingsModel()\ .pretrained('urduvec_140M_300d', 'ur')\ .setInputCols(["sentence",'token'])\ .setOutputCol("word_embeddings") sentence_embeddings = SentenceEmbeddings() \ .setInputCols(["sentence", "word_embeddings"]) \ .setOutputCol("sentence_embeddings") \ .setPoolingStrategy("AVERAGE") classifier = SentimentDLModel.pretrained('sentimentdl_urduvec_imdb', 'ur' )\ .setInputCols(['sentence_embeddings'])\ .setOutputCol('sentiment') nlpPipeline = Pipeline( stages=[ document_assembler, sentence_detector, tokenizer, word_embeddings, sentence_embeddings, classifier ]) return nlpPipeline def fit_data(pipeline, data): empty_df = spark.createDataFrame([['']]).toDF('text') pipeline_model = pipeline.fit(empty_df) model = LightPipeline(pipeline_model) results = model.fullAnnotate(data)[0] return results['sentiment'][0].result # Set up the page layout st.markdown('
State-of-the-Art Urdu Sentiment Detection with Spark NLP
', unsafe_allow_html=True) # Sidebar content model = st.sidebar.selectbox( "Choose the pretrained model", ["sentimentdl_urduvec_imdb"], help="For more info about the models visit: https://sparknlp.org/models" ) # Reference notebook link in sidebar link = """ Open In Colab """ st.sidebar.markdown('Reference notebook:') st.sidebar.markdown(link, unsafe_allow_html=True) # Load examples folder_path = f"inputs/{model}" examples = [ lines[1].strip() for filename in os.listdir(folder_path) if filename.endswith('.txt') for lines in [open(os.path.join(folder_path, filename), 'r', encoding='utf-8').readlines()] if len(lines) >= 2 ] selected_text = st.selectbox("Select a sample", examples) custom_input = st.text_input("Try it for yourself!") if custom_input: selected_text = custom_input elif selected_text: selected_text = selected_text st.subheader('Selected Text') st.write(selected_text) # Initialize Spark and create pipeline spark = init_spark() pipeline = create_pipeline(model) output = fit_data(pipeline, selected_text) # Display output sentence if output.lower() in ['pos', 'positive']: st.markdown("""

This seems like a {} text. 😃

""".format('positive'), unsafe_allow_html=True) elif output.lower() in ['neg', 'negative']: st.markdown("""

This seems like a {} text. 😠""".format('negative'), unsafe_allow_html=True)