app.py

import gradio as gr

import os
import pandas as pd
import matplotlib.pyplot as plt
import numpy as np
import nltk, spacy, gensim
from sklearn.decomposition import LatentDirichletAllocation
from sklearn.feature_extraction.text import CountVectorizer
from pprint import pprint
import matplotlib
matplotlib.use('agg')

def concat_comments(*kwargs):
    return ['\n'.join(ele) for ele in zip(*kwargs)]

def sent_to_words(sentences):
    for sentence in sentences:
        yield(gensim.utils.simple_preprocess(str(sentence), deacc=True))  # deacc=True removes punctuations
        
def lemmatization(texts, allowed_postags=['NOUN', 'ADJ', 'VERB', 'ADV'], nlp=None): #'NOUN', 'ADJ', 'VERB', 'ADV'
    texts_out = []
    for sent in texts:
        doc = nlp(" ".join(sent))
        texts_out.append(" ".join([
            token.lemma_ if token.lemma_ not in ['-PRON-'] else '' for token in doc if token.pos_ in allowed_postags
        ]))
    return texts_out

def get_lda(n_components):
    df = pd.read_csv('./data/results.csv', index_col=0)
    data = concat_comments(df.subreddit, df.sup_comment, df.comment)
    data_words = list(sent_to_words(data))

    
    if not spacy.util.is_package("en_core_web_sm"):
        print('[x] en_core_web_sm not found, downloading...')
        os.system("python -m spacy download en_core_web_sm")
        print('[x] en_core_web_sm downloaded')
    
    print('[x] Lemmatization begins')
    nlp = spacy.load("en_core_web_sm", disable=["parser", "ner"])
    data_lemmatized = lemmatization(data_words, allowed_postags=["NOUN", "ADJ"], nlp=nlp) #select noun and verb

    print('[x] Vectorizing')
    vectorizer = CountVectorizer(
        analyzer='word',
        min_df=10,
        stop_words='english',
        lowercase=True,
        token_pattern='[a-zA-Z0-9]{3,}'
    )

    print('[x] Fitting vectorized data on lemmatization')
    data_vectorized = vectorizer.fit_transform(data_lemmatized)

    print('[x] Init LDA model')
    lda_model = LatentDirichletAllocation(
        n_components=5,
        max_iter=10,
        learning_method='online',
        random_state=100,
        batch_size=128,
        evaluate_every = -1,
        n_jobs = -1,
        verbose=1,
    )
    
    print('[x] Fitting LDA model')
    lda_output = lda_model.fit_transform(data_vectorized)
    print(lda_model)    # Model attributes

    print('[x] Getting performances')
    performances = lda_model.score(data_vectorized), lda_model.perplexity(data_vectorized)
    # Log Likelyhood: Higher the better
    print("Log Likelihood: ", performances[0])
    # Perplexity: Lower the better. Perplexity = exp(-1. * log-likelihood per word)
    print("Perplexity: ",  performances[1])

    print('[x] Check parameters if they look correct')
    # See model parameters
    pprint(lda_model.get_params())

    # switching to the best model
    best_lda_model = lda_model

    print('[x] Getting LDA output')
    lda_output = best_lda_model.transform(data_vectorized)

    topicnames = ["Topic" + str(i) for i in range(best_lda_model.n_components)]
    docnames = ["Doc" + str(i) for i in range(len(data))]
    df_document_topic = pd.DataFrame(np.round(lda_output, 2), columns=topicnames, index=docnames)

    dominant_topic = np.argmax(df_document_topic.values, axis=1)
    df_document_topic["dominant_topic"] = dominant_topic

    # Topic-Keyword Matrix
    df_topic_keywords = pd.DataFrame(best_lda_model.components_)
    df_topic_keywords
    # Assign Column and Index
    df_topic_keywords.columns = vectorizer.get_feature_names_out()
    df_topic_keywords.index = topicnames

    # Show top n keywords for each topic
    def show_topics(vectorizer=vectorizer, lda_model=lda_model, n_words=20):
        keywords = np.array(vectorizer.get_feature_names_out())
        topic_keywords = []
        for topic_weights in lda_model.components_:
            top_keyword_locs = (-topic_weights).argsort()[:n_words]
            topic_keywords.append(keywords.take(top_keyword_locs))
        return topic_keywords
    topic_keywords = show_topics(vectorizer=vectorizer, lda_model=best_lda_model, n_words=15)
    # Topic - Keywords Dataframe
    df_topic_keywords = pd.DataFrame(topic_keywords)
    df_topic_keywords.columns = ['Word '+str(i) for i in range(df_topic_keywords.shape[1])]
    df_topic_keywords.index = ['Topic '+str(i) for i in range(df_topic_keywords.shape[0])]
    df_topic_keywords

    topics = [
        f'Topic {i}' for i in range(len(df_topic_keywords))
    ]
    df_topic_keywords["Topics"] = topics
    df_topic_keywords

    # Define function to predict topic for a given text document.
    def predict_topic(text, nlp=nlp):
        global sent_to_words
        global lemmatization
        # Step 1: Clean with simple_preprocess
        mytext_2 = list(sent_to_words(text))
        # Step 2: Lemmatize
        mytext_3 = lemmatization(mytext_2, allowed_postags=['NOUN', 'ADJ', 'VERB', 'ADV'], nlp=nlp)
        # Step 3: Vectorize transform
        mytext_4 = vectorizer.transform(mytext_3)
        # Step 4: LDA Transform
        topic_probability_scores = best_lda_model.transform(mytext_4)
        topic = df_topic_keywords.iloc[np.argmax(topic_probability_scores), 1:14].values.tolist()
        
        # Step 5: Infer Topic
        infer_topic = df_topic_keywords.iloc[np.argmax(topic_probability_scores), -1]
        
        #topic_guess = df_topic_keywords.iloc[np.argmax(topic_probability_scores), Topics]
        return infer_topic, topic, topic_probability_scores

    # Predict the topic
    mytext = ["This is a test of a random topic where I talk about politics"]
    infer_topic, topic, prob_scores = predict_topic(text = mytext, nlp=nlp)

    def apply_predict_topic(text):
        text = [text]
        infer_topic, topic, prob_scores = predict_topic(text = text, nlp=nlp)
        return(infer_topic)

    df["Topic_key_word"] = df['comment'].apply(apply_predict_topic)


    # plot
    subreddits = df.subreddit.value_counts().index[:22]

    weight_counts = {
        t: [
            df[df.Topic_key_word == t].subreddit.value_counts()[subreddit] / df.subreddit.value_counts()[subreddit] for subreddit in subreddits
        ] for t in topics
    }

    irony_percs = {
        t: [
            len(
                df[df.subreddit == subreddit][(df[df.subreddit == subreddit].Topic_key_word == t) & (df[df.subreddit == subreddit].label == 1)]
            ) / 
            len(
                df[df.subreddit == subreddit]
            ) for subreddit in subreddits
        ] for t in topics
    }
    width = 0.9

    fig, ax = plt.subplots(figsize = (10, 7))
    plt.axhline(0.5, color = 'red', ls=":", alpha = .3)

    bottom = np.zeros(len(subreddits))

    for k, v in weight_counts.items():
        p = ax.bar(subreddits, v, width, label=k, bottom=bottom)
        ax.bar(subreddits, irony_percs[k], width - 0.01, bottom=bottom, color = 'black', edgecolor = 'white', alpha = .2, hatch = '\\')
        bottom += v

    ax.set_title("Perc of topics for each subreddit")
    ax.legend(loc="upper right")
    plt.xticks(rotation=70)

    return fig


# def main():

    


with gr.Blocks() as demo:
    gr.Markdown("# Dashboard per l'analisi con LDA")
    gr.Markdown("### Questo è un sottotitolo")
    # gradio.Dataframe(···)
    
    n_comp = gr.Slider(2, 25, value=5, step = 1, label="N components", info="Scegli il numero di componenti per LDA"),

    btn = gr.Button(value="Submit")
    
    plot = gr.Plot(label="Plot")

    btn.click(get_lda, inputs=[n_comp[0]], outputs=[plot])

    
    

    # demo.load(main, inputs=[], outputs=[plot])


# iface = gr.Interface(fn=greet, inputs="text", outputs="text")
if __name__ == "__main__":
    demo.launch()