import gradio as gr import os import pandas as pd import matplotlib.pyplot as plt import numpy as np import nltk, spacy, gensim from sklearn.decomposition import LatentDirichletAllocation from sklearn.feature_extraction.text import CountVectorizer from pprint import pprint import matplotlib matplotlib.use('agg') def concat_comments(*kwargs): return ['\n'.join(ele) for ele in zip(*kwargs)] def sent_to_words(sentences): for sentence in sentences: yield(gensim.utils.simple_preprocess(str(sentence), deacc=True)) # deacc=True removes punctuations def lemmatization(texts, allowed_postags=['NOUN', 'ADJ', 'VERB', 'ADV'], nlp=None): #'NOUN', 'ADJ', 'VERB', 'ADV' texts_out = [] for sent in texts: doc = nlp(" ".join(sent)) texts_out.append(" ".join([ token.lemma_ if token.lemma_ not in ['-PRON-'] else '' for token in doc if token.pos_ in allowed_postags ])) return texts_out def get_lda(n_components, n_top_subreddit_to_analyse, what_label_to_use): df = pd.read_csv('./data/results.csv', index_col=0) data = concat_comments(df.subreddit, df.sup_comment, df.comment) data_words = list(sent_to_words(data)) if what_label_to_use == 'Use True label': label = 'label' else: label = 'prediction' if not spacy.util.is_package("en_core_web_sm"): print('[x] en_core_web_sm not found, downloading...') os.system("python -m spacy download en_core_web_sm") print('[x] en_core_web_sm downloaded') print('[x] Lemmatization begins') nlp = spacy.load("en_core_web_sm", disable=["parser", "ner"]) data_lemmatized = lemmatization(data_words, allowed_postags=["NOUN", "ADJ"], nlp=nlp) #select noun and verb print('[x] Vectorizing') vectorizer = CountVectorizer( analyzer='word', min_df=10, stop_words='english', lowercase=True, token_pattern='[a-zA-Z0-9]{3,}' ) print('[x] Fitting vectorized data on lemmatization') data_vectorized = vectorizer.fit_transform(data_lemmatized) print('[x] Init LDA model') lda_model = LatentDirichletAllocation( n_components=n_components, max_iter=10, learning_method='online', random_state=100, batch_size=128, evaluate_every = -1, n_jobs = -1, verbose=1, ) print('[x] Fitting LDA model') lda_output = lda_model.fit_transform(data_vectorized) print(lda_model) # Model attributes print('[x] Getting performances') performances = lda_model.score(data_vectorized), lda_model.perplexity(data_vectorized) # Log Likelyhood: Higher the better print("Log Likelihood: ", performances[0]) # Perplexity: Lower the better. Perplexity = exp(-1. * log-likelihood per word) print("Perplexity: ", performances[1]) print('[x] Check parameters if they look correct') # See model parameters pprint(lda_model.get_params()) # switching to the best model best_lda_model = lda_model print('[x] Getting LDA output') lda_output = best_lda_model.transform(data_vectorized) print('[x] Assigning topics') topicnames = ["Topic" + str(i) for i in range(best_lda_model.n_components)] docnames = ["Doc" + str(i) for i in range(len(data))] df_document_topic = pd.DataFrame(np.round(lda_output, 2), columns=topicnames, index=docnames) print('[x] Checking dominant topics') dominant_topic = np.argmax(df_document_topic.values, axis=1) df_document_topic["dominant_topic"] = dominant_topic # Topic-Keyword Matrix df_topic_keywords = pd.DataFrame(best_lda_model.components_) df_topic_keywords # Assign Column and Index df_topic_keywords.columns = vectorizer.get_feature_names_out() df_topic_keywords.index = topicnames print('[x] Computing word-topic association') # Show top n keywords for each topic def show_topics(vectorizer=vectorizer, lda_model=lda_model, n_words=20): keywords = np.array(vectorizer.get_feature_names_out()) topic_keywords = [] for topic_weights in lda_model.components_: top_keyword_locs = (-topic_weights).argsort()[:n_words] topic_keywords.append(keywords.take(top_keyword_locs)) return topic_keywords topic_keywords = show_topics(vectorizer=vectorizer, lda_model=best_lda_model, n_words=15) # Topic - Keywords Dataframe df_topic_keywords = pd.DataFrame(topic_keywords) df_topic_keywords.columns = ['Word '+str(i) for i in range(df_topic_keywords.shape[1])] df_topic_keywords.index = ['Topic '+str(i) for i in range(df_topic_keywords.shape[0])] df_topic_keywords topics = [ f'Topic {i}' for i in range(len(df_topic_keywords)) ] df_topic_keywords["Topics"] = topics df_topic_keywords print('[x] Predicting dominant topic for each document') # Define function to predict topic for a given text document. def predict_topic(text, nlp=nlp): global sent_to_words global lemmatization # Step 1: Clean with simple_preprocess mytext_2 = list(sent_to_words(text)) # Step 2: Lemmatize mytext_3 = lemmatization(mytext_2, allowed_postags=['NOUN', 'ADJ', 'VERB', 'ADV'], nlp=nlp) # Step 3: Vectorize transform mytext_4 = vectorizer.transform(mytext_3) # Step 4: LDA Transform topic_probability_scores = best_lda_model.transform(mytext_4) topic = df_topic_keywords.iloc[np.argmax(topic_probability_scores), 1:14].values.tolist() # Step 5: Infer Topic infer_topic = df_topic_keywords.iloc[np.argmax(topic_probability_scores), -1] #topic_guess = df_topic_keywords.iloc[np.argmax(topic_probability_scores), Topics] return infer_topic, topic, topic_probability_scores # # Predict the topic # mytext = ["This is a test of a random topic where I talk about politics"] # infer_topic, topic, prob_scores = predict_topic(text = mytext, nlp=nlp) def apply_predict_topic(text): text = [text] infer_topic, topic, prob_scores = predict_topic(text = text, nlp=nlp) return(infer_topic) df["Topic_key_word"] = df['comment'].apply(apply_predict_topic) print('[x] Generating plot [1]') print('Percentuale di commenti ironici per ogni topic') perc_topic_irony = {} for t in topics: total_0label = sum((df[label] == 1) & (df.Topic_key_word == t)) if total_0label != 0: total_X_topic = df.Topic_key_word.value_counts()[t] else: total_0label, total_X_topic = 0, 0.001 # Non ci cono topic nel dataset perc_topic_irony[t] = total_0label / total_X_topic print(f'{t} w/ label 1: {total_0label}/{total_X_topic} ({total_0label / total_X_topic * 100 :.2f}%)') fig1, ax = plt.subplots(figsize = (10, 7)) bottom = np.zeros(len(perc_topic_irony)) width = 0.9 ax.bar(perc_topic_irony.keys(), perc_topic_irony.values(), width, label = 'sarcastic') comp = list(map(lambda x: 1 - x if x > 0 else 0, perc_topic_irony.values())) ax.bar(perc_topic_irony.keys(), comp, width, bottom=list(perc_topic_irony.values()), label = 'not sarcastic') ax.set_title("% of sarcastic comments for each topic") plt.xticks(rotation=70) plt.legend() plt.axhline(0.5, color = 'red', ls=":") # probably not necessary (?) To drop eventually if log are to much cluttered! print('Percentage of each topic for each subreddit') weight_counts = {} for t in topics: weight_counts[t] = [] for subreddit in df['subreddit'].value_counts().index[:n_top_subreddit_to_analyse]: # first 10 big subreddits if sum(df[df.Topic_key_word == t].subreddit == subreddit) > 0: # se ci sono subreddit per il topic t (almeno una riga nel df) perc_sub = df[df.Topic_key_word == t]['subreddit'].value_counts()[subreddit] / df['subreddit'].value_counts()[subreddit] else: perc_sub = 0 weight_counts[t].append(perc_sub) print(f'Perc of topic {t} in subreddit {subreddit}: {perc_sub * 100:.2f}') print() print('[x] Generating plot [2]') # plot subreddits = list(df.subreddit.value_counts().index)[:n_top_subreddit_to_analyse] irony_percs = { t: [ len( df[df.subreddit == subreddit][(df[df.subreddit == subreddit].Topic_key_word == t) & (df[df.subreddit == subreddit][label] == 1)] ) / len( df[df.subreddit == subreddit] ) for subreddit in subreddits ] for t in topics } width = 0.9 fig2, ax = plt.subplots(figsize = (10, 7)) plt.axhline(0.5, color = 'red', ls=":", alpha = .3) bottom = np.zeros(len(subreddits)) for k, v in weight_counts.items(): p = ax.bar(subreddits, v, width, label=k, bottom=bottom) ax.bar(subreddits, irony_percs[k], width - 0.01, bottom=bottom, color = 'black', edgecolor = 'white', alpha = .2, hatch = '\\') bottom += v ax.set_title("% of topics for each subreddit") ax.legend(loc="upper right") plt.xticks(rotation=50) print('[v] All looking good!') return df_topic_keywords, fig1, fig2 # def main(): with gr.Blocks() as demo: gr.Markdown("# Dashboard per l'analisi con LDA") gr.Markdown("### La dashboard permette l'addestramento di un modello LDA per controllare se e quali topic sono più propensi a commenti di tipo sarcastico") # gradio.Dataframe(···) inputs = [] with gr.Row(): inputs.append(gr.Slider(2, 25, value=5, step = 1, label="LDA N components", info="Scegli il numero di componenti per LDA")) inputs.append(gr.Slider(2, 20, value=5, step = 1, label="Subreddit dal dataset", info="Numero di subreddit da analizzare")) inputs.append(gr.Radio( choices = ['Use True label', 'Use BERT prediction'], value = 'Use True label', label = "Scegliere quali label sull'ironia utilizzare:", ) ) btn = gr.Button(value="Submit") gr.Markdown("## Risulati ottenuti") gr.Markdown("#### Top 15 parole che più contribuiscono al topic di riferimento (utlima colonna):") btn.click( get_lda, inputs=inputs, outputs=[ gr.DataFrame(), gr.Plot(label="Quanto i topic trovati portano ironia?"), gr.Plot(label="Come i topic sono correlati ai diversi subreddit del dataset?"), ] ) # iface = gr.Interface(fn=greet, inputs="text", outputs="text") if __name__ == "__main__": demo.launch()