first commit

app.py

@@ -0,0 +1,171 @@
+import streamlit as st
+import preprocessor
+import helper
+import matplotlib.pyplot as plt
+import seaborn as sns
+
+
+
+def main():
+    st.sidebar.title("Whatsapp Chat Analyzer")
+    uploaded_file = st.sidebar.file_uploader("Choose a file")
+    if uploaded_file is not None:
+        # To read file as bytes:
+        bytes_data = uploaded_file.getvalue()
+        data = bytes_data.decode("utf-8")
+        df = preprocessor.preprocess(data)
+
+        #fetch unique users
+        user_list = df['user'].unique().tolist()
+        user_list.remove('group_notification')
+        user_list.sort()
+        user_list.insert(0,"Overall")
+        selected_user = st.sidebar.selectbox("Show analysis wrt",user_list)
+        if st.sidebar.button("Show Analysis"):
+            num_messages,words, num_media_messages ,num_links = helper.fetch_stats(selected_user,df)
+            st.title("Top Statistics")
+            col1, col2 , col3, col4 = st.columns(4)
+
+            with col1:
+                st.header("Total Messages")
+                st.title(num_messages)
+            with col2:
+                st.header("Total Words")
+                st.title(words)
+            with col3:
+                st.header("Media shared")
+                st.title(num_media_messages)
+            with col4:
+                st.header("Links shared")
+                st.title(num_links)
+
+            #monthly_timeline
+            st.title("Monthly Timeline")
+            timeline=helper.monthly_timeline(selected_user,df)
+            fig = plt.figure()
+            sns.set_style('darkgrid')
+            sns.lineplot(x=timeline['time'],y=timeline['message'],color='red')
+            plt.xticks(rotation='vertical')
+            st.pyplot(fig)
+
+            # daily timeline
+            st.title("Daily Timeline")
+            daily_timeline = helper.daily_timeline(selected_user, df)
+            fig = plt.figure(figsize=(10, 3))
+            sns.set_style('whitegrid')
+            sns.lineplot(x=daily_timeline['only_date'],y=daily_timeline['message'], color='purple')
+            plt.xticks(rotation='vertical')
+            plt.xlabel("date")
+            st.pyplot(fig)
+
+            #activity map
+            st.title('Activity Map')
+            col1,col2 = st.columns(2)
+
+            with col1:
+                st.header("Most busy day")
+                busy_day=helper.week_activity_map(selected_user,df)
+                fig = plt.figure()
+                sns.set_style('ticks')
+                pal=sns.cubehelix_palette(start=2, rot=0, dark=0.5, light=0.9, reverse=True)
+                sns.barplot(x=busy_day.index,y=busy_day.values,palette=pal)
+                plt.ylabel("messages")
+                plt.xticks(rotation='vertical')
+                st.pyplot(fig)
+
+            with col2:
+                st.header("Most busy month")
+                busy_month=helper.month_activity_map(selected_user,df)
+                fig=plt.figure()
+                sns.set_style('ticks')
+                pal=sns.cubehelix_palette(start=0, rot=0, dark=0.2, light=0.9, reverse=True)
+                sns.barplot(x=busy_month.index,y=busy_month.values,palette=pal)
+                plt.ylabel("messages")
+                plt.xticks(rotation='vertical')
+                st.pyplot(fig)
+
+            st.title("Weekly Activity Map")
+            user_heatmap = helper.activity_heatmap(selected_user, df)
+            fig = plt.figure(figsize=(13,4))
+            cmap = sns.color_palette("viridis", as_cmap=True)
+            sns.heatmap(user_heatmap,cmap=cmap,square=True)
+            st.pyplot(fig)
+
+            #finding active users
+            if(selected_user=='Overall'):
+                st.title('Most active users')
+                x,new_df=helper.most_busy_users(df)
+                fig=plt.figure()
+
+                col1,col2 = st.columns(2)
+
+                with col1:
+                    pal = sns.color_palette("cubehelix")
+                    sns.barplot(x=x.index,y=x.values,palette=pal)
+                    plt.xticks(rotation='vertical')
+                    plt.ylabel('messages')
+                    st.pyplot(fig)
+                with col2:
+                    st.dataframe(new_df)
+            #word cloud
+            st.title('WordCloud')
+            df_wc = helper.create_wordcloud(selected_user,df)
+            fig,ax = plt.subplots()
+            ax.imshow(df_wc)
+            st.pyplot(fig)
+
+            #most common words
+            st.title('Most Common Words')
+            most_common_df=helper.most_common_words(selected_user,df)
+            colors = ['#e6194b', '#3cb44b', '#ffe119', '#4363d8', '#f58231',
+                      '#911eb4', '#46f0f0', '#f032e6', '#bcf60c', '#fabebe',
+                      '#008080', '#e6beff', '#9a6324', '#fffac8', '#aaffc3',
+                      '#808000', '#ffd8b1', '#808080', 'lightgreen', 'lightblue']
+            # explosion
+
+            fig = plt.figure()
+            # Pie Chart
+            plt.pie(most_common_df[1], labels=most_common_df[0], colors=colors,
+                    autopct='%0.1f%%', pctdistance=0.9, labeldistance=1, rotatelabels=270, startangle=180,
+                    counterclock=False)
+            # draw circle
+            centre_circle = plt.Circle((0, 0), 0.50, fc='white')
+            fig2 = plt.gcf()
+            # Adding Circle in Pie chart
+            fig2.gca().add_artist(centre_circle)
+            st.pyplot(fig)
+
+            #emoji analysys
+
+            emoji_df = helper.emoji_helper(selected_user,df)
+            if(emoji_df.shape[0]):
+                st.title("Emoji Analysis")
+                col1,col2 =st.columns(2)
+                with col1:
+                    st.dataframe(emoji_df)
+                with col2:
+                    fig,ax = plt.subplots()
+                    plt.rcParams['font.family'] = 'Segoe UI Emoji'
+                    ax.pie(emoji_df[1].head(min(5,emoji_df.shape[0])),labels=emoji_df[0].head(min(5,emoji_df.shape[0])),autopct="%0.2f")
+                    st.pyplot(fig)
+
+            #birth_dates
+            if(selected_user=='Overall'):
+                birth_data = helper.birth_dates(df)
+                if(birth_data.shape[0]):
+                    st.title("Birth dates of some users.")
+                    st.dataframe(birth_data)
+
+            #Sentiment-analysis
+            if (selected_user != 'Overall'):
+                st.title("Sentiment Analysis")
+                sentiment_data,number=helper.sentiment_analysis(selected_user,df)
+                fig = plt.figure()
+                sns.set_style('ticks')
+                pal = sns.cubehelix_palette(start=0.5, rot=0, dark=0.2, light=0.9, reverse=True)
+                sns.barplot(x=sentiment_data.index,y=sentiment_data.values,palette=pal)
+                plt.xticks(rotation='vertical')
+                st.pyplot(fig)
+                st.header("Based on random "+str(number)+" messages.")
+                st.text("Note : Sentiment Analysis give good results if messages \nare in hinglish (hindi or english or both).")
+main()

helper.py

@@ -0,0 +1,161 @@
+from urlextract import URLExtract
+from wordcloud import WordCloud
+import pandas as pd
+from collections import Counter
+import emoji
+import re
+import numpy as np
+import torch
+
+extract = URLExtract()
+def fetch_stats(selected_user,df):
+    if(selected_user!='Overall'):
+        df=df[df['user']==selected_user]
+    num_messages = df.shape[0]
+    words = []
+    for message in df['message']:
+        words.extend(message.split())
+    num_media_messages = df[df['message']=='<Media omitted>\n'].shape[0]
+    links=[]
+    for message in df['message']:
+        links.extend(extract.find_urls(message))
+    return num_messages, len(words), num_media_messages ,len(links)
+
+def most_busy_users(df):
+    x = df['user'].value_counts()
+    x = x.head(min(10, len(x)))
+    new_df = round((df['user'].value_counts()/df.shape[0])*100,2).reset_index().rename(columns={'user':'name','count':'percent'})
+    return x,new_df
+
+def create_wordcloud(selected_user,df):
+
+    f = open('stop_hinglish.txt', 'r')
+    stop_words = f.read()
+
+    if selected_user != 'Overall':
+        df = df[df['user'] == selected_user]
+
+    temp = df[df['user'] != 'group_notification']
+    temp = temp[temp['message'] != '<Media omitted>\n']
+
+    def remove_stop_words(message):
+        y = []
+        for word in message.lower().split():
+            if word not in stop_words:
+                y.append(word)
+        return " ".join(y)
+
+    wc = WordCloud(width=500,height=500,min_font_size=10,background_color='white')
+    temp['message'] = temp['message'].apply(remove_stop_words)
+    df_wc = wc.generate(temp['message'].str.cat(sep=" "))
+    return df_wc
+
+def most_common_words(selected_user,df):
+
+    f = open('stop_hinglish.txt','r')
+    stop_words = f.read()
+
+    if selected_user != 'Overall':
+        df = df[df['user'] == selected_user]
+
+    temp = df[df['user'] != 'group_notification']
+    temp = temp[temp['message'] != '<Media omitted>\n']
+
+    words = []
+
+    for message in temp['message']:
+            for word in message.lower().split():
+                if (word not in stop_words):
+                    for c in word:
+                        if c not in emoji.UNICODE_EMOJI_ENGLISH:
+                            words.append(word)
+                            break
+
+    most_common_df = pd.DataFrame(Counter(words).most_common(20))
+    return most_common_df
+
+def emoji_helper(selected_user,df):
+    if (selected_user != 'Overall'):
+        df = df[df['user'] == selected_user]
+    emojis=[]
+    for message in df['message']:
+        emojis.extend([c for c in message if c in emoji.UNICODE_EMOJI_ENGLISH])
+    emoji_df = pd.DataFrame(Counter(emojis).most_common(len(Counter(emojis))))
+    return emoji_df
+
+def monthly_timeline(selected_user,df):
+    if (selected_user != 'Overall'):
+        df = df[df['user'] == selected_user]
+
+    timeline = df.groupby(['year','month_num','month']).count()['message'].reset_index()
+    time=[]
+    for i in range(timeline.shape[0]):
+        time.append(timeline['month'][i]+"-"+str(timeline['year'][i]))
+    timeline['time'] =time
+    return timeline
+
+def daily_timeline(selected_user,df):
+
+    if selected_user != 'Overall':
+        df = df[df['user'] == selected_user]
+
+    daily_timeline = df.groupby('only_date').count()['message'].reset_index()
+
+    return daily_timeline
+
+def week_activity_map(selected_user,df):
+    if selected_user != 'Overall':
+        df = df[df['user'] == selected_user]
+    return df['day_name'].value_counts()
+
+def month_activity_map(selected_user,df):
+    if selected_user != 'Overall':
+        df = df[df['user'] == selected_user]
+    return df['month'].value_counts()
+
+def activity_heatmap(selected_user,df):
+
+    if selected_user != 'Overall':
+        df = df[df['user'] == selected_user]
+
+    user_heatmap = df.pivot_table(index='day_name', columns='period', values='message', aggfunc='count').fillna(0)
+
+    return user_heatmap
+def birth_dates(df):
+    birthdates = []
+    names = []
+    for i in range(df.shape[0]):
+        msg = df['message'][i].lower()
+        if (re.search('happy birthday', msg)):
+            if (re.findall('@[A-Za-z0-9]+', df['message'][i])):
+                users = re.findall('@[A-Za-z0-9]+', df['message'][i])
+                for user in users:
+                    if user[1:] not in names:
+                        names.append(user[1:])
+                        birthdates.append(str(df['month'][i]) + " " + str(df['day'][i]))
+    return pd.DataFrame({'contacts':names,'birthdates':birthdates})
+
+def sentiment_analysis(selected_user,df):
+    if selected_user != 'Overall':
+        df = df[df['user'] == selected_user]
+    # sample code
+    from transformers import BertTokenizer, BertForSequenceClassification
+    tokenizer = BertTokenizer.from_pretrained("ganeshkharad/gk-hinglish-sentiment")
+    model = BertForSequenceClassification.from_pretrained("ganeshkharad/gk-hinglish-sentiment")
+    if df.shape[0]>600:
+        df=df.sample(n=600)
+
+    ans = []
+    for i in range(df.shape[0]):
+        encoded_input = tokenizer(df['message'].iloc[i], return_tensors='pt')
+        output = model(**encoded_input)
+        output = np.argmax(output.logits.detach().numpy())
+        if (output == 0):
+            ans.append('Negative-messages')
+        elif (output == 1):
+            ans.append('Neutral-messages')
+        else:
+            ans.append('Positive-messages')
+
+        # output contains 3 lables LABEL_0 = Negative ,LABEL_1 = Nuetral ,LABEL_2 = Positive
+    return pd.Series(Counter(ans)),df.shape[0]

preprocessor.py

@@ -0,0 +1,50 @@
+import re
+import pandas as pd
+def preprocess (data):
+    pattern = '\d\d/\d\d/\d\d,\s[0-9]+:\d\d\s[a-z]m\s-\s'
+    messages = re.split(pattern, data)
+    messages = messages[1:]
+    dates = re.findall(pattern, data)
+    for i in range(len(dates)):
+        dates[i] = re.sub('am', 'AM', dates[i])
+        dates[i] = re.sub('pm', 'PM', dates[i])
+    df = pd.DataFrame({'user_message': messages, 'message-date': dates})
+    df['message-date'] = pd.to_datetime(df['message-date'], format="%d/%m/%y, %I:%M %p - ")
+    df.rename(columns={'message-date': 'date'}, inplace=True)
+
+    # separate users and messages
+    users = []
+    messages = []
+    for message in df['user_message']:
+        entry = re.split(':\s', message)
+        if entry[1:]:
+            users.append(entry[0])
+            messages.append(entry[1])
+        else:
+            users.append('group_notification')
+            messages.append(entry[0])
+    df['user'] = users
+    df['message'] = messages
+    df.drop(columns=['user_message'], inplace=True)
+
+    df['year'] = df['date'].dt.year
+    df['month'] = df['date'].dt.month_name()
+    df['month_num'] = df['date'].dt.month
+    df['only_date']=df['date'].dt.date
+    df['day'] = df['date'].dt.day
+    df['day_name']=df['date'].dt.day_name()
+    df['hour'] = df['date'].dt.hour
+    df['minute'] = df['date'].dt.minute
+
+    period = []
+    for hour in df[['day_name', 'hour']]['hour']:
+        if hour == 23:
+            period.append(str(hour) + "-" + str('00'))
+        elif hour == 0:
+            period.append(str('00') + "-" + str(hour + 1))
+        else:
+            period.append(str(hour) + "-" + str(hour + 1))
+
+    df['period'] = period
+
+    return df

requirements.txt

@@ -0,0 +1,10 @@
+streamlit
+matplotlib
+seaborn
+urlextract
+wordcloud
+pandas
+numpy
+torch
+transformers
+emoji==1.7.0