app.py

import streamlit as st
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from sklearn.cluster import DBSCAN
from sklearn.decomposition import PCA
from sklearn.preprocessing import StandardScaler
from sklearn.preprocessing import LabelEncoder
from sklearn.cluster import SpectralClustering
from sklearn.cluster import KMeans
from sklearn.metrics import silhouette_score

@st.cache_resource
def load_dataset():
    """
    Loads the `.csv` dataset
    
    Returns:
        pd.DataFrame
    """
    return pd.read_csv('./Country-data.csv'), pd.read_csv('./data-dictionary.csv')

df, df_dict = load_dataset()

@st.cache_resource
def preprocess_data(df: pd.DataFrame,
                    pca_n_components: int = 2,
                    pca: bool = True,
                    scale: bool = True) -> np.array:
    """
    Preprocess the `Country-data.csv`.
    
    Args:
        df (pd.DataFrame): The dataset to be preprocessed
        pca_n_components (int): Number of components to be passed on PCA. Ignore if `pca` == False
        pca (bool): Implement PCA on the dataset
        scale (bool): Scale the dataset with Standard Scaler
    """
    df = df.copy()

    encoder = LabelEncoder()
    df['country'] = encoder.fit_transform(df['country'].values)
    
    if scale:
        scaler = StandardScaler()
        df = scaler.fit_transform(df)
        
    if pca:
        pca_model = PCA(n_components=pca_n_components)
        df = pca_model.fit_transform(df)
        
    return np.array(df)

@st.cache_resource
def preprocess_input_data(input_data: np.array,
                          pca_n_components: int,
                          pca: bool = True,
                          scale: bool = True) -> np.array:
    
    """
    Preprocess the a single instance of input data.
    
    Args:
        df (pd.DataFrame): The dataset to be preprocessed
        pca_n_components (int): Number of components to be passed on PCA. Ignore if `pca` == False
        pca (bool): Implement PCA on the dataset
        scale (bool): Scale the dataset with Standard Scaler
    """
    input_data = input_data.copy()
    df_copy = df.copy()
    
    encoder = LabelEncoder()
    df_copy['country'] = encoder.fit_transform(df_copy['country'].values)
    
    if scale:
        scaler = StandardScaler()
        df_copy = scaler.fit_transform(df_copy)
        input_data = scaler.transform(input_data)
        
    if pca:
        pca_model = PCA(n_components=pca_n_components)
        df_copy = pca_model.fit_transform(df_copy)
        input_data = pca_model.transform(input_data)
        
    return input_data[0]
    
def main():
    st.header('Country Development Clustering')
    
    col1, col2 = st.columns(2)

    with col1:
        pca = st.toggle('Train with PCA', value=True)

    with col2:
        scaler = st.toggle('Train with Scaled Data', value=True)
    
    preprocessed_df = preprocess_data(df=df, pca_n_components=2, pca=pca, scale=scaler)
    
    data, kmeans, dbscan, specclus = st.tabs(['About the Data', 'KMeans Algorithm', 'DBSCAN', 'Spectral Clustering'])
    
    with data:
        st.header('Country Development Data')
        st.write('**Clustering the Countries by using Unsupervised Learning for HELP International**')
    
        col1, col2 = st.columns(2)
    
        with col1:
            st.write('**OBJECTIVE**:')
            st.caption('To categorise the countries using socio-economic and health factors that determine the overall development of the country.')
    
        with col2:
            st.write('**About Organization**')
            st.caption('HELP International is an international humanitarian NGO that is committed to fighting poverty and providing the people of backward countries with basic amenities and relief during the time of disasters and natural calamities.')
    
        st.write('**Problem Statement:**')
        st.caption('HELP International have been able to raise around $ 10 million. Now the CEO of the NGO needs to decide how to use this money strategically and effectively. So, CEO has to make decision to choose the countries that are in the direst need of aid. Hence, your Job as a Data scientist is to categorise the countries using some socio-economic and health factors that determine the overall development of the country. Then you need to suggest the countries which the CEO needs to focus on the most.')
    
        st.dataframe(df)
        
        st.dataframe(df_dict.set_index('Column Name'), width=1000)
        
        url = 'https://www.kaggle.com/datasets/rohan0301/unsupervised-learning-on-country-data?select=Country-data.csv'
        st.caption('For more details about the dataset, visit [Kaggle](%s).' % url)
    
    with kmeans:
        st.write("In this tab, you get to train a KMeans model through hyperparameter tuning. With prediction and visualization!")
        st.write('Experiment with the hyperparameters to tune out the best KMeans model.')
        
        col1, col2 = st.columns(2)
        
        with col1:
            n_clusters = st.slider('`n_clusters`', min_value=1, max_value=20, value=2)
            
        with col2:
            init = st.selectbox('`init`', ['k-means++', 'random'], index=0)
            
            with st.popover('Input data for prediction'):
                st.caption('**NOTE**: The input data will only be used in KMeans algorithm.')
                country = st.selectbox('Which country are you from?', df['country'].unique())
                country_index = list(df['country'].unique()).index(country)
                child_mort = st.slider('Child Mortality Rate', min_value=1.00, max_value=df['child_mort'].max())
                exports = st.slider('Exports', min_value=1.00, max_value=df['exports'].max())
                imports = st.slider('Imports', min_value=1.00, max_value=df['imports'].max())
                health = st.slider('Health', min_value=1.00, max_value=df['health'].max())
                income = st.slider('Income', min_value=1, max_value=df['income'].max())
                inflation = st.slider('Inflation', min_value=1.00, max_value=df['inflation'].max())
                life_expec = st.slider('Health', min_value=1.00, max_value=df['life_expec'].max())
                total_fer = st.slider('Fertility Rate', min_value=1.00, max_value=df['total_fer'].max())
                gdpp = st.slider('GDPP', min_value=1, max_value=df['gdpp'].max())
            
            input_data = np.array([[country_index, child_mort, exports,
                                   health, imports, income, inflation,
                                   life_expec, total_fer, gdpp
                                   ]])
            input_data = preprocess_input_data(input_data, 2, pca, scaler)
                
        km = KMeans(n_clusters=n_clusters, init=init)
        km.fit(preprocessed_df)
        
        km_centroids = km.cluster_centers_
        km_preds = km.predict(preprocessed_df)
        
        st.write(f'Silhouette Score of K-Means: {silhouette_score(preprocessed_df, km_preds):2f}%')
        
        tab1, tab2 = st.tabs(['Matplotlib Plot', 'Interactive Plot'])
        
        with tab1:
            fig, ax = plt.subplots(figsize=(10,7))
            ax.set_title('KMeans Algorithm')
            ax.scatter(km_centroids[:,0], km_centroids[:,1], s=200, c='black', alpha=0.5, label='Centroids')
            scatter = ax.scatter(preprocessed_df[:,0], preprocessed_df[:,1], c=km_preds, s=20)
            plt.scatter(input_data[0], input_data[1], s=150, c='red', marker='X', label='Input Data')
            plt.legend()
            plt.colorbar(scatter, ax=ax, label='Cluster Labels')
            ax.grid(True)
            st.pyplot(fig)
            
        with tab2:
            data_df = pd.DataFrame({
                'x': preprocessed_df[:, 0],
                'y': preprocessed_df[:, 1],
                'Type': ['Data Point'] * len(preprocessed_df),
                'Cluster': km_preds
                })

            centroids_df = pd.DataFrame({
                'x': km_centroids[:, 0],
                'y': km_centroids[:, 1],
                'Type': ['Centroid'] * len(km_centroids),
                'Cluster': ['Centroid'] * len(km_centroids)
                })

            input_df = pd.DataFrame({
                'x': [input_data[0]],
                'y': [input_data[1]],
                'Type': ['Input Data'],
                'Cluster': ['Input Data']
                })
            
            plot_df = pd.concat([data_df, centroids_df, input_df])
            
            st.scatter_chart(plot_df, x='x', y='y', color='Cluster', size='Type')
        
        url = 'https://scikit-learn.org/stable/modules/generated/sklearn.cluster.KMeans.html'
        st.caption('For more details of the hyperparameters, check out the the documentation of KMeans [source](%s) of the dataset.' % url)
    
    with dbscan:
        st.write('In this tab, you get to train a DBSCAN model through hyperparameter tuning, and visualize its results!')
        st.caption('**NOTE**: DBSCAN is not a predictive algorithm, hence no input data for prediction.')
        st.write('Experiment with the hyperparameters to tune out the best DBSCAN model.')
        
        col1, col2 = st.columns(2)
        
        with col1:
            eps = st.slider('`eps`', min_value=0.1, max_value=10.0, value=0.5)
            min_samples = st.slider('`min_samples`', min_value=2, max_value=20, value=5)
        
        with col2:
            metric = st.selectbox('`metric`', ['euclidean', 'manhattan', 'chebyshev'], index=0)
            algorithm = st.selectbox('`algorithm`', ['auto', 'ball_tree', 'kd_tree', 'brute'], index=0)
            
        dbs = DBSCAN(eps=eps, min_samples=min_samples, metric=metric, algorithm=algorithm)
        dbs.fit(preprocessed_df)
        dbs_labels = dbs.labels_
        
        if scaler:
            silhouette = silhouette_score(preprocessed_df, dbs_labels)
            st.write(f'Silhouette Score of DBSCAN: {silhouette:.2f}')
        else:
            st.error('Cannot calculate Silhouette Score with unscaled data.')
        
        tab1, tab2 = st.tabs(['Matplotlib Plot', 'Interactive Plot'])
        
        with tab1: 
            fig, ax = plt.subplots(figsize=(10,7))
            ax.set_title('DBSCAN Model')
            scatter = ax.scatter(preprocessed_df[:, 0], preprocessed_df[:, 1], s=20, c=dbs_labels, cmap='viridis')
            ax.grid(True)
            plt.colorbar(scatter, ax=ax, label='Cluster Labels')
            st.pyplot(fig)
            
        with tab2:
            scatter_df = pd.DataFrame({
                'x': preprocessed_df[:, 0],
                'y': preprocessed_df[:, 1],
                'Cluster': dbs_labels
            })
            
            st.scatter_chart(scatter_df, x='x', y='y', color='Cluster')
        
        url = 'https://scikit-learn.org/stable/modules/generated/sklearn.cluster.DBSCAN.html'
        st.caption('For more details of the hyperparameters, check out the the documentation of DBSCAN [source](%s) of the dataset.' % url)
        
    with specclus:
        st.write('In this tab, you get to train a Spectral Clustering model through hyperparameter tuning, and visualize its results!')
        st.write('Experiment with the hyperparameters to tune out the best Spectral Clustering model.')
        st.caption('**NOTE**: Spectral Clustering is not a predictive algorithm, hence no input data for prediction.')
        
        col1, col2 = st.columns(2)
        
        gamma = 1.0
        n_neighbors = 2
        
        with col1:
            affinity = st.selectbox('`affinity`', ['nearest_neighbors', 'rbf', 'precomputed', 'cosine'], index=1)
            eigen_solver = st.selectbox('`eigen_solver`', ['arpack', 'lobpcg', 'amg', None], index=3)
            assign_labels = st.selectbox('`degree`', ['kmeans', 'discretize'], index=0)
        
        with col2:
            n_clusters = st.slider('`n_clusters`', min_value=2, max_value=20, value=8)
            if affinity == 'rbf':
                gamma = st.slider('`gamma`', min_value=0.01, max_value=0.1, value=1.0)
            elif affinity == 'nearest_neighbors':
                n_neighbors = st.slider('`n_neighbors`', min_value=2, max_value=20, value=10)
            
        
        sc = SpectralClustering(n_clusters=n_clusters, affinity=affinity,
                                gamma=gamma, n_neighbors=n_neighbors,
                                eigen_solver=eigen_solver, assign_labels=assign_labels
                                )
        sc_labels = sc.fit_predict(preprocessed_df)
        
        if scaler:
            st.write(f'Silhouette Score of Spectral Clustering: {silhouette_score(preprocessed_df, km_preds):2f}%')
        else:
            st.error('Cannot calculate Silhouette Score with unscaled data.')
            
        tab1, tab2 = st.tabs(['Matplotlib Plot', 'Interactive Plot'])
        
        with tab1:
            fig, ax = plt.subplots(figsize=(10,7))
            scatter = ax.scatter(preprocessed_df[:, 0], preprocessed_df[:, 1], c=sc_labels, s=20)
            ax.set_title('Spectral Clustering')
            plt.colorbar(scatter, ax=ax, label='Cluster Labels')
            plt.grid(True)
            st.pyplot(fig)
            
        with tab2:
            scatter_df = pd.DataFrame({
                'x': preprocessed_df[:, 0],
                'y': preprocessed_df[:, 1],
                'Cluster': sc_labels
            })
            
            st.scatter_chart(scatter_df, x='x', y='y', color='Cluster')
        
        url = 'https://scikit-learn.org/stable/modules/generated/sklearn.cluster.SpectralClustering.html'
        st.caption('For more details of the hyperparameters, check out the the documentation of Spectral Clustering [source](%s) of the dataset.' % url)
        
if __name__ == "__main__":
    main()