src/models/util_model_comparison.py

from typing import OrderedDict
import streamlit as st
from sklearn.metrics import roc_auc_score
from  features.util_build_features import SplitDataset
from  visualization.graphs_settings import (
    streamlit_chart_setting_height_width
)

from  visualization.graphs_test import (
    roc_auc_compare_n_models,
    calibration_curve_report_commented_n
)


from  models.util_model_class import ModelClass


def roc_auc_for_model(split_dataset: SplitDataset, model_view: ModelClass):
    roc_auc_model = roc_auc_score(
        split_dataset.y_test, model_view.predicted_default_status
    )

    if roc_auc_model > 0.9:
        roc_auc_lvl = f'Very good {"{:.2f}".format(roc_auc_model)} > 0.9)'
    elif 0.8 < roc_auc_model < 0.9:
        roc_auc_lvl = f'Good (0.8 < {"{:.2f}".format(roc_auc_model)} <0.9)'
    elif 0.7 < roc_auc_model < 0.8:
        roc_auc_lvl = f'Fair (0.7 <  {"{:.2f}".format(roc_auc_model)} < 0.8)'
    elif 0.6 < roc_auc_model < 0.7:
        roc_auc_lvl = f'Poor (0.6 <  {"{:.2f}".format(roc_auc_model)} < 0.7)'
    else:
        roc_auc_lvl = f'Fail ( {"{:.2f}".format(roc_auc_model)} < 0.6)'

    return roc_auc_model, roc_auc_lvl


def model_comparison_view(
    split_dataset: SplitDataset,
    model_views: OrderedDict[str, ModelClass],
):
    st.header("Model Comparison")

    for model_name, model_view in model_views.items():
        roc_auc_model, roc_auc_lvl = roc_auc_for_model(
            split_dataset, model_view
        )
        st.subheader(
            f"Receiver Operating Characteristic (ROC) Curve - {model_name}"
        )
        st.markdown(
            f'Area Under the Receiver Operating Characteristic Curve from prediction scores from {model_name} model is {roc_auc_model}.\n'
        )
        st.markdown(
            f'The score of {"{:.2f}".format(roc_auc_model)} is in the {roc_auc_lvl} ROC AUC score category.'
        )
    fig1 = roc_auc_compare_n_models(
        split_dataset.y_test,
        model_views,
    )

    fig1 = fig1.figure

    (xsize_roc, ysize_roc) = streamlit_chart_setting_height_width(
        "Chart Settings", 7, 7, "xsize_roc", "ysize_roc"
    )

    fig1.set_size_inches(xsize_roc, ysize_roc)

    st.pyplot(fig1)

    st.subheader("Models Calibration Curve")

    fig2 = calibration_curve_report_commented_n(
        split_dataset.y_test,
        model_views,
        10,
    )
    fig2 = fig2.figure

    (xsize_cal, ysize_cal) = streamlit_chart_setting_height_width(
        "Chart Settings", 7, 7, "xsize_cal", "ysize_cal"
    )

    fig2.set_size_inches(xsize_cal, ysize_cal)

    st.pyplot(fig2)