{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"import pandas as pd\n",
"import json\n",
"import matplotlib.pyplot as plt\n",
"import seaborn as sns\n",
"# display settings to show all columns\n",
"pd.set_option('display.max_columns', None)\n",
"# display settings to show full text in columns\n",
"pd.set_option('display.max_colwidth', None)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Model run on 7/2/2024 using split_SMOTE_crossval.py"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
"# Load results\n",
"model_results = pd.read_csv('split_SMOTE_crossval_results.csv')\n",
"#model_results.columns"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [],
"source": [
"# Load predictions\n",
"model_predictions = pd.read_csv('bequest_prediciton_alive_20240702.csv')\n",
"#model_predictions.columns"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [],
"source": [
"# Load full dataset used for training and testing\n",
"bequests_clean = pd.read_csv('bequests_clean.csv')\n",
"#bequests_clean.columns"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"1376"
]
},
"execution_count": 5,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"# Count unique ROI_FAMILY_IDs in the model results\n",
"model_predictions['ROI_FAMILY_ID'].nunique()"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"4520004"
]
},
"execution_count": 6,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"# Count unique ROI_FAMILY_IDs in the full dataset\n",
"bequests_clean['ROI_FAMILY_ID'].nunique()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Identify all \"alive\" bequestors"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"33"
]
},
"execution_count": 7,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"# Unique bequests_clean['ROI_FAMILY_ID'] where 'BEQUEST_RECEIVED' = 1 and DEATH_FLAG = 0\n",
"bequests_clean[(bequests_clean['BEQUEST_RECEIVED'] == 1) & \n",
" (bequests_clean['DEATH_FLAG'] == 0)]['ROI_FAMILY_ID'].nunique()"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [],
"source": [
"# Create list of unique ROI_FAMILY_IDs where 'BEQUEST_RECEIVED' = 1 and DEATH_FLAG = 0\n",
"bequest_alive = bequests_clean[(bequests_clean['BEQUEST_RECEIVED'] == 1) & \n",
" (bequests_clean['DEATH_FLAG'] == 0)]['ROI_FAMILY_ID'].unique()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### How many \"alive\" bequestors were found by the model?"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"30"
]
},
"execution_count": 9,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"# How many unique ROI_FAMILY_IDs are in both model_predictions and bequest_alive\n",
"len(set(model_predictions['ROI_FAMILY_ID']).intersection(set(bequest_alive)))"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### What was the observed recall?"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"0.9090909090909091"
]
},
"execution_count": 10,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"30/33"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Mark the predicted bequestors in the bequests_clean dataset"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {},
"outputs": [],
"source": [
"# Create bequests_clean['FUTURE_BEQUESTOR'] = 1 when model_predictions['ROI_FAMILY_ID'] is in bequests_clean['ROI_FAMILY_ID']\n",
"bequests_clean['FUTURE_BEQUESTOR'] = bequests_clean['ROI_FAMILY_ID'].isin(model_predictions['ROI_FAMILY_ID']).astype(int)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Restructure report df to describe modeling metrics"
]
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {},
"outputs": [],
"source": [
"# Function to flatten a JSON column (classification_report)\n",
"def flatten_json_column(df, column):\n",
" # Parse the JSON data\n",
" json_data = df[column].apply(json.loads)\n",
" # Create a new DataFrame with the flattened data\n",
" json_df = pd.json_normalize(json_data)\n",
" # Concatenate the original DataFrame with the new flattened DataFrame\n",
" df = pd.concat([df, json_df], axis=1)\n",
" # Drop the original JSON column\n",
" df.drop(column, axis=1, inplace=True)\n",
" return df"
]
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {},
"outputs": [],
"source": [
"# List of columns containing JSON arrays to be flattened\n",
"json_columns = ['classification_report']"
]
},
{
"cell_type": "code",
"execution_count": 14,
"metadata": {},
"outputs": [],
"source": [
"# Flatten each JSON column\n",
"for column in json_columns:\n",
" model_results = flatten_json_column(model_results, column)"
]
},
{
"cell_type": "code",
"execution_count": 15,
"metadata": {},
"outputs": [],
"source": [
"# Function to flatten the feature_importance column\n",
"def flatten_feature_importance(df, column):\n",
" # Parse the JSON data\n",
" flattened_data = []\n",
" for entry in df[column]:\n",
" try:\n",
" feature_data = json.loads(entry.replace(\"'\", '\"'))\n",
" flattened_row = {}\n",
" for idx, (feature, importance) in enumerate(zip(feature_data['Feature'], feature_data['Importance'])):\n",
" flattened_row[f\"feature_{idx+1}\"] = feature\n",
" flattened_row[f\"feature_{idx+1}_score\"] = importance\n",
" flattened_data.append(flattened_row)\n",
" except json.JSONDecodeError:\n",
" print(f\"Failed to parse JSON: {entry}\")\n",
" flattened_data.append({})\n",
" # Convert the flattened data into a DataFrame\n",
" feature_df = pd.DataFrame(flattened_data)\n",
" # Concatenate the original DataFrame with the new flattened DataFrame\n",
" df = pd.concat([df, feature_df], axis=1)\n",
" # Drop the original feature_importance column\n",
" df.drop(column, axis=1, inplace=True)\n",
" return df\n"
]
},
{
"cell_type": "code",
"execution_count": 16,
"metadata": {},
"outputs": [],
"source": [
"# Flatten the feature_importance column\n",
"model_results = flatten_feature_importance(model_results, 'feature_importance')"
]
},
{
"cell_type": "code",
"execution_count": 17,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"\n",
"\n",
"
\n",
" \n",
" \n",
" | \n",
" imputer | \n",
" accuracy | \n",
" auc_pr | \n",
" accuracy | \n",
" 0.precision | \n",
" 0.recall | \n",
" 0.f1-score | \n",
" 0.support | \n",
" 1.precision | \n",
" 1.recall | \n",
" 1.f1-score | \n",
" 1.support | \n",
" macro avg.precision | \n",
" macro avg.recall | \n",
" macro avg.f1-score | \n",
" macro avg.support | \n",
" weighted avg.precision | \n",
" weighted avg.recall | \n",
" weighted avg.f1-score | \n",
" weighted avg.support | \n",
" feature_1 | \n",
" feature_1_score | \n",
" feature_2 | \n",
" feature_2_score | \n",
" feature_3 | \n",
" feature_3_score | \n",
" feature_4 | \n",
" feature_4_score | \n",
" feature_5 | \n",
" feature_5_score | \n",
" feature_6 | \n",
" feature_6_score | \n",
" feature_7 | \n",
" feature_7_score | \n",
" feature_8 | \n",
" feature_8_score | \n",
" feature_9 | \n",
" feature_9_score | \n",
" feature_10 | \n",
" feature_10_score | \n",
" feature_11 | \n",
" feature_11_score | \n",
" feature_12 | \n",
" feature_12_score | \n",
" feature_13 | \n",
" feature_13_score | \n",
" feature_14 | \n",
" feature_14_score | \n",
" feature_15 | \n",
" feature_15_score | \n",
" feature_16 | \n",
" feature_16_score | \n",
" feature_17 | \n",
" feature_17_score | \n",
" feature_18 | \n",
" feature_18_score | \n",
" feature_19 | \n",
" feature_19_score | \n",
"
\n",
" \n",
" \n",
" \n",
" | 2 | \n",
" mice | \n",
" 0.998411 | \n",
" 0.880103 | \n",
" 0.998411 | \n",
" 0.999521 | \n",
" 0.998884 | \n",
" 0.999202 | \n",
" 94057.0 | \n",
" 0.755814 | \n",
" 0.878378 | \n",
" 0.8125 | \n",
" 370.0 | \n",
" 0.877668 | \n",
" 0.938631 | \n",
" 0.905851 | \n",
" 94427.0 | \n",
" 0.998566 | \n",
" 0.998411 | \n",
" 0.998471 | \n",
" 94427.0 | \n",
" HPC_AMOUNT | \n",
" 0.228298 | \n",
" MRC_AMOUNT | \n",
" 0.201433 | \n",
" YEARS_SINCE_HPC_GIFT | \n",
" 0.146419 | \n",
" TOTAL_AMOUNT | \n",
" 0.143256 | \n",
" YEARS_SINCE_MRC_GIFT | \n",
" 0.112135 | \n",
" YEARS_SINCE_FIRST_GIFT | \n",
" 0.039562 | \n",
" FIRST_GIFT_AMOUNT | \n",
" 0.038579 | \n",
" TOTAL_TRANSACTIONS | \n",
" 0.034014 | \n",
" BIRTH_YEAR | \n",
" 0.027341 | \n",
" REGION_CODE_PARO | \n",
" 0.003979 | \n",
" REGION_CODE_MARO | \n",
" 0.003973 | \n",
" REGION_CODE_MWRO | \n",
" 0.003866 | \n",
" REGION_CODE_NERO | \n",
" 0.003536 | \n",
" REGION_CODE_SWRO | \n",
" 0.003177 | \n",
" REGION_CODE_SERO | \n",
" 0.002632 | \n",
" REGION_CODE_SCRO | \n",
" 0.002519 | \n",
" REGION_CODE_TXRO | \n",
" 0.001848 | \n",
" REGION_CODE_NWRO | \n",
" 0.001833 | \n",
" REGION_CODE_NRRO | \n",
" 0.001599 | \n",
"
\n",
" \n",
"
\n",
"
"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"# Data for feature importance from the MICE imputer\n",
"data = {\n",
" 'Feature': ['HPC_AMOUNT', 'MRC_AMOUNT', 'YEARS_SINCE_HPC_GIFT', 'TOTAL_AMOUNT', 'YEARS_SINCE_MRC_GIFT',\n",
" 'YEARS_SINCE_FIRST_GIFT', 'FIRST_GIFT_AMOUNT', 'TOTAL_TRANSACTIONS', 'BIRTH_YEAR', \n",
" 'REGION_CODE_PARO', 'REGION_CODE_MARO', 'REGION_CODE_MWRO', 'REGION_CODE_NERO', \n",
" 'REGION_CODE_SWRO', 'REGION_CODE_SERO', 'REGION_CODE_SCRO', 'REGION_CODE_TXRO', \n",
" 'REGION_CODE_NWRO', 'REGION_CODE_NRRO'],\n",
" 'Importance': [0.228298, 0.201433, 0.146419, 0.143256, 0.112135, 0.039562, 0.038579, 0.034014, \n",
" 0.027341, 0.003979, 0.003973, 0.003866, 0.003536, 0.003177, 0.002632, 0.002519, \n",
" 0.001848, 0.001833, 0.001599]\n",
"}\n",
"\n",
"mice_results = pd.DataFrame(data)\n",
"\n",
"# Create a custom palette with 11 colors from the icefire palette\n",
"custom_palette = sns.color_palette(\"tab20\", 20)\n",
"\n",
"# Use the custom color palette\n",
"sns.set_palette(custom_palette)\n",
"\n",
"# Plotting the feature importance\n",
"plt.figure(figsize=(10, 6))\n",
"plt.barh(mice_results['Feature'], mice_results['Importance'], color=custom_palette)\n",
"plt.xlabel('Importance')\n",
"plt.ylabel('Feature')\n",
"plt.title('Feature Importance (MICE Imputer)')\n",
"plt.gca().invert_yaxis()\n",
"plt.tight_layout()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Summarize info about predicted future bequestors"
]
},
{
"cell_type": "code",
"execution_count": 19,
"metadata": {},
"outputs": [],
"source": [
"# Create subset of bequests_clean showing just FUTURE_BEQUESTOR = 1\n",
"bequests_clean_future_bequestors = bequests_clean[bequests_clean['FUTURE_BEQUESTOR'] == 1]"
]
},
{
"cell_type": "code",
"execution_count": 20,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"BIRTH_YEAR 0.124909\n",
"ROI_ID 0.000000\n",
"DEATH_FLAG 0.000000\n",
"LE 0.000000\n",
"BEQUEST_RECEIVED 0.000000\n",
"YEARS_SINCE_HPC_GIFT 0.000000\n",
"YEARS_SINCE_MRC_GIFT 0.000000\n",
"YEARS_SINCE_FIRST_GIFT 0.000000\n",
"HPC_AMOUNT 0.000000\n",
"ROI_FAMILY_ID 0.000000\n",
"MRC_AMOUNT 0.000000\n",
"FIRST_GIFT_AMOUNT 0.000000\n",
"TOTAL_AMOUNT 0.000000\n",
"TOTAL_TRANSACTIONS 0.000000\n",
"REGION_CODE 0.000000\n",
"STATE 0.000000\n",
"FUTURE_BEQUESTOR 0.000000\n",
"dtype: float64"
]
},
"execution_count": 20,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"# Percentage of blank values in each column\n",
"bequests_clean_future_bequestors.isnull().mean().sort_values(ascending=False)"
]
},
{
"cell_type": "code",
"execution_count": 21,
"metadata": {},
"outputs": [],
"source": [
"# Create AGE column in bequests_clean_future_bequestor BIRTH_YEAR\n",
"# Ensure bequests_clean_future_bequestors is a standalone DataFrame to avoid SettingWithCopyWarning\n",
"bequests_clean_future_bequestors = bequests_clean_future_bequestors.copy()\n",
"# Create AGE column\n",
"bequests_clean_future_bequestors['AGE'] = 2024 - bequests_clean_future_bequestors['BIRTH_YEAR']"
]
},
{
"cell_type": "code",
"execution_count": 22,
"metadata": {},
"outputs": [],
"source": [
"# Drop records where DEATH_FLAG = 1\n",
"bequests_clean_future_bequestors = bequests_clean_future_bequestors[bequests_clean_future_bequestors['DEATH_FLAG'] == 0]"
]
},
{
"cell_type": "code",
"execution_count": 23,
"metadata": {},
"outputs": [],
"source": [
"# Drop records where AGE is greater than 85\n",
"bequests_clean_future_bequestors = bequests_clean_future_bequestors[bequests_clean_future_bequestors['AGE'] <= 85]"
]
},
{
"cell_type": "code",
"execution_count": 24,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Number of predicted future bequestors: 1019\n",
"Predicted future bequestors summary:\n"
]
},
{
"data": {
"text/html": [
"\n",
"\n",
"
\n",
" \n",
" \n",
" | \n",
" HPC_AMOUNT | \n",
" MRC_AMOUNT | \n",
" YEARS_SINCE_HPC_GIFT | \n",
" TOTAL_AMOUNT | \n",
" YEARS_SINCE_MRC_GIFT | \n",
" YEARS_SINCE_FIRST_GIFT | \n",
" FIRST_GIFT_AMOUNT | \n",
" TOTAL_TRANSACTIONS | \n",
" AGE | \n",
"
\n",
" \n",
" \n",
" \n",
" | mean | \n",
" 17537.72 | \n",
" 7829.05 | \n",
" 3.10 | \n",
" 85546.73 | \n",
" 1.98 | \n",
" 16.16 | \n",
" 2131.21 | \n",
" 18.20 | \n",
" 68.74 | \n",
"
\n",
" \n",
" | std | \n",
" 59532.58 | \n",
" 22394.29 | \n",
" 3.37 | \n",
" 521091.79 | \n",
" 2.86 | \n",
" 11.35 | \n",
" 6441.81 | \n",
" 23.11 | \n",
" 12.18 | \n",
"
\n",
" \n",
" | min | \n",
" 420.00 | \n",
" 10.00 | \n",
" 0.00 | \n",
" 600.00 | \n",
" 0.00 | \n",
" 0.00 | \n",
" 5.00 | \n",
" 1.00 | \n",
" 29.00 | \n",
"
\n",
" \n",
" | 25% | \n",
" 2500.00 | \n",
" 2000.00 | \n",
" 0.00 | \n",
" 5000.00 | \n",
" 0.00 | \n",
" 7.00 | \n",
" 25.00 | \n",
" 3.00 | \n",
" 62.00 | \n",
"
\n",
" \n",
" | 50% | \n",
" 5000.00 | \n",
" 2500.00 | \n",
" 2.00 | \n",
" 10932.00 | \n",
" 1.00 | \n",
" 12.00 | \n",
" 50.00 | \n",
" 10.00 | \n",
" 72.00 | \n",
"
\n",
" \n",
" | 75% | \n",
" 10000.00 | \n",
" 5000.00 | \n",
" 5.00 | \n",
" 33106.25 | \n",
" 3.00 | \n",
" 27.00 | \n",
" 1262.50 | \n",
" 25.00 | \n",
" 78.00 | \n",
"
\n",
" \n",
" | max | \n",
" 993532.50 | \n",
" 480000.00 | \n",
" 24.00 | \n",
" 13796611.00 | \n",
" 12.00 | \n",
" 41.00 | \n",
" 82390.72 | \n",
" 229.00 | \n",
" 85.00 | \n",
"
\n",
" \n",
"
\n",
"