{ "cells": [ { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [], "source": [ "# Data visualization\n", "import numpy as np\n", "import pandas as pd \n", "\n", "# Keras\n", "from keras.models import Sequential\n", "from keras.layers import Dense\n", "from keras.layers import Dropout\n", "from keras.optimizers import Adam\n", "from keras.utils.np_utils import to_categorical\n", "from keras.callbacks import EarlyStopping\n", "import keras_tuner as kt\n", "\n", "# Train-Test\n", "from sklearn.model_selection import train_test_split\n", "# Classification Report\n", "from sklearn.metrics import confusion_matrix, precision_recall_fscore_support\n", "\n", "import pickle\n", "\n", "import warnings\n", "warnings.filterwarnings('ignore')" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## 1. Describe Dataset" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [], "source": [ "# Determine important landmarks for plank\n", "IMPORTANT_LMS = [\n", " \"NOSE\",\n", " \"LEFT_SHOULDER\",\n", " \"RIGHT_SHOULDER\",\n", " \"LEFT_ELBOW\",\n", " \"RIGHT_ELBOW\",\n", " \"LEFT_WRIST\",\n", " \"RIGHT_WRIST\",\n", " \"LEFT_HIP\",\n", " \"RIGHT_HIP\",\n", " \"LEFT_KNEE\",\n", " \"RIGHT_KNEE\",\n", " \"LEFT_ANKLE\",\n", " \"RIGHT_ANKLE\",\n", " \"LEFT_HEEL\",\n", " \"RIGHT_HEEL\",\n", " \"LEFT_FOOT_INDEX\",\n", " \"RIGHT_FOOT_INDEX\",\n", "]\n", "\n", "# Generate all columns of the data frame\n", "\n", "HEADERS = [\"label\"] # Label column\n", "\n", "for lm in IMPORTANT_LMS:\n", " HEADERS += [f\"{lm.lower()}_x\", f\"{lm.lower()}_y\", f\"{lm.lower()}_z\", f\"{lm.lower()}_v\"]" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Headers: ['label', 'nose_x', 'nose_y', 'nose_z', 'nose_v', 'left_shoulder_x', 'left_shoulder_y', 'left_shoulder_z', 'left_shoulder_v', 'right_shoulder_x', 'right_shoulder_y', 'right_shoulder_z', 'right_shoulder_v', 'left_elbow_x', 'left_elbow_y', 'left_elbow_z', 'left_elbow_v', 'right_elbow_x', 'right_elbow_y', 'right_elbow_z', 'right_elbow_v', 'left_wrist_x', 'left_wrist_y', 'left_wrist_z', 'left_wrist_v', 'right_wrist_x', 'right_wrist_y', 'right_wrist_z', 'right_wrist_v', 'left_hip_x', 'left_hip_y', 'left_hip_z', 'left_hip_v', 'right_hip_x', 'right_hip_y', 'right_hip_z', 'right_hip_v', 'left_knee_x', 'left_knee_y', 'left_knee_z', 'left_knee_v', 'right_knee_x', 'right_knee_y', 'right_knee_z', 'right_knee_v', 'left_ankle_x', 'left_ankle_y', 'left_ankle_z', 'left_ankle_v', 'right_ankle_x', 'right_ankle_y', 'right_ankle_z', 'right_ankle_v', 'left_heel_x', 'left_heel_y', 'left_heel_z', 'left_heel_v', 'right_heel_x', 'right_heel_y', 'right_heel_z', 'right_heel_v', 'left_foot_index_x', 'left_foot_index_y', 'left_foot_index_z', 'left_foot_index_v', 'right_foot_index_x', 'right_foot_index_y', 'right_foot_index_z', 'right_foot_index_v']\n", "Number of rows: 28520 \n", "Number of columns: 69\n", "\n", "Labels: \n", "C 9904\n", "L 9546\n", "H 9070\n", "Name: label, dtype: int64\n", "\n", "Missing values: False\n", "\n", "Duplicate Rows : 0\n" ] } ], "source": [ "def describe_dataset(dataset_path: str):\n", " '''\n", " Describe dataset\n", " '''\n", "\n", " data = pd.read_csv(dataset_path)\n", " print(f\"Headers: {list(data.columns.values)}\")\n", " print(f'Number of rows: {data.shape[0]} \\nNumber of columns: {data.shape[1]}\\n')\n", " print(f\"Labels: \\n{data['label'].value_counts()}\\n\")\n", " print(f\"Missing values: {data.isnull().values.any()}\\n\")\n", " \n", " duplicate = data[data.duplicated()]\n", " print(f\"Duplicate Rows : {len(duplicate.sum(axis=1))}\")\n", "\n", " return data\n", "\n", "\n", "# Remove duplicate rows (optional)\n", "def remove_duplicate_rows(dataset_path: str):\n", " '''\n", " Remove duplicated data from the dataset then save it to another files\n", " '''\n", " \n", " df = pd.read_csv(dataset_path)\n", " df.drop_duplicates(keep=\"first\", inplace=True)\n", " df.to_csv(f\"cleaned_train.csv\", sep=',', encoding='utf-8', index=False)\n", "\n", "\n", "def round_up_metric_results(results) -> list:\n", " '''Round up metrics results such as precision score, recall score, ...'''\n", " return list(map(lambda el: round(el, 3), results))\n", "\n", "\n", "df = describe_dataset(\"./train.csv\")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## 2. Preprocess Dataset" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Number of rows: 28520 \n", "Number of columns: 69\n", "\n", "Labels: \n", "0 9904\n", "2 9546\n", "1 9070\n", "Name: label, dtype: int64\n", "\n" ] } ], "source": [ "# load dataset\n", "df = pd.read_csv(\"./train.csv\")\n", "\n", "# Categorizing label\n", "df.loc[df[\"label\"] == \"C\", \"label\"] = 0\n", "df.loc[df[\"label\"] == \"H\", \"label\"] = 1\n", "df.loc[df[\"label\"] == \"L\", \"label\"] = 2\n", "\n", "print(f'Number of rows: {df.shape[0]} \\nNumber of columns: {df.shape[1]}\\n')\n", "print(f\"Labels: \\n{df['label'].value_counts()}\\n\")" ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [], "source": [ "# Standard Scaling of features\n", "# Dump input scaler\n", "with open(\"./model/input_scaler.pkl\", \"rb\") as f2:\n", " sc = pickle.load(f2)\n", "\n", "x = df.drop(\"label\", axis = 1)\n", "x = pd.DataFrame(sc.transform(x))\n", "\n", "y = df[\"label\"]\n", "\n", "# # Converting prediction to categorical\n", "y_cat = to_categorical(y)" ] }, { "cell_type": "code", "execution_count": 7, "metadata": {}, "outputs": [], "source": [ "x_train, x_test, y_train, y_test = train_test_split(x.values, y_cat, test_size=0.2, random_state=1234)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## 3. Build Model" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### 3.1. Set up" ] }, { "cell_type": "code", "execution_count": 8, "metadata": {}, "outputs": [], "source": [ "stop_early = EarlyStopping(monitor='loss', patience=3)\n", "\n", "# Final Results\n", "final_models = {}" ] }, { "cell_type": "code", "execution_count": 10, "metadata": {}, "outputs": [], "source": [ "def describe_model(model):\n", " '''\n", " Describe Model architecture\n", " '''\n", " print(f\"Describe models architecture\")\n", " for i, layer in enumerate(model.layers):\n", " number_of_units = layer.units if hasattr(layer, 'units') else 0\n", "\n", " if hasattr(layer, \"activation\"):\n", " print(f\"Layer-{i + 1}: {number_of_units} units, func: \", layer.activation)\n", " else:\n", " print(f\"Layer-{i + 1}: {number_of_units} units, func: None\")\n", " \n", "\n", "def get_best_model(tuner):\n", " '''\n", " Describe and return the best model found from keras tuner\n", " '''\n", " best_hps = tuner.get_best_hyperparameters(num_trials=1)[0]\n", " best_model = tuner.hypermodel.build(best_hps)\n", "\n", " describe_model(best_model)\n", "\n", " print(\"\\nOther params:\")\n", " ignore_params = [\"tuner\", \"activation\", \"layer\"]\n", " for param, value in best_hps.values.items():\n", " if not any(word in param for word in ignore_params):\n", " print(f\"{param}: {value}\")\n", "\n", " return best_model" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### 3.2. Model with 3 layers" ] }, { "cell_type": "code", "execution_count": 10, "metadata": {}, "outputs": [], "source": [ "def model_3l_builder(hp):\n", " model = Sequential()\n", " model.add(Dense(68, input_dim = 68, activation = \"relu\"))\n", "\n", " hp_activation = hp.Choice('activation', values=['relu', 'tanh'])\n", " hp_layer_1 = hp.Int('layer_1', min_value=32, max_value=512, step=32)\n", " hp_learning_rate = hp.Choice('learning_rate', values=[1e-2, 1e-3, 1e-4])\n", "\n", " model.add(Dense(units=hp_layer_1, activation=hp_activation))\n", " model.add(Dense(3, activation = \"softmax\"))\n", "\n", " model.compile(optimizer=Adam(learning_rate=hp_learning_rate), loss=\"categorical_crossentropy\", metrics = [\"accuracy\"])\n", " \n", " return model" ] }, { "cell_type": "code", "execution_count": 11, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Trial 30 Complete [00h 00m 55s]\n", "accuracy: 0.9989480972290039\n", "\n", "Best accuracy So Far: 0.9990357756614685\n", "Total elapsed time: 00h 11m 35s\n", "INFO:tensorflow:Oracle triggered exit\n" ] } ], "source": [ "tuner_3l = kt.Hyperband(\n", " model_3l_builder,\n", " objective='accuracy',\n", " max_epochs=10,\n", " directory='keras_tuner_dir',\n", " project_name='keras_tuner_demo'\n", ")\n", "tuner_3l.search(x_train, y_train, epochs=10, callbacks=[stop_early])" ] }, { "cell_type": "code", "execution_count": 12, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Describe models architecture\n", "Layer-1: 68 units, func: \n", "Layer-2: 32 units, func: \n", "Layer-3: 3 units, func: \n", "learning_rate: 0.001\n", "Epoch 1/100\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "2022-11-22 22:34:40.154500: I tensorflow/core/grappler/optimizers/custom_graph_optimizer_registry.cc:113] Plugin optimizer for device_type GPU is enabled.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "2282/2282 [==============================] - ETA: 0s - loss: 0.0489 - accuracy: 0.9877" ] }, { "name": "stderr", "output_type": "stream", "text": [ "2022-11-22 22:35:00.478855: I tensorflow/core/grappler/optimizers/custom_graph_optimizer_registry.cc:113] Plugin optimizer for device_type GPU is enabled.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "2282/2282 [==============================] - 24s 10ms/step - loss: 0.0489 - accuracy: 0.9877 - val_loss: 0.0067 - val_accuracy: 0.9986\n", "Epoch 2/100\n", "2282/2282 [==============================] - 22s 10ms/step - loss: 0.0096 - accuracy: 0.9975 - val_loss: 0.0045 - val_accuracy: 0.9989\n", "Epoch 3/100\n", "2282/2282 [==============================] - 22s 10ms/step - loss: 0.0100 - accuracy: 0.9976 - val_loss: 0.0076 - val_accuracy: 0.9979\n", "Epoch 4/100\n", "2282/2282 [==============================] - 22s 10ms/step - loss: 0.0063 - accuracy: 0.9982 - val_loss: 0.0038 - val_accuracy: 0.9986\n", "Epoch 5/100\n", "2282/2282 [==============================] - 22s 10ms/step - loss: 0.0066 - accuracy: 0.9981 - val_loss: 0.0012 - val_accuracy: 0.9996\n", "Epoch 6/100\n", "2282/2282 [==============================] - 22s 10ms/step - loss: 0.0058 - accuracy: 0.9985 - val_loss: 0.0014 - val_accuracy: 0.9993\n", "Epoch 7/100\n", "2282/2282 [==============================] - 22s 10ms/step - loss: 0.0073 - accuracy: 0.9981 - val_loss: 0.0034 - val_accuracy: 0.9989\n", "Epoch 8/100\n", "2282/2282 [==============================] - 22s 10ms/step - loss: 0.0045 - accuracy: 0.9987 - val_loss: 9.3191e-04 - val_accuracy: 0.9996\n", "Epoch 9/100\n", "2282/2282 [==============================] - 22s 10ms/step - loss: 0.0052 - accuracy: 0.9989 - val_loss: 0.0023 - val_accuracy: 0.9993\n", "Epoch 10/100\n", "2282/2282 [==============================] - 23s 10ms/step - loss: 0.0054 - accuracy: 0.9989 - val_loss: 0.0023 - val_accuracy: 0.9995\n", "Epoch 11/100\n", "2282/2282 [==============================] - 24s 10ms/step - loss: 0.0045 - accuracy: 0.9988 - val_loss: 0.0012 - val_accuracy: 0.9995\n", "Epoch 12/100\n", "2282/2282 [==============================] - 23s 10ms/step - loss: 0.0048 - accuracy: 0.9986 - val_loss: 0.0034 - val_accuracy: 0.9988\n", "Epoch 13/100\n", "2282/2282 [==============================] - 23s 10ms/step - loss: 0.0040 - accuracy: 0.9990 - val_loss: 0.0021 - val_accuracy: 0.9991\n", "Epoch 14/100\n", "2282/2282 [==============================] - 23s 10ms/step - loss: 0.0044 - accuracy: 0.9988 - val_loss: 0.0014 - val_accuracy: 0.9996\n", "Epoch 15/100\n", "2282/2282 [==============================] - 24s 11ms/step - loss: 0.0040 - accuracy: 0.9990 - val_loss: 0.0046 - val_accuracy: 0.9982\n", "Epoch 16/100\n", "2282/2282 [==============================] - 23s 10ms/step - loss: 0.0049 - accuracy: 0.9989 - val_loss: 9.1994e-04 - val_accuracy: 0.9998\n", "Epoch 17/100\n", "2282/2282 [==============================] - 23s 10ms/step - loss: 0.0037 - accuracy: 0.9992 - val_loss: 0.0042 - val_accuracy: 0.9984\n", "Epoch 18/100\n", "2282/2282 [==============================] - 23s 10ms/step - loss: 0.0035 - accuracy: 0.9991 - val_loss: 9.2398e-04 - val_accuracy: 0.9996\n", "Epoch 19/100\n", "2282/2282 [==============================] - 23s 10ms/step - loss: 0.0040 - accuracy: 0.9990 - val_loss: 0.0063 - val_accuracy: 0.9982\n", "Epoch 20/100\n", "2282/2282 [==============================] - 23s 10ms/step - loss: 0.0027 - accuracy: 0.9994 - val_loss: 0.0015 - val_accuracy: 0.9995\n", "Epoch 21/100\n", "2282/2282 [==============================] - 24s 10ms/step - loss: 0.0033 - accuracy: 0.9991 - val_loss: 0.0011 - val_accuracy: 0.9996\n", "Epoch 22/100\n", "2282/2282 [==============================] - 23s 10ms/step - loss: 0.0034 - accuracy: 0.9991 - val_loss: 0.0040 - val_accuracy: 0.9991\n", "Epoch 23/100\n", "2282/2282 [==============================] - 23s 10ms/step - loss: 0.0034 - accuracy: 0.9990 - val_loss: 0.0019 - val_accuracy: 0.9996\n" ] }, { "data": { "text/plain": [ "" ] }, "execution_count": 12, "metadata": {}, "output_type": "execute_result" } ], "source": [ "model_3l = get_best_model(tuner_3l)\n", "model_3l.fit(x_train, y_train, epochs=100, batch_size=10, validation_data=(x_test, y_test), callbacks=[stop_early])" ] }, { "cell_type": "code", "execution_count": 13, "metadata": {}, "outputs": [], "source": [ "final_models[\"3_layers\"] = model_3l" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### 3.3. Model with 5 layers" ] }, { "cell_type": "code", "execution_count": 14, "metadata": {}, "outputs": [], "source": [ "def model_5l_builder(hp):\n", " model = Sequential()\n", " model.add(Dense(68, input_dim = 68, activation = \"relu\"))\n", "\n", " hp_activation = hp.Choice('activation', values=['relu', 'tanh'])\n", " hp_layer_1 = hp.Int('layer_1', min_value=32, max_value=512, step=32)\n", " hp_layer_2 = hp.Int('layer_2', min_value=32, max_value=512, step=32)\n", " hp_layer_3 = hp.Int('layer_3', min_value=32, max_value=512, step=32)\n", " hp_learning_rate = hp.Choice('learning_rate', values=[1e-2, 1e-3, 1e-4])\n", "\n", " model.add(Dense(units=hp_layer_1, activation=hp_activation))\n", " model.add(Dense(units=hp_layer_2, activation=hp_activation))\n", " model.add(Dense(units=hp_layer_3, activation=hp_activation))\n", " model.add(Dense(3, activation = \"softmax\"))\n", "\n", " model.compile(optimizer=Adam(learning_rate=hp_learning_rate), loss=\"categorical_crossentropy\", metrics = [\"accuracy\"])\n", " \n", " return model" ] }, { "cell_type": "code", "execution_count": 15, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Trial 30 Complete [00h 01m 07s]\n", "accuracy: 0.9977647066116333\n", "\n", "Best accuracy So Far: 0.9995178580284119\n", "Total elapsed time: 00h 14m 00s\n", "INFO:tensorflow:Oracle triggered exit\n" ] } ], "source": [ "tuner_5l = kt.Hyperband(\n", " model_5l_builder,\n", " objective='accuracy',\n", " max_epochs=10,\n", " directory='keras_tuner_dir',\n", " project_name='keras_tuner_demo_1'\n", ")\n", "tuner_5l.search(x_train, y_train, epochs=10, callbacks=[stop_early])" ] }, { "cell_type": "code", "execution_count": 17, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Describe models architecture\n", "Layer-1: 68 units, func: \n", "Layer-2: 96 units, func: \n", "Layer-3: 416 units, func: \n", "Layer-4: 320 units, func: \n", "Layer-5: 3 units, func: \n", "learning_rate: 0.0001\n", "Epoch 1/100\n", " 1/2282 [..............................] - ETA: 12:30 - loss: 1.1718 - accuracy: 0.3000" ] }, { "name": "stderr", "output_type": "stream", "text": [ "2022-11-22 23:03:14.751713: I tensorflow/core/grappler/optimizers/custom_graph_optimizer_registry.cc:113] Plugin optimizer for device_type GPU is enabled.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "2282/2282 [==============================] - ETA: 0s - loss: 0.0861 - accuracy: 0.9754" ] }, { "name": "stderr", "output_type": "stream", "text": [ "2022-11-22 23:03:38.016530: I tensorflow/core/grappler/optimizers/custom_graph_optimizer_registry.cc:113] Plugin optimizer for device_type GPU is enabled.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "2282/2282 [==============================] - 28s 12ms/step - loss: 0.0861 - accuracy: 0.9754 - val_loss: 0.0077 - val_accuracy: 0.9988\n", "Epoch 2/100\n", "2282/2282 [==============================] - 28s 12ms/step - loss: 0.0099 - accuracy: 0.9979 - val_loss: 0.0114 - val_accuracy: 0.9977\n", "Epoch 3/100\n", "2282/2282 [==============================] - 26s 12ms/step - loss: 0.0077 - accuracy: 0.9978 - val_loss: 0.0027 - val_accuracy: 0.9993\n", "Epoch 4/100\n", "2282/2282 [==============================] - 26s 11ms/step - loss: 0.0066 - accuracy: 0.9982 - val_loss: 0.0021 - val_accuracy: 0.9993\n", "Epoch 5/100\n", "2282/2282 [==============================] - 27s 12ms/step - loss: 0.0046 - accuracy: 0.9990 - val_loss: 9.3464e-04 - val_accuracy: 0.9996\n", "Epoch 6/100\n", "2282/2282 [==============================] - 27s 12ms/step - loss: 0.0051 - accuracy: 0.9986 - val_loss: 0.0013 - val_accuracy: 0.9995\n", "Epoch 7/100\n", "2282/2282 [==============================] - 27s 12ms/step - loss: 0.0045 - accuracy: 0.9989 - val_loss: 0.0021 - val_accuracy: 0.9991\n", "Epoch 8/100\n", "2282/2282 [==============================] - 27s 12ms/step - loss: 0.0034 - accuracy: 0.9992 - val_loss: 0.0031 - val_accuracy: 0.9986\n", "Epoch 9/100\n", "2282/2282 [==============================] - 27s 12ms/step - loss: 0.0038 - accuracy: 0.9990 - val_loss: 9.4086e-04 - val_accuracy: 0.9996\n", "Epoch 10/100\n", "2282/2282 [==============================] - 27s 12ms/step - loss: 0.0028 - accuracy: 0.9993 - val_loss: 7.7891e-04 - val_accuracy: 0.9996\n", "Epoch 11/100\n", "2282/2282 [==============================] - 27s 12ms/step - loss: 0.0029 - accuracy: 0.9995 - val_loss: 0.0013 - val_accuracy: 0.9995\n", "Epoch 12/100\n", "2282/2282 [==============================] - 27s 12ms/step - loss: 0.0030 - accuracy: 0.9993 - val_loss: 0.0017 - val_accuracy: 0.9993\n", "Epoch 13/100\n", "2282/2282 [==============================] - 28s 12ms/step - loss: 0.0019 - accuracy: 0.9995 - val_loss: 0.0016 - val_accuracy: 0.9995\n", "Epoch 14/100\n", "2282/2282 [==============================] - 27s 12ms/step - loss: 0.0021 - accuracy: 0.9995 - val_loss: 0.0015 - val_accuracy: 0.9995\n", "Epoch 15/100\n", "2282/2282 [==============================] - 27s 12ms/step - loss: 0.0031 - accuracy: 0.9992 - val_loss: 8.1616e-04 - val_accuracy: 0.9996\n", "Epoch 16/100\n", "2282/2282 [==============================] - 27s 12ms/step - loss: 0.0019 - accuracy: 0.9994 - val_loss: 6.3959e-04 - val_accuracy: 0.9998\n", "Epoch 17/100\n", "2282/2282 [==============================] - 27s 12ms/step - loss: 0.0022 - accuracy: 0.9994 - val_loss: 0.0017 - val_accuracy: 0.9995\n", "Epoch 18/100\n", "2282/2282 [==============================] - 27s 12ms/step - loss: 0.0014 - accuracy: 0.9996 - val_loss: 0.0015 - val_accuracy: 0.9993\n", "Epoch 19/100\n", "2282/2282 [==============================] - 27s 12ms/step - loss: 0.0021 - accuracy: 0.9995 - val_loss: 0.0013 - val_accuracy: 0.9996\n", "Epoch 20/100\n", "2282/2282 [==============================] - 27s 12ms/step - loss: 0.0018 - accuracy: 0.9996 - val_loss: 0.0025 - val_accuracy: 0.9988\n", "Epoch 21/100\n", "2282/2282 [==============================] - 27s 12ms/step - loss: 0.0017 - accuracy: 0.9995 - val_loss: 0.0017 - val_accuracy: 0.9996\n" ] }, { "data": { "text/plain": [ "" ] }, "execution_count": 17, "metadata": {}, "output_type": "execute_result" } ], "source": [ "model_5l = get_best_model(tuner_5l)\n", "model_5l.fit(x_train, y_train, epochs=100, batch_size=10, validation_data=(x_test, y_test), callbacks=[stop_early])" ] }, { "cell_type": "code", "execution_count": 18, "metadata": {}, "outputs": [], "source": [ "final_models[\"5_layers\"] = model_5l" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### 3.4. Model with 7 layers (along with Dropout)" ] }, { "cell_type": "code", "execution_count": 23, "metadata": {}, "outputs": [], "source": [ "def model_7lD_builder(hp):\n", " model = Sequential()\n", " model.add(Dense(68, input_dim = 68, activation = \"relu\"))\n", "\n", " hp_activation = hp.Choice('activation', values=['relu', 'tanh'])\n", " hp_layer_1 = hp.Int('layer_1', min_value=32, max_value=512, step=32)\n", " hp_layer_2 = hp.Int('layer_2', min_value=32, max_value=512, step=32)\n", " hp_layer_3 = hp.Int('layer_3', min_value=32, max_value=512, step=32)\n", " hp_dropout_1 = hp.Float('dropout_1', min_value=0.1, max_value=0.5, step=0.1)\n", " hp_dropout_2 = hp.Float('dropout_2', min_value=0.1, max_value=0.5, step=0.1)\n", " hp_learning_rate = hp.Choice('learning_rate', values=[1e-2, 1e-3, 1e-4])\n", "\n", " model.add(Dense(units=hp_layer_1, activation=hp_activation))\n", " model.add(Dropout(rate=hp_dropout_1))\n", " model.add(Dense(units=hp_layer_2, activation=hp_activation))\n", " model.add(Dropout(rate=hp_dropout_2))\n", " model.add(Dense(units=hp_layer_3, activation=hp_activation))\n", " model.add(Dense(3, activation = \"softmax\"))\n", "\n", " model.compile(optimizer=Adam(learning_rate=hp_learning_rate), loss=\"categorical_crossentropy\", metrics = [\"accuracy\"])\n", " \n", " return model" ] }, { "cell_type": "code", "execution_count": 39, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow:Reloading Oracle from existing project keras_tuner_dir/keras_tuner_demo_4/oracle.json\n", "INFO:tensorflow:Reloading Tuner from keras_tuner_dir/keras_tuner_demo_4/tuner0.json\n", "INFO:tensorflow:Oracle triggered exit\n" ] } ], "source": [ "tuner_7lD = kt.Hyperband(\n", " model_7lD_builder,\n", " objective='accuracy',\n", " max_epochs=10,\n", " directory='keras_tuner_dir',\n", " project_name='keras_tuner_demo_4'\n", ")\n", "tuner_7lD.search(x_train, y_train, epochs=10, callbacks=[stop_early])" ] }, { "cell_type": "code", "execution_count": 41, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Describe models architecture\n", "Layer-1: 68 units, func: \n", "Layer-2: 480 units, func: \n", "Layer-3: 0 units, func: None\n", "Layer-4: 256 units, func: \n", "Layer-5: 0 units, func: None\n", "Layer-6: 480 units, func: \n", "Layer-7: 3 units, func: \n", "\n", "Other params:\n", "dropout_1: 0.30000000000000004\n", "dropout_2: 0.4\n", "learning_rate: 0.0001\n", "Epoch 1/100\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "2022-11-22 23:59:17.522675: I tensorflow/core/grappler/optimizers/custom_graph_optimizer_registry.cc:113] Plugin optimizer for device_type GPU is enabled.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "2282/2282 [==============================] - ETA: 0s - loss: 0.0917 - accuracy: 0.9722" ] }, { "name": "stderr", "output_type": "stream", "text": [ "2022-11-22 23:59:44.220298: I tensorflow/core/grappler/optimizers/custom_graph_optimizer_registry.cc:113] Plugin optimizer for device_type GPU is enabled.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "2282/2282 [==============================] - 31s 13ms/step - loss: 0.0917 - accuracy: 0.9722 - val_loss: 0.0082 - val_accuracy: 0.9984\n", "Epoch 2/100\n", "2282/2282 [==============================] - 31s 14ms/step - loss: 0.0121 - accuracy: 0.9968 - val_loss: 0.0067 - val_accuracy: 0.9988\n", "Epoch 3/100\n", "2282/2282 [==============================] - 29s 13ms/step - loss: 0.0096 - accuracy: 0.9975 - val_loss: 0.0134 - val_accuracy: 0.9965\n", "Epoch 4/100\n", "2282/2282 [==============================] - 30s 13ms/step - loss: 0.0083 - accuracy: 0.9981 - val_loss: 0.0051 - val_accuracy: 0.9989\n", "Epoch 5/100\n", "2282/2282 [==============================] - 30s 13ms/step - loss: 0.0060 - accuracy: 0.9986 - val_loss: 0.0025 - val_accuracy: 0.9996\n", "Epoch 6/100\n", "2282/2282 [==============================] - 30s 13ms/step - loss: 0.0072 - accuracy: 0.9983 - val_loss: 0.0046 - val_accuracy: 0.9995\n", "Epoch 7/100\n", "2282/2282 [==============================] - 30s 13ms/step - loss: 0.0047 - accuracy: 0.9990 - val_loss: 0.0024 - val_accuracy: 0.9993\n", "Epoch 8/100\n", "2282/2282 [==============================] - 30s 13ms/step - loss: 0.0042 - accuracy: 0.9992 - val_loss: 0.0041 - val_accuracy: 0.9995\n", "Epoch 9/100\n", "2282/2282 [==============================] - 31s 13ms/step - loss: 0.0048 - accuracy: 0.9988 - val_loss: 0.0014 - val_accuracy: 0.9996\n", "Epoch 10/100\n", "2282/2282 [==============================] - 31s 14ms/step - loss: 0.0042 - accuracy: 0.9990 - val_loss: 0.0014 - val_accuracy: 0.9996\n", "Epoch 11/100\n", "2282/2282 [==============================] - 30s 13ms/step - loss: 0.0041 - accuracy: 0.9989 - val_loss: 0.0021 - val_accuracy: 0.9993\n", "Epoch 12/100\n", "2282/2282 [==============================] - 30s 13ms/step - loss: 0.0034 - accuracy: 0.9992 - val_loss: 0.0013 - val_accuracy: 0.9995\n", "Epoch 13/100\n", "2282/2282 [==============================] - 30s 13ms/step - loss: 0.0029 - accuracy: 0.9994 - val_loss: 0.0012 - val_accuracy: 0.9995\n", "Epoch 14/100\n", "2282/2282 [==============================] - 30s 13ms/step - loss: 0.0031 - accuracy: 0.9994 - val_loss: 0.0058 - val_accuracy: 0.9977\n", "Epoch 15/100\n", "2282/2282 [==============================] - 30s 13ms/step - loss: 0.0026 - accuracy: 0.9993 - val_loss: 0.0015 - val_accuracy: 0.9996\n", "Epoch 16/100\n", "2282/2282 [==============================] - 31s 13ms/step - loss: 0.0043 - accuracy: 0.9989 - val_loss: 0.0014 - val_accuracy: 0.9995\n", "Epoch 17/100\n", "2282/2282 [==============================] - 30s 13ms/step - loss: 0.0021 - accuracy: 0.9996 - val_loss: 6.3795e-04 - val_accuracy: 0.9996\n", "Epoch 18/100\n", "2282/2282 [==============================] - 30s 13ms/step - loss: 0.0021 - accuracy: 0.9995 - val_loss: 0.0015 - val_accuracy: 0.9996\n", "Epoch 19/100\n", "2282/2282 [==============================] - 30s 13ms/step - loss: 0.0030 - accuracy: 0.9993 - val_loss: 0.0028 - val_accuracy: 0.9993\n", "Epoch 20/100\n", "2282/2282 [==============================] - 30s 13ms/step - loss: 0.0020 - accuracy: 0.9995 - val_loss: 9.1208e-04 - val_accuracy: 0.9996\n", "Epoch 21/100\n", "2282/2282 [==============================] - 30s 13ms/step - loss: 0.0036 - accuracy: 0.9991 - val_loss: 2.6149e-04 - val_accuracy: 0.9998\n", "Epoch 22/100\n", "2282/2282 [==============================] - 30s 13ms/step - loss: 0.0025 - accuracy: 0.9993 - val_loss: 0.0011 - val_accuracy: 0.9996\n", "Epoch 23/100\n", "2282/2282 [==============================] - 30s 13ms/step - loss: 0.0021 - accuracy: 0.9994 - val_loss: 6.3920e-04 - val_accuracy: 0.9996\n" ] }, { "data": { "text/plain": [ "" ] }, "execution_count": 41, "metadata": {}, "output_type": "execute_result" } ], "source": [ "model_7lD = get_best_model(tuner_7lD)\n", "model_7lD.fit(x_train, y_train, epochs=100, batch_size=10, validation_data=(x_test, y_test), callbacks=[stop_early])" ] }, { "cell_type": "code", "execution_count": 42, "metadata": {}, "outputs": [], "source": [ "final_models[\"7_layers_with_dropout\"] = model_7lD" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### 3.5. Model with 7 layers" ] }, { "cell_type": "code", "execution_count": 44, "metadata": {}, "outputs": [], "source": [ "def model_7l_builder(hp):\n", " model = Sequential()\n", " model.add(Dense(68, input_dim = 68, activation = \"relu\"))\n", "\n", " hp_activation = hp.Choice('activation', values=['relu', 'tanh'])\n", " hp_layer_1 = hp.Int('layer_1', min_value=32, max_value=512, step=32)\n", " hp_layer_2 = hp.Int('layer_2', min_value=32, max_value=512, step=32)\n", " hp_layer_3 = hp.Int('layer_3', min_value=32, max_value=512, step=32)\n", " hp_layer_4 = hp.Int('layer_4', min_value=32, max_value=512, step=32)\n", " hp_layer_5 = hp.Int('layer_5', min_value=32, max_value=512, step=32)\n", " hp_learning_rate = hp.Choice('learning_rate', values=[1e-2, 1e-3, 1e-4])\n", "\n", " model.add(Dense(units=hp_layer_1, activation=hp_activation))\n", " model.add(Dense(units=hp_layer_2, activation=hp_activation))\n", " model.add(Dense(units=hp_layer_3, activation=hp_activation))\n", " model.add(Dense(units=hp_layer_4, activation=hp_activation))\n", " model.add(Dense(units=hp_layer_5, activation=hp_activation))\n", " model.add(Dense(3, activation = \"softmax\"))\n", "\n", " model.compile(optimizer=Adam(learning_rate=hp_learning_rate), loss=\"categorical_crossentropy\", metrics = [\"accuracy\"])\n", " \n", " return model" ] }, { "cell_type": "code", "execution_count": 46, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Trial 30 Complete [00h 01m 26s]\n", "accuracy: 0.9993863701820374\n", "\n", "Best accuracy So Far: 0.9993863701820374\n", "Total elapsed time: 00h 17m 16s\n", "INFO:tensorflow:Oracle triggered exit\n" ] } ], "source": [ "tuner_7l = kt.Hyperband(\n", " model_7l_builder,\n", " objective='accuracy',\n", " max_epochs=10,\n", " directory='keras_tuner_dir',\n", " project_name='keras_tuner_demo_5'\n", ")\n", "tuner_7l.search(x_train, y_train, epochs=10, callbacks=[stop_early])" ] }, { "cell_type": "code", "execution_count": 47, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Describe models architecture\n", "Layer-1: 68 units, func: \n", "Layer-2: 160 units, func: \n", "Layer-3: 416 units, func: \n", "Layer-4: 288 units, func: \n", "Layer-5: 512 units, func: \n", "Layer-6: 256 units, func: \n", "Layer-7: 3 units, func: \n", "\n", "Other params:\n", "learning_rate: 0.0001\n", "Epoch 1/100\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "2022-11-23 00:37:07.916975: I tensorflow/core/grappler/optimizers/custom_graph_optimizer_registry.cc:113] Plugin optimizer for device_type GPU is enabled.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "2282/2282 [==============================] - ETA: 0s - loss: 0.0573 - accuracy: 0.9844" ] }, { "name": "stderr", "output_type": "stream", "text": [ "2022-11-23 00:37:36.882232: I tensorflow/core/grappler/optimizers/custom_graph_optimizer_registry.cc:113] Plugin optimizer for device_type GPU is enabled.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "2282/2282 [==============================] - 34s 15ms/step - loss: 0.0573 - accuracy: 0.9844 - val_loss: 0.0053 - val_accuracy: 0.9982\n", "Epoch 2/100\n", "2282/2282 [==============================] - 33s 15ms/step - loss: 0.0104 - accuracy: 0.9974 - val_loss: 0.0030 - val_accuracy: 0.9991\n", "Epoch 3/100\n", "2282/2282 [==============================] - 35s 15ms/step - loss: 0.0089 - accuracy: 0.9979 - val_loss: 0.0014 - val_accuracy: 0.9996\n", "Epoch 4/100\n", "2282/2282 [==============================] - 34s 15ms/step - loss: 0.0090 - accuracy: 0.9979 - val_loss: 0.0016 - val_accuracy: 0.9995\n", "Epoch 5/100\n", "2282/2282 [==============================] - 34s 15ms/step - loss: 0.0067 - accuracy: 0.9983 - val_loss: 0.0011 - val_accuracy: 0.9998\n", "Epoch 6/100\n", "2282/2282 [==============================] - 34s 15ms/step - loss: 0.0043 - accuracy: 0.9990 - val_loss: 0.0036 - val_accuracy: 0.9989\n", "Epoch 7/100\n", "2282/2282 [==============================] - 34s 15ms/step - loss: 0.0053 - accuracy: 0.9988 - val_loss: 0.0059 - val_accuracy: 0.9982\n", "Epoch 8/100\n", "2282/2282 [==============================] - 34s 15ms/step - loss: 0.0033 - accuracy: 0.9990 - val_loss: 0.0025 - val_accuracy: 0.9995\n", "Epoch 9/100\n", "2282/2282 [==============================] - 34s 15ms/step - loss: 0.0046 - accuracy: 0.9990 - val_loss: 0.0020 - val_accuracy: 0.9995\n", "Epoch 10/100\n", "2282/2282 [==============================] - 34s 15ms/step - loss: 0.0058 - accuracy: 0.9986 - val_loss: 0.0013 - val_accuracy: 0.9995\n", "Epoch 11/100\n", "2282/2282 [==============================] - 34s 15ms/step - loss: 0.0027 - accuracy: 0.9994 - val_loss: 0.0038 - val_accuracy: 0.9989\n", "Epoch 12/100\n", "2282/2282 [==============================] - 34s 15ms/step - loss: 0.0027 - accuracy: 0.9995 - val_loss: 0.0021 - val_accuracy: 0.9995\n", "Epoch 13/100\n", "2282/2282 [==============================] - 34s 15ms/step - loss: 0.0024 - accuracy: 0.9995 - val_loss: 0.0017 - val_accuracy: 0.9995\n", "Epoch 14/100\n", "2282/2282 [==============================] - 34s 15ms/step - loss: 0.0032 - accuracy: 0.9992 - val_loss: 6.6669e-04 - val_accuracy: 0.9998\n", "Epoch 15/100\n", "2282/2282 [==============================] - 34s 15ms/step - loss: 0.0038 - accuracy: 0.9992 - val_loss: 0.0014 - val_accuracy: 0.9996\n", "Epoch 16/100\n", "2282/2282 [==============================] - 34s 15ms/step - loss: 0.0028 - accuracy: 0.9993 - val_loss: 0.0019 - val_accuracy: 0.9996\n" ] }, { "data": { "text/plain": [ "" ] }, "execution_count": 47, "metadata": {}, "output_type": "execute_result" } ], "source": [ "model_7l = get_best_model(tuner_7l)\n", "model_7l.fit(x_train, y_train, epochs=100, batch_size=10, validation_data=(x_test, y_test), callbacks=[stop_early])" ] }, { "cell_type": "code", "execution_count": 48, "metadata": {}, "outputs": [], "source": [ "final_models[\"7_layers\"] = model_7l" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### 3.6. Describe Final Models" ] }, { "cell_type": "code", "execution_count": 11, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "3_layers: Describe models architecture\n", "Layer-1: 68 units, func: \n", "Layer-2: 32 units, func: \n", "Layer-3: 3 units, func: \n", "\n", "5_layers: Describe models architecture\n", "Layer-1: 68 units, func: \n", "Layer-2: 96 units, func: \n", "Layer-3: 416 units, func: \n", "Layer-4: 320 units, func: \n", "Layer-5: 3 units, func: \n", "\n", "7_layers_with_dropout: Describe models architecture\n", "Layer-1: 68 units, func: \n", "Layer-2: 480 units, func: \n", "Layer-3: 0 units, func: None\n", "Layer-4: 256 units, func: \n", "Layer-5: 0 units, func: None\n", "Layer-6: 480 units, func: \n", "Layer-7: 3 units, func: \n", "\n", "7_layers: Describe models architecture\n", "Layer-1: 68 units, func: \n", "Layer-2: 160 units, func: \n", "Layer-3: 416 units, func: \n", "Layer-4: 288 units, func: \n", "Layer-5: 512 units, func: \n", "Layer-6: 256 units, func: \n", "Layer-7: 3 units, func: \n", "\n" ] } ], "source": [ "for name, model in final_models.items():\n", " print(f\"{name}: \", end=\"\")\n", " describe_model(model)\n", " print()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## 4. Evaluate model with Train set" ] }, { "cell_type": "code", "execution_count": 26, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
ModelPrecision ScoreRecall ScoreF1 scoreConfusion Matrix
03_layers[1.0, 1.0, 0.999][0.999, 1.0, 1.0][0.999, 1.0, 0.999][[1913, 0, 2], [0, 1844, 0], [0, 0, 1945]]
15_layers[0.999, 1.0, 0.999][0.999, 1.0, 0.999][0.999, 1.0, 0.999][[1914, 0, 1], [0, 1844, 0], [1, 0, 1944]]
27_layers_with_dropout[0.999, 1.0, 0.999][0.999, 1.0, 0.999][0.999, 1.0, 0.999][[1914, 0, 1], [0, 1844, 0], [1, 0, 1944]]
37_layers[0.999, 1.0, 0.999][0.999, 1.0, 0.999][0.999, 1.0, 0.999][[1914, 0, 1], [0, 1844, 0], [1, 0, 1944]]
\n", "
" ], "text/plain": [ " Model Precision Score Recall Score \\\n", "0 3_layers [1.0, 1.0, 0.999] [0.999, 1.0, 1.0] \n", "1 5_layers [0.999, 1.0, 0.999] [0.999, 1.0, 0.999] \n", "2 7_layers_with_dropout [0.999, 1.0, 0.999] [0.999, 1.0, 0.999] \n", "3 7_layers [0.999, 1.0, 0.999] [0.999, 1.0, 0.999] \n", "\n", " F1 score Confusion Matrix \n", "0 [0.999, 1.0, 0.999] [[1913, 0, 2], [0, 1844, 0], [0, 0, 1945]] \n", "1 [0.999, 1.0, 0.999] [[1914, 0, 1], [0, 1844, 0], [1, 0, 1944]] \n", "2 [0.999, 1.0, 0.999] [[1914, 0, 1], [0, 1844, 0], [1, 0, 1944]] \n", "3 [0.999, 1.0, 0.999] [[1914, 0, 1], [0, 1844, 0], [1, 0, 1944]] " ] }, "execution_count": 26, "metadata": {}, "output_type": "execute_result" } ], "source": [ "train_set_results = []\n", "\n", "for name, model in final_models.items():\n", " # Evaluate model\n", " predict_x = model.predict(x_test, verbose=False) \n", " y_pred_class = np.argmax(predict_x, axis=1)\n", " y_test_class = np.argmax(y_test, axis=1)\n", "\n", " cm = confusion_matrix(y_test_class, y_pred_class, labels=[0, 1, 2])\n", " (p_score, r_score, f_score, _) = precision_recall_fscore_support(y_test_class, y_pred_class, labels=[0, 1, 2])\n", " \n", " train_set_results.append(( name, round_up_metric_results(p_score), round_up_metric_results(r_score), round_up_metric_results(f_score), cm ))\n", "\n", "train_set_results.sort(key=lambda k: sum(k[3]), reverse=True)\n", "pd.DataFrame(train_set_results, columns=[\"Model\", \"Precision Score\", \"Recall Score\", \"F1 score\", \"Confusion Matrix\"])" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## 5. Evaluation with test set" ] }, { "cell_type": "code", "execution_count": 22, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Number of rows: 710 \n", "Number of columns: 69\n", "\n", "Labels: \n", "1 241\n", "2 235\n", "0 234\n", "Name: label, dtype: int64\n", "\n" ] } ], "source": [ "# load dataset\n", "test_df = pd.read_csv(\"./test.csv\")\n", "\n", "# Categorizing label\n", "test_df.loc[test_df[\"label\"] == \"C\", \"label\"] = 0\n", "test_df.loc[test_df[\"label\"] == \"H\", \"label\"] = 1\n", "test_df.loc[test_df[\"label\"] == \"L\", \"label\"] = 2\n", "\n", "print(f'Number of rows: {test_df.shape[0]} \\nNumber of columns: {test_df.shape[1]}\\n')\n", "print(f\"Labels: \\n{test_df['label'].value_counts()}\\n\")" ] }, { "cell_type": "code", "execution_count": 23, "metadata": {}, "outputs": [], "source": [ "# Standard Scaling of features\n", "test_x = test_df.drop(\"label\", axis = 1)\n", "test_x = pd.DataFrame(sc.transform(test_x))\n", "\n", "test_y = test_df[\"label\"]\n", "\n", "# # Converting prediction to categorical\n", "test_y_cat = to_categorical(test_y)" ] }, { "cell_type": "code", "execution_count": 27, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
ModelPrecision ScoreRecall ScoreF1 scoreConfusion Matrix
07_layers_with_dropout[0.983, 1.0, 1.0][1.0, 0.992, 0.991][0.992, 0.996, 0.996][[234, 0, 0], [2, 239, 0], [2, 0, 233]]
15_layers[0.984, 0.937, 0.883][0.803, 0.992, 0.991][0.885, 0.964, 0.934][[188, 16, 30], [1, 239, 1], [2, 0, 233]]
27_layers[0.984, 0.825, 1.0][0.782, 0.996, 0.991][0.871, 0.902, 0.996][[183, 51, 0], [1, 240, 0], [2, 0, 233]]
33_layers[0.88, 0.732, 1.0][0.624, 0.996, 0.919][0.73, 0.844, 0.958][[146, 88, 0], [1, 240, 0], [19, 0, 216]]
\n", "
" ], "text/plain": [ " Model Precision Score Recall Score \\\n", "0 7_layers_with_dropout [0.983, 1.0, 1.0] [1.0, 0.992, 0.991] \n", "1 5_layers [0.984, 0.937, 0.883] [0.803, 0.992, 0.991] \n", "2 7_layers [0.984, 0.825, 1.0] [0.782, 0.996, 0.991] \n", "3 3_layers [0.88, 0.732, 1.0] [0.624, 0.996, 0.919] \n", "\n", " F1 score Confusion Matrix \n", "0 [0.992, 0.996, 0.996] [[234, 0, 0], [2, 239, 0], [2, 0, 233]] \n", "1 [0.885, 0.964, 0.934] [[188, 16, 30], [1, 239, 1], [2, 0, 233]] \n", "2 [0.871, 0.902, 0.996] [[183, 51, 0], [1, 240, 0], [2, 0, 233]] \n", "3 [0.73, 0.844, 0.958] [[146, 88, 0], [1, 240, 0], [19, 0, 216]] " ] }, "execution_count": 27, "metadata": {}, "output_type": "execute_result" } ], "source": [ "test_set_results = []\n", "\n", "for name, model in final_models.items():\n", " # Evaluate model\n", " predict_x = model.predict(test_x, verbose=False) \n", " y_pred_class = np.argmax(predict_x, axis=1)\n", " y_test_class = np.argmax(test_y_cat, axis=1)\n", "\n", " cm = confusion_matrix(y_test_class, y_pred_class, labels=[0, 1, 2])\n", " (p_score, r_score, f_score, _) = precision_recall_fscore_support(y_test_class, y_pred_class, labels=[0, 1, 2])\n", " \n", " test_set_results.append(( name, round_up_metric_results(p_score), round_up_metric_results(r_score), round_up_metric_results(f_score), cm ))\n", "\n", "test_set_results.sort(key=lambda k: sum(k[3]), reverse=True)\n", "pd.DataFrame(test_set_results, columns=[\"Model\", \"Precision Score\", \"Recall Score\", \"F1 score\", \"Confusion Matrix\"])" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## 6. Conclusion & Dumped model\n", "\n", "Best model according to evaluation is model with 7 layers include some dropout layers" ] }, { "cell_type": "code", "execution_count": 54, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow:Assets written to: ram://72fd27ad-5e1d-4761-a5f0-b2b4682c6a65/assets\n" ] } ], "source": [ "# Dump the best model to a pickle file\n", "with open(\"./model/plank_dp.pkl\", \"wb\") as f:\n", " pickle.dump(final_models[\"7_layers_with_dropout\"], f)" ] }, { "cell_type": "code", "execution_count": 55, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "INFO:tensorflow:Assets written to: ram://cc39563d-8dfe-4f23-9dc6-71462ea5ef5b/assets\n", "INFO:tensorflow:Assets written to: ram://5855831e-e2d4-4bb1-9799-a65f926cefb3/assets\n", "INFO:tensorflow:Assets written to: ram://bf30ae68-3e13-4624-8b29-c9c341693ed8/assets\n", "INFO:tensorflow:Assets written to: ram://9b787241-d173-4542-88d3-5551c1cbb80d/assets\n" ] } ], "source": [ "# Dump final results\n", "with open(\"./model/all_dp.pkl\", \"wb\") as f:\n", " pickle.dump(final_models, f)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3.8.13", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.8.13" }, "orig_nbformat": 4, "vscode": { "interpreter": { "hash": "9260f401923fb5c4108c543a7d176de9733d378b3752e49535ad7c43c2271b65" } } }, "nbformat": 4, "nbformat_minor": 2 }