upload models

models/FullyConectedModels/Grid_SearchCV.ipynb

@@ -0,0 +1,1213 @@
+{
+  "nbformat": 4,
+  "nbformat_minor": 0,
+  "metadata": {
+    "accelerator": "GPU",
+    "colab": {
+      "name": "Grid_SearchCV.ipynb",
+      "provenance": [],
+      "collapsed_sections": [],
+      "toc_visible": true
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "language": "python",
+      "name": "python3"
+    },
+    "language_info": {
+      "codemirror_mode": {
+        "name": "ipython",
+        "version": 3
+      },
+      "file_extension": ".py",
+      "mimetype": "text/x-python",
+      "name": "python",
+      "nbconvert_exporter": "python",
+      "pygments_lexer": "ipython3",
+      "version": "3.7.4"
+    }
+  },
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "o7mJMiThKvtT"
+      },
+      "source": [
+        "# <font color=\"purple\"><b>Grid Search CV Algorithm for Fully Connected Networks</b></font>"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "kUrakNvqKvtU"
+      },
+      "source": [
+        "Using the grid search CV algorithm, the hyperparameters of this model is sought.\n",
+        "<li><b> Learning Rate:</b> 0.1, 0.01</li>\n",
+        "<li><b> Regularization Penalty:</b>0.01, 0.001, 0.0001</li>\n",
+        "<li><b> Batch Size:</b> 64, 128</li>\n",
+        "<li><b> Epochs:</b> 50, 100, 150</li>"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "9rFQbEcDKvtV"
+      },
+      "source": [
+        "## <font color=\"blue\">Import Libraries</font>"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "metadata": {
+        "id": "6nhsKKJZ02AK"
+      },
+      "source": [
+        "import gzip\n",
+        "import pickle\n",
+        "import numpy as np\n",
+        "import pandas as pd\n",
+        "import numpy as np\n",
+        "from sklearn.preprocessing import LabelEncoder\n",
+        "from tensorflow.keras.utils import to_categorical\n",
+        "from tensorflow.keras import backend as K\n",
+        "from itertools import product\n",
+        "from sklearn.model_selection import train_test_split\n",
+        "from sklearn.model_selection import KFold\n",
+        "\n",
+        "from tensorflow.keras.regularizers import l2\n",
+        "from tensorflow.keras.optimizers import SGD\n",
+        "import tensorflow\n",
+        "import json\n",
+        "import cv2\n",
+        "import io\n",
+        "from sklearn.metrics import accuracy_score\n",
+        "from sklearn.metrics import precision_score\n",
+        "from sklearn.metrics import recall_score\n",
+        "try:\n",
+        "    to_unicode = unicode\n",
+        "except NameError:\n",
+        "    to_unicode = str\n",
+        "from sklearn.preprocessing import LabelEncoder\n",
+        "from tensorflow.keras.utils import to_categorical"
+      ],
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "metadata": {
+        "id": "TdoFKvR-m04D"
+      },
+      "source": [
+        "!pip install hickle\n",
+        "import hickle as hkl"
+      ],
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "metadata": {
+        "id": "XqmnAXulmtWm"
+      },
+      "source": [
+        "data = hkl.load(\"data.hkl\")\n",
+        "X_train, X_test, Y_train, y_test = data['xtrain'], data['xtest'], data['ytrain'], data['ytest']\n",
+        "x_train, x_val, y_train, y_val = train_test_split(X_train, Y_train, test_size=0.1)"
+      ],
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "metadata": {
+        "id": "u4zwcqNAnEoE"
+      },
+      "source": [
+        "from tensorflow.data import Dataset\n",
+        "import tensorflow.keras as keras\n",
+        "from tensorflow.keras.optimizers import Adam\n",
+        "from tensorflow.keras.layers import Conv2D,Input,MaxPooling2D, Dense, Dropout, MaxPool1D, Flatten, AveragePooling1D, BatchNormalization\n",
+        "from tensorflow.keras import Model\n",
+        "import numpy as np\n",
+        "import tensorflow as tf\n",
+        "from tensorflow.keras.models import Sequential\n"
+      ],
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "metadata": {
+        "id": "pW_D-_Dqm5mg"
+      },
+      "source": [
+        "def model_1(weight_decay):\n",
+        "  model = Sequential()\n",
+        "  model.add(Conv2D(32, kernel_size=(3, 3),activation='relu', input_shape=(32, 32, 1), kernel_regularizer=l2(weight_decay)))\n",
+        "  model.add(Conv2D(64, kernel_size=(3, 3), activation='relu', kernel_regularizer=l2(weight_decay)))\n",
+        "  model.add(MaxPooling2D(pool_size=(2, 2)))\n",
+        "  model.add(BatchNormalization())\n",
+        "  model.add(Flatten())\n",
+        "  model.add(Dense(4, activation='softmax', kernel_regularizer=l2(weight_decay)))\n",
+        "  return model\n",
+        "\n",
+        "\n",
+        "def model_2(weight_decay):\n",
+        "  model = Sequential()\n",
+        "  model.add(Conv2D(32, kernel_size=(3, 3),activation='relu', input_shape=(32, 32, 1), kernel_regularizer=l2(weight_decay)))\n",
+        "  model.add(Conv2D(64, kernel_size=(3, 3), activation='relu', kernel_regularizer=l2(weight_decay)))\n",
+        "  model.add(MaxPooling2D(pool_size=(2, 2)))\n",
+        "  model.add(BatchNormalization())\n",
+        "  model.add(Conv2D(128, kernel_size=(3, 3), activation='relu', kernel_regularizer=l2(weight_decay)))\n",
+        "  model.add(MaxPooling2D(pool_size=(2, 2)))\n",
+        "  model.add(BatchNormalization())\n",
+        "  model.add(Flatten())\n",
+        "  model.add(Dense(4, activation='softmax', kernel_regularizer=l2(weight_decay)))\n",
+        "  return model\n",
+        "\n",
+        "\n",
+        "def model_3(weight_decay):\n",
+        "  model = Sequential()\n",
+        "  model.add(Conv2D(32, kernel_size=(3, 3),activation='relu', input_shape=(32, 32, 1),kernel_regularizer=l2(weight_decay)))\n",
+        "  model.add(Conv2D(64, kernel_size=(3, 3), activation='relu',kernel_regularizer=l2(weight_decay)))\n",
+        "  model.add(MaxPooling2D(pool_size=(2, 2)))\n",
+        "  model.add(BatchNormalization())\n",
+        "  model.add(Conv2D(128, kernel_size=(3, 3), activation='relu',kernel_regularizer=l2(weight_decay)))\n",
+        "  model.add(MaxPooling2D(pool_size=(2, 2)))\n",
+        "  model.add(BatchNormalization())\n",
+        "  model.add(Conv2D(256, kernel_size=(3, 3), activation='relu',kernel_regularizer=l2(weight_decay)))\n",
+        "  model.add(MaxPooling2D(pool_size=(2, 2)))\n",
+        "  model.add(BatchNormalization())\n",
+        "  model.add(Flatten())\n",
+        "  model.add(Dense(4, activation='softmax',kernel_regularizer=l2(weight_decay)))\n",
+        "  return model\n"
+      ],
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "p9X1E2wybnZj"
+      },
+      "source": [
+        "<font color=\"blue\"> The algorithm below is that ... </font>"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "metadata": {
+        "id": "V8U0Vk9t0l3V"
+      },
+      "source": [
+        "from sklearn.metrics import confusion_matrix\n",
+        "def encoded_label(y_predict):\n",
+        "  y_list = [] \n",
+        "  for y_hat in y_predict:\n",
+        "    y_hat = np.argmax(y_hat)\n",
+        "    y_list.append(to_categorical(y_hat))\n",
+        "  return y_list\n",
+        "\n",
+        "\n",
+        "def KFold_GridSearchCV(input_dim, X, Y, X_test, y_test, combinations, filename=\"log.csv\", acc_loss_json=\"hist.json\"):\n",
+        "    \"\"\"Summary: Grid Search CV for 3 Folds Cross Validation\n",
+        "    \"\"\"\n",
+        "    res_df = pd.DataFrame(columns=['momentum','learning rate','batch size',\n",
+        "                                      'loss1', 'acc1','loss2', 'acc2','loss3', 'acc3', 'widing factor',\n",
+        "                                      'prec1', 'prec2', 'prec3', 'recall1', 'recall2', 'recall3'])\n",
+        "    generator = tensorflow.keras.preprocessing.image.ImageDataGenerator(rotation_range=10,\n",
+        "                               width_shift_range=5./32,\n",
+        "                               height_shift_range=5./32,)\n",
+        "    hist_dict_global = {}\n",
+        "\n",
+        "    for i, combination in enumerate(combinations):\n",
+        "        kf = KFold(n_splits=3, random_state=42, shuffle=False)\n",
+        "        metrics_dict = {}\n",
+        "  \n",
+        "        for j, (train_index, test_index) in enumerate(kf.split(X)):\n",
+        "            X_train, X_val = X[train_index], X[test_index]\n",
+        "            y_train, y_val = Y[train_index], Y[test_index]\n",
+        "            model = model_1(combination[2])\n",
+        "            opt = tensorflow.keras.optimizers.SGD(learning_rate=combination[0])\n",
+        "            model.compile(loss='categorical_crossentropy', optimizer=opt, metrics= ['accuracy'])\n",
+        "            hist = model.fit(generator.flow(X_train, y_train, batch_size=combination[1]), steps_per_epoch=len(X_train) // combination[1], epochs=combination[3],\n",
+        "                                      validation_data=(X_val, y_val),\n",
+        "                                      validation_steps=len(X_val) // combination[1],)\n",
+        "            loss, acc = model.evaluate(X_test, y_test)\n",
+        "            #yhat_classes = encoded_label( model.predict(X_test))\n",
+        "            predict =  model.predict(X_test)\n",
+        "            yhat_classes = np.argmax(predict, axis=1)\n",
+        "            print(yhat_classes)\n",
+        "            print(y_test)\n",
+        "            cm = confusion_matrix(np.argmax(y_test, axis=1), yhat_classes)\n",
+        "            recall = np.diag(cm) / np.sum(cm, axis = 1)\n",
+        "            precision = np.diag(cm) / np.sum(cm, axis = 0)\n",
+        "\n",
+        "            recall_avg = np.mean(recall)\n",
+        "            precision_avg = np.mean(precision)\n",
+        "            metrics_dict[j+1] = {\"loss\": loss, \"acc\": acc, \"epoch_stopped\": combination[3], \"precision_avg\":precision_avg,\n",
+        "                                 \"avg_recall\":recall_avg}\n",
+        "            graph_loss_acc = {\"id\": i, \"com\":j+1, \"val_acc\":hist.history[\"val_accuracy\"], \"train_acc\":hist.history[\"accuracy\"],\n",
+        "                                    \"val_loss\":hist.history[\"val_loss\"], \"train_loss\":hist.history[\"loss\"], \"epoch_stopped\": combination[3], 'learning rate': combination[0],\n",
+        "                                    'batch size': combination[1], 'reg_penalty': combination[2]}\n",
+        "\n",
+        "              # Write JSON file\n",
+        "            with io.open(acc_loss_json, 'a+', encoding='utf8') as outfile:\n",
+        "                str_ = json.dumps(graph_loss_acc)\n",
+        "                outfile.write(to_unicode(str_))\n",
+        "        row = {'momentum': combination[4],'learning rate': combination[0],\n",
+        "               'batch size': combination[1],\n",
+        "               'reg_penalty': combination[2],\n",
+        "               'epoch_stopped': metrics_dict[1][\"epoch_stopped\"],\n",
+        "               'widing factor' : 1,\n",
+        "               'loss1': metrics_dict[1][\"loss\"],\n",
+        "               'acc1': metrics_dict[1][\"acc\"],\n",
+        "               'loss2': metrics_dict[2][\"loss\"],\n",
+        "               'acc2': metrics_dict[2][\"acc\"],\n",
+        "               'loss3': metrics_dict[3][\"loss\"],\n",
+        "               'acc3': metrics_dict[3][\"acc\"],\n",
+        "               'prec1':metrics_dict[1][\"precision_avg\"],\n",
+        "               'prec2':metrics_dict[2][\"precision_avg\"],\n",
+        "               'prec3':metrics_dict[3][\"precision_avg\"],\n",
+        "               'recall1':metrics_dict[1][\"avg_recall\"],\n",
+        "               'recall2':metrics_dict[2][\"avg_recall\"],\n",
+        "               'recall3':metrics_dict[3][\"avg_recall\"]}\n",
+        "        res_df = res_df.append(row , ignore_index=True)\n",
+        "        res_df.to_csv(filename, sep=\";\")"
+      ],
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "metadata": {
+        "id": "tgO8xJ0Fe93I"
+      },
+      "source": [
+        "\n",
+        "if __name__ == \"__main__\":\n",
+        "    learning_rate = [0.1,0.01]\n",
+        "    batch_size = [64,128]\n",
+        "    reg_penalty = [0,0.01, 0.001, 0.0001]\n",
+        "    epochs = [50,100,150]\n",
+        "    momentum = [0.9]\n",
+        "    in_dim = (32,32,1)\n",
+        "    grid_result = \"grid_model_1.csv\"\n",
+        "    acc_loss_json = \"history.json\"\n",
+        "    # create list of all different parameter combinations\n",
+        "    param_grid = dict(learning_rate = learning_rate, batch_size = batch_size, \n",
+        "                      reg_penalty = reg_penalty, epochs = epochs, momentum=momentum)\n",
+        "    combinations = list(product(*param_grid.values()))\n",
+        "    KFold_GridSearchCV(in_dim,X_train,Y_train,X_test, y_test, combinations, grid_result, acc_loss_json)"
+      ],
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "metadata": {
+        "id": "iVp5lMuhpAMW"
+      },
+      "source": [
+        "data  = pd.read_csv(\"grid_model_1.csv\", sep=\";\")"
+      ],
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/",
+          "height": 244
+        },
+        "id": "rru6HHyrsgbl",
+        "outputId": "01496643-5865-41a6-85c2-f69242b5912b"
+      },
+      "source": [
+        "data.head(5)"
+      ],
+      "execution_count": null,
+      "outputs": [
+        {
+          "output_type": "execute_result",
+          "data": {
+            "text/html": [
+              "<div>\n",
+              "<style scoped>\n",
+              "    .dataframe tbody tr th:only-of-type {\n",
+              "        vertical-align: middle;\n",
+              "    }\n",
+              "\n",
+              "    .dataframe tbody tr th {\n",
+              "        vertical-align: top;\n",
+              "    }\n",
+              "\n",
+              "    .dataframe thead th {\n",
+              "        text-align: right;\n",
+              "    }\n",
+              "</style>\n",
+              "<table border=\"1\" class=\"dataframe\">\n",
+              "  <thead>\n",
+              "    <tr style=\"text-align: right;\">\n",
+              "      <th></th>\n",
+              "      <th>momentum</th>\n",
+              "      <th>learning rate</th>\n",
+              "      <th>batch size</th>\n",
+              "      <th>loss1</th>\n",
+              "      <th>acc1</th>\n",
+              "      <th>loss2</th>\n",
+              "      <th>acc2</th>\n",
+              "      <th>loss3</th>\n",
+              "      <th>acc3</th>\n",
+              "      <th>widing factor</th>\n",
+              "      <th>prec1</th>\n",
+              "      <th>prec2</th>\n",
+              "      <th>prec3</th>\n",
+              "      <th>recall1</th>\n",
+              "      <th>recall2</th>\n",
+              "      <th>recall3</th>\n",
+              "      <th>epoch_stopped</th>\n",
+              "      <th>reg_penalty</th>\n",
+              "    </tr>\n",
+              "  </thead>\n",
+              "  <tbody>\n",
+              "    <tr>\n",
+              "      <th>0</th>\n",
+              "      <td>0.9</td>\n",
+              "      <td>0.1</td>\n",
+              "      <td>64.0</td>\n",
+              "      <td>1.078664</td>\n",
+              "      <td>0.541012</td>\n",
+              "      <td>1.087765</td>\n",
+              "      <td>0.560209</td>\n",
+              "      <td>1.140540</td>\n",
+              "      <td>0.495637</td>\n",
+              "      <td>1.0</td>\n",
+              "      <td>0.545625</td>\n",
+              "      <td>0.570303</td>\n",
+              "      <td>0.540447</td>\n",
+              "      <td>0.549296</td>\n",
+              "      <td>0.561227</td>\n",
+              "      <td>0.506039</td>\n",
+              "      <td>50.0</td>\n",
+              "      <td>0.00</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <th>1</th>\n",
+              "      <td>0.9</td>\n",
+              "      <td>0.1</td>\n",
+              "      <td>64.0</td>\n",
+              "      <td>1.090459</td>\n",
+              "      <td>0.544503</td>\n",
+              "      <td>1.052103</td>\n",
+              "      <td>0.568935</td>\n",
+              "      <td>0.991915</td>\n",
+              "      <td>0.586387</td>\n",
+              "      <td>1.0</td>\n",
+              "      <td>0.567745</td>\n",
+              "      <td>0.581882</td>\n",
+              "      <td>0.587054</td>\n",
+              "      <td>0.549632</td>\n",
+              "      <td>0.571559</td>\n",
+              "      <td>0.584749</td>\n",
+              "      <td>100.0</td>\n",
+              "      <td>0.00</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <th>2</th>\n",
+              "      <td>0.9</td>\n",
+              "      <td>0.1</td>\n",
+              "      <td>64.0</td>\n",
+              "      <td>1.203270</td>\n",
+              "      <td>0.549738</td>\n",
+              "      <td>1.024475</td>\n",
+              "      <td>0.607330</td>\n",
+              "      <td>1.099551</td>\n",
+              "      <td>0.542757</td>\n",
+              "      <td>1.0</td>\n",
+              "      <td>0.597314</td>\n",
+              "      <td>0.629707</td>\n",
+              "      <td>0.583353</td>\n",
+              "      <td>0.558664</td>\n",
+              "      <td>0.612029</td>\n",
+              "      <td>0.532126</td>\n",
+              "      <td>150.0</td>\n",
+              "      <td>0.00</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <th>3</th>\n",
+              "      <td>0.9</td>\n",
+              "      <td>0.1</td>\n",
+              "      <td>64.0</td>\n",
+              "      <td>1.331875</td>\n",
+              "      <td>0.450262</td>\n",
+              "      <td>1.357389</td>\n",
+              "      <td>0.425829</td>\n",
+              "      <td>1.136434</td>\n",
+              "      <td>0.542757</td>\n",
+              "      <td>1.0</td>\n",
+              "      <td>0.520104</td>\n",
+              "      <td>0.556441</td>\n",
+              "      <td>0.541896</td>\n",
+              "      <td>0.467984</td>\n",
+              "      <td>0.441258</td>\n",
+              "      <td>0.549730</td>\n",
+              "      <td>50.0</td>\n",
+              "      <td>0.01</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <th>4</th>\n",
+              "      <td>0.9</td>\n",
+              "      <td>0.1</td>\n",
+              "      <td>64.0</td>\n",
+              "      <td>1.286225</td>\n",
+              "      <td>0.490401</td>\n",
+              "      <td>1.218342</td>\n",
+              "      <td>0.539267</td>\n",
+              "      <td>1.207052</td>\n",
+              "      <td>0.537522</td>\n",
+              "      <td>1.0</td>\n",
+              "      <td>0.552838</td>\n",
+              "      <td>0.569478</td>\n",
+              "      <td>0.586226</td>\n",
+              "      <td>0.494587</td>\n",
+              "      <td>0.541650</td>\n",
+              "      <td>0.547532</td>\n",
+              "      <td>100.0</td>\n",
+              "      <td>0.01</td>\n",
+              "    </tr>\n",
+              "  </tbody>\n",
+              "</table>\n",
+              "</div>"
+            ],
+            "text/plain": [
+              "   momentum  learning rate  batch size  ...   recall3  epoch_stopped  reg_penalty\n",
+              "0       0.9            0.1        64.0  ...  0.506039           50.0         0.00\n",
+              "1       0.9            0.1        64.0  ...  0.584749          100.0         0.00\n",
+              "2       0.9            0.1        64.0  ...  0.532126          150.0         0.00\n",
+              "3       0.9            0.1        64.0  ...  0.549730           50.0         0.01\n",
+              "4       0.9            0.1        64.0  ...  0.547532          100.0         0.01\n",
+              "\n",
+              "[5 rows x 18 columns]"
+            ]
+          },
+          "metadata": {
+            "tags": []
+          },
+          "execution_count": 17
+        }
+      ]
+    },
+    {
+      "cell_type": "code",
+      "metadata": {
+        "id": "LeVPJJQQt36n"
+      },
+      "source": [
+        "data[\"loss_mean\"] = (data[\"loss1\"]+data[\"loss2\"]+data[\"loss3\"])/3\n",
+        "data[\"acc_mean\"] = (data[\"acc1\"]+data[\"acc2\"]+data[\"acc3\"])/3"
+      ],
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "metadata": {
+        "id": "XPIKRbPMuUOr"
+      },
+      "source": [
+        "data['epoch'] = data['epoch_stopped']\n",
+        "data['weight_decay'] = data['reg_penalty']"
+      ],
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "metadata": {
+        "id": "5_2tMeO6x9Uq"
+      },
+      "source": [
+        "data['recall_mean'] = (data['recall1']+data['recall2']+data['recall3'])/3\n",
+        "data['prec_mean'] = (data['prec1']+data['prec2']+data['prec3'])/3"
+      ],
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "metadata": {
+        "id": "JA0eRctYuWl-"
+      },
+      "source": [
+        "column_list = [\"momentum\", \"learning rate\", \"epoch\",\"batch size\",\"weight_decay\",\"loss_mean\", \"acc_mean\",\"recall_mean\", \"prec_mean\"]"
+      ],
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/",
+          "height": 143
+        },
+        "id": "McsMaiLZuZaa",
+        "outputId": "d04090a8-b295-45ad-d64f-e5965741cd80"
+      },
+      "source": [
+        "data.sort_values(axis=0, by=\"loss_mean\", ascending=True)[column_list].head(3)"
+      ],
+      "execution_count": null,
+      "outputs": [
+        {
+          "output_type": "execute_result",
+          "data": {
+            "text/html": [
+              "<div>\n",
+              "<style scoped>\n",
+              "    .dataframe tbody tr th:only-of-type {\n",
+              "        vertical-align: middle;\n",
+              "    }\n",
+              "\n",
+              "    .dataframe tbody tr th {\n",
+              "        vertical-align: top;\n",
+              "    }\n",
+              "\n",
+              "    .dataframe thead th {\n",
+              "        text-align: right;\n",
+              "    }\n",
+              "</style>\n",
+              "<table border=\"1\" class=\"dataframe\">\n",
+              "  <thead>\n",
+              "    <tr style=\"text-align: right;\">\n",
+              "      <th></th>\n",
+              "      <th>momentum</th>\n",
+              "      <th>learning rate</th>\n",
+              "      <th>epoch</th>\n",
+              "      <th>batch size</th>\n",
+              "      <th>weight_decay</th>\n",
+              "      <th>loss_mean</th>\n",
+              "      <th>acc_mean</th>\n",
+              "      <th>recall_mean</th>\n",
+              "      <th>prec_mean</th>\n",
+              "    </tr>\n",
+              "  </thead>\n",
+              "  <tbody>\n",
+              "    <tr>\n",
+              "      <th>24</th>\n",
+              "      <td>0.9</td>\n",
+              "      <td>0.01</td>\n",
+              "      <td>50.0</td>\n",
+              "      <td>64.0</td>\n",
+              "      <td>0.0000</td>\n",
+              "      <td>1.012071</td>\n",
+              "      <td>0.612565</td>\n",
+              "      <td>0.614360</td>\n",
+              "      <td>0.615845</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <th>34</th>\n",
+              "      <td>0.9</td>\n",
+              "      <td>0.01</td>\n",
+              "      <td>100.0</td>\n",
+              "      <td>64.0</td>\n",
+              "      <td>0.0001</td>\n",
+              "      <td>1.026681</td>\n",
+              "      <td>0.614311</td>\n",
+              "      <td>0.612435</td>\n",
+              "      <td>0.619908</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <th>8</th>\n",
+              "      <td>0.9</td>\n",
+              "      <td>0.10</td>\n",
+              "      <td>150.0</td>\n",
+              "      <td>64.0</td>\n",
+              "      <td>0.0010</td>\n",
+              "      <td>1.027309</td>\n",
+              "      <td>0.603258</td>\n",
+              "      <td>0.601262</td>\n",
+              "      <td>0.606730</td>\n",
+              "    </tr>\n",
+              "  </tbody>\n",
+              "</table>\n",
+              "</div>"
+            ],
+            "text/plain": [
+              "    momentum  learning rate  epoch  ...  acc_mean  recall_mean  prec_mean\n",
+              "24       0.9           0.01   50.0  ...  0.612565     0.614360   0.615845\n",
+              "34       0.9           0.01  100.0  ...  0.614311     0.612435   0.619908\n",
+              "8        0.9           0.10  150.0  ...  0.603258     0.601262   0.606730\n",
+              "\n",
+              "[3 rows x 9 columns]"
+            ]
+          },
+          "metadata": {
+            "tags": []
+          },
+          "execution_count": 26
+        }
+      ]
+    },
+    {
+      "cell_type": "code",
+      "metadata": {
+        "id": "hCgRAh43udBu"
+      },
+      "source": [
+        "data[\"loss_na\"] = data.loc[:,[\"loss1\",\"loss2\", \"loss3\"]].isnull().sum(1)"
+      ],
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/",
+          "height": 181
+        },
+        "id": "-ptzEmSHudkM",
+        "outputId": "72a8ca57-603a-451f-8702-962b4be4f91a"
+      },
+      "source": [
+        "data.head(3)"
+      ],
+      "execution_count": null,
+      "outputs": [
+        {
+          "output_type": "execute_result",
+          "data": {
+            "text/html": [
+              "<div>\n",
+              "<style scoped>\n",
+              "    .dataframe tbody tr th:only-of-type {\n",
+              "        vertical-align: middle;\n",
+              "    }\n",
+              "\n",
+              "    .dataframe tbody tr th {\n",
+              "        vertical-align: top;\n",
+              "    }\n",
+              "\n",
+              "    .dataframe thead th {\n",
+              "        text-align: right;\n",
+              "    }\n",
+              "</style>\n",
+              "<table border=\"1\" class=\"dataframe\">\n",
+              "  <thead>\n",
+              "    <tr style=\"text-align: right;\">\n",
+              "      <th></th>\n",
+              "      <th>momentum</th>\n",
+              "      <th>learning rate</th>\n",
+              "      <th>batch size</th>\n",
+              "      <th>loss1</th>\n",
+              "      <th>acc1</th>\n",
+              "      <th>loss2</th>\n",
+              "      <th>acc2</th>\n",
+              "      <th>loss3</th>\n",
+              "      <th>acc3</th>\n",
+              "      <th>widing factor</th>\n",
+              "      <th>prec1</th>\n",
+              "      <th>prec2</th>\n",
+              "      <th>prec3</th>\n",
+              "      <th>recall1</th>\n",
+              "      <th>recall2</th>\n",
+              "      <th>recall3</th>\n",
+              "      <th>epoch_stopped</th>\n",
+              "      <th>reg_penalty</th>\n",
+              "      <th>loss_mean</th>\n",
+              "      <th>acc_mean</th>\n",
+              "      <th>epoch</th>\n",
+              "      <th>weight_decay</th>\n",
+              "      <th>loss_na</th>\n",
+              "    </tr>\n",
+              "  </thead>\n",
+              "  <tbody>\n",
+              "    <tr>\n",
+              "      <th>0</th>\n",
+              "      <td>0.9</td>\n",
+              "      <td>0.1</td>\n",
+              "      <td>64.0</td>\n",
+              "      <td>1.078664</td>\n",
+              "      <td>0.541012</td>\n",
+              "      <td>1.087765</td>\n",
+              "      <td>0.560209</td>\n",
+              "      <td>1.140540</td>\n",
+              "      <td>0.495637</td>\n",
+              "      <td>1.0</td>\n",
+              "      <td>0.545625</td>\n",
+              "      <td>0.570303</td>\n",
+              "      <td>0.540447</td>\n",
+              "      <td>0.549296</td>\n",
+              "      <td>0.561227</td>\n",
+              "      <td>0.506039</td>\n",
+              "      <td>50.0</td>\n",
+              "      <td>0.0</td>\n",
+              "      <td>1.102323</td>\n",
+              "      <td>0.532286</td>\n",
+              "      <td>50.0</td>\n",
+              "      <td>0.0</td>\n",
+              "      <td>0</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <th>1</th>\n",
+              "      <td>0.9</td>\n",
+              "      <td>0.1</td>\n",
+              "      <td>64.0</td>\n",
+              "      <td>1.090459</td>\n",
+              "      <td>0.544503</td>\n",
+              "      <td>1.052103</td>\n",
+              "      <td>0.568935</td>\n",
+              "      <td>0.991915</td>\n",
+              "      <td>0.586387</td>\n",
+              "      <td>1.0</td>\n",
+              "      <td>0.567745</td>\n",
+              "      <td>0.581882</td>\n",
+              "      <td>0.587054</td>\n",
+              "      <td>0.549632</td>\n",
+              "      <td>0.571559</td>\n",
+              "      <td>0.584749</td>\n",
+              "      <td>100.0</td>\n",
+              "      <td>0.0</td>\n",
+              "      <td>1.044826</td>\n",
+              "      <td>0.566609</td>\n",
+              "      <td>100.0</td>\n",
+              "      <td>0.0</td>\n",
+              "      <td>0</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <th>2</th>\n",
+              "      <td>0.9</td>\n",
+              "      <td>0.1</td>\n",
+              "      <td>64.0</td>\n",
+              "      <td>1.203270</td>\n",
+              "      <td>0.549738</td>\n",
+              "      <td>1.024475</td>\n",
+              "      <td>0.607330</td>\n",
+              "      <td>1.099551</td>\n",
+              "      <td>0.542757</td>\n",
+              "      <td>1.0</td>\n",
+              "      <td>0.597314</td>\n",
+              "      <td>0.629707</td>\n",
+              "      <td>0.583353</td>\n",
+              "      <td>0.558664</td>\n",
+              "      <td>0.612029</td>\n",
+              "      <td>0.532126</td>\n",
+              "      <td>150.0</td>\n",
+              "      <td>0.0</td>\n",
+              "      <td>1.109098</td>\n",
+              "      <td>0.566608</td>\n",
+              "      <td>150.0</td>\n",
+              "      <td>0.0</td>\n",
+              "      <td>0</td>\n",
+              "    </tr>\n",
+              "  </tbody>\n",
+              "</table>\n",
+              "</div>"
+            ],
+            "text/plain": [
+              "   momentum  learning rate  batch size  ...  epoch  weight_decay  loss_na\n",
+              "0       0.9            0.1        64.0  ...   50.0           0.0        0\n",
+              "1       0.9            0.1        64.0  ...  100.0           0.0        0\n",
+              "2       0.9            0.1        64.0  ...  150.0           0.0        0\n",
+              "\n",
+              "[3 rows x 23 columns]"
+            ]
+          },
+          "metadata": {
+            "tags": []
+          },
+          "execution_count": 23
+        }
+      ]
+    },
+    {
+      "cell_type": "code",
+      "metadata": {
+        "id": "YBBXsz_rzQl-"
+      },
+      "source": [
+        "generator = tensorflow.keras.preprocessing.image.ImageDataGenerator(rotation_range=10,\n",
+        "                               width_shift_range=5./32,\n",
+        "                               height_shift_range=5./32,)"
+      ],
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "metadata": {
+        "id": "ShIJn_53mawD"
+      },
+      "source": [
+        "kf = KFold(n_splits=3, random_state=42, shuffle=False)\n",
+        "result = []\n",
+        "for j, (train_index, test_index) in enumerate(kf.split(X_train)):\n",
+        "  x_train, x_val = X_train[train_index], X_train[test_index]\n",
+        "  y_train, y_val = Y_train[train_index], Y_train[test_index]\n",
+        "  model = model_2(0)\n",
+        "  opt = tensorflow.keras.optimizers.SGD(learning_rate=0.01)\n",
+        "  model.compile(loss='categorical_crossentropy', optimizer=opt, metrics= ['accuracy'])\n",
+        "  hist = model.fit(generator.flow(x_train, y_train, batch_size=64), steps_per_epoch=len(x_train) //64 , epochs=50,\n",
+        "                   validation_data=(x_val, y_val),\n",
+        "                  validation_steps=len(x_val) //64 ,)\n",
+        "\n",
+        "  test = model.evaluate(X_test, y_test)\n",
+        "  result.append(test)"
+      ],
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "Rj6hnUVRzjX-",
+        "outputId": "41b04e1e-d756-437a-afff-ca9f3fcff46c"
+      },
+      "source": [
+        "mean_acc = (result[0][1]+result[1][1]+result[2][1])/3;mean_acc"
+      ],
+      "execution_count": null,
+      "outputs": [
+        {
+          "output_type": "execute_result",
+          "data": {
+            "text/plain": [
+              "0.6305991808573405"
+            ]
+          },
+          "metadata": {
+            "tags": []
+          },
+          "execution_count": 47
+        }
+      ]
+    },
+    {
+      "cell_type": "code",
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "v_AccCit1De3",
+        "outputId": "a6299f58-4389-4f39-f318-e2371e94f4e5"
+      },
+      "source": [
+        "mean_loss = (result[0][0]+result[1][0]+result[2][0])/3;mean_loss"
+      ],
+      "execution_count": null,
+      "outputs": [
+        {
+          "output_type": "execute_result",
+          "data": {
+            "text/plain": [
+              "0.9891296029090881"
+            ]
+          },
+          "metadata": {
+            "tags": []
+          },
+          "execution_count": 48
+        }
+      ]
+    },
+    {
+      "cell_type": "code",
+      "metadata": {
+        "id": "8gdDigrPzYM5"
+      },
+      "source": [
+        "kf = KFold(n_splits=3, random_state=42, shuffle=False)\n",
+        "result_2 = []\n",
+        "for j, (train_index, test_index) in enumerate(kf.split(X_train)):\n",
+        "  x_train, x_val = X_train[train_index], X_train[test_index]\n",
+        "  y_train, y_val = Y_train[train_index], Y_train[test_index]\n",
+        "  model = model_3(0.0001)\n",
+        "  opt = tensorflow.keras.optimizers.SGD(learning_rate=0.01)\n",
+        "  model.compile(loss='categorical_crossentropy', optimizer=opt, metrics= ['accuracy'])\n",
+        "  hist = model.fit(generator.flow(x_train, y_train, batch_size=64), steps_per_epoch=len(x_train) //64 , epochs=100,\n",
+        "                   validation_data=(x_val, y_val),\n",
+        "                  validation_steps=len(x_val) //64 ,)\n",
+        "\n",
+        "  test = model.evaluate(X_test, y_test)\n",
+        "  result_2.append(test)"
+      ],
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "FYVQINoA2F4l",
+        "outputId": "ac29f686-9b48-4193-bf01-4a17ab589d91"
+      },
+      "source": [
+        "mean_acc = (result_2[0][1]+result_2[1][1]+result_2[2][1])/3;mean_acc"
+      ],
+      "execution_count": null,
+      "outputs": [
+        {
+          "output_type": "execute_result",
+          "data": {
+            "text/plain": [
+              "0.7108784119288126"
+            ]
+          },
+          "metadata": {
+            "tags": []
+          },
+          "execution_count": 56
+        }
+      ]
+    },
+    {
+      "cell_type": "code",
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "guueg5Wo2IE8",
+        "outputId": "986e139c-1224-43a7-b8fc-59d57e72d3a8"
+      },
+      "source": [
+        "mean_loss = (result_2[0][0]+result_2[1][0]+result_2[2][0])/3;mean_loss"
+      ],
+      "execution_count": null,
+      "outputs": [
+        {
+          "output_type": "execute_result",
+          "data": {
+            "text/plain": [
+              "0.9208946625391642"
+            ]
+          },
+          "metadata": {
+            "tags": []
+          },
+          "execution_count": 57
+        }
+      ]
+    },
+    {
+      "cell_type": "code",
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "luS42Hil0H5R",
+        "outputId": "91c28154-8014-42a9-fbf2-8935cb4c521a"
+      },
+      "source": [
+        "mean_acc = (result_2[0][1]+result_2[1][1]+result_2[2][1])/3;mean_acc"
+      ],
+      "execution_count": null,
+      "outputs": [
+        {
+          "output_type": "execute_result",
+          "data": {
+            "text/plain": [
+              "0.6783013343811035"
+            ]
+          },
+          "metadata": {
+            "tags": []
+          },
+          "execution_count": 50
+        }
+      ]
+    },
+    {
+      "cell_type": "code",
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "F9Zg3pI61W_J",
+        "outputId": "fe5f7861-58e9-47f6-8134-8872e494e268"
+      },
+      "source": [
+        "mean_loss = (result_2[0][0]+result_2[1][0]+result_2[2][0])/3;mean_loss"
+      ],
+      "execution_count": null,
+      "outputs": [
+        {
+          "output_type": "execute_result",
+          "data": {
+            "text/plain": [
+              "0.8747362097104391"
+            ]
+          },
+          "metadata": {
+            "tags": []
+          },
+          "execution_count": 51
+        }
+      ]
+    },
+    {
+      "cell_type": "code",
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "OWx9Jej11f_V",
+        "outputId": "a5b72e6c-de64-4bee-da66-d48a5e6e1870"
+      },
+      "source": [
+        "model = model_1(0)\n",
+        "opt = tensorflow.keras.optimizers.SGD(learning_rate=0.01)\n",
+        "model.compile(loss='categorical_crossentropy', optimizer=opt, metrics= ['accuracy'])\n",
+        "model.summary()"
+      ],
+      "execution_count": null,
+      "outputs": [
+        {
+          "output_type": "stream",
+          "text": [
+            "Model: \"sequential_162\"\n",
+            "_________________________________________________________________\n",
+            "Layer (type)                 Output Shape              Param #   \n",
+            "=================================================================\n",
+            "conv2d_342 (Conv2D)          (None, 30, 30, 32)        320       \n",
+            "_________________________________________________________________\n",
+            "conv2d_343 (Conv2D)          (None, 28, 28, 64)        18496     \n",
+            "_________________________________________________________________\n",
+            "max_pooling2d_183 (MaxPoolin (None, 14, 14, 64)        0         \n",
+            "_________________________________________________________________\n",
+            "batch_normalization_183 (Bat (None, 14, 14, 64)        256       \n",
+            "_________________________________________________________________\n",
+            "flatten_162 (Flatten)        (None, 12544)             0         \n",
+            "_________________________________________________________________\n",
+            "dense_162 (Dense)            (None, 4)                 50180     \n",
+            "=================================================================\n",
+            "Total params: 69,252\n",
+            "Trainable params: 69,124\n",
+            "Non-trainable params: 128\n",
+            "_________________________________________________________________\n"
+          ],
+          "name": "stdout"
+        }
+      ]
+    },
+    {
+      "cell_type": "code",
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "l8mg2Bup1lPJ",
+        "outputId": "6a2348f9-f6d8-47d3-a5d7-9169d3906d76"
+      },
+      "source": [
+        "model = model_2(0)\n",
+        "opt = tensorflow.keras.optimizers.SGD(learning_rate=0.01)\n",
+        "model.compile(loss='categorical_crossentropy', optimizer=opt, metrics= ['accuracy'])\n",
+        "model.summary()"
+      ],
+      "execution_count": null,
+      "outputs": [
+        {
+          "output_type": "stream",
+          "text": [
+            "Model: \"sequential_154\"\n",
+            "_________________________________________________________________\n",
+            "Layer (type)                 Output Shape              Param #   \n",
+            "=================================================================\n",
+            "conv2d_320 (Conv2D)          (None, 30, 30, 32)        320       \n",
+            "_________________________________________________________________\n",
+            "conv2d_321 (Conv2D)          (None, 28, 28, 64)        18496     \n",
+            "_________________________________________________________________\n",
+            "max_pooling2d_166 (MaxPoolin (None, 14, 14, 64)        0         \n",
+            "_________________________________________________________________\n",
+            "batch_normalization_166 (Bat (None, 14, 14, 64)        256       \n",
+            "_________________________________________________________________\n",
+            "conv2d_322 (Conv2D)          (None, 12, 12, 128)       73856     \n",
+            "_________________________________________________________________\n",
+            "max_pooling2d_167 (MaxPoolin (None, 6, 6, 128)         0         \n",
+            "_________________________________________________________________\n",
+            "batch_normalization_167 (Bat (None, 6, 6, 128)         512       \n",
+            "_________________________________________________________________\n",
+            "flatten_154 (Flatten)        (None, 4608)              0         \n",
+            "_________________________________________________________________\n",
+            "dense_154 (Dense)            (None, 4)                 18436     \n",
+            "=================================================================\n",
+            "Total params: 111,876\n",
+            "Trainable params: 111,492\n",
+            "Non-trainable params: 384\n",
+            "_________________________________________________________________\n"
+          ],
+          "name": "stdout"
+        }
+      ]
+    },
+    {
+      "cell_type": "code",
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "vZImk4zB1mxk",
+        "outputId": "805f844e-1312-423a-b309-35944f11a210"
+      },
+      "source": [
+        "model = model_3(0)\n",
+        "opt = tensorflow.keras.optimizers.SGD(learning_rate=0.01)\n",
+        "model.compile(loss='categorical_crossentropy', optimizer=opt, metrics= ['accuracy'])\n",
+        "model.summary()"
+      ],
+      "execution_count": null,
+      "outputs": [
+        {
+          "output_type": "stream",
+          "text": [
+            "Model: \"sequential_155\"\n",
+            "_________________________________________________________________\n",
+            "Layer (type)                 Output Shape              Param #   \n",
+            "=================================================================\n",
+            "conv2d_323 (Conv2D)          (None, 30, 30, 32)        320       \n",
+            "_________________________________________________________________\n",
+            "conv2d_324 (Conv2D)          (None, 28, 28, 64)        18496     \n",
+            "_________________________________________________________________\n",
+            "max_pooling2d_168 (MaxPoolin (None, 14, 14, 64)        0         \n",
+            "_________________________________________________________________\n",
+            "batch_normalization_168 (Bat (None, 14, 14, 64)        256       \n",
+            "_________________________________________________________________\n",
+            "conv2d_325 (Conv2D)          (None, 12, 12, 128)       73856     \n",
+            "_________________________________________________________________\n",
+            "max_pooling2d_169 (MaxPoolin (None, 6, 6, 128)         0         \n",
+            "_________________________________________________________________\n",
+            "batch_normalization_169 (Bat (None, 6, 6, 128)         512       \n",
+            "_________________________________________________________________\n",
+            "conv2d_326 (Conv2D)          (None, 4, 4, 256)         295168    \n",
+            "_________________________________________________________________\n",
+            "max_pooling2d_170 (MaxPoolin (None, 2, 2, 256)         0         \n",
+            "_________________________________________________________________\n",
+            "batch_normalization_170 (Bat (None, 2, 2, 256)         1024      \n",
+            "_________________________________________________________________\n",
+            "flatten_155 (Flatten)        (None, 1024)              0         \n",
+            "_________________________________________________________________\n",
+            "dense_155 (Dense)            (None, 4)                 4100      \n",
+            "=================================================================\n",
+            "Total params: 393,732\n",
+            "Trainable params: 392,836\n",
+            "Non-trainable params: 896\n",
+            "_________________________________________________________________\n"
+          ],
+          "name": "stdout"
+        }
+      ]
+    }
+  ]
+}

models/FullyConectedModels/gridsearchcv/grid_model_1.csv

@@ -0,0 +1,49 @@
+;momentum;learning rate;batch size;loss1;acc1;loss2;acc2;loss3;acc3;widing factor;prec1;prec2;prec3;recall1;recall2;recall3;epoch_stopped;reg_penalty
+0;0.9;0.1;64.0;1.0786635875701904;0.5410122275352478;1.087765097618103;0.5602094531059265;1.1405400037765503;0.4956369996070862;1.0;0.5456249725117649;0.5703025971923495;0.5404468631355052;0.549295511252033;0.561227419923072;0.5060386473429952;50.0;0.0
+1;0.9;0.1;64.0;1.0904589891433716;0.5445026159286499;1.0521031618118286;0.5689354538917542;0.9919149279594421;0.5863874554634094;1.0;0.5677450027120325;0.5818822696361521;0.5870538777082881;0.5496315278923974;0.5715594791681748;0.5847485847485847;100.0;0.0
+2;0.9;0.1;64.0;1.203269600868225;0.5497382283210754;1.0244745016098022;0.6073298454284668;1.0995512008666992;0.5427573919296265;1.0;0.5973140968206758;0.6297070150268066;0.5833525841963643;0.5586641619250315;0.6120292750727534;0.5321260864739126;150.0;0.0
+3;0.9;0.1;64.0;1.3318754434585571;0.45026177167892456;1.3573890924453735;0.4258289635181427;1.1364340782165527;0.5427573919296265;1.0;0.520103685420754;0.5564409752732054;0.5418956991063357;0.4679843103756147;0.44125801734497383;0.5497298595124682;50.0;0.01
+4;0.9;0.1;64.0;1.2862250804901123;0.49040138721466064;1.2183420658111572;0.5392670035362244;1.2070523500442505;0.5375218391418457;1.0;0.5528382317603805;0.5694776622208874;0.586225988700565;0.4945869348043261;0.5416503786069004;0.5475315747054877;100.0;0.01
+5;0.9;0.1;64.0;1.1847248077392578;0.5445026159286499;1.4426966905593872;0.43106457591056824;1.2306617498397827;0.47993019223213196;1.0;0.5312498290025648;0.5339512821095386;0.5311977318393986;0.544515870602827;0.4474462735332301;0.49295993861211257;150.0;0.01
+6;0.9;0.1;64.0;1.224439263343811;0.5095986127853394;1.1083790063858032;0.5584642291069031;1.042079210281372;0.584642231464386;1.0;0.5344176263987584;0.5580401772808269;0.5725199160342243;0.5072463768115942;0.564412640499597;0.5839504698200351;50.0;0.001
+7;0.9;0.1;64.0;1.1233032941818237;0.5671902298927307;1.279549241065979;0.49389180541038513;1.0785572528839111;0.5811518430709839;1.0;0.5835013620894915;0.5600592746287815;0.5944705565842157;0.5647160810204288;0.4860936165283991;0.5825973543364849;100.0;0.001
+8;0.9;0.1;64.0;1.0618174076080322;0.5898778438568115;1.0041369199752808;0.6178010702133179;1.0159738063812256;0.6020942330360413;1.0;0.5958448974789733;0.6197077143812308;0.6046378435659604;0.5848010684967206;0.6172076715554976;0.6017768463420637;150.0;0.001
+9;0.9;0.1;64.0;1.1275826692581177;0.5095986127853394;1.037006139755249;0.547993004322052;1.1576757431030273;0.4712041914463043;1.0;0.5163513023182834;0.5527921101786539;0.4780335054814982;0.5184676434676434;0.5468667805624328;0.47072553050813926;50.0;0.0001
+10;0.9;0.1;64.0;1.0660938024520874;0.5811518430709839;1.0170561075210571;0.5828970074653625;1.0196107625961304;0.5811518430709839;1.0;0.5900164662084766;0.5999640724915478;0.5950983436203666;0.5901320901320901;0.5829641373119634;0.5774551535421101;100.0;0.0001
+11;0.9;0.1;64.0;1.0177899599075317;0.5881326198577881;1.0439162254333496;0.5654450058937073;1.1021149158477783;0.5759162306785583;1.0;0.6036141210870313;0.5902555674010898;0.6140637358081388;0.5845627802149542;0.561161664422534;0.5720825068651156;150.0;0.0001
+12;0.9;0.1;128.0;1.0818946361541748;0.5253053903579712;1.0099669694900513;0.5968586206436157;1.0918453931808472;0.5410122275352478;1.0;0.521080315961989;0.6039058290508166;0.5263435769673295;0.5298210243862418;0.6008387747518182;0.5431760268716791;50.0;0.0
+13;0.9;0.1;128.0;1.164839744567871;0.45724257826805115;0.9867958426475525;0.5933682322502136;1.0679926872253418;0.554973840713501;1.0;0.5123214368077382;0.5988966761697891;0.5643833075235515;0.4695654586958935;0.597261434217956;0.5518648018648019;100.0;0.0
+14;0.9;0.1;128.0;1.113759160041809;0.5567190051078796;1.046524167060852;0.584642231464386;0.9935498833656311;0.5724258422851562;1.0;0.6122945895111208;0.5897342892147153;0.5820350700747354;0.553781966825445;0.58606007519051;0.5708771904424078;150.0;0.0
+15;0.9;0.1;128.0;1.1370242834091187;0.5619546175003052;1.1299653053283691;0.5462478399276733;1.1480666399002075;0.5322862267494202;1.0;0.5756164609818162;0.5500235261699395;0.5222666160817157;0.5605360822752127;0.5444175389827564;0.53630849826502;50.0;0.01
+16;0.9;0.1;128.0;1.1349637508392334;0.5636998414993286;1.1408828496932983;0.5410122275352478;1.3376970291137695;0.47993019223213196;1.0;0.5815890943000785;0.5523155608073429;0.5205896541198826;0.5641309173917869;0.5482174830000918;0.4919084538649756;100.0;0.01
+17;0.9;0.1;128.0;1.1322417259216309;0.5881326198577881;1.1545387506484985;0.5602094531059265;1.2242604494094849;0.5654450058937073;1.0;0.599198600670369;0.5865846668946237;0.5603340273492765;0.5861801242236024;0.5607828162175988;0.5646859179467875;150.0;0.01
+18;0.9;0.1;128.0;1.1220964193344116;0.5514833927154541;1.1653169393539429;0.5270506143569946;1.1562250852584839;0.5235602259635925;1.0;0.5715813891898507;0.5249991236833342;0.5223967978570353;0.5467980087545306;0.5330587287109027;0.5233751755490886;50.0;0.001
+19;0.9;0.1;128.0;1.153067708015442;0.5881326198577881;1.1445388793945312;0.5602094531059265;1.1189345121383667;0.5357766151428223;1.0;0.6072767352919373;0.5604236873448876;0.5574203282247268;0.5862139068660808;0.5572374485417964;0.5373183579705318;100.0;0.001
+20;0.9;0.1;128.0;1.2125698328018188;0.5410122275352478;1.1219922304153442;0.5602094531059265;1.0713446140289307;0.5828970074653625;1.0;0.5605981222253124;0.5931479015961776;0.5828110117010443;0.5311391507043681;0.5714448594883378;0.5890498390498391;150.0;0.001
+21;0.9;0.1;128.0;1.1447523832321167;0.5567190051078796;1.1324081420898438;0.518324613571167;1.1090422868728638;0.49738219380378723;1.0;0.596030303030303;0.5490280922716101;0.5121340146227983;0.5638238573021181;0.5146260744086831;0.4972123287340679;50.0;0.0001
+22;0.9;0.1;128.0;0.9825822710990906;0.584642231464386;1.108230471611023;0.5218150019645691;1.1356145143508911;0.5200698375701904;1.0;0.5948660954866036;0.5665316277437249;0.5656796449319814;0.5828724415680937;0.5286633656198874;0.5303995521386826;100.0;0.0001
+23;0.9;0.1;128.0;1.0473191738128662;0.5951134562492371;1.0678406953811646;0.518324613571167;1.0169429779052734;0.5881326198577881;1.0;0.6219409183078248;0.5693268693732438;0.593293789725479;0.5926543263499786;0.5205706129619173;0.5907293189901885;150.0;0.0001
+24;0.9;0.01;64.0;1.0598105192184448;0.5968586206436157;1.0257790088653564;0.6038394570350647;0.9506248235702515;0.6369982361793518;1.0;0.6051437947470779;0.602723696484381;0.6396683984470244;0.5915582002538524;0.609172831998919;0.6423491966970227;50.0;0.0
+25;0.9;0.01;64.0;1.1066389083862305;0.6108202338218689;1.0695348978042603;0.6195462346076965;1.1887136697769165;0.5741710066795349;1.0;0.6467146374525526;0.6245149204071099;0.5971208662939392;0.6051062464105943;0.6201624462494028;0.5657778212126039;100.0;0.0
+26;0.9;0.01;64.0;1.076154351234436;0.6404886841773987;1.1792618036270142;0.6073298454284668;1.4671285152435303;0.547993004322052;1.0;0.6529888091081401;0.6461774006977742;0.6066030792539256;0.6368685662163923;0.6003965840922363;0.537592841940668;150.0;0.0
+27;0.9;0.01;64.0;1.2690242528915405;0.5968586206436157;1.2563925981521606;0.5968586206436157;1.2163962125778198;0.5968586206436157;1.0;0.5916521579476596;0.6248306369232546;0.5985189460592925;0.5966666184057489;0.6007983562331388;0.5971992982862548;50.0;0.01
+28;0.9;0.01;64.0;1.116417646408081;0.6160558462142944;1.1469776630401611;0.6178010702133179;1.1717371940612793;0.584642231464386;1.0;0.6261048220941269;0.6335991775298133;0.610479317595543;0.6109440076831381;0.6120992534036012;0.5811808963982876;100.0;0.01
+29;0.9;0.01;64.0;1.214686393737793;0.5811518430709839;1.2714550495147705;0.5584642291069031;1.2080868482589722;0.6073298454284668;1.0;0.6001486481081634;0.5946394895073237;0.6365187860446218;0.5804340586949283;0.5500978490108924;0.6034834730486904;150.0;0.01
+30;0.9;0.01;64.0;1.1514617204666138;0.5881326198577881;1.0881938934326172;0.5881326198577881;1.053907871246338;0.6020942330360413;1.0;0.5834891320869433;0.6065228770652975;0.6088152818711143;0.5858362651840913;0.5805794447098794;0.6013455143889928;50.0;0.001
+31;0.9;0.01;64.0;1.22362220287323;0.5968586206436157;1.0609272718429565;0.6300174593925476;1.1912355422973633;0.5968586206436157;1.0;0.621725687037072;0.6327141334647053;0.6186631707154095;0.5975027388070866;0.6298701298701299;0.5883030013464796;100.0;0.001
+32;0.9;0.01;64.0;1.1824991703033447;0.5986038446426392;1.226974368095398;0.6090750694274902;1.2909026145935059;0.6038394570350647;1.0;0.6073992988417198;0.6445184510325356;0.619910049538771;0.5955710955710956;0.6017605582822975;0.5952151713021279;150.0;0.001
+33;0.9;0.01;64.0;1.1622414588928223;0.5287958383560181;1.0493906736373901;0.5794066190719604;1.2138421535491943;0.5497382283210754;1.0;0.5809122116993566;0.6019885476001634;0.5910039233578294;0.5309117211291124;0.5840572471007254;0.5480895915678524;50.0;0.0001
+34;0.9;0.01;64.0;1.0573968887329102;0.6073298454284668;1.0098820924758911;0.62129145860672;1.0127638578414917;0.614310622215271;1.0;0.6070338521099506;0.62878369303158;0.6239062815043207;0.603968495272843;0.6201449516666908;0.6131923631923631;100.0;0.0001
+35;0.9;0.01;64.0;1.0984822511672974;0.657940685749054;1.0174990892410278;0.6561954617500305;1.1236447095870972;0.5986038446426392;1.0;0.6531590453460197;0.6671137560770346;0.5932537237439376;0.6610702099832534;0.6503381883816667;0.5962919930311235;150.0;0.0001
+36;0.9;0.01;128.0;1.0621216297149658;0.584642231464386;1.0866228342056274;0.5497382283210754;1.073479413986206;0.5671902298927307;1.0;0.5977450132718186;0.5629235472659129;0.5687495323396486;0.5795900958944438;0.545400251921991;0.565356141443098;50.0;0.0
+37;0.9;0.01;128.0;1.0547319650650024;0.6125654578208923;1.1377087831497192;0.5776614546775818;1.127797245979309;0.5636998414993286;1.0;0.6235207292967124;0.5720610223175946;0.5682047407187032;0.6082697495740974;0.5770853542592673;0.5593319723754506;100.0;0.0
+38;0.9;0.01;128.0;0.9883608222007751;0.6265270709991455;1.316406488418579;0.5567190051078796;1.1691138744354248;0.5828970074653625;1.0;0.6431009815575375;0.5975566000144896;0.6062137814624874;0.624762918241179;0.5490433479563914;0.577417148069322;150.0;0.0
+39;0.9;0.01;128.0;1.443315863609314;0.5253053903579712;1.3692501783370972;0.5828970074653625;1.342460036277771;0.5933682322502136;1.0;0.5716634678107281;0.597909057945944;0.5906145139282292;0.5164768806073153;0.57604231517275;0.5995459854155506;50.0;0.01
+40;0.9;0.01;128.0;1.2021657228469849;0.6265270709991455;1.3414463996887207;0.5619546175003052;1.2604358196258545;0.5828970074653625;1.0;0.6285688058844148;0.6186485688650869;0.6176072056565729;0.6251604675517719;0.5610753980319197;0.5805716023107328;100.0;0.01
+41;0.9;0.01;128.0;1.1669517755508423;0.6003490686416626;1.195756435394287;0.6020942330360413;1.1677254438400269;0.614310622215271;1.0;0.6203690104168971;0.6203570670428672;0.6432026808004327;0.5943404421665291;0.596520025867852;0.6156590993547515;150.0;0.01
+42;0.9;0.01;128.0;1.1474894285202026;0.5497382283210754;1.1794898509979248;0.5357766151428223;1.0964933633804321;0.5759162306785583;1.0;0.5853768536604357;0.5617280741536463;0.5739665220002184;0.5477795151708195;0.5463829648612257;0.5779051866008388;50.0;0.001
+43;0.9;0.01;128.0;1.253674864768982;0.5253053903579712;1.160112977027893;0.5776614546775818;1.1979146003723145;0.5602094531059265;1.0;0.5522893772893773;0.5946065751075158;0.609541492289036;0.5178848928848929;0.5693081073515855;0.5646515320428364;100.0;0.001
+44;0.9;0.01;128.0;1.0433732271194458;0.62129145860672;1.1583247184753418;0.6055846214294434;1.2541570663452148;0.5584642291069031;1.0;0.6268407792270415;0.6419238559031413;0.5858269376442662;0.6190530484008745;0.5973609723609723;0.5483924288272114;150.0;0.001
+45;0.9;0.01;128.0;1.1987037658691406;0.5375218391418457;1.1068445444107056;0.5445026159286499;1.1949487924575806;0.518324613571167;1.0;0.5710625395113904;0.5581101609004185;0.5326604175732083;0.5442896475505171;0.5378775813558422;0.5108786141394838;50.0;0.0001
+46;0.9;0.01;128.0;1.1030073165893555;0.5863874554634094;1.0547511577606201;0.6195462346076965;1.066347599029541;0.5881326198577881;1.0;0.6079505660141936;0.6188059698281521;0.5895490996780354;0.5875187614318049;0.6249137336093857;0.5872448807231415;100.0;0.0001
+47;0.9;0.01;128.0;1.132765293121338;0.5898778438568115;1.2393956184387207;0.5340313911437988;1.0497409105300903;0.5881326198577881;1.0;0.5911296865320181;0.566333885194723;0.6075930260346379;0.5854749115618681;0.5218344457474892;0.5897791821704865;150.0;0.0001

models/FullyConectedModels/model.py

@@ -0,0 +1,120 @@
+from tensorflow.data import Dataset
+import tensorflow.keras as keras
+from tensorflow.keras.optimizers import Adam
+from tensorflow.keras.layers import (
+    Conv2D,
+    Input,
+    MaxPooling2D,
+    Dense,
+    Dropout,
+    MaxPool1D,
+    Flatten,
+    AveragePooling1D,
+    BatchNormalization,
+)
+from tensorflow.keras import Model
+import numpy as np
+import tensorflow as tf
+from tensorflow.keras.models import Sequential
+from tensorflow.keras.models import Model
+from tensorflow.keras.layers import Input, Add, Activation, Dropout, Flatten, Dense
+from tensorflow.keras.layers import Convolution2D, MaxPooling2D, AveragePooling2D
+from tensorflow.keras.layers import BatchNormalization
+from tensorflow.keras.regularizers import l2
+from tensorflow.keras import backend as K
+from tensorflow.keras.optimizers import SGD
+import warnings
+
+warnings.filterwarnings("ignore")
+
+
+def basemodel(weight_decay):
+    # 2 hidden layers
+    model_input = Input(
+        shape=(
+            32,
+            32,
+            1,
+        )
+    )
+    model = Conv2D(
+        32,
+        kernel_size=(3, 3),
+        kernel_regularizer=l2(weight_decay),
+        activation="relu",
+    )(model_input)
+    model = Conv2D(
+        64, kernel_size=(3, 3), kernel_regularizer=l2(weight_decay), activation="relu"
+    )(model)
+    model = MaxPooling2D(pool_size=(2, 2))(model)
+    model = BatchNormalization()(model)
+    model = Flatten()(model)
+    model = Dense(4, kernel_regularizer=l2(weight_decay), activation="softmax")(model)
+    model = Model(inputs=model_input, outputs=model)
+    return model
+
+
+def model_2(weight_decay):
+    model_input = Input(
+        shape=(
+            32,
+            32,
+            1,
+        )
+    )
+    model = Conv2D(
+        32,
+        kernel_size=(3, 3),
+        kernel_regularizer=l2(weight_decay),
+        activation="relu",
+    )(model_input)
+    model = Conv2D(
+        64, kernel_size=(3, 3), kernel_regularizer=l2(weight_decay), activation="relu"
+    )(model)
+    model = MaxPooling2D(pool_size=(2, 2))(model)
+    model = BatchNormalization()(model)
+    model = Conv2D(
+        128, kernel_size=(3, 3), kernel_regularizer=l2(weight_decay), activation="relu"
+    )(model)
+    model = MaxPooling2D(pool_size=(2, 2))(model)
+    model = BatchNormalization()(model)
+    model = Flatten()(model)
+    model = Dense(4, kernel_regularizer=l2(weight_decay), activation="softmax")(model)
+    model = Model(inputs=model_input, outputs=model)
+    return model
+
+
+def model_3(weight_decay):
+    # 4 hidden layers
+    model_input = Input(
+        shape=(
+            32,
+            32,
+            1,
+        )
+    )
+    model = Conv2D(
+        32,
+        kernel_size=(3, 3),
+        kernel_regularizer=l2(weight_decay),
+        activation="relu",
+    )(model_input)
+    model = Conv2D(
+        64, kernel_size=(3, 3), kernel_regularizer=l2(weight_decay), activation="relu"
+    )(model)
+    model = MaxPooling2D(pool_size=(2, 2))(model)
+    model = BatchNormalization()(model)
+    model = Conv2D(
+        128, kernel_size=(3, 3), kernel_regularizer=l2(weight_decay), activation="relu"
+    )(model)
+    model = MaxPooling2D(pool_size=(2, 2))(model)
+    model = BatchNormalization()(model)
+    model = Conv2D(
+        256, kernel_size=(3, 3), kernel_regularizer=l2(weight_decay), activation="relu"
+    )(model)
+    model = MaxPooling2D(pool_size=(2, 2))(model)
+    model = BatchNormalization()(model)
+    model = Flatten()(model)
+    model = Dense(4, kernel_regularizer=l2(weight_decay), activation="softmax")(model)
+    model = Model(inputs=model_input, outputs=model)
+    return model

models/FullyConectedModels/parseval.py

@@ -0,0 +1,83 @@
+from tensorflow.data import Dataset
+import tensorflow.keras as keras
+from tensorflow.keras.optimizers import Adam
+from tensorflow.keras.layers import (
+    Conv2D,
+    Input,
+    MaxPooling2D,
+    Dense,
+    Dropout,
+    MaxPool1D,
+    Flatten,
+    AveragePooling1D,
+    BatchNormalization,
+)
+from tensorflow.keras import Model
+import numpy as np
+import tensorflow as tf
+from tensorflow.keras.models import Sequential
+from tensorflow.keras.models import Model
+from tensorflow.keras.layers import Input, Add, Activation, Dropout, Flatten, Dense
+from tensorflow.keras.layers import Convolution2D, MaxPooling2D, AveragePooling2D
+from tensorflow.keras.layers import BatchNormalization
+from tensorflow.keras.regularizers import l2
+from tensorflow.keras import backend as K
+from tensorflow.keras.optimizers import SGD
+import warnings
+from constraint import tight_frame
+
+warnings.filterwarnings("ignore")
+
+
+def model_parseval(weight_decay):
+
+    model_input = Input(
+        shape=(
+            32,
+            32,
+            1,
+        )
+    )
+    model = Conv2D(
+        32,
+        kernel_size=(3, 3),
+        activation="relu",
+        input_shape=(32, 32, 1),
+        kernel_regularizer=l2(weight_decay),
+        kernel_constraint=tight_frame(0.001),
+        kernel_initializer="Orthogonal",
+    )(model_input)
+    model = Conv2D(
+        64,
+        kernel_size=(3, 3),
+        activation="relu",
+        kernel_regularizer=l2(weight_decay),
+        kernel_initializer="Orthogonal",
+        kernel_constraint=tight_frame(0.001),
+    )(model)
+    model = MaxPooling2D(pool_size=(2, 2))(model)
+    model = BatchNormalization()(model)
+    model = Conv2D(
+        128,
+        kernel_size=(3, 3),
+        activation="relu",
+        kernel_initializer="Orthogonal",
+        kernel_regularizer=l2(weight_decay),
+        kernel_constraint=tight_frame(0.001),
+    )(model)
+    model = MaxPooling2D(pool_size=(2, 2))(model)
+    model = BatchNormalization()(model)
+    model = Conv2D(
+        256,
+        kernel_size=(3, 3),
+        activation="relu",
+        kernel_initializer="Orthogonal",
+        kernel_regularizer=l2(weight_decay),
+        kernel_constraint=tight_frame(0.001),
+    )(model)
+    model = MaxPooling2D(pool_size=(2, 2))(model)
+    model = BatchNormalization()(model)
+    model = Flatten()(model)
+    model = Dense(4, activation="softmax", kernel_regularizer=l2(weight_decay))(model)
+    model = Model(inputs=model_input, outputs=model)
+    return model

models/Parseval_Networks/README.md

@@ -0,0 +1,3 @@
+## ParsevalNetworks
+* Orthogonality Constraint
+* Convexity Constraint

models/Parseval_Networks/constraint.py

@@ -0,0 +1,81 @@
+from tensorflow.python.keras.constraints import Constraint
+from tensorflow.python.ops import math_ops, array_ops
+
+
+class TightFrame(Constraint):
+    """
+    Parseval (tight) frame contstraint, as introduced in https://arxiv.org/abs/1704.08847
+
+    Constraints the weight matrix to be a tight frame, so that the Lipschitz
+    constant of the layer is <= 1. This increases the robustness of the network
+    to adversarial noise.
+
+    Warning: This constraint simply performs the update step on the weight matrix
+    (or the unfolded weight matrix for convolutional layers). Thus, it does not
+    handle the necessary scalings for convolutional layers.
+
+    Args:
+        scale (float):    Retraction parameter (length of retraction step).
+        num_passes (int): Number of retraction steps.
+
+    Returns:
+        Weight matrix after applying regularizer.
+    """
+
+    def __init__(self, scale, num_passes=1):
+        """[summary]
+
+        Args:
+            scale ([type]): [description]
+            num_passes (int, optional): [description]. Defaults to 1.
+
+        Raises:
+            ValueError: [description]
+        """
+        self.scale = scale
+
+        if num_passes < 1:
+            raise ValueError(
+                "Number of passes cannot be non-positive! (got {})".format(num_passes)
+            )
+        self.num_passes = num_passes
+
+    def __call__(self, w):
+        """[summary]
+
+        Args:
+            w ([type]): weight of conv or linear layers
+
+        Returns:
+            [type]: returns new weights
+        """
+        transpose_channels = len(w.shape) == 4
+
+        # Move channels_num to the front in order to make the dimensions correct for matmul
+        if transpose_channels:
+            w_reordered = array_ops.reshape(w, (-1, w.shape[3]))
+
+        else:
+            w_reordered = w
+
+        last = w_reordered
+        for i in range(self.num_passes):
+            temp1 = math_ops.matmul(last, last, transpose_a=True)
+            temp2 = (1 + self.scale) * w_reordered - self.scale * math_ops.matmul(
+                w_reordered, temp1
+            )
+
+            last = temp2
+
+        # Move channels_num to the back again
+        if transpose_channels:
+            return array_ops.reshape(last, w.shape)
+        else:
+            return last
+
+    def get_config(self):
+        return {"scale": self.scale, "num_passes": self.num_passes}
+
+
+# Alias
+tight_frame = TightFrame

models/Parseval_Networks/convexity_constraint.py

@@ -0,0 +1,53 @@
+from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import variables
+from tensorflow.python.framework import dtypes
+import numpy as _np
+
+
+def convex_add(input_layer, layer_3, initial_convex_par=0.5, trainable=False):
+    """
+    Do a convex combination of input_layer and layer_3. That is, return the output of
+
+        lamda* input_layer + (1 - lamda) * layer_3
+
+
+    Args:
+            input_layer (tf.Tensor):	Input to take convex combinatio of
+            layer_3 (tf.Tensor):		Input to take convex combinatio of
+            initial_convex_par (float):	Initial value for convex parameter. Must be
+                                                                    in [0, 1].
+            trainable (bool):			Whether convex parameter should be trainable
+                                                                    or not.
+
+    Returns:
+            tf.Tensor: Result of convex combination
+    """
+    # Will implement this as sigmoid(p)*input_layer + (1-sigmoid(p))*layer_3 to ensure
+    # convex parameter to be in the unit interval without constraints during
+    # optimization
+
+    # Find value for p, also check for legal initial_convex_par
+    if initial_convex_par < 0:
+        raise ValueError("Convex parameter must be >=0")
+
+    elif initial_convex_par == 0:
+        # sigmoid(-16) is approximately a 32bit roundoff error, practically 0
+        initial_p_value = -16
+
+    elif initial_convex_par < 1:
+        # Compute inverse of sigmoid to find initial p value
+        initial_p_value = -_np.log(1 / initial_convex_par - 1)
+
+    elif initial_convex_par == 1:
+        # Same argument as for 0
+        initial_p_value = 16
+
+    else:
+        raise ValueError("Convex parameter must be <=1")
+
+    p = variables.Variable(
+        initial_value=initial_p_value, dtype=dtypes.float32, trainable=trainable
+    )
+
+    lam = math_ops.sigmoid(p)
+    return input_layer * lam + (1 - lam) * layer_3

models/Parseval_Networks/parsevalnet.py

@@ -0,0 +1,328 @@
+from tensorflow.keras.models import Model
+from tensorflow.keras.layers import Input, Add, Activation, Dropout, Flatten, Dense
+from tensorflow.keras.layers import Convolution2D, MaxPooling2D, AveragePooling2D
+from tensorflow.keras.layers import BatchNormalization
+from tensorflow.keras.regularizers import l2
+from tensorflow.keras import backend as K
+from tensorflow.keras.optimizers import SGD
+import warnings
+from constraint import tight_frame
+from convexity_constraint import convex_add
+
+warnings.filterwarnings("ignore")
+
+
+class ParsevalNetwork(Model):
+    def __init__(
+        self,
+        input_dim,
+        weight_decay,
+        momentum,
+        nb_classes=4,
+        N=2,
+        k=1,
+        dropout=0.0,
+        verbose=1,
+    ):
+        """[Assign the initial parameters of the wide residual network]
+
+        Args:
+            weight_decay ([float]): [description]
+            input_dim ([tuple]): [input dimension]
+            nb_classes (int, optional): [output class]. Defaults to 4.
+            N (int, optional): [the number of blocks]. Defaults to 2.
+            k (int, optional): [network width]. Defaults to 1.
+            dropout (float, optional): [dropout value to prevent overfitting]. Defaults to 0.0.
+            verbose (int, optional): [description]. Defaults to 1.
+
+        Returns:
+            [Model]: [parsevalnetwork]
+        """
+        self.weight_decay = weight_decay
+        self.input_dim = input_dim
+        self.nb_classes = nb_classes
+        self.N = N
+        self.k = k
+        self.dropout = dropout
+        self.verbose = verbose
+
+    def initial_conv(self, input):
+        """[summary]
+
+        Args:
+            input ([type]): [description]
+
+        Returns:
+            [type]: [description]
+        """
+        x = Convolution2D(
+            16,
+            (3, 3),
+            padding="same",
+            kernel_initializer="orthogonal",
+            kernel_regularizer=l2(self.weight_decay),
+            kernel_constraint=tight_frame(0.001),
+            use_bias=False,
+        )(input)
+
+        channel_axis = 1 if K.image_data_format() == "channels_first" else -1
+
+        x = BatchNormalization(
+            axis=channel_axis, momentum=0.1, epsilon=1e-5, gamma_initializer="uniform"
+        )(x)
+        x = Activation("relu")(x)
+        return x
+
+    def expand_conv(self, init, base, k, strides=(1, 1)):
+        """[summary]
+
+        Args:
+            init ([type]): [description]
+            base ([type]): [description]
+            k ([type]): [description]
+            strides (tuple, optional): [description]. Defaults to (1, 1).
+
+        Returns:
+            [type]: [description]
+        """
+        x = Convolution2D(
+            base * k,
+            (3, 3),
+            padding="same",
+            strides=strides,
+            kernel_initializer="Orthogonal",
+            kernel_regularizer=l2(self.weight_decay),
+            kernel_constraint=tight_frame(0.001),
+            use_bias=False,
+        )(init)
+
+        channel_axis = 1 if K.image_data_format() == "channels_first" else -1
+
+        x = BatchNormalization(
+            axis=channel_axis, momentum=0.1, epsilon=1e-5, gamma_initializer="uniform"
+        )(x)
+        x = Activation("relu")(x)
+
+        x = Convolution2D(
+            base * k,
+            (3, 3),
+            padding="same",
+            kernel_initializer="Orthogonal",
+            kernel_regularizer=l2(self.weight_decay),
+            kernel_constraint=tight_frame(0.001),
+            use_bias=False,
+        )(x)
+
+        skip = Convolution2D(
+            base * k,
+            (1, 1),
+            padding="same",
+            strides=strides,
+            kernel_initializer="Orthogonal",
+            kernel_regularizer=l2(self.weight_decay),
+            kernel_constraint=tight_frame(0.001),
+            use_bias=False,
+        )(init)
+
+        m = Add()([x, skip])
+
+        return m
+
+    def conv1_block(self, input, k=1, dropout=0.0):
+        """[summary]
+
+        Args:
+            input ([type]): [description]
+            k (int, optional): [description]. Defaults to 1.
+            dropout (float, optional): [description]. Defaults to 0.0.
+
+        Returns:
+            [type]: [description]
+        """
+        init = input
+
+        channel_axis = 1 if K.image_data_format() == "channels_first" else -1
+
+        x = BatchNormalization(
+            axis=channel_axis, momentum=0.1, epsilon=1e-5, gamma_initializer="uniform"
+        )(input)
+        x = Activation("relu")(x)
+        x = Convolution2D(
+            16 * k,
+            (3, 3),
+            padding="same",
+            kernel_initializer="Orthogonal",
+            kernel_regularizer=l2(self.weight_decay),
+            kernel_constraint=tight_frame(0.001),
+            use_bias=False,
+        )(x)
+
+        if dropout > 0.0:
+            x = Dropout(dropout)(x)
+
+        x = BatchNormalization(
+            axis=channel_axis, momentum=0.1, epsilon=1e-5, gamma_initializer="uniform"
+        )(x)
+        x = Activation("relu")(x)
+        x = Convolution2D(
+            16 * k,
+            (3, 3),
+            padding="same",
+            kernel_initializer="Orthogonal",
+            kernel_regularizer=l2(self.weight_decay),
+            kernel_constraint=tight_frame(0.001),
+            use_bias=False,
+        )(x)
+        m = convex_add(init, x, initial_convex_par=0.5, trainable=True)
+        return m
+
+    def conv2_block(self, input, k=1, dropout=0.0):
+        """[summary]
+
+        Args:
+            input ([type]): [description]
+            k (int, optional): [description]. Defaults to 1.
+            dropout (float, optional): [description]. Defaults to 0.0.
+
+        Returns:
+            [type]: [description]
+        """
+        init = input
+
+        channel_axis = 1 if K.image_data_format() == "channels_first" else -1
+        x = BatchNormalization(
+            axis=channel_axis, momentum=0.1, epsilon=1e-5, gamma_initializer="uniform"
+        )(input)
+        x = Activation("relu")(x)
+        x = Convolution2D(
+            32 * k,
+            (3, 3),
+            padding="same",
+            kernel_initializer="Orthogonal",
+            kernel_regularizer=l2(self.weight_decay),
+            kernel_constraint=tight_frame(0.001),
+            use_bias=False,
+        )(x)
+
+        if dropout > 0.0:
+            x = Dropout(dropout)(x)
+
+        x = BatchNormalization(
+            axis=channel_axis, momentum=0.1, epsilon=1e-5, gamma_initializer="uniform"
+        )(x)
+        x = Activation("relu")(x)
+        x = Convolution2D(
+            32 * k,
+            (3, 3),
+            padding="same",
+            kernel_initializer="Orthogonal",
+            kernel_regularizer=l2(self.weight_decay),
+            kernel_constraint=tight_frame(0.001),
+            use_bias=False,
+        )(x)
+
+        m = convex_add(init, x, initial_convex_par=0.5, trainable=True)
+        return m
+
+    def conv3_block(self, input, k=1, dropout=0.0):
+        init = input
+
+        channel_axis = 1 if K.image_data_format() == "channels_first" else -1
+        x = BatchNormalization(
+            axis=channel_axis, momentum=0.1, epsilon=1e-5, gamma_initializer="uniform"
+        )(input)
+        x = Activation("relu")(x)
+        x = Convolution2D(
+            64 * k,
+            (3, 3),
+            padding="same",
+            kernel_initializer="Orthogonal",
+            kernel_constraint=tight_frame(0.001),
+            kernel_regularizer=l2(self.weight_decay),
+            use_bias=False,
+        )(x)
+
+        if dropout > 0.0:
+            x = Dropout(dropout)(x)
+
+        x = BatchNormalization(
+            axis=channel_axis, momentum=0.1, epsilon=1e-5, gamma_initializer="uniform"
+        )(x)
+        x = Activation("relu")(x)
+        x = Convolution2D(
+            64 * k,
+            (3, 3),
+            padding="same",
+            kernel_initializer="Orthogonal",
+            kernel_constraint=tight_frame(0.001),
+            kernel_regularizer=l2(self.weight_decay),
+            use_bias=False,
+        )(x)
+
+        m = convex_add(init, x, initial_convex_par=0.5, trainable=True)
+        return m
+
+    def create_wide_residual_network(self):
+        """create a wide residual network model
+
+
+        Returns:
+            [Model]: [wide residual network]
+        """
+        channel_axis = 1 if K.image_data_format() == "channels_first" else -1
+
+        ip = Input(shape=self.input_dim)
+
+        x = self.initial_conv(ip)
+        nb_conv = 4
+
+        x = self.expand_conv(x, 16, self.k)
+        nb_conv += 2
+
+        for i in range(self.N - 1):
+            x = self.conv1_block(x, self.k, self.dropout)
+            nb_conv += 2
+
+        x = BatchNormalization(
+            axis=channel_axis, momentum=0.1, epsilon=1e-5, gamma_initializer="uniform"
+        )(x)
+        x = Activation("relu")(x)
+
+        x = self.expand_conv(x, 32, self.k, strides=(2, 2))
+        nb_conv += 2
+
+        for i in range(self.N - 1):
+            x = self.conv2_block(x, self.k, self.dropout)
+            nb_conv += 2
+
+        x = BatchNormalization(
+            axis=channel_axis, momentum=0.1, epsilon=1e-5, gamma_initializer="uniform"
+        )(x)
+        x = Activation("relu")(x)
+
+        x = self.expand_conv(x, 64, self.k, strides=(2, 2))
+        nb_conv += 2
+
+        for i in range(self.N - 1):
+            x = self.conv3_block(x, self.k, self.dropout)
+            nb_conv += 2
+
+        x = BatchNormalization(
+            axis=channel_axis, momentum=0.1, epsilon=1e-5, gamma_initializer="uniform"
+        )(x)
+        x = Activation("relu")(x)
+
+        x = AveragePooling2D((8, 8))(x)
+        x = Flatten()(x)
+
+        x = Dense(
+            self.nb_classes,
+            kernel_regularizer=l2(self.weight_decay),
+            activation="softmax",
+        )(x)
+
+        model = Model(ip, x)
+
+        if self.verbose:
+            print("Parseval  Network-%d-%d created." % (nb_conv, self.k))
+        return model

models/README.md

@@ -0,0 +1,15 @@
+## Models
+
+````
+├── Parseval_network
+│   ├── __init__.py
+│   └── Parseval_resnet.py
+├── Parseval_Networks_OC
+│   ├── constraint.py
+│   ├── parsnet_oc.py
+│   └── README.md
+├── README.md
+├── _utility.py
+└── wideresnet
+    └── wresnet.py
+````

models/_utility.py

@@ -0,0 +1,109 @@
+from tensorflow.keras.callbacks import LearningRateScheduler
+
+# Define configuration parameters
+import math
+import cleverhans
+from cleverhans.tf2.attacks.fast_gradient_method import fast_gradient_method
+import tensorflow as tf
+
+import numpy as np
+
+
+def step_decay(epoch):
+    """[summary]
+
+    Args:
+        epoch (int): epoch number
+
+    Returns:
+        lrate(float): new learning rate
+    """
+    initial_lrate = 0.1
+    factor = 0.1
+    if epoch < 10:
+        lrate = initial_lrate
+    elif epoch < 20:
+        lrate = initial_lrate * math.pow(factor, 1)
+    elif epoch < 30:
+        lrate = initial_lrate * math.pow(factor, 2)
+    elif epoch < 40:
+        lrate = initial_lrate * math.pow(factor, 3)
+    else:
+        lrate = initial_lrate * math.pow(factor, 4)
+    return lrate
+
+
+def step_decay_conv(epoch):
+    """step decay for learning rate in convolutional networks
+
+    Args:
+        epoch (int): epoch number
+
+    Returns:
+        lrate(float): new learning rate
+    """
+    initial_lrate = 0.01
+    factor = 0.1
+    if epoch < 10:
+        lrate = initial_lrate
+    elif epoch < 20:
+        lrate = initial_lrate * math.pow(factor, 1)
+    elif epoch < 30:
+        lrate = initial_lrate * math.pow(factor, 2)
+    elif epoch < 40:
+        lrate = initial_lrate * math.pow(factor, 3)
+    else:
+        lrate = initial_lrate * math.pow(factor, 4)
+    return lrate
+
+
+def print_test(model, X_adv, X_test, y_test, epsilon):
+    """
+    returns the test results and show the SNR and evaluation results
+    """
+    loss, acc = model.evaluate(X_adv, y_test)
+    print("epsilon: {} and test evaluation : {}, {}".format(epsilon, loss, acc))
+    SNR = 20 * np.log10(np.linalg.norm(X_test) / np.linalg.norm(X_test - X_adv))
+    print("SNR: {}".format(SNR))
+    return loss, acc
+
+
+def get_adversarial_examples(pretrained_model, X_true, y_true, epsilon):
+    """
+    The attack requires the model to ouput the logits
+    returns the adversarial example/s of a given image/s for epsilon value using
+    fast gradient sign method
+    """
+    logits_model = tf.keras.Model(
+        pretrained_model.input, pretrained_model.layers[-1].output
+    )
+    X_adv = []
+
+    for i in range(len(X_true)):
+
+        random_index = i
+
+        original_image = X_true[random_index]
+        original_image = tf.convert_to_tensor(
+            original_image.reshape((1, 32, 32))
+        )  # The .reshape just gives it the proper form to input into the model, a batch of 1 a.k.a a tensor
+        original_label = y_true[random_index]
+        original_label = np.reshape(np.argmax(original_label), (1,)).astype("int64")
+
+        adv_example_targeted_label = fast_gradient_method(
+            logits_model,
+            original_image,
+            epsilon,
+            np.inf,
+            y=original_label,
+            targeted=False,
+        )
+        X_adv.append(np.array(adv_example_targeted_label).reshape(32, 32, 1))
+
+    X_adv = np.array(X_adv)
+
+    return X_adv
+
+
+lrate_conv = LearningRateScheduler(step_decay_conv)
+lrate = LearningRateScheduler(step_decay)

models/wideresnet/wresnet.py

@@ -0,0 +1,329 @@
+from tensorflow.keras.models import Model
+from tensorflow.keras.layers import Input, Add, Activation, Dropout, Flatten, Dense
+from tensorflow.keras.layers import Convolution2D, MaxPooling2D, AveragePooling2D
+from tensorflow.keras.layers import BatchNormalization
+from tensorflow.keras.regularizers import l2
+from tensorflow.keras import backend as K
+from tensorflow.keras.optimizers import SGD
+import warnings
+
+warnings.filterwarnings("ignore")
+
+
+class WideResidualNetwork(object):
+    def __init__(
+        self,
+        input_dim,
+        weight_decay,
+        momentum,
+        nb_classes=100,
+        N=2,
+        k=1,
+        dropout=0.0,
+        verbose=1,
+    ):
+        """[Assign the initial parameters of the wide residual network]
+
+        Args:
+            weight_decay ([float]): [description]
+            input_dim ([tuple]): [input dimension]
+            nb_classes (int, optional): [output class]. Defaults to 100.
+            N (int, optional): [the number of blocks]. Defaults to 2.
+            k (int, optional): [network width]. Defaults to 1.
+            dropout (float, optional): [dropout value to prevent overfitting]. Defaults to 0.0.
+            verbose (int, optional): [description]. Defaults to 1.
+
+        Returns:
+            [Model]: [wideresnet]
+        """
+        self.weight_decay = weight_decay
+        self.input_dim = input_dim
+        self.nb_classes = nb_classes
+        self.N = N
+        self.k = k
+        self.dropout = dropout
+        self.verbose = verbose
+
+    def initial_conv(self, input):
+        """[summary]
+
+        Args:
+            input ([type]): [description]
+
+        Returns:
+            [type]: [description]
+        """
+        x = Convolution2D(
+            16,
+            (3, 3),
+            padding="same",
+            kernel_initializer="he_normal",
+            kernel_regularizer=l2(self.weight_decay),
+            use_bias=False,
+        )(input)
+
+        channel_axis = 1 if K.image_data_format() == "channels_first" else -1
+
+        x = BatchNormalization(
+            axis=channel_axis, momentum=0.1, epsilon=1e-5, gamma_initializer="uniform"
+        )(x)
+        x = Activation("relu")(x)
+        return x
+
+    def expand_conv(self, init, base, k, strides=(1, 1)):
+        """[summary]
+
+        Args:
+            init ([type]): [description]
+            base ([type]): [description]
+            k ([type]): [description]
+            strides (tuple, optional): [description]. Defaults to (1, 1).
+
+        Returns:
+            [type]: [description]
+        """
+        x = Convolution2D(
+            base * k,
+            (3, 3),
+            padding="same",
+            strides=strides,
+            kernel_initializer="he_normal",
+            kernel_regularizer=l2(self.weight_decay),
+            use_bias=False,
+        )(init)
+
+        channel_axis = 1 if K.image_data_format() == "channels_first" else -1
+
+        x = BatchNormalization(
+            axis=channel_axis, momentum=0.1, epsilon=1e-5, gamma_initializer="uniform"
+        )(x)
+        x = Activation("relu")(x)
+
+        x = Convolution2D(
+            base * k,
+            (3, 3),
+            padding="same",
+            kernel_initializer="he_normal",
+            kernel_regularizer=l2(self.weight_decay),
+            use_bias=False,
+        )(x)
+
+        skip = Convolution2D(
+            base * k,
+            (1, 1),
+            padding="same",
+            strides=strides,
+            kernel_initializer="he_normal",
+            kernel_regularizer=l2(self.weight_decay),
+            use_bias=False,
+        )(init)
+
+        m = Add()([x, skip])
+
+        return m
+
+    def conv1_block(self, input, k=1, dropout=0.0):
+        """[summary]
+
+        Args:
+            input ([type]): [description]
+            k (int, optional): [description]. Defaults to 1.
+            dropout (float, optional): [description]. Defaults to 0.0.
+
+        Returns:
+            [type]: [description]
+        """
+        init = input
+
+        channel_axis = 1 if K.image_data_format() == "channels_first" else -1
+
+        x = BatchNormalization(
+            axis=channel_axis, momentum=0.1, epsilon=1e-5, gamma_initializer="uniform"
+        )(input)
+        x = Activation("relu")(x)
+        x = Convolution2D(
+            16 * k,
+            (3, 3),
+            padding="same",
+            kernel_initializer="he_normal",
+            kernel_regularizer=l2(self.weight_decay),
+            use_bias=False,
+        )(x)
+
+        if dropout > 0.0:
+            x = Dropout(dropout)(x)
+
+        x = BatchNormalization(
+            axis=channel_axis, momentum=0.1, epsilon=1e-5, gamma_initializer="uniform"
+        )(x)
+        x = Activation("relu")(x)
+        x = Convolution2D(
+            16 * k,
+            (3, 3),
+            padding="same",
+            kernel_initializer="he_normal",
+            kernel_regularizer=l2(self.weight_decay),
+            use_bias=False,
+        )(x)
+
+        m = Add()([init, x])
+        return m
+
+    def conv2_block(self, input, k=1, dropout=0.0):
+        """[summary]
+
+        Args:
+            input ([type]): [description]
+            k (int, optional): [description]. Defaults to 1.
+            dropout (float, optional): [description]. Defaults to 0.0.
+
+        Returns:
+            [type]: [description]
+        """
+        init = input
+
+        channel_axis = 1 if K.image_data_format() == "channels_first" else -1
+        print("conv2:channel:  {}".format(channel_axis))
+        x = BatchNormalization(
+            axis=channel_axis, momentum=0.1, epsilon=1e-5, gamma_initializer="uniform"
+        )(input)
+        x = Activation("relu")(x)
+        x = Convolution2D(
+            32 * k,
+            (3, 3),
+            padding="same",
+            kernel_initializer="he_normal",
+            kernel_regularizer=l2(self.weight_decay),
+            use_bias=False,
+        )(x)
+
+        if dropout > 0.0:
+            x = Dropout(dropout)(x)
+
+        x = BatchNormalization(
+            axis=channel_axis, momentum=0.1, epsilon=1e-5, gamma_initializer="uniform"
+        )(x)
+        x = Activation("relu")(x)
+        x = Convolution2D(
+            32 * k,
+            (3, 3),
+            padding="same",
+            kernel_initializer="he_normal",
+            kernel_regularizer=l2(self.weight_decay),
+            use_bias=False,
+        )(x)
+
+        m = Add()([init, x])
+        return m
+
+    def conv3_block(self, input, k=1, dropout=0.0):
+        """[summary]
+
+        Args:
+            input ([type]): [description]
+            k (int, optional): [description]. Defaults to 1.
+            dropout (float, optional): [description]. Defaults to 0.0.
+
+        Returns:
+            [type]: [description]
+        """
+        init = input
+
+        channel_axis = 1 if K.image_data_format() == "channels_first" else -1
+
+        x = BatchNormalization(
+            axis=channel_axis, momentum=0.1, epsilon=1e-5, gamma_initializer="uniform"
+        )(input)
+        x = Activation("relu")(x)
+        x = Convolution2D(
+            64 * k,
+            (3, 3),
+            padding="same",
+            kernel_initializer="he_normal",
+            kernel_regularizer=l2(self.weight_decay),
+            use_bias=False,
+        )(x)
+
+        if dropout > 0.0:
+            x = Dropout(dropout)(x)
+
+        x = BatchNormalization(
+            axis=channel_axis, momentum=0.1, epsilon=1e-5, gamma_initializer="uniform"
+        )(x)
+        x = Activation("relu")(x)
+        x = Convolution2D(
+            64 * k,
+            (3, 3),
+            padding="same",
+            kernel_initializer="he_normal",
+            kernel_regularizer=l2(self.weight_decay),
+            use_bias=False,
+        )(x)
+
+        m = Add()([init, x])
+        return m
+
+    def create_wide_residual_network(self):
+        """create a wide residual network model
+
+
+        Returns:
+            [Model]: [wide residual network]
+        """
+        channel_axis = 1 if K.image_data_format() == "channels_first" else -1
+
+        ip = Input(shape=self.input_dim)
+
+        x = self.initial_conv(ip)
+        nb_conv = 4
+
+        x = self.expand_conv(x, 16, self.k)
+        nb_conv += 2
+
+        for i in range(self.N - 1):
+            x = self.conv1_block(x, self.k, self.dropout)
+            nb_conv += 2
+
+        x = BatchNormalization(
+            axis=channel_axis, momentum=0.1, epsilon=1e-5, gamma_initializer="uniform"
+        )(x)
+        x = Activation("relu")(x)
+
+        x = self.expand_conv(x, 32, self.k, strides=(2, 2))
+        nb_conv += 2
+
+        for i in range(self.N - 1):
+            x = self.conv2_block(x, self.k, self.dropout)
+            nb_conv += 2
+
+        x = BatchNormalization(
+            axis=channel_axis, momentum=0.1, epsilon=1e-5, gamma_initializer="uniform"
+        )(x)
+        x = Activation("relu")(x)
+
+        x = self.expand_conv(x, 64, self.k, strides=(2, 2))
+        nb_conv += 2
+
+        for i in range(self.N - 1):
+            x = self.conv3_block(x, self.k, self.dropout)
+            nb_conv += 2
+
+        x = BatchNormalization(
+            axis=channel_axis, momentum=0.1, epsilon=1e-5, gamma_initializer="uniform"
+        )(x)
+        x = Activation("relu")(x)
+
+        x = AveragePooling2D((8, 8))(x)
+        x = Flatten()(x)
+
+        x = Dense(
+            self.nb_classes,
+            kernel_regularizer=l2(self.weight_decay),
+            activation="softmax",
+        )(x)
+
+        model = Model(ip, x)
+
+        if self.verbose:
+            print("Wide Residual Network-%d-%d created." % (nb_conv, self.k))
+        return model