stock_predictor.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "authorship_tag": "ABX9TyOc5/oQ0Z2ie5dOI46PpyV0",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "gpuClass": "standard",
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/jsebdev/Stock_Predictor/blob/main/stock_predictor.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')\n",
        "project_path = '/content/drive/MyDrive/projects/Stock_Predicter'\n",
        "%cd $project_path"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Xr3Qozgfktoc",
        "outputId": "b4ce9a19-4dc1-43e9-b09e-af91a2b07343"
      },
      "execution_count": 90,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n",
            "/content/drive/MyDrive/projects/Stock_Predicter\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 91,
      "metadata": {
        "id": "e8SQqogMQYLh"
      },
      "outputs": [],
      "source": [
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "import pandas as pd\n",
        "import pandas_datareader as web\n",
        "import datetime as dt\n",
        "import yfinance as yfin\n",
        "import tensorflow as tf\n",
        "import os\n",
        "import re\n",
        "\n",
        "from sklearn.preprocessing import MinMaxScaler\n",
        "from tensorflow.keras.models import Sequential\n",
        "from tensorflow.keras.layers import Dense, Dropout, LSTM\n"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Get Data"
      ],
      "metadata": {
        "id": "5vO8pty3VwkG"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Select a company for now\n",
        "ticker = 'AAPL'\n",
        "\n",
        "start = dt.datetime(2013,1,1)\n",
        "end = dt.datetime(2023,4,5)"
      ],
      "metadata": {
        "id": "O6dtJpJwS5Eg"
      },
      "execution_count": 92,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "yfin.pdr_override()\n",
        "data = web.data.get_data_yahoo(ticker, start, end)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "LwPyk8Uh-Zz_",
        "outputId": "2217df50-87e9-48e3-e096-71163331f570"
      },
      "execution_count": 93,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\r[*********************100%***********************]  1 of 1 completed\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Preprocess_data"
      ],
      "metadata": {
        "id": "SSuS9OONV5-a"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def create_remove_columns(data):\n",
        "  # create jump column\n",
        "  data = pd.DataFrame.copy(data)\n",
        "  data['Jump'] = data['Open'] - data['Close'].shift(1)\n",
        "  data['Jump'].fillna(0, inplace=True)\n",
        "  # data = data.reindex(columns=['Open', 'High', 'Low', 'Close', 'Adj Close', 'Jump'])\n",
        "  data.insert(0,'Jump', data.pop('Jump'))\n",
        "  return data"
      ],
      "metadata": {
        "id": "Bpym8x-Kxf0p"
      },
      "execution_count": 94,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def normalize_data(data, scaler=None):\n",
        "  the_data = pd.DataFrame.copy(data)\n",
        "  # substract the open value to all columns but the first one and the last one which are \"Jump\" and \"Volume\"\n",
        "  the_data.iloc[:, 1:-1] = the_data.iloc[:,1:-1] - the_data['Open'].values[:, np.newaxis]\n",
        "  # print('the_data')\n",
        "  # print(the_data)\n",
        "\n",
        "  the_data.pop('Open')\n",
        "  # todo save an csv with the values for the scaler\n",
        "  if scaler is None:\n",
        "    # Create the scaler\n",
        "    values = np.abs(the_data.values)\n",
        "    max_value = np.max(values[:,:-1])\n",
        "    max_volume = np.max(values[:,-1])\n",
        "    def scaler(d):\n",
        "      data = pd.DataFrame.copy(d)\n",
        "      print('max_value: ', max_value)\n",
        "      print('max_volume: ', max_volume)\n",
        "      data.iloc[:, :-1] = data.iloc[:,:-1].apply(lambda x: x/max_value)\n",
        "      data.iloc[:, -1] = data.iloc[:,-1].apply(lambda x: x/max_volume)\n",
        "      return data\n",
        "    def decoder(values):\n",
        "      decoded_values = values * max_value\n",
        "      return decoded_values\n",
        "  else:\n",
        "    decoder = None\n",
        "  \n",
        "  normalized_data = scaler(the_data)\n",
        "\n",
        "  return normalized_data, scaler, decoder\n",
        "\n",
        "\n"
      ],
      "metadata": {
        "id": "v9RoqzBvtrOb"
      },
      "execution_count": 95,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def create_training_data(norm_data):\n",
        "  prediction_days = 500\n",
        "  \n",
        "  x_train_list = []\n",
        "  y_train_list = []\n",
        "  \n",
        "  for i in range(prediction_days, len(norm_data)):\n",
        "    x_train_list.append(norm_data[i-prediction_days:i])\n",
        "    y_train_list.append(norm_data.iloc[i].values[0:4])\n",
        "  \n",
        "  x_train = np.array(x_train_list)\n",
        "  y_train = np.array(y_train_list)\n",
        "  return x_train, y_train"
      ],
      "metadata": {
        "id": "jMXkRAYFomHM"
      },
      "execution_count": 96,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#Make all the preprocesing\n",
        "def preprocessing(data, scaler=None):\n",
        "  # print(data.head(3))\n",
        "  data_0 = create_remove_columns(data)\n",
        "  # print(data_0.head(3))\n",
        "  #todo: save the_scaler somehow to use in new runtimes\n",
        "  norm_data, scaler, decoder = normalize_data(data_0, scaler=scaler)\n",
        "  # print(norm_data.head(3))\n",
        "  x_train, y_train = create_training_data(norm_data)\n",
        "  # print(x_train.shape, y_train.shape)\n",
        "  return x_train, y_train, scaler, decoder"
      ],
      "metadata": {
        "id": "YZWMfusT-I7Z"
      },
      "execution_count": 97,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "x_train, y_train, scaler, decoder = preprocessing(data)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "PeJjDC0VBG_6",
        "outputId": "aff2cf0b-e630-4727-bbec-3cf5be4e53a0"
      },
      "execution_count": 98,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "max_value:  10.589996337890625\n",
            "max_volume:  1460852400.0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(x_train.shape)\n",
        "x_train[1,499,:]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "YkI8vSguuS8A",
        "outputId": "6e5eeaa2-12de-4dd2-b17c-1d61e415fbd8"
      },
      "execution_count": 99,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(2082, 500, 6)\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([ 0.00212456,  0.05712934, -0.00212456,  0.04461756, -0.22778379,\n",
              "        0.09233239])"
            ]
          },
          "metadata": {},
          "execution_count": 99
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "td = data.iloc[498:501]\n",
        "# print('td:\\n',td)\n",
        "td0 = create_remove_columns(td)\n",
        "print('td0:\\n',td0)\n",
        "print(decoder(y_train[0]))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QaO34uSds2wJ",
        "outputId": "af5d9a04-214c-4a2d-c706-5af3a2a1ea5a"
      },
      "execution_count": 100,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "td0:\n",
            "                 Jump       Open       High        Low      Close  Adj Close  \\\n",
            "Date                                                                          \n",
            "2014-12-23  0.000000  28.307501  28.332500  28.115000  28.135000  25.286961   \n",
            "2014-12-24  0.010000  28.145000  28.177500  28.002501  28.002501  25.167873   \n",
            "2014-12-26  0.022499  28.025000  28.629999  28.002501  28.497499  25.612770   \n",
            "\n",
            "               Volume  \n",
            "Date                   \n",
            "2014-12-23  104113600  \n",
            "2014-12-24   57918400  \n",
            "2014-12-26  134884000  \n",
            "[ 0.02249908  0.60499954 -0.02249908  0.47249985]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Model"
      ],
      "metadata": {
        "id": "Z3N2WMYNV-qX"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Create Model"
      ],
      "metadata": {
        "id": "emDyvzVUp5KJ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def create_model():\n",
        "  model = Sequential()\n",
        "  # model.add(LSTM(units=112, return_sequences=True, input_shape=(x_train.shape[1:])))\n",
        "  model.add(LSTM(units=1000, return_sequences=True, input_shape=(None,x_train.shape[-1],)))\n",
        "  model.add(Dropout(0.2))\n",
        "  model.add(LSTM(units=1000, return_sequences=True))\n",
        "  model.add(Dropout(0.2))\n",
        "  model.add(LSTM(units=1000))\n",
        "  model.add(Dropout(0.2))\n",
        "  model.add(Dense(units=4))\n",
        "  return model\n",
        "\n",
        "model = create_model()\n",
        "print(model.summary())"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GXhYAKzXVfku",
        "outputId": "bbf96ec2-84fc-4246-9003-32c2f8083bb6"
      },
      "execution_count": 101,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"sequential_2\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " lstm_6 (LSTM)               (None, None, 1000)        4028000   \n",
            "                                                                 \n",
            " dropout_6 (Dropout)         (None, None, 1000)        0         \n",
            "                                                                 \n",
            " lstm_7 (LSTM)               (None, None, 1000)        8004000   \n",
            "                                                                 \n",
            " dropout_7 (Dropout)         (None, None, 1000)        0         \n",
            "                                                                 \n",
            " lstm_8 (LSTM)               (None, 1000)              8004000   \n",
            "                                                                 \n",
            " dropout_8 (Dropout)         (None, 1000)              0         \n",
            "                                                                 \n",
            " dense_2 (Dense)             (None, 4)                 4004      \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 20,040,004\n",
            "Trainable params: 20,040,004\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n",
            "None\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model.compile(optimizer='adam', loss='mean_squared_error')"
      ],
      "metadata": {
        "id": "ZhoWj_XeXQws"
      },
      "execution_count": 102,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Model Train"
      ],
      "metadata": {
        "id": "65QbfffusPoJ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "print(x_train.shape)\n",
        "print(y_train.shape)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "HDT9XPXHvqyN",
        "outputId": "6bc3a3e9-a7ae-48ff-e64a-fb529c5e1f75"
      },
      "execution_count": 103,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(2082, 500, 6)\n",
            "(2082, 4)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Change to False to avoid trainging the model\n",
        "# if False:\n",
        "if True:\n",
        "  # Directory where the checkpoints will be saved\n",
        "  checkpoint_dir = './training_checkpoints_'+dt.datetime.now().strftime(\"%Y%m%d%H%M%S\")\n",
        "  # Name of the checkpoint files\n",
        "  checkpoint_prefix = os.path.join(checkpoint_dir, \"ckpt_epoch{epoch}_loss{loss}\")\n",
        "  \n",
        "  checkpoint_callback=tf.keras.callbacks.ModelCheckpoint(\n",
        "      filepath=checkpoint_prefix,\n",
        "      save_weights_only=True,\n",
        "      monitor=\"loss\", mode=\"min\",\n",
        "      save_best_only=True)\n",
        "  model.fit(x_train, y_train, epochs=25, batch_size=32, callbacks=[checkpoint_callback])\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "9Ccc_Ej2TmYO",
        "outputId": "4e7fe210-6cbb-4a9d-f856-829cfa6bced5"
      },
      "execution_count": 104,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/25\n",
            "66/66 [==============================] - 58s 773ms/step - loss: 0.0125\n",
            "Epoch 2/25\n",
            "66/66 [==============================] - 54s 816ms/step - loss: 0.0115\n",
            "Epoch 3/25\n",
            "66/66 [==============================] - 55s 841ms/step - loss: 0.0113\n",
            "Epoch 4/25\n",
            "66/66 [==============================] - 56s 845ms/step - loss: 0.0114\n",
            "Epoch 5/25\n",
            "66/66 [==============================] - 57s 859ms/step - loss: 0.0113\n",
            "Epoch 6/25\n",
            "66/66 [==============================] - 58s 886ms/step - loss: 0.0112\n",
            "Epoch 7/25\n",
            "66/66 [==============================] - 59s 889ms/step - loss: 0.0112\n",
            "Epoch 8/25\n",
            "66/66 [==============================] - 59s 890ms/step - loss: 0.0111\n",
            "Epoch 9/25\n",
            "66/66 [==============================] - 58s 875ms/step - loss: 0.0112\n",
            "Epoch 10/25\n",
            "66/66 [==============================] - 58s 880ms/step - loss: 0.0112\n",
            "Epoch 11/25\n",
            "66/66 [==============================] - 58s 881ms/step - loss: 0.0111\n",
            "Epoch 12/25\n",
            "66/66 [==============================] - 59s 892ms/step - loss: 0.0111\n",
            "Epoch 13/25\n",
            "66/66 [==============================] - 59s 895ms/step - loss: 0.0110\n",
            "Epoch 14/25\n",
            "66/66 [==============================] - 58s 880ms/step - loss: 0.0111\n",
            "Epoch 15/25\n",
            "66/66 [==============================] - 58s 882ms/step - loss: 0.0111\n",
            "Epoch 16/25\n",
            "66/66 [==============================] - 59s 896ms/step - loss: 0.0110\n",
            "Epoch 17/25\n",
            "66/66 [==============================] - 58s 882ms/step - loss: 0.0112\n",
            "Epoch 18/25\n",
            "66/66 [==============================] - 58s 882ms/step - loss: 0.0110\n",
            "Epoch 19/25\n",
            "66/66 [==============================] - 58s 882ms/step - loss: 0.0111\n",
            "Epoch 20/25\n",
            "24/66 [=========>....................] - ETA: 37s - loss: 0.0099"
          ]
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-104-78bd1a1c9ef9>\u001b[0m in \u001b[0;36m<cell line: 5>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     12\u001b[0m       \u001b[0mmonitor\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"loss\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmode\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"min\"\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     13\u001b[0m       save_best_only=True)\n\u001b[0;32m---> 14\u001b[0;31m   \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m25\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m32\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcallbacks\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mcheckpoint_callback\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m/usr/local/lib/python3.9/dist-packages/keras/utils/traceback_utils.py\u001b[0m in \u001b[0;36merror_handler\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     63\u001b[0m         \u001b[0mfiltered_tb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     64\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 65\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     66\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     67\u001b[0m             \u001b[0mfiltered_tb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_process_traceback_frames\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__traceback__\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.9/dist-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_batch_size, validation_freq, max_queue_size, workers, use_multiprocessing)\u001b[0m\n\u001b[1;32m   1683\u001b[0m                         ):\n\u001b[1;32m   1684\u001b[0m                             \u001b[0mcallbacks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_train_batch_begin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1685\u001b[0;31m                             \u001b[0mtmp_logs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0miterator\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1686\u001b[0m                             \u001b[0;32mif\u001b[0m \u001b[0mdata_handler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshould_sync\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1687\u001b[0m                                 \u001b[0mcontext\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0masync_wait\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.9/dist-packages/tensorflow/python/util/traceback_utils.py\u001b[0m in \u001b[0;36merror_handler\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    148\u001b[0m     \u001b[0mfiltered_tb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    149\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 150\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    151\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    152\u001b[0m       \u001b[0mfiltered_tb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_process_traceback_frames\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__traceback__\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.9/dist-packages/tensorflow/python/eager/polymorphic_function/polymorphic_function.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwds)\u001b[0m\n\u001b[1;32m    892\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    893\u001b[0m       \u001b[0;32mwith\u001b[0m \u001b[0mOptionalXlaContext\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_jit_compile\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 894\u001b[0;31m         \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    895\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    896\u001b[0m       \u001b[0mnew_tracing_count\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexperimental_get_tracing_count\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.9/dist-packages/tensorflow/python/eager/polymorphic_function/polymorphic_function.py\u001b[0m in \u001b[0;36m_call\u001b[0;34m(self, *args, **kwds)\u001b[0m\n\u001b[1;32m    924\u001b[0m       \u001b[0;31m# In this case we have created variables on the first call, so we run the\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    925\u001b[0m       \u001b[0;31m# defunned version which is guaranteed to never create variables.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 926\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_no_variable_creation_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# pylint: disable=not-callable\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    927\u001b[0m     \u001b[0;32melif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_variable_creation_fn\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    928\u001b[0m       \u001b[0;31m# Release the lock early so that multiple threads can perform the call\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.9/dist-packages/tensorflow/python/eager/polymorphic_function/tracing_compiler.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m    141\u001b[0m       (concrete_function,\n\u001b[1;32m    142\u001b[0m        filtered_flat_args) = self._maybe_define_function(args, kwargs)\n\u001b[0;32m--> 143\u001b[0;31m     return concrete_function._call_flat(\n\u001b[0m\u001b[1;32m    144\u001b[0m         filtered_flat_args, captured_inputs=concrete_function.captured_inputs)  # pylint: disable=protected-access\n\u001b[1;32m    145\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.9/dist-packages/tensorflow/python/eager/polymorphic_function/monomorphic_function.py\u001b[0m in \u001b[0;36m_call_flat\u001b[0;34m(self, args, captured_inputs, cancellation_manager)\u001b[0m\n\u001b[1;32m   1755\u001b[0m         and executing_eagerly):\n\u001b[1;32m   1756\u001b[0m       \u001b[0;31m# No tape is watching; skip to running the function.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1757\u001b[0;31m       return self._build_call_outputs(self._inference_function.call(\n\u001b[0m\u001b[1;32m   1758\u001b[0m           ctx, args, cancellation_manager=cancellation_manager))\n\u001b[1;32m   1759\u001b[0m     forward_backward = self._select_forward_and_backward_functions(\n",
            "\u001b[0;32m/usr/local/lib/python3.9/dist-packages/tensorflow/python/eager/polymorphic_function/monomorphic_function.py\u001b[0m in \u001b[0;36mcall\u001b[0;34m(self, ctx, args, cancellation_manager)\u001b[0m\n\u001b[1;32m    379\u001b[0m       \u001b[0;32mwith\u001b[0m \u001b[0m_InterpolateFunctionError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    380\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mcancellation_manager\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 381\u001b[0;31m           outputs = execute.execute(\n\u001b[0m\u001b[1;32m    382\u001b[0m               \u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msignature\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    383\u001b[0m               \u001b[0mnum_outputs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_num_outputs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.9/dist-packages/tensorflow/python/eager/execute.py\u001b[0m in \u001b[0;36mquick_execute\u001b[0;34m(op_name, num_outputs, inputs, attrs, ctx, name)\u001b[0m\n\u001b[1;32m     50\u001b[0m   \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     51\u001b[0m     \u001b[0mctx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mensure_initialized\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 52\u001b[0;31m     tensors = pywrap_tfe.TFE_Py_Execute(ctx._handle, device_name, op_name,\n\u001b[0m\u001b[1;32m     53\u001b[0m                                         inputs, attrs, num_outputs)\n\u001b[1;32m     54\u001b[0m   \u001b[0;32mexcept\u001b[0m \u001b[0mcore\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_NotOkStatusException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Testing a model"
      ],
      "metadata": {
        "id": "dbSKl47vZvpe"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#print trainings directories to pick one\n",
        "!ls -ld training_checkpoints_*/"
      ],
      "metadata": {
        "id": "59CDDB0i4yTx"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "test_model = create_model()"
      ],
      "metadata": {
        "id": "tpmru7nG9kbW"
      },
      "execution_count": 105,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# if checkpoint_dir does not exists, select the one stated in the except block\n",
        "try:\n",
        "  checkpoint_dir\n",
        "except NameError: \n",
        "  checkpoint_dir = './training_checkpoints_20230406214431'\n",
        "\n",
        "print(checkpoint_dir)\n",
        "\n",
        "def load_weights(epoch=None):\n",
        "  if epoch is None:\n",
        "    weights_file = tf.train.latest_checkpoint(checkpoint_dir)\n",
        "  else:\n",
        "    with os.scandir(checkpoint_dir) as entries:\n",
        "      for entry in entries:\n",
        "        if re.search(f'^ckpt_epoch{epoch}_.*\\.index', entry.name):\n",
        "          weights_file = checkpoint_dir + '/'+ entry.name[:-6]\n",
        "\n",
        "  print(weights_file)\n",
        "  test_model.load_weights(weights_file)\n",
        "  return test_model\n",
        "\n",
        "test_model = load_weights()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "wQ0JTXsp4VKF",
        "outputId": "2b25d414-f188-4c14-ef43-f7af5566a3be"
      },
      "execution_count": 107,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "./training_checkpoints_20230406230143\n",
            "./training_checkpoints_20230406230143/ckpt_epoch16_loss0.01097947172820568\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "test_start = dt.datetime(2013,1,1)\n",
        "end = dt.datetime(2023,4,5)\n",
        "\n",
        "yfin.pdr_override()\n",
        "test_data = web.data.get_data_yahoo(ticker, test_start, test_end)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Mf4q97pfaSCA",
        "outputId": "7355b53f-c879-4296-d24a-bb8bd739e5d8"
      },
      "execution_count": 114,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\r[*********************100%***********************]  1 of 1 completed\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# def close_tester(model, test_data, scaler=None):\n",
        "model = test_model\n",
        "scaler = scaler\n",
        "test_x_train, test_y_train, _, _ = preprocessing(data, scaler=scaler)\n",
        "print(test_x_train.shape)\n",
        "print(test_y_train.shape)\n",
        "results = model.predict(test_x_train)\n",
        "# the results are tensors of 4 numbers, Jump, High, Low, and Close respectively\n",
        "\n",
        "# close_tester(test_model, test_data, scaler=the_scaler)\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MqCeMf3UoxZm",
        "outputId": "a0591f06-f804-41e9-b973-627a7693ff89"
      },
      "execution_count": 115,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "max_value:  10.589996337890625\n",
            "max_volume:  1460852400.0\n",
            "(2082, 500, 6)\n",
            "(2082, 4)\n",
            "66/66 [==============================] - 18s 275ms/step\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "right_counter = 0\n",
        "wrong_counter = 0\n",
        "no_action_counter = 0\n",
        "# for result, expected in zip(results[:2], test_y_train[:2]):\n",
        "for result, expected in zip(results[:], test_y_train[:]):\n",
        "  # print(result)\n",
        "  # print(expected)\n",
        "  comparer = result[3] * expected[3]\n",
        "  if comparer > 0:\n",
        "    right_counter += 1\n",
        "  elif comparer == 0:\n",
        "    no_action_counter\n",
        "  elif comparer < 0:\n",
        "    wrong_counter += 1\n",
        "\n",
        "  # print('expected: ', decoder(expected))\n",
        "  # print('result: ', decoder(result))\n",
        "\n",
        "print('right_counter :', right_counter)\n",
        "print('no_action_counter :',no_action_counter)\n",
        "print('wrong_counter :', wrong_counter)\n",
        "print('success rate: {}%'.format(right_counter*100/len(results)))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "AVYFQZnqEqhx",
        "outputId": "7353f76c-f2e6-4a48-ba31-74ab67bb73ea"
      },
      "execution_count": 120,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "right_counter : 1118\n",
            "no_action_counter : 0\n",
            "wrong_counter : 959\n",
            "success rate: 53.6983669548511%\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "test_data.iloc[500,:]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "gyhzy_l6sAvi",
        "outputId": "78f5d2cf-cd21-47b1-b58e-6b3321a802bd"
      },
      "execution_count": 123,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Open         2.802500e+01\n",
              "High         2.863000e+01\n",
              "Low          2.800250e+01\n",
              "Close        2.849750e+01\n",
              "Adj Close    2.561277e+01\n",
              "Volume       1.348840e+08\n",
              "Name: 2014-12-26 00:00:00, dtype: float64"
            ]
          },
          "metadata": {},
          "execution_count": 123
        }
      ]
    }
  ]
}