{
"nbformat": 4,
"nbformat_minor": 0,
"metadata": {
"colab": {
"provenance": [],
"authorship_tag": "ABX9TyOc5/oQ0Z2ie5dOI46PpyV0",
"include_colab_link": true
},
"kernelspec": {
"name": "python3",
"display_name": "Python 3"
},
"language_info": {
"name": "python"
},
"gpuClass": "standard",
"accelerator": "GPU"
},
"cells": [
{
"cell_type": "markdown",
"metadata": {
"id": "view-in-github",
"colab_type": "text"
},
"source": [
"![\"Open](\"https://colab.research.google.com/assets/colab-badge.svg\")
"
]
},
{
"cell_type": "code",
"source": [
"from google.colab import drive\n",
"drive.mount('/content/drive')\n",
"project_path = '/content/drive/MyDrive/projects/Stock_Predicter'\n",
"%cd $project_path"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "Xr3Qozgfktoc",
"outputId": "b4ce9a19-4dc1-43e9-b09e-af91a2b07343"
},
"execution_count": 90,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n",
"/content/drive/MyDrive/projects/Stock_Predicter\n"
]
}
]
},
{
"cell_type": "code",
"execution_count": 91,
"metadata": {
"id": "e8SQqogMQYLh"
},
"outputs": [],
"source": [
"import numpy as np\n",
"import matplotlib.pyplot as plt\n",
"import pandas as pd\n",
"import pandas_datareader as web\n",
"import datetime as dt\n",
"import yfinance as yfin\n",
"import tensorflow as tf\n",
"import os\n",
"import re\n",
"\n",
"from sklearn.preprocessing import MinMaxScaler\n",
"from tensorflow.keras.models import Sequential\n",
"from tensorflow.keras.layers import Dense, Dropout, LSTM\n"
]
},
{
"cell_type": "markdown",
"source": [
"# Get Data"
],
"metadata": {
"id": "5vO8pty3VwkG"
}
},
{
"cell_type": "code",
"source": [
"# Select a company for now\n",
"ticker = 'AAPL'\n",
"\n",
"start = dt.datetime(2013,1,1)\n",
"end = dt.datetime(2023,4,5)"
],
"metadata": {
"id": "O6dtJpJwS5Eg"
},
"execution_count": 92,
"outputs": []
},
{
"cell_type": "code",
"source": [
"yfin.pdr_override()\n",
"data = web.data.get_data_yahoo(ticker, start, end)\n"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "LwPyk8Uh-Zz_",
"outputId": "2217df50-87e9-48e3-e096-71163331f570"
},
"execution_count": 93,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"\r[*********************100%***********************] 1 of 1 completed\n"
]
}
]
},
{
"cell_type": "markdown",
"source": [
"# Preprocess_data"
],
"metadata": {
"id": "SSuS9OONV5-a"
}
},
{
"cell_type": "code",
"source": [
"def create_remove_columns(data):\n",
" # create jump column\n",
" data = pd.DataFrame.copy(data)\n",
" data['Jump'] = data['Open'] - data['Close'].shift(1)\n",
" data['Jump'].fillna(0, inplace=True)\n",
" # data = data.reindex(columns=['Open', 'High', 'Low', 'Close', 'Adj Close', 'Jump'])\n",
" data.insert(0,'Jump', data.pop('Jump'))\n",
" return data"
],
"metadata": {
"id": "Bpym8x-Kxf0p"
},
"execution_count": 94,
"outputs": []
},
{
"cell_type": "code",
"source": [
"def normalize_data(data, scaler=None):\n",
" the_data = pd.DataFrame.copy(data)\n",
" # substract the open value to all columns but the first one and the last one which are \"Jump\" and \"Volume\"\n",
" the_data.iloc[:, 1:-1] = the_data.iloc[:,1:-1] - the_data['Open'].values[:, np.newaxis]\n",
" # print('the_data')\n",
" # print(the_data)\n",
"\n",
" the_data.pop('Open')\n",
" # todo save an csv with the values for the scaler\n",
" if scaler is None:\n",
" # Create the scaler\n",
" values = np.abs(the_data.values)\n",
" max_value = np.max(values[:,:-1])\n",
" max_volume = np.max(values[:,-1])\n",
" def scaler(d):\n",
" data = pd.DataFrame.copy(d)\n",
" print('max_value: ', max_value)\n",
" print('max_volume: ', max_volume)\n",
" data.iloc[:, :-1] = data.iloc[:,:-1].apply(lambda x: x/max_value)\n",
" data.iloc[:, -1] = data.iloc[:,-1].apply(lambda x: x/max_volume)\n",
" return data\n",
" def decoder(values):\n",
" decoded_values = values * max_value\n",
" return decoded_values\n",
" else:\n",
" decoder = None\n",
" \n",
" normalized_data = scaler(the_data)\n",
"\n",
" return normalized_data, scaler, decoder\n",
"\n",
"\n"
],
"metadata": {
"id": "v9RoqzBvtrOb"
},
"execution_count": 95,
"outputs": []
},
{
"cell_type": "code",
"source": [
"def create_training_data(norm_data):\n",
" prediction_days = 500\n",
" \n",
" x_train_list = []\n",
" y_train_list = []\n",
" \n",
" for i in range(prediction_days, len(norm_data)):\n",
" x_train_list.append(norm_data[i-prediction_days:i])\n",
" y_train_list.append(norm_data.iloc[i].values[0:4])\n",
" \n",
" x_train = np.array(x_train_list)\n",
" y_train = np.array(y_train_list)\n",
" return x_train, y_train"
],
"metadata": {
"id": "jMXkRAYFomHM"
},
"execution_count": 96,
"outputs": []
},
{
"cell_type": "code",
"source": [
"#Make all the preprocesing\n",
"def preprocessing(data, scaler=None):\n",
" # print(data.head(3))\n",
" data_0 = create_remove_columns(data)\n",
" # print(data_0.head(3))\n",
" #todo: save the_scaler somehow to use in new runtimes\n",
" norm_data, scaler, decoder = normalize_data(data_0, scaler=scaler)\n",
" # print(norm_data.head(3))\n",
" x_train, y_train = create_training_data(norm_data)\n",
" # print(x_train.shape, y_train.shape)\n",
" return x_train, y_train, scaler, decoder"
],
"metadata": {
"id": "YZWMfusT-I7Z"
},
"execution_count": 97,
"outputs": []
},
{
"cell_type": "code",
"source": [
"x_train, y_train, scaler, decoder = preprocessing(data)"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "PeJjDC0VBG_6",
"outputId": "aff2cf0b-e630-4727-bbec-3cf5be4e53a0"
},
"execution_count": 98,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"max_value: 10.589996337890625\n",
"max_volume: 1460852400.0\n"
]
}
]
},
{
"cell_type": "code",
"source": [
"print(x_train.shape)\n",
"x_train[1,499,:]"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "YkI8vSguuS8A",
"outputId": "6e5eeaa2-12de-4dd2-b17c-1d61e415fbd8"
},
"execution_count": 99,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"(2082, 500, 6)\n"
]
},
{
"output_type": "execute_result",
"data": {
"text/plain": [
"array([ 0.00212456, 0.05712934, -0.00212456, 0.04461756, -0.22778379,\n",
" 0.09233239])"
]
},
"metadata": {},
"execution_count": 99
}
]
},
{
"cell_type": "code",
"source": [
"td = data.iloc[498:501]\n",
"# print('td:\\n',td)\n",
"td0 = create_remove_columns(td)\n",
"print('td0:\\n',td0)\n",
"print(decoder(y_train[0]))"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "QaO34uSds2wJ",
"outputId": "af5d9a04-214c-4a2d-c706-5af3a2a1ea5a"
},
"execution_count": 100,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"td0:\n",
" Jump Open High Low Close Adj Close \\\n",
"Date \n",
"2014-12-23 0.000000 28.307501 28.332500 28.115000 28.135000 25.286961 \n",
"2014-12-24 0.010000 28.145000 28.177500 28.002501 28.002501 25.167873 \n",
"2014-12-26 0.022499 28.025000 28.629999 28.002501 28.497499 25.612770 \n",
"\n",
" Volume \n",
"Date \n",
"2014-12-23 104113600 \n",
"2014-12-24 57918400 \n",
"2014-12-26 134884000 \n",
"[ 0.02249908 0.60499954 -0.02249908 0.47249985]\n"
]
}
]
},
{
"cell_type": "markdown",
"source": [
"# Model"
],
"metadata": {
"id": "Z3N2WMYNV-qX"
}
},
{
"cell_type": "markdown",
"source": [
"## Create Model"
],
"metadata": {
"id": "emDyvzVUp5KJ"
}
},
{
"cell_type": "code",
"source": [
"def create_model():\n",
" model = Sequential()\n",
" # model.add(LSTM(units=112, return_sequences=True, input_shape=(x_train.shape[1:])))\n",
" model.add(LSTM(units=1000, return_sequences=True, input_shape=(None,x_train.shape[-1],)))\n",
" model.add(Dropout(0.2))\n",
" model.add(LSTM(units=1000, return_sequences=True))\n",
" model.add(Dropout(0.2))\n",
" model.add(LSTM(units=1000))\n",
" model.add(Dropout(0.2))\n",
" model.add(Dense(units=4))\n",
" return model\n",
"\n",
"model = create_model()\n",
"print(model.summary())"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "GXhYAKzXVfku",
"outputId": "bbf96ec2-84fc-4246-9003-32c2f8083bb6"
},
"execution_count": 101,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"Model: \"sequential_2\"\n",
"_________________________________________________________________\n",
" Layer (type) Output Shape Param # \n",
"=================================================================\n",
" lstm_6 (LSTM) (None, None, 1000) 4028000 \n",
" \n",
" dropout_6 (Dropout) (None, None, 1000) 0 \n",
" \n",
" lstm_7 (LSTM) (None, None, 1000) 8004000 \n",
" \n",
" dropout_7 (Dropout) (None, None, 1000) 0 \n",
" \n",
" lstm_8 (LSTM) (None, 1000) 8004000 \n",
" \n",
" dropout_8 (Dropout) (None, 1000) 0 \n",
" \n",
" dense_2 (Dense) (None, 4) 4004 \n",
" \n",
"=================================================================\n",
"Total params: 20,040,004\n",
"Trainable params: 20,040,004\n",
"Non-trainable params: 0\n",
"_________________________________________________________________\n",
"None\n"
]
}
]
},
{
"cell_type": "code",
"source": [
"model.compile(optimizer='adam', loss='mean_squared_error')"
],
"metadata": {
"id": "ZhoWj_XeXQws"
},
"execution_count": 102,
"outputs": []
},
{
"cell_type": "markdown",
"source": [
"## Model Train"
],
"metadata": {
"id": "65QbfffusPoJ"
}
},
{
"cell_type": "code",
"source": [
"print(x_train.shape)\n",
"print(y_train.shape)"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "HDT9XPXHvqyN",
"outputId": "6bc3a3e9-a7ae-48ff-e64a-fb529c5e1f75"
},
"execution_count": 103,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"(2082, 500, 6)\n",
"(2082, 4)\n"
]
}
]
},
{
"cell_type": "code",
"source": [
"# Change to False to avoid trainging the model\n",
"# if False:\n",
"if True:\n",
" # Directory where the checkpoints will be saved\n",
" checkpoint_dir = './training_checkpoints_'+dt.datetime.now().strftime(\"%Y%m%d%H%M%S\")\n",
" # Name of the checkpoint files\n",
" checkpoint_prefix = os.path.join(checkpoint_dir, \"ckpt_epoch{epoch}_loss{loss}\")\n",
" \n",
" checkpoint_callback=tf.keras.callbacks.ModelCheckpoint(\n",
" filepath=checkpoint_prefix,\n",
" save_weights_only=True,\n",
" monitor=\"loss\", mode=\"min\",\n",
" save_best_only=True)\n",
" model.fit(x_train, y_train, epochs=25, batch_size=32, callbacks=[checkpoint_callback])\n"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/",
"height": 1000
},
"id": "9Ccc_Ej2TmYO",
"outputId": "4e7fe210-6cbb-4a9d-f856-829cfa6bced5"
},
"execution_count": 104,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"Epoch 1/25\n",
"66/66 [==============================] - 58s 773ms/step - loss: 0.0125\n",
"Epoch 2/25\n",
"66/66 [==============================] - 54s 816ms/step - loss: 0.0115\n",
"Epoch 3/25\n",
"66/66 [==============================] - 55s 841ms/step - loss: 0.0113\n",
"Epoch 4/25\n",
"66/66 [==============================] - 56s 845ms/step - loss: 0.0114\n",
"Epoch 5/25\n",
"66/66 [==============================] - 57s 859ms/step - loss: 0.0113\n",
"Epoch 6/25\n",
"66/66 [==============================] - 58s 886ms/step - loss: 0.0112\n",
"Epoch 7/25\n",
"66/66 [==============================] - 59s 889ms/step - loss: 0.0112\n",
"Epoch 8/25\n",
"66/66 [==============================] - 59s 890ms/step - loss: 0.0111\n",
"Epoch 9/25\n",
"66/66 [==============================] - 58s 875ms/step - loss: 0.0112\n",
"Epoch 10/25\n",
"66/66 [==============================] - 58s 880ms/step - loss: 0.0112\n",
"Epoch 11/25\n",
"66/66 [==============================] - 58s 881ms/step - loss: 0.0111\n",
"Epoch 12/25\n",
"66/66 [==============================] - 59s 892ms/step - loss: 0.0111\n",
"Epoch 13/25\n",
"66/66 [==============================] - 59s 895ms/step - loss: 0.0110\n",
"Epoch 14/25\n",
"66/66 [==============================] - 58s 880ms/step - loss: 0.0111\n",
"Epoch 15/25\n",
"66/66 [==============================] - 58s 882ms/step - loss: 0.0111\n",
"Epoch 16/25\n",
"66/66 [==============================] - 59s 896ms/step - loss: 0.0110\n",
"Epoch 17/25\n",
"66/66 [==============================] - 58s 882ms/step - loss: 0.0112\n",
"Epoch 18/25\n",
"66/66 [==============================] - 58s 882ms/step - loss: 0.0110\n",
"Epoch 19/25\n",
"66/66 [==============================] - 58s 882ms/step - loss: 0.0111\n",
"Epoch 20/25\n",
"24/66 [=========>....................] - ETA: 37s - loss: 0.0099"
]
},
{
"output_type": "error",
"ename": "KeyboardInterrupt",
"evalue": "ignored",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)",
"\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 12\u001b[0m \u001b[0mmonitor\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"loss\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmode\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"min\"\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 13\u001b[0m save_best_only=True)\n\u001b[0;32m---> 14\u001b[0;31m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m25\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m32\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcallbacks\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mcheckpoint_callback\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
"\u001b[0;32m/usr/local/lib/python3.9/dist-packages/keras/utils/traceback_utils.py\u001b[0m in \u001b[0;36merror_handler\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 63\u001b[0m \u001b[0mfiltered_tb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 64\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 65\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 66\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 67\u001b[0m \u001b[0mfiltered_tb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_process_traceback_frames\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__traceback__\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m/usr/local/lib/python3.9/dist-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_batch_size, validation_freq, max_queue_size, workers, use_multiprocessing)\u001b[0m\n\u001b[1;32m 1683\u001b[0m ):\n\u001b[1;32m 1684\u001b[0m \u001b[0mcallbacks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_train_batch_begin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1685\u001b[0;31m \u001b[0mtmp_logs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0miterator\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1686\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mdata_handler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshould_sync\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1687\u001b[0m \u001b[0mcontext\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0masync_wait\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m/usr/local/lib/python3.9/dist-packages/tensorflow/python/util/traceback_utils.py\u001b[0m in \u001b[0;36merror_handler\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 148\u001b[0m \u001b[0mfiltered_tb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 149\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 150\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 151\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 152\u001b[0m \u001b[0mfiltered_tb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_process_traceback_frames\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__traceback__\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m/usr/local/lib/python3.9/dist-packages/tensorflow/python/eager/polymorphic_function/polymorphic_function.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwds)\u001b[0m\n\u001b[1;32m 892\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 893\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mOptionalXlaContext\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_jit_compile\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 894\u001b[0;31m \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 895\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 896\u001b[0m \u001b[0mnew_tracing_count\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexperimental_get_tracing_count\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m/usr/local/lib/python3.9/dist-packages/tensorflow/python/eager/polymorphic_function/polymorphic_function.py\u001b[0m in \u001b[0;36m_call\u001b[0;34m(self, *args, **kwds)\u001b[0m\n\u001b[1;32m 924\u001b[0m \u001b[0;31m# In this case we have created variables on the first call, so we run the\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 925\u001b[0m \u001b[0;31m# defunned version which is guaranteed to never create variables.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 926\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_no_variable_creation_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m# pylint: disable=not-callable\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 927\u001b[0m \u001b[0;32melif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_variable_creation_fn\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 928\u001b[0m \u001b[0;31m# Release the lock early so that multiple threads can perform the call\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m/usr/local/lib/python3.9/dist-packages/tensorflow/python/eager/polymorphic_function/tracing_compiler.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m 141\u001b[0m (concrete_function,\n\u001b[1;32m 142\u001b[0m filtered_flat_args) = self._maybe_define_function(args, kwargs)\n\u001b[0;32m--> 143\u001b[0;31m return concrete_function._call_flat(\n\u001b[0m\u001b[1;32m 144\u001b[0m filtered_flat_args, captured_inputs=concrete_function.captured_inputs) # pylint: disable=protected-access\n\u001b[1;32m 145\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m/usr/local/lib/python3.9/dist-packages/tensorflow/python/eager/polymorphic_function/monomorphic_function.py\u001b[0m in \u001b[0;36m_call_flat\u001b[0;34m(self, args, captured_inputs, cancellation_manager)\u001b[0m\n\u001b[1;32m 1755\u001b[0m and executing_eagerly):\n\u001b[1;32m 1756\u001b[0m \u001b[0;31m# No tape is watching; skip to running the function.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1757\u001b[0;31m return self._build_call_outputs(self._inference_function.call(\n\u001b[0m\u001b[1;32m 1758\u001b[0m ctx, args, cancellation_manager=cancellation_manager))\n\u001b[1;32m 1759\u001b[0m forward_backward = self._select_forward_and_backward_functions(\n",
"\u001b[0;32m/usr/local/lib/python3.9/dist-packages/tensorflow/python/eager/polymorphic_function/monomorphic_function.py\u001b[0m in \u001b[0;36mcall\u001b[0;34m(self, ctx, args, cancellation_manager)\u001b[0m\n\u001b[1;32m 379\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0m_InterpolateFunctionError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 380\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mcancellation_manager\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 381\u001b[0;31m outputs = execute.execute(\n\u001b[0m\u001b[1;32m 382\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msignature\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 383\u001b[0m \u001b[0mnum_outputs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_num_outputs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;32m/usr/local/lib/python3.9/dist-packages/tensorflow/python/eager/execute.py\u001b[0m in \u001b[0;36mquick_execute\u001b[0;34m(op_name, num_outputs, inputs, attrs, ctx, name)\u001b[0m\n\u001b[1;32m 50\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 51\u001b[0m \u001b[0mctx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mensure_initialized\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 52\u001b[0;31m tensors = pywrap_tfe.TFE_Py_Execute(ctx._handle, device_name, op_name,\n\u001b[0m\u001b[1;32m 53\u001b[0m inputs, attrs, num_outputs)\n\u001b[1;32m 54\u001b[0m \u001b[0;32mexcept\u001b[0m \u001b[0mcore\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_NotOkStatusException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
"\u001b[0;31mKeyboardInterrupt\u001b[0m: "
]
}
]
},
{
"cell_type": "markdown",
"source": [
"# Testing a model"
],
"metadata": {
"id": "dbSKl47vZvpe"
}
},
{
"cell_type": "code",
"source": [
"#print trainings directories to pick one\n",
"!ls -ld training_checkpoints_*/"
],
"metadata": {
"id": "59CDDB0i4yTx"
},
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"source": [
"test_model = create_model()"
],
"metadata": {
"id": "tpmru7nG9kbW"
},
"execution_count": 105,
"outputs": []
},
{
"cell_type": "code",
"source": [
"# if checkpoint_dir does not exists, select the one stated in the except block\n",
"try:\n",
" checkpoint_dir\n",
"except NameError: \n",
" checkpoint_dir = './training_checkpoints_20230406214431'\n",
"\n",
"print(checkpoint_dir)\n",
"\n",
"def load_weights(epoch=None):\n",
" if epoch is None:\n",
" weights_file = tf.train.latest_checkpoint(checkpoint_dir)\n",
" else:\n",
" with os.scandir(checkpoint_dir) as entries:\n",
" for entry in entries:\n",
" if re.search(f'^ckpt_epoch{epoch}_.*\\.index', entry.name):\n",
" weights_file = checkpoint_dir + '/'+ entry.name[:-6]\n",
"\n",
" print(weights_file)\n",
" test_model.load_weights(weights_file)\n",
" return test_model\n",
"\n",
"test_model = load_weights()"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "wQ0JTXsp4VKF",
"outputId": "2b25d414-f188-4c14-ef43-f7af5566a3be"
},
"execution_count": 107,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"./training_checkpoints_20230406230143\n",
"./training_checkpoints_20230406230143/ckpt_epoch16_loss0.01097947172820568\n"
]
}
]
},
{
"cell_type": "code",
"source": [
"test_start = dt.datetime(2013,1,1)\n",
"end = dt.datetime(2023,4,5)\n",
"\n",
"yfin.pdr_override()\n",
"test_data = web.data.get_data_yahoo(ticker, test_start, test_end)"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "Mf4q97pfaSCA",
"outputId": "7355b53f-c879-4296-d24a-bb8bd739e5d8"
},
"execution_count": 114,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"\r[*********************100%***********************] 1 of 1 completed\n"
]
}
]
},
{
"cell_type": "code",
"source": [
"# def close_tester(model, test_data, scaler=None):\n",
"model = test_model\n",
"scaler = scaler\n",
"test_x_train, test_y_train, _, _ = preprocessing(data, scaler=scaler)\n",
"print(test_x_train.shape)\n",
"print(test_y_train.shape)\n",
"results = model.predict(test_x_train)\n",
"# the results are tensors of 4 numbers, Jump, High, Low, and Close respectively\n",
"\n",
"# close_tester(test_model, test_data, scaler=the_scaler)\n"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "MqCeMf3UoxZm",
"outputId": "a0591f06-f804-41e9-b973-627a7693ff89"
},
"execution_count": 115,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"max_value: 10.589996337890625\n",
"max_volume: 1460852400.0\n",
"(2082, 500, 6)\n",
"(2082, 4)\n",
"66/66 [==============================] - 18s 275ms/step\n"
]
}
]
},
{
"cell_type": "code",
"source": [
"right_counter = 0\n",
"wrong_counter = 0\n",
"no_action_counter = 0\n",
"# for result, expected in zip(results[:2], test_y_train[:2]):\n",
"for result, expected in zip(results[:], test_y_train[:]):\n",
" # print(result)\n",
" # print(expected)\n",
" comparer = result[3] * expected[3]\n",
" if comparer > 0:\n",
" right_counter += 1\n",
" elif comparer == 0:\n",
" no_action_counter\n",
" elif comparer < 0:\n",
" wrong_counter += 1\n",
"\n",
" # print('expected: ', decoder(expected))\n",
" # print('result: ', decoder(result))\n",
"\n",
"print('right_counter :', right_counter)\n",
"print('no_action_counter :',no_action_counter)\n",
"print('wrong_counter :', wrong_counter)\n",
"print('success rate: {}%'.format(right_counter*100/len(results)))"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "AVYFQZnqEqhx",
"outputId": "7353f76c-f2e6-4a48-ba31-74ab67bb73ea"
},
"execution_count": 120,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"right_counter : 1118\n",
"no_action_counter : 0\n",
"wrong_counter : 959\n",
"success rate: 53.6983669548511%\n"
]
}
]
},
{
"cell_type": "code",
"source": [
"test_data.iloc[500,:]"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "gyhzy_l6sAvi",
"outputId": "78f5d2cf-cd21-47b1-b58e-6b3321a802bd"
},
"execution_count": 123,
"outputs": [
{
"output_type": "execute_result",
"data": {
"text/plain": [
"Open 2.802500e+01\n",
"High 2.863000e+01\n",
"Low 2.800250e+01\n",
"Close 2.849750e+01\n",
"Adj Close 2.561277e+01\n",
"Volume 1.348840e+08\n",
"Name: 2014-12-26 00:00:00, dtype: float64"
]
},
"metadata": {},
"execution_count": 123
}
]
}
]
} |