feat: Crafting Data pipeline, Models, and Restful API

.github/workflows/pipeline.yaml

@@ -0,0 +1,57 @@
+name: pipeline
+
+on:
+  push:
+    branches:
+      - main
+    tags:
+      - '*'   
+  schedule:
+    - cron: "0 9 * * *"
+    # 16 - 7 = 9
+
+jobs:
+  extraction_train_modeling:
+    name: Data Extraction, Training, and Modeling
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Set global directory
+        run: git config --global --add safe.directory /github/workspace
+
+      - uses: actions/checkout@v3     
+        with:
+         persist-credentials: false
+         fetch-depth: 1
+
+      - name: Scraping Yahoo Finance
+        run:  go run scraper.go
+
+      - name: Install Libraries
+        run: pip install -r requirements.txt
+
+      - name: Modeling and Training
+        run: python training.py
+
+      - name: Commit changes
+        run: |
+          git config --local user.email "[email protected]"
+          git config --local user.name "belajarqywok"
+          git add -A
+          git commit -m "Get Cryptocurrencies Data every 4:00 PM"
+
+      - name: Push changes
+        uses: ad-m/github-push-action@master
+        with:
+          github_token: ${{ secrets.GH_TOKEN }}
+          branch: main
+
+  deployment:
+    name: Deployment
+    runs-on: ubuntu-latest
+    needs: extraction_train_modeling
+    environment: Production
+
+    steps:
+      - name: Deployment
+        run: echo "coming soon..."

.gitignore

@@ -0,0 +1,13 @@
+# Postman
+/postman/dataset.url
+
+# Environments
+/bin
+/Lib
+/lib64
+/Include
+/Scripts
+
+# Pycache
+/__pycache__
+/restful/__pycache__

.vercelignore

@@ -0,0 +1,7 @@
+/.github
+
+/bin
+/include
+/lib
+
+/postman

README.md

@@ -0,0 +1,73 @@
+<p align="center">
+  <a href="http://nestjs.com/" target="blank"><img src="https://nestjs.com/img/logo-small.svg" width="200" alt="Nest Logo" /></a>
+</p>
+
+[circleci-image]: https://img.shields.io/circleci/build/github/nestjs/nest/master?token=abc123def456
+[circleci-url]: https://circleci.com/gh/nestjs/nest
+
+  <p align="center">A progressive <a href="http://nodejs.org" target="_blank">Node.js</a> framework for building efficient and scalable server-side applications.</p>
+    <p align="center">
+<a href="https://www.npmjs.com/~nestjscore" target="_blank"><img src="https://img.shields.io/npm/v/@nestjs/core.svg" alt="NPM Version" /></a>
+<a href="https://www.npmjs.com/~nestjscore" target="_blank"><img src="https://img.shields.io/npm/l/@nestjs/core.svg" alt="Package License" /></a>
+<a href="https://www.npmjs.com/~nestjscore" target="_blank"><img src="https://img.shields.io/npm/dm/@nestjs/common.svg" alt="NPM Downloads" /></a>
+<a href="https://circleci.com/gh/nestjs/nest" target="_blank"><img src="https://img.shields.io/circleci/build/github/nestjs/nest/master" alt="CircleCI" /></a>
+<a href="https://coveralls.io/github/nestjs/nest?branch=master" target="_blank"><img src="https://coveralls.io/repos/github/nestjs/nest/badge.svg?branch=master#9" alt="Coverage" /></a>
+<a href="https://discord.gg/G7Qnnhy" target="_blank"><img src="https://img.shields.io/badge/discord-online-brightgreen.svg" alt="Discord"/></a>
+<a href="https://opencollective.com/nest#backer" target="_blank"><img src="https://opencollective.com/nest/backers/badge.svg" alt="Backers on Open Collective" /></a>
+<a href="https://opencollective.com/nest#sponsor" target="_blank"><img src="https://opencollective.com/nest/sponsors/badge.svg" alt="Sponsors on Open Collective" /></a>
+  <a href="https://paypal.me/kamilmysliwiec" target="_blank"><img src="https://img.shields.io/badge/Donate-PayPal-ff3f59.svg"/></a>
+    <a href="https://opencollective.com/nest#sponsor"  target="_blank"><img src="https://img.shields.io/badge/Support%20us-Open%20Collective-41B883.svg" alt="Support us"></a>
+  <a href="https://twitter.com/nestframework" target="_blank"><img src="https://img.shields.io/twitter/follow/nestframework.svg?style=social&label=Follow"></a>
+</p>
+  <!--[![Backers on Open Collective](https://opencollective.com/nest/backers/badge.svg)](https://opencollective.com/nest#backer)
+  [![Sponsors on Open Collective](https://opencollective.com/nest/sponsors/badge.svg)](https://opencollective.com/nest#sponsor)-->
+
+## Description
+
+[Nest](https://github.com/nestjs/nest) framework TypeScript starter repository.
+
+## Installation
+
+```bash
+$ npm install
+```
+
+## Running the app
+
+```bash
+# development
+$ npm run start
+
+# watch mode
+$ npm run start:dev
+
+# production mode
+$ npm run start:prod
+```
+
+## Test
+
+```bash
+# unit tests
+$ npm run test
+
+# e2e tests
+$ npm run test:e2e
+
+# test coverage
+$ npm run test:cov
+```
+
+## Support
+
+Nest is an MIT-licensed open source project. It can grow thanks to the sponsors and support by the amazing backers. If you'd like to join them, please [read more here](https://docs.nestjs.com/support).
+
+## Stay in touch
+
+- Author - [Kamil Myśliwiec](https://kamilmysliwiec.com)
+- Website - [https://nestjs.com](https://nestjs.com/)
+- Twitter - [@nestframework](https://twitter.com/nestframework)
+
+## License
+
+Nest is [MIT licensed](LICENSE).

converter.py

@@ -0,0 +1,40 @@
+import json
+
+"""
+
+    Data Mining Assignment - Group 5
+
+"""
+
+class JSONProcessor:
+    def __init__(self, input_file: str, output_file: str) -> None:
+        self.input_file:  str = input_file
+        self.output_file: str = output_file
+        self.data = None
+
+    def load_json(self) -> None:
+        with open(self.input_file, 'r') as file:
+            self.data = json.load(file)
+
+    def extract_symbols(self) -> list:
+        if self.data is None:
+            raise ValueError("data not loaded. call load_json() first.")
+        quotes = self.data['finance']['result'][0]['quotes']
+        return [quote['symbol'] for quote in quotes]
+
+    def save_json(self, data: list) -> None:
+        with open(self.output_file, 'w') as file:
+            json.dump({'symbols': data}, file, indent = 4)
+            print(f'saved: {self.output_file}')
+
+def main():
+    input_file  = './postman/response.json'
+    output_file = './postman/symbols.json'
+    
+    processor = JSONProcessor(input_file, output_file)
+    processor.load_json()
+    symbols = processor.extract_symbols()
+    processor.save_json(symbols)
+
+
+if __name__ == "__main__": main()

go.mod

@@ -0,0 +1,3 @@
+module cryptocurrency_prediction
+
+go 1.20

main.py

@@ -0,0 +1,23 @@
+import os
+from fastapi import FastAPI
+from fastapi.middleware.cors import CORSMiddleware
+from restful.routes import route
+
+REST = FastAPI(
+    title = "Cryptocurency Prediction Service",
+    version = "1.0"
+)
+
+# CORS Middleware
+REST.add_middleware(
+    CORSMiddleware,
+    allow_origins = ["*"],
+    allow_methods = ["*"],
+    allow_headers = ["*"],
+    allow_credentials = True,
+)
+
+REST.include_router(
+    router = route,
+    prefix = '/crypto'
+)

postman/Yahoo Finance.postman_collection.json

@@ -0,0 +1,69 @@
+{
+	"info": {
+		"_postman_id": "249fd388-44f6-45c2-9ad5-37da9c2af089",
+		"name": "Yahoo Finance",
+		"schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json"
+	},
+	"item": [
+		{
+			"name": "cryptocurrencies",
+			"request": {
+				"method": "POST",
+				"header": [
+					{
+						"key": "Cookie",
+						"value": "GUC=AQEBCAFmWUlmh0IaaAQw&s=AQAAAH-PIsT_&g=ZlgBjg; A1=d=AQABBBR-S2YCEF6h7KkHtT6kUMd5eQmdvDIFEgEBCAFJWWaHZlpOb2UB_eMBAAcIFH5LZgmdvDI&S=AQAAAge4BvAFwzWWdJFVm5Wyq9k; A3=d=AQABBBR-S2YCEF6h7KkHtT6kUMd5eQmdvDIFEgEBCAFJWWaHZlpOb2UB_eMBAAcIFH5LZgmdvDI&S=AQAAAge4BvAFwzWWdJFVm5Wyq9k; axids=gam=y-BdfSS7lE2uLV0LrGZqbRPm.8FUDjf.82~A&dv360=eS1EdjNSYkpGRTJ1R2RYQTAwYnNhcFJmQ0ZZN3BtTmNGan5B&ydsp=y-wmHAUIFE2uKC4PXfccNh1ff.Lz1oO0cj~A&tbla=y-gt8RDdJE2uKuvojQP3_mil11ZyoZelyw~A; tbla_id=f1c3e4ae-853f-47af-ba52-d13fe18de92e-tuctd4c1d85; PRF=t%3DBTC-USD%252BETH-USD%252BLTC-USD%252BLTC-INR%252BCU%253DF%26newChartbetateaser%3D0%252C1718255372183; A1S=d=AQABBBR-S2YCEF6h7KkHtT6kUMd5eQmdvDIFEgEBCAFJWWaHZlpOb2UB_eMBAAcIFH5LZgmdvDI&S=AQAAAge4BvAFwzWWdJFVm5Wyq9k; cmp=t=1717308407&j=0&u=1---; gpp=DBAA; gpp_sid=-1",
+						"type": "text"
+					}
+				],
+				"body": {
+					"mode": "raw",
+					"raw": "{\"offset\":0,\"size\":50,\"sortType\":\"DESC\",\"sortField\":\"intradaymarketcap\",\"quoteType\":\"CRYPTOCURRENCY\",\"query\":{\"operator\":\"and\",\"operands\":[{\"operator\":\"eq\",\"operands\":[\"currency\",\"USD\"]},{\"operator\":\"eq\",\"operands\":[\"exchange\",\"CCC\"]}]},\"userId\":\"\",\"userIdType\":\"guid\"}",
+					"options": {
+						"raw": {
+							"language": "json"
+						}
+					}
+				},
+				"url": {
+					"raw": "https://query2.finance.yahoo.com/v1/finance/screener?crumb=55ovV9srjcg&lang=en-US&region=US&formatted=true&corsDomain=finance.yahoo.com",
+					"protocol": "https",
+					"host": [
+						"query2",
+						"finance",
+						"yahoo",
+						"com"
+					],
+					"path": [
+						"v1",
+						"finance",
+						"screener"
+					],
+					"query": [
+						{
+							"key": "crumb",
+							"value": "55ovV9srjcg"
+						},
+						{
+							"key": "lang",
+							"value": "en-US"
+						},
+						{
+							"key": "region",
+							"value": "US"
+						},
+						{
+							"key": "formatted",
+							"value": "true"
+						},
+						{
+							"key": "corsDomain",
+							"value": "finance.yahoo.com"
+						}
+					]
+				}
+			},
+			"response": []
+		}
+	]
+}

postman/symbols.json

@@ -0,0 +1,54 @@
+{
+    "symbols": [
+        "BTC-USD",
+        "ETH-USD",
+        "USDT-USD",
+        "BNB-USD",
+        "SOL-USD",
+        "STETH-USD",
+        "USDC-USD",
+        "XRP-USD",
+        "DOGE-USD",
+        "ADA-USD",
+        "TON11419-USD",
+        "SHIB-USD",
+        "AVAX-USD",
+        "WSTETH-USD",
+        "WETH-USD",
+        "LINK-USD",
+        "WBTC-USD",
+        "DOT-USD",
+        "TRX-USD",
+        "WTRX-USD",
+        "BCH-USD",
+        "NEAR-USD",
+        "MATIC-USD",
+        "LTC-USD",
+        "PEPE24478-USD",
+        "EETH-USD",
+        "UNI7083-USD",
+        "ICP-USD",
+        "LEO-USD",
+        "DAI-USD",
+        "WEETH-USD",
+        "ETC-USD",
+        "EZETH-USD",
+        "APT21794-USD",
+        "RNDR-USD",
+        "BTCB-USD",
+        "HBAR-USD",
+        "WHBAR-USD",
+        "WBETH-USD",
+        "IMX10603-USD",
+        "KAS-USD",
+        "ATOM-USD",
+        "ARB11841-USD",
+        "MNT27075-USD",
+        "FIL-USD",
+        "WIF-USD",
+        "XLM-USD",
+        "USDE29470-USD",
+        "CRO-USD",
+        "AR-USD"
+    ]
+}

pyproject.toml

@@ -0,0 +1,15 @@
+[tool.poetry]
+name = "cryptocurrency-prediction"
+version = "0.1.0"
+description = "Data Mining Assignment - Group 5"
+authors = ["belajarqywok <[email protected]>"]
+license = "MIT"
+readme = "README.md"
+
+[tool.poetry.dependencies]
+python = "^3.9"
+
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"

pyvenv.cfg

@@ -0,0 +1,3 @@
+home = /usr/bin
+include-system-site-packages = false
+version = 3.10.12

requirements.txt

@@ -0,0 +1,74 @@
+absl-py==2.1.0
+annotated-types==0.7.0
+anyio==4.4.0
+astunparse==1.6.3
+certifi==2024.2.2
+charset-normalizer==3.3.2
+click==8.1.7
+dnspython==2.6.1
+email_validator==2.1.1
+exceptiongroup==1.2.1
+fastapi==0.111.0
+fastapi-cli==0.0.4
+flatbuffers==24.3.25
+gast==0.5.4
+google-pasta==0.2.0
+grpcio==1.64.0
+h11==0.14.0
+h5py==3.11.0
+httpcore==1.0.5
+httptools==0.6.1
+httpx==0.27.0
+idna==3.7
+importlib_metadata==7.1.0
+Jinja2==3.1.4
+joblib==1.4.2
+keras==3.3.3
+libclang==18.1.1
+Markdown==3.6
+markdown-it-py==3.0.0
+MarkupSafe==2.1.5
+mdurl==0.1.2
+ml-dtypes==0.3.2
+namex==0.0.8
+numpy==1.26.4
+opt-einsum==3.3.0
+optree==0.11.0
+orjson==3.10.3
+packaging==24.0
+pandas==2.2.2
+protobuf==4.25.3
+pydantic==2.7.2
+pydantic_core==2.18.3
+Pygments==2.18.0
+python-dateutil==2.9.0.post0
+python-dotenv==1.0.1
+python-multipart==0.0.9
+pytz==2024.1
+PyYAML==6.0.1
+requests==2.32.3
+rich==13.7.1
+scikit-learn==1.5.0
+scipy==1.13.1
+shellingham==1.5.4
+six==1.16.0
+sniffio==1.3.1
+starlette==0.37.2
+tensorboard==2.16.2
+tensorboard-data-server==0.7.2
+tensorflow==2.16.1
+tensorflow-io-gcs-filesystem==0.31.0
+termcolor==2.4.0
+threadpoolctl==3.5.0
+typer==0.12.3
+typing_extensions==4.12.1
+tzdata==2024.1
+ujson==5.10.0
+urllib3==2.2.1
+uvicorn==0.30.1
+uvloop==0.19.0
+watchfiles==0.22.0
+websockets==12.0
+Werkzeug==3.0.3
+wrapt==1.16.0
+zipp==3.19.1

restful/controllers.py

@@ -0,0 +1,50 @@
+from http import HTTPStatus
+from fastapi.responses import JSONResponse
+from restful.services import cryptocurrency_svc
+from restful.schemas import CryptocurrencyPredictionSchema
+
+
+# Cryptocurrency Controller
+class cryptocurrency_controller:
+    # Cryptocurrency Service
+    __SERVICE = cryptocurrency_svc()
+
+    # Cryptocurrency Controller
+    def prediction(self, payload: CryptocurrencyPredictionSchema) -> JSONResponse:
+        try: 
+            prediction: list = self.__SERVICE.prediction(
+                payload = payload
+            )
+
+            if not prediction :
+                return JSONResponse(
+                    content = {
+                        'message': 'Request Failed',
+                        'status_code': HTTPStatus.BAD_REQUEST,
+                        'data': None
+                    },
+                    status_code = HTTPStatus.BAD_REQUEST
+                )
+
+            return JSONResponse(
+                content = {
+                    'message': 'Prediction Success',
+                    'status_code': HTTPStatus.OK,
+                    'data': {
+						'currency': payload.currency,
+						'predictions': prediction
+					}
+                },
+                status_code = HTTPStatus.OK
+            )
+
+        except Exception as error_message:
+            print(error_message)
+            return JSONResponse(
+                content = {
+                    'message': 'Internal Server Error',
+                    'status_code': HTTPStatus.INTERNAL_SERVER_ERROR,
+                    'data': None
+                },
+                status_code = HTTPStatus.INTERNAL_SERVER_ERROR
+            )

restful/routes.py

@@ -0,0 +1,19 @@
+from fastapi import APIRouter, Body
+from fastapi.responses import JSONResponse
+from restful.controllers import cryptocurrency_controller
+from restful.schemas import CryptocurrencyPredictionSchema
+
+# Route
+route = APIRouter()
+
+# Controller
+__CONTROLLER = cryptocurrency_controller()
+
+# Cryptocurrency Prediction
+@route.post(path = '/prediction', tags = ['machine_learning'])
+async def cryptocurrency_pred_route(
+    payload: CryptocurrencyPredictionSchema = Body(...)
+) -> JSONResponse:
+    # Cryptocurrency Controller
+    return __CONTROLLER.prediction(payload = payload)
+

restful/schemas.py

@@ -0,0 +1,6 @@
+from pydantic import BaseModel
+
+class CryptocurrencyPredictionSchema(BaseModel) :
+	days: int
+	currency: str
+	

restful/services.py

@@ -0,0 +1,19 @@
+from restful.utilities import Utilities
+from restful.schemas import CryptocurrencyPredictionSchema
+
+class cryptocurrency_svc:
+	# Prediction Utilities
+	__PRED_UTILS = Utilities()
+
+	# Prediction Service
+	def prediction(self, payload: CryptocurrencyPredictionSchema) -> list:
+		days: int = payload.days
+		currency: str = payload.currency
+
+		result: list = self.__PRED_UTILS.cryptocurrency_prediction_utils(
+			days = days,
+			model_name = currency,
+			sequence_length = 60
+		)
+
+		return result

restful/utilities.py

@@ -0,0 +1,44 @@
+import os
+from joblib import load
+from numpy import append, expand_dims
+from pandas import read_json, to_datetime, Timedelta
+
+from tensorflow.keras.models import load_model
+
+
+class Utilities:
+	def __init__(self) -> None:
+		self.model_path = './models'
+		self.posttrained_path = './posttrained'
+		self.scaler_path = './pickles'
+
+	def cryptocurrency_prediction_utils(self,
+		days: int,sequence_length: int, model_name: str) -> list:
+		model_path = os.path.join(self.model_path, f'{model_name}.keras')
+		model = load_model(model_path)
+
+		dataframe_path = os.path.join(self.posttrained_path, f'{model_name}-posttrained.json')
+		dataframe = read_json(dataframe_path)
+		dataframe.set_index('Date', inplace = True)
+
+		minmax_scaler = load(os.path.join(self.scaler_path, f'{model_name}_minmax_scaler.pickle'))
+		standard_scaler = load(os.path.join(self.scaler_path, f'{model_name}_standard_scaler.pickle'))
+
+		lst_seq = dataframe[-sequence_length:].values
+		lst_seq = expand_dims(lst_seq, axis = 0)
+
+		predicted_prices = {}
+		last_date = to_datetime(dataframe.index[-1])
+
+		for _ in range(days):
+			predicted_price = model.predict(lst_seq)
+			last_date       = last_date + Timedelta(days = 1)
+
+			predicted_prices[last_date] = minmax_scaler.inverse_transform(predicted_price)
+			predicted_prices[last_date] = standard_scaler.inverse_transform(predicted_prices[last_date])
+
+			lst_seq = append(lst_seq[:, 1:, :], [predicted_price], axis = 1)
+
+		values = [{'date': date.strftime('%Y-%m-%d'), 'price': float(price)} for date, price in predicted_prices.items()]
+        
+		return values

scraper.go

@@ -0,0 +1,116 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"os"
+	"strconv"
+	"sync"
+	"time"
+)
+
+
+/*
+
+    Data Mining Assignment - Group 5
+     
+*/
+
+
+type Symbols struct {
+	Symbols []string `json:"symbols"`
+}
+
+type Downloader struct {
+	symbols []string
+}
+
+
+/*
+ *  New downloader
+*/
+func NewDownloader(symbols []string) *Downloader {
+	return &Downloader{
+		symbols: symbols,
+	}
+}
+
+
+/*
+ *  Download dataset
+*/
+func (d *Downloader) Download(symbol string, wg *sync.WaitGroup) {
+	defer wg.Done()
+
+	unixTimestamp := getCurrentUnixTimestamp()
+	endpoint := fmt.Sprintf(
+		"https://query1.finance.yahoo.com/v7/finance/download/" +
+		"%s?period1=1410912000&period2=%s&interval=1d&events=history&includeAdjustedClose=true",
+		symbol, strconv.FormatInt(unixTimestamp, 10),
+	)
+
+	resp, err := http.Get(endpoint)
+	if err != nil {
+		log.Printf("[ERROR] error downloading %s: %v\n", symbol, err)
+		return
+	}
+	defer resp.Body.Close()
+
+	filename := fmt.Sprintf("./datasets/%s.csv", symbol)
+	file, err := os.Create(filename)
+	if err != nil {
+		log.Printf("[ERROR] error creating file for %s: %v\n", symbol, err)
+		return
+	}
+	defer file.Close()
+
+	_, err = io.Copy(file, resp.Body)
+	if err != nil {
+		log.Printf("[ERROR] error writing data for %s: %v\n", symbol, err)
+		return
+	}
+
+	fmt.Printf("[SUCCESS] saved: %s\n", symbol)
+}
+
+
+/*
+ *  Get current UNIX timetamp
+*/
+func getCurrentUnixTimestamp() int64 {
+	now := time.Now().UTC()
+	return now.Unix()
+}
+
+
+func main() {
+	jsonFile, err := os.Open("./postman/symbols.json")
+	if err != nil {
+		log.Fatalf("[ERROR] failed to open JSON file: %v", err)
+	}
+	defer jsonFile.Close()
+
+	byteValue, err := ioutil.ReadAll(jsonFile)
+	if err != nil {
+		log.Fatalf("[ERROR] failed to read JSON file: %v", err)
+	}
+
+	var symbols Symbols
+	if err := json.Unmarshal(byteValue, &symbols); err != nil {
+		log.Fatalf("[ERROR] failed to unmarshal JSON: %v", err)
+	}
+
+	var wg sync.WaitGroup
+	downloader := NewDownloader(symbols.Symbols)
+
+	for _, symbol := range symbols.Symbols {
+		wg.Add(1)
+		go downloader.Download(symbol, &wg)
+	}
+
+	wg.Wait()
+}

training.py

@@ -0,0 +1,159 @@
+import os
+import json
+import joblib
+import numpy as np
+import pandas as pd
+from sklearn.preprocessing import StandardScaler, MinMaxScaler
+from tensorflow.keras.models import Sequential
+from tensorflow.keras.layers import LSTM, Dense, Dropout
+from tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint
+
+
+"""
+
+    Data Mining Assignment - Group 5
+
+"""
+
+
+from warnings import filterwarnings
+filterwarnings('ignore')
+
+class DataProcessor:
+    def __init__(self, datasets_path):
+        self.datasets_path = datasets_path
+        self.datasets = self._get_datasets()
+
+    def _get_datasets(self):
+        return sorted([
+            item for item in os.listdir(self.datasets_path)
+            if os.path.isfile(os.path.join(self.datasets_path, item)) and item.endswith('.csv')
+        ])
+
+    @staticmethod
+    def create_sequences(df, sequence_length):
+        labels, sequences = [], []
+        for i in range(len(df) - sequence_length):
+            seq = df.iloc[i:i + sequence_length].values
+            label = df.iloc[i + sequence_length].values[0]
+            sequences.append(seq)
+            labels.append(label)
+        return np.array(sequences), np.array(labels)
+
+    @staticmethod
+    def preprocess_data(dataframe):
+        for col in dataframe.columns:
+            if dataframe[col].isnull().any():
+                if dataframe[col].dtype == 'object':
+                    dataframe[col].fillna(dataframe[col].mode()[0], inplace = True)
+                else:
+                    dataframe[col].fillna(dataframe[col].mean(), inplace = True)
+        return dataframe
+
+    @staticmethod
+    def scale_data(dataframe, scaler_cls):
+        scaler = scaler_cls()
+        dataframe['Close'] = scaler.fit_transform(dataframe[['Close']])
+        return scaler, dataframe
+
+class ModelBuilder:
+    @staticmethod
+    def build_model(input_shape):
+        model = Sequential([
+            LSTM(50, return_sequences = True, input_shape = input_shape),
+            Dropout(0.2),
+            LSTM(50, return_sequences = False),
+            Dropout(0.2),
+            Dense(1)
+        ])
+        model.compile(optimizer = 'adam', loss = 'mean_squared_error')
+        return model
+
+class Trainer:
+    def __init__(self, model, model_file, sequence_length, epochs, batch_size):
+        self.model = model
+        self.model_file = model_file
+        self.sequence_length = sequence_length
+        self.epochs = epochs
+        self.batch_size = batch_size
+
+    def train(self, X_train, y_train, X_test, y_test):
+        early_stopping = EarlyStopping(monitor = 'val_loss', patience = 5, mode = 'min')
+
+        model_checkpoint = ModelCheckpoint(
+          filepath       = self.model_file,
+          save_best_only = True,
+          monitor        = 'val_loss',
+          mode           = 'min'
+        )
+
+        history = self.model.fit(
+            X_train, y_train,
+            epochs          = self.epochs,
+            batch_size      = self.batch_size,
+            validation_data = (X_test, y_test),
+            callbacks       = [early_stopping, model_checkpoint]
+        )
+
+        return history
+
+class PostProcessor:
+    @staticmethod
+    def inverse_transform(scaler, data):
+        return scaler.inverse_transform(data)
+
+    @staticmethod
+    def save_json(filename, data):
+        with open(filename, 'w') as f:
+            json.dump(data, f)
+
+def main():
+    datasets_path = './datasets'
+    models_path   = './models'
+    posttrained   = './posttrained'
+    pickle_file   = './pickles'
+
+    sequence_length = 60
+    epochs = 200
+    batch_size = 32
+
+    data_processor = DataProcessor(datasets_path)
+
+    for dataset in data_processor.datasets:
+        print(f"[TRAINING] {dataset.replace('.csv', '')} ")
+
+        dataframe = pd.read_csv(os.path.join(datasets_path, dataset), index_col='Date')[['Close']]
+        model_file = os.path.join(models_path, f"{dataset.replace('.csv', '')}.keras")
+
+        dataframe = data_processor.preprocess_data(dataframe)
+        standard_scaler, dataframe = data_processor.scale_data(dataframe, StandardScaler)
+        minmax_scaler, dataframe = data_processor.scale_data(dataframe, MinMaxScaler)
+
+        sequences, labels = data_processor.create_sequences(dataframe, sequence_length)
+        input_shape = (sequences.shape[1], sequences.shape[2])
+        model = ModelBuilder.build_model(input_shape)
+
+        train_size = int(len(sequences) * 0.8)
+        X_train, X_test = sequences[:train_size], sequences[train_size:]
+        y_train, y_test = labels[:train_size], labels[train_size:]
+
+        trainer = Trainer(model, model_file, sequence_length, epochs, batch_size)
+        trainer.train(X_train, y_train, X_test, y_test)
+
+        dataframe_json = {'Date': dataframe.index.tolist(), 'Close': dataframe['Close'].tolist()}
+
+        PostProcessor.save_json(
+          os.path.join(posttrained, f'{dataset.replace(".csv", "")}-posttrained.json'),
+          dataframe_json
+        )
+
+        joblib.dump(minmax_scaler, os.path.join(pickle_file, f'{dataset.replace(".csv", "")}_minmax_scaler.pickle'))
+        joblib.dump(standard_scaler, os.path.join(pickle_file, f'{dataset.replace(".csv", "")}_standard_scaler.pickle'))
+
+        model.load_weights(model_file)
+        model.save(model_file)
+
+        print("\n\n")
+
+if __name__ == "__main__":
+    main()

vercel.json

@@ -0,0 +1,14 @@
+{
+  "builds": [
+    {
+      "src": "main.py",
+      "use": "@vercel/python"
+    }
+  ],
+  "routes": [
+    {
+      "src": "/(.*)",
+      "dest": "main.py"
+    }
+  ]
+}