-
-
-
-
-
-
-
- Double-click (or enter) to edit
-
- Skip to main content
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- Notebook unstarred
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- Table of contents
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- Notebook
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- -
-
-
-
-
-
-
-
-
-
- Import Packages
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
-
-
-
-
-
- /usr/local/lib/python3.7/dist-packages/statsmodels/tools/_testing.py:19: FutureWarning: pandas.util.testing is deprecated. Use the functions in the public API at pandas.testing instead. - import pandas.util.testing as tm -
-
-
-
-
-
-
-
-
-
- Install Yahoo Finance
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 - 2 - 3 - 4 -utc_now = pytz.utc.localize(datetime.utcnow())
today = utc_now.astimezone(pytz.timezone("America/Chicago"))
new_today = today.strftime("%m-%d-%Y")
print(today, new_today)
-
-
-
-
- 2022-04-10 12:49:41.860788-05:00 04-10-2022 -
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 - 2 - 3 - 4 -!pip install yfinance
import yfinance as yf
-
-
-
-
- Requirement already satisfied: yfinance in /usr/local/lib/python3.7/dist-packages (0.1.70) -Requirement already satisfied: requests>=2.26 in /usr/local/lib/python3.7/dist-packages (from yfinance) (2.27.1) -Requirement already satisfied: numpy>=1.15 in /usr/local/lib/python3.7/dist-packages (from yfinance) (1.21.5) -Requirement already satisfied: pandas>=0.24.0 in /usr/local/lib/python3.7/dist-packages (from yfinance) (1.3.5) -Requirement already satisfied: multitasking>=0.0.7 in /usr/local/lib/python3.7/dist-packages (from yfinance) (0.0.10) -Requirement already satisfied: lxml>=4.5.1 in /usr/local/lib/python3.7/dist-packages (from yfinance) (4.8.0) -Requirement already satisfied: pytz>=2017.3 in /usr/local/lib/python3.7/dist-packages (from pandas>=0.24.0->yfinance) (2018.9) -Requirement already satisfied: python-dateutil>=2.7.3 in /usr/local/lib/python3.7/dist-packages (from pandas>=0.24.0->yfinance) (2.8.2) -Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.7/dist-packages (from python-dateutil>=2.7.3->pandas>=0.24.0->yfinance) (1.15.0) -Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests>=2.26->yfinance) (1.24.3) -Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests>=2.26->yfinance) (2.10) -Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests>=2.26->yfinance) (2021.10.8) -Requirement already satisfied: charset-normalizer~=2.0.0 in /usr/local/lib/python3.7/dist-packages (from requests>=2.26->yfinance) (2.0.12) -
-
-
-
-
-
-
-
-
-
- Import Coinbase Tickers
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 - 2 - 3 - 4 - 5 - 6 - 7 - 8 - 9 - 10 - 11 - 12 - 13 - 14 - 15 - 16 - 17 - 18 - 19 - 20 - 21 - 22 - 23 - 24 - 25 - 26 - 27 -##############################################
# Use this code to read and write coins to DB from Google Sheets
##############################################
from google.colab import auth
auth.authenticate_user()
print('Google Authenticated')
# ##############################################
# # read in historical data previously created #
# ##############################################
# Here the data is stored in G drive and then written to BQ
# Only run this if needed
# drive.mount('/content/drive')
# coinsDF = pd.read_excel("drive/My Drive/Data/coinbase_coins.xlsx")
# drive.flush_and_unmount()
# ############################
# # Write to BigQuery
# # Run this only write new coins file to G drive
# ############################
# table_id = 'my_dataset.coinbase_coins'
# coinsDF.to_gbq(table_id, project_id = 'sports-betting1', if_exists = 'replace')
-
-
-
-
- Google Authenticated -
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 - 2 - 3 - 4 - 5 - 6 - 7 - 8 - 9 - 10 - 11 - 12 - 13 - 14 - 15 - 16 - 17 -# variable_name = "" #@param [""]
###########################
# Let's test a form field #
###########################
# import ipywidgets as widgets
# fruit_list = coinbase_coins['Coin']
# fruit_picker = widgets.Dropdown(options=fruit_list)
# fruit_picker
# fruit_picker.value
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 -## fruit_picker.value
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 - 2 - 3 - 4 - 5 - 6 - 7 - 8 - 9 - 10 - 11 -##############################################
# read in coins from DB
##############################################
# %%bigquery --project sports-betting1 coinbase_coins
%%bigquery --project crypto1-307915 coinbase_coins
SELECT
*
# FROM `my_dataset.coinbase_coins`
FROM `my_dataset.coins`
--order by Coin asc
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 -coinbase_coins['Coin']
-
-
-
-
- 0 TBTC -1 KRL -2 IDEX -3 KNC -4 BAND -5 TRU -6 SYN -7 CTSI -8 WBTC -9 OGN -10 CTX -11 ICP -12 ARPA -13 COVAL -14 LOOM -15 ZEN -16 BOND -17 CLV -18 XRP -19 CRO -20 LCX -21 RLY -22 AGLD -23 PRO -24 DASH -25 IMX -26 COMP -27 NKN -28 PERP -29 ORN -30 DDX -31 LTC -32 UNI -33 FOX -34 BTC -35 APE -36 MKR -37 CVC -38 MASK -39 ASM -40 GFI -41 FIDA -42 QNT -43 IOTX -44 CGLD -45 SUPER -46 AMP -47 CHZ -48 FARM -49 RGT -50 USDC -51 BNT -52 BAT -53 LQTY -54 RAI -55 SHIB -56 TRAC -57 GODS -58 SNX -59 AXS -60 REN -61 MANA -62 SOL -63 UPI -64 SPELL -65 BCH -66 CRV -67 BADGER -68 TRIBE -69 ALICE -70 DOGE -71 STX -72 ENJ -73 PAX -74 PLA -75 ADA -76 RBN -77 REQ -78 OXT -79 ETH -80 FORTH -81 PLU -82 JASMY -83 UST -84 USDT -85 FIL -86 STORJ -87 ERN -88 GTC -89 SHPING -90 UNFI -91 COTI -92 RARI -93 SUKU -94 MUSD -95 MDT -96 MATIC -97 BICO -98 ZRX -99 ACH -100 BAL -101 AAVE -102 ANKR -103 XLM -104 LPT -105 AVT -106 AUCTION -107 FX -108 GYEN -109 DIA -110 SUSHI -111 BTRST -112 REP -113 XYO -114 AVAX -115 DOT -116 ZEC -117 GRT -118 OMG -119 AERGO -120 FET -121 ATOM -122 WCFG -123 CRPT -124 YFI -125 YFII -126 DNT -127 POWR -128 NMR -129 MIR -130 EOS -131 UMA -132 KEEP -133 BLZ -134 TRB -135 LRC -136 QUICK -137 MPL -138 SNT -139 ALGO -140 ETC -141 POLS -142 NU -143 NCT -144 MINA -145 MCO2 -146 INV -147 RNDR -148 API3 -149 ORCA -150 QSP -151 DAI -152 GALA -153 SKL -154 RAD -155 VGX -156 POLY -157 RLC -158 GLM -159 WLUNA -160 1INCH -161 ENS -162 GNT -163 MLN -164 HIGH -165 LINK -166 DESO -167 XTZ -168 ALCX -169 AIOZ -Name: Coin, dtype: object
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 - 2 -# from google.colab import drive
# drive.mount('/content/drive')
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 - 2 - 3 - 4 - 5 - 6 -# coinbase_coins.iloc[0,0]
# test_coin = (str(fruit_picker.value) + '-' + str('USD'))
# # test_coin = ('SHIB' + '-' + str('USD'))
# test_coin
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 - 2 -# tmp = yf.download(tickers=test_coin, period = '90d', interval = '1d')
# tmp
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 - 2 - 3 - 4 - 5 - 6 - 7 - 8 - 9 - 10 - 11 - 12 - 13 - 14 - 15 - 16 - 17 - 18 - 19 - 20 - 21 - 22 - 23 - 24 - 25 - 26 - 27 - 28 - 29 - 30 - 31 - 32 - 33 - 34 -########################
# test a single ticker #
########################
# tmp = yf.download(tickers=test_coin, period = '90d', interval = '1d')
# tmp.reset_index(inplace=True)
# # pd.datetime(np.where(tmp['Date'] > (datetime.today() - timedelta(days = 1)), datetime.today(), tmp['Date']))
# # tmp
# tmp.loc[tmp['Date'] > (datetime.today() - timedelta(days = 1)),'Date'] = (datetime.today() - timedelta(days = 1)).strftime('%Y-%m-%d 00:00:00')
# # (datetime.today() - timedelta(days = 1)).strftime('%Y-%m-%d')
# ## tmp.set_index('Date', inplace = True)
# tmp.set_index(pd.to_datetime(tmp['Date']), inplace = True)
# tmp.drop(columns = ['Date', inplace = True)
# tmp
########################
# Hourly data
########################
# tmp = yf.download(tickers=test_coin, period = '24h', interval = '30m', progress = True)
# .sort_values(ascending = False, by = 'Datetime')
# tmp = (yf.download(tickers=(str(coinbase_coins.iloc[x, 0]) + '-' + str('USD')), period = '72h', interval = '1h', progress=False))
# tmp = tmp[tmp['Volume'] != 0].tail(1)
## tmp.reset_index(inplace=True)
# tmp
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 - 2 - 3 - 4 - 5 -# Reset datetime
# utc_now = pytz.utc.localize(datetime.utcnow())
# today = utc_now.astimezone(pytz.timezone("America/Chicago"))
# new_today = today.strftime("%m-%d-%Y")
# print(today, new_today)
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 - 2 -# pd.DatetimeIndex
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 - 2 - 3 - 4 - 5 - 6 - 7 - 8 - 9 - 10 - 11 -############################
# here we take in UTC time and convert to local time
############################
# tmp.set_index(pd.DatetimeIndex(tmp['Datetime'])).tz_convert('US/Central')
# plt.figure(figsize=(16,8))
# plt.plot(tmp['Datetime'], tmp['Adj Close'])
# plt.xticks(rotation = 90)
# plt.grid()
# plt.show()
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 - 2 - 3 - 4 - 5 - 6 - 7 - 8 - 9 -# from vega_datasets import data
# stocks = data.stocks()
# import altair as alt
# alt.Chart(tmp).mark_line().encode(
# x='Datetime:T',
# y='Adj Close'
# ).interactive(bind_y=True)
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 - 2 - 3 -# tmp['Date'] = pd.DatetimeIndex(tmp['Datetime'])
# tmp = tmp.set_index(tmp['Date']).tz_convert('US/Central')
# tmp.drop(['Datetime', 'Date'], axis = 1)
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 -# tmp.head()
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 -# tmp.shape
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 - 2 -# tmp2 = tmp[['Open', 'High', 'Low', 'Close']]
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 -# tmp3 = (tmp2).diff(1).dropna()
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 -# tmp3.tail()
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 -# tmp3_var = VAR(tmp3, freq='-30T')
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 -# (tmp3_var.fit(5)).summary()
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 -#tmp3_results = tmp3_var.fit(5)
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 -#tmp3.values[-1:]
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 -#pd.DataFrame(tmp3_results.forecast(tmp3.values[-5:], 2))
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 - 2 - 3 -# pull min/max CST + 30 min
#(tmp['Date'].min()).astimezone(pytz.timezone("America/Chicago")) + timedelta(minutes = 30)
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
-
-
-
-
-
- 2022-04-10 12:50:58.623903-05:00 04-10-2022 -
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 - 2 - 3 - 4 - 5 - 6 - 7 - 8 - 9 -# (tmp.index).apply(pytz.utc.localize)
# df15['Type of day'] = np.where(df15.index.weekday > 5, "Weekend", "Working Day")
# tmp['Type of day'] =
# np.where(tmp.index > (datetime.today() - timedelta(days = 0)), datetime.today(), tmp.index)
# tmp[tmp.index > datetime.today()].index = datetime.today()
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 -#tmp.reset_index(inplace=True)
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 -#tmp.dtypes
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 -# tmp[tmp.index > datetime.today()].index
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 -# coinbase_coins.shape
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 -# coinbase_coins.iloc[1, 0]
-
-
-
-
-
-
-
-
-
- Grab Data
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 - 2 - 3 - 4 - 5 - 6 - 7 - 8 - 9 - 10 - 11 - 12 - 13 - 14 - 15 - 16 - 17 - 18 - 19 - 20 - 21 - 22 - 23 - 24 - 25 - 26 - 27 - 28 - 29 - 30 - 31 - 32 - 33 - 34 - 35 - 36 - 37 - 38 - 39 - 40 -dfOut = []
dfOut = pd.DataFrame()
for x in range(len(coinbase_coins)):
# print(coinbase_coins.iloc[x, 0])
# print(coinbase_coins.iloc[x, 0])
########################
# Hourly data
########################
## tmp = (yf.download(tickers=(str(coinbase_coins.iloc[x, 0]) + '-' + str('USD')), period = '72h', interval = '1h', progress=False))
## tmp = tmp[tmp['Volume'] != 0].tail(1)
########################
# Daily Data
########################
# tmp = (yf.download(tickers=(str(coinbase_coins.iloc[x, 0]) + '-' + str('USD')), period = '24h', interval = '30m', progress = False))
tmp = (yf.download(tickers=(str(coinbase_coins.iloc[x, 0]) + '-' + str('USD')), period = '180d', interval = '1d', progress=False))
tmp = tmp[tmp['Volume'] != 0]
# print(tmp)
###########################
# Clean up data alignment #
###########################
# tmp.reset_index(inplace=True)
# tmp.loc[tmp['Date'] > (datetime.today() - timedelta(days = 1)),'Date'] = (datetime.today() - timedelta(days = 1)).strftime('%Y-%m-%d 00:00:00')
# tmp.set_index(pd.to_datetime(tmp['Date']), inplace = True)
# tmp.drop(columns = ['Date'], inplace = True)
###########################
###########################
# print(tmp.shape)
tmp['Coin'] = coinbase_coins.iloc[x, 0]
if (tmp.shape[0] != 0):
print(coinbase_coins.iloc[x, 0])
dfOut = dfOut.append(tmp)
print(dfOut.shape)
# dfOut = dfOut.append(tmp)
-
-
-
-
- TBTC -(180, 7) -KRL -(360, 7) -IDEX -(540, 7) -KNC -(720, 7) -BAND -(900, 7) -TRU -(1080, 7) -
/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:34: SettingWithCopyWarning: -A value is trying to be set on a copy of a slice from a DataFrame. -Try using .loc[row_indexer,col_indexer] = value instead - -See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy -
SYN -(1172, 7) -CTSI -(1352, 7) -WBTC -(1532, 7) -OGN -(1712, 7) -CTX -(1892, 7) -ICP -(1893, 7) -ARPA -(2005, 7) -COVAL -(2185, 7) -LOOM -(2365, 7) -ZEN -(2545, 7) -BOND -(2725, 7) -CLV -(2905, 7) -XRP -(3085, 7) -CRO -(3265, 7) -LCX -(3445, 7) -RLY -(3625, 7) -AGLD -(3805, 7) -PRO -(3985, 7) -DASH -(4165, 7) -IMX -(4343, 7) -COMP -(4432, 7) -NKN -(4612, 7) -PERP -(4792, 7) -ORN -(4972, 7) -DDX -(5152, 7) -LTC -(5332, 7) -UNI -(5511, 7) -FOX -(5691, 7) -BTC -(5871, 7) -APE -(6042, 7) -MKR -(6222, 7) -CVC -(6402, 7) -MASK -(6462, 7) -ASM -(6642, 7) -GFI -(6643, 7) -FIDA -(6751, 7) -QNT -(6931, 7) -IOTX -(7111, 7) - -1 Failed download: -- CGLD-USD: No data found, symbol may be delisted -SUPER -(7145, 7) -AMP -(7325, 7) -CHZ -(7480, 7) -FARM -(7660, 7) -RGT -(7840, 7) -USDC -(8020, 7) -BNT -(8200, 7) -BAT -(8380, 7) -LQTY -(8560, 7) -RAI -(8740, 7) -SHIB -(8920, 7) -TRAC -(9100, 7) -GODS -(9264, 7) -SNX -(9444, 7) -AXS -(9624, 7) -REN -(9804, 7) -MANA -(9984, 7) -SOL -(10164, 7) -UPI -(10344, 7) -SPELL -(10524, 7) -BCH -(10704, 7) -CRV -(10884, 7) -BADGER -(11064, 7) -TRIBE -(11244, 7) -ALICE -(11424, 7) -DOGE -(11604, 7) -STX -(11784, 7) -ENJ -(11964, 7) - -1 Failed download: -- PAX-USD: No data found, symbol may be delisted -PLA -(12144, 7) -ADA -(12324, 7) -RBN -(12504, 7) -REQ -(12684, 7) -OXT -(12864, 7) -ETH -(13044, 7) -FORTH -(13224, 7) -PLU -(13404, 7) -JASMY -(13584, 7) -UST -(13764, 7) -USDT -(13944, 7) -FIL -(14124, 7) -STORJ -(14304, 7) -ERN -(14484, 7) -GTC -(14664, 7) -SHPING -(14844, 7) -UNFI -(15024, 7) -COTI -(15204, 7) -RARI -(15384, 7) -SUKU -(15564, 7) -MUSD -(15742, 7) -MDT -(15922, 7) -MATIC -(16102, 7) -BICO -(16233, 7) -ZRX -(16413, 7) -ACH -(16593, 7) -BAL -(16773, 7) -AAVE -(16953, 7) -ANKR -(17133, 7) -XLM -(17313, 7) -LPT -(17493, 7) -AVT -(17668, 7) -AUCTION -(17848, 7) -FX -(18028, 7) -GYEN -(18208, 7) -DIA -(18388, 7) -SUSHI -(18568, 7) -BTRST -(18748, 7) -REP -(18928, 7) -XYO -(19108, 7) -AVAX -(19288, 7) -DOT -(19410, 7) -ZEC -(19590, 7) -GRT -(19761, 7) -OMG -(19935, 7) -AERGO -(20115, 7) -FET -(20295, 7) -ATOM -(20475, 7) -WCFG -(20655, 7) -CRPT -(20835, 7) -YFI -(21015, 7) -YFII -(21195, 7) -DNT -(21375, 7) -POWR -(21555, 7) -NMR -(21735, 7) -MIR -(21915, 7) -EOS -(22095, 7) -UMA -(22275, 7) -KEEP -(22455, 7) -BLZ -(22635, 7) -TRB -(22815, 7) -LRC -(22995, 7) -QUICK -(23175, 7) -MPL -(23355, 7) -SNT -(23535, 7) -ALGO -(23636, 7) -ETC -(23816, 7) -POLS -(23996, 7) -NU -(24176, 7) -NCT -(24356, 7) -MINA -(24536, 7) -MCO2 -(24716, 7) -INV -(24896, 7) -RNDR -(25076, 7) -API3 -(25256, 7) -ORCA -(25436, 7) -QSP -(25616, 7) -DAI -(25796, 7) -GALA -(25797, 7) -SKL -(25977, 7) -RAD -(26157, 7) -VGX -(26337, 7) -POLY -(26517, 7) -RLC -(26697, 7) -GLM -(26877, 7) -WLUNA -(27057, 7) -1INCH -(27237, 7) -ENS -(27390, 7) -GNT -(27570, 7) -MLN -(27750, 7) -HIGH -(27929, 7) -LINK -(28109, 7) -DESO -(28110, 7) -XTZ -(28290, 7) -ALCX -(28470, 7) -AIOZ -(28650, 7) -
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 - 2 -dfOut.tail()
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 - 2 - 3 - 4 - 5 - 6 - 7 - 8 - 9 - 10 -##########################
# make dates locak #
##########################
# dfOut.reset_index(inplace=True)
# dfOut['Date'] = pd.DatetimeIndex(dfOut['Datetime'])
# dfOut = dfOut.set_index(dfOut['Date']).tz_convert('US/Central')
# dfOut.drop(['Datetime', 'Date'], axis = 1, inplace=True)
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 -dfOut[dfOut['Coin'] == 'ADA'].tail()
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 -dfOut.tail()
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 -dfOut['Coin'].unique()
-
-
-
-
- array(['TBTC', 'KRL', 'IDEX', 'KNC', 'BAND', 'TRU', 'SYN', 'CTSI', 'WBTC', - 'OGN', 'CTX', 'ICP', 'ARPA', 'COVAL', 'LOOM', 'ZEN', 'BOND', 'CLV', - 'XRP', 'CRO', 'LCX', 'RLY', 'AGLD', 'PRO', 'DASH', 'IMX', 'COMP', - 'NKN', 'PERP', 'ORN', 'DDX', 'LTC', 'UNI', 'FOX', 'BTC', 'APE', - 'MKR', 'CVC', 'MASK', 'ASM', 'GFI', 'FIDA', 'QNT', 'IOTX', 'SUPER', - 'AMP', 'CHZ', 'FARM', 'RGT', 'USDC', 'BNT', 'BAT', 'LQTY', 'RAI', - 'SHIB', 'TRAC', 'GODS', 'SNX', 'AXS', 'REN', 'MANA', 'SOL', 'UPI', - 'SPELL', 'BCH', 'CRV', 'BADGER', 'TRIBE', 'ALICE', 'DOGE', 'STX', - 'ENJ', 'PLA', 'ADA', 'RBN', 'REQ', 'OXT', 'ETH', 'FORTH', 'PLU', - 'JASMY', 'UST', 'USDT', 'FIL', 'STORJ', 'ERN', 'GTC', 'SHPING', - 'UNFI', 'COTI', 'RARI', 'SUKU', 'MUSD', 'MDT', 'MATIC', 'BICO', - 'ZRX', 'ACH', 'BAL', 'AAVE', 'ANKR', 'XLM', 'LPT', 'AVT', - 'AUCTION', 'FX', 'GYEN', 'DIA', 'SUSHI', 'BTRST', 'REP', 'XYO', - 'AVAX', 'DOT', 'ZEC', 'GRT', 'OMG', 'AERGO', 'FET', 'ATOM', 'WCFG', - 'CRPT', 'YFI', 'YFII', 'DNT', 'POWR', 'NMR', 'MIR', 'EOS', 'UMA', - 'KEEP', 'BLZ', 'TRB', 'LRC', 'QUICK', 'MPL', 'SNT', 'ALGO', 'ETC', - 'POLS', 'NU', 'NCT', 'MINA', 'MCO2', 'INV', 'RNDR', 'API3', 'ORCA', - 'QSP', 'DAI', 'GALA', 'SKL', 'RAD', 'VGX', 'POLY', 'RLC', 'GLM', - 'WLUNA', '1INCH', 'ENS', 'GNT', 'MLN', 'HIGH', 'LINK', 'DESO', - 'XTZ', 'ALCX', 'AIOZ'], dtype=object)
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 -dfOut[dfOut['Coin'] == 'ADA'].tail()
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 -# dfOut.loc[:,'Coin'].head()
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 - 2 - 3 -# Reference Date
## datetime.now() - timedelta(days = (120 + 0))
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 - 2 - 3 - 4 -# Max Date
## datetime.now() - timedelta(days = 0)
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 - 2 - 3 - 4 -# Min Date
## datetime.now() - timedelta(days = (0 - 1))
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 - 2 - 3 - 4 - 5 - 6 - 7 -###################################################
# Need logic to handle date changes in data
###################################################
## dfOut[dfOut['Coin'] == 'ADA']['Date'].max() > date.today()
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 - 2 - 3 -# dfOut[dfOut['Coin'] == 'ADA'][(dfOut[dfOut['Coin'] == 'ADA']['Date'] >= (datetime.now() - timedelta(days = (120 + 0)))) & (dfOut[dfOut['Coin'] == 'ADA']['Date'] < (datetime.now() - timedelta(days = 0)))]
# dfOut[dfOut['Coin'] == 'KNC']
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 -(datetime.today() - timedelta(days = 1))
-
-
-
-
- datetime.datetime(2022, 4, 9, 17, 54, 34, 110184)
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 - 2 - 3 - 4 - 5 - 6 -#############################
# address time difference
#############################
## dfOut[dfOut['Date'] > (datetime.today() - timedelta(hours = 5))]['Date'] = (datetime.today() - timedelta(days = 1))
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 -dfOut[dfOut['Date'] > (datetime.today() - timedelta(hours = 5))]['Date']
-
-
-
-
- Series([], Name: Date, dtype: datetime64[ns])
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 -# dfOut[(dfOut['Coin'].isin(['BTC', 'ADA', 'MATIC', 'STORJ', 'XLM']))]
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 - 2 -# dfOut[dfOut['Date'] > datetime.today().strftime('%Y-%m-%d')]
# datetime.today().strftime('%Y-%m-%d')
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 -# dfOut[dfOut['Date'] == dfOut['Date'].max()]
-
-
-
-
-
-
-
-
-
- Process AI
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 - 2 - 3 - 4 - 5 - 6 - 7 - 8 - 9 - 10 - 11 - 12 - 13 - 14 - 15 - 16 - 17 - 18 - 19 - 20 - 21 - 22 - 23 - 24 - 25 - 26 - 27 - 28 - 29 - 30 - 31 - 32 - 33 - 34 - 35 - 36 - 37 - 38 - 39 - 40 - 41 - 42 - 43 - 44 - 45 - 46 - 47 - 48 - 49 - 50 - 51 - 52 - 53 - 54 - 55 - 56 - 57 - 58 - 59 - 60 - 61 - 62 - 63 - 64 - 65 - 66 - 67 - 68 - 69 - 70 - 71 - 72 - 73 - 74 - 75 - 76 - 77 - 78 - 79 - 80 - 81 - 82 - 83 - 84 - 85 - 86 - 87 - 88 - 89 - 90 - 91 - 92 - 93 - 94 - 95 - 96 - 97 - 98 - 99 -100 -101 -102 -103 -104 -105 -106 -107 -108 -109 -110 -111 -112 -113 -114 -115 -116 -117 -118 -119 -120 -121 -122 -123 -124 -125 -126 -127 -128 -129 -130 -131 -132 -133 -134 -135 -136 -137 -138 -139 -140 -141 -142 -143 -144 -145 -146 -147 -148 -149 -150 -151 -152 -153 -154 -155 -156 -157 -158 -159 -160 -161 -162 -163 -164 -165 -166 -167 -168 -169 -170 -171 -172 -173 -174 -175 -176 -177 -178 -179 -180 -181 -182 -183 -184 -185 -###########################
# assume long data
###########################
def get_sim(inputData, histDays, backcastDays, numLags, forecastDays):
resultsOut = pd.DataFrame()
tmp = []
###########################
# Loop over backcast days
###########################
for day in range(backcastDays):
print(day)
###########################
# partition data by last X days
# and previous Y days
###########################
ref_date = datetime.now() - timedelta(days = (histDays + day))
print("Reference Date:" )
print(ref_date)
max_date = datetime.now() - timedelta(days = day)
print("Max Date:")
print(max_date)
###############################
#
# IMPORTANT!!
#
###############################
fcst_date = datetime.now() - timedelta(days = (day - 1))
## fcst_date = datetime.now() - timedelta(days = (day))
print("Forecast Date:")
print(fcst_date)
# max_date = pd.Timestamp('now').floor('D')
# + pd.Timedelta(-1, unit='D')
#################################
# Take last Z days
# loop over backCast days
#################################
tmp = inputData[(inputData['Date'] >= ref_date) & (inputData['Date'] < max_date)]
print("Observed Min Date:")
print(tmp['Date'].min())
print("Observed Max Date:")
print(tmp['Date'].max())
tmp = tmp.set_index(['Date'])
#################################
# Loop over Coins
#################################
for coin in inputData['Coin'].unique():
print(coin)
###########################
# ETL long data to wide
# add Log(Volume)
###########################
mytmp = tmp[tmp['Coin'] == coin]
# yf.download(tickers='DOGE-USD', period = '90d', interval = '1d')
## print(mytmp.head(), mytmp.tail())
myBTC = tmp[tmp['Coin'] == 'BTC']
myBTC.columns = ('BTC-' + myBTC.columns)
mytmp2 = mytmp.merge(myBTC, how = 'left', on='Date')
mytmp2['logVolume'] = np.log(mytmp2.loc[:, 'Volume'])
mytmp2['BTC-logVolume'] = np.log(mytmp2.loc[:, 'BTC-Volume'])
# print(mytmp2)
# yf.download(tickers='BTC-USD', period = '90d', interval = '1d')
# tmp1 = tmp[tmp['Coin'] == coin]
# tmp2 = tmp1.pivot_table(index=['Date', 'Coin'], columns=['Variable'], values='Value').reset_index()
# mytmp2 = mytmp2.set_index(['Date'])
# tmp2 = tmp2.sort_values(['Coin', 'Date'], ascending = True)
# tmp2['logVolume'] = np.log(tmp2.loc[:,'5. volume'])
###########################
# Create Daily High-Low Diff
###########################
mytmp2['dailyHighLow'] = mytmp2['High'] - mytmp2['Low']
# mytmp2_use = mytmp2[['Open', 'High', 'Low', 'Close', 'Adj Close', 'BTC-Open', 'BTC-High', 'BTC-Low', 'BTC-Close', 'BTC-Adj Close', 'logVolume', 'BTC-logVolume', 'dailyHighLow']]
mytmp2_use = mytmp2[['Open', 'High', 'Low', 'Close', 'logVolume', 'dailyHighLow', 'BTC-Close', 'BTC-logVolume']]
# print((mytmp2_use.shape))
if mytmp2_use.shape[0] > 9:
# tmp2_out = tmp2[['1b. open (USD)', '2b. high (USD)', '3b. low (USD)', '4b. close (USD)', 'logVolume', 'Coin']]
#########################
# Remove Closing Price. Lag1Close = Lag0Open
#########################
# tmp2_use = tmp2[['1b. open (USD)', '2b. high (USD)', '3b. low (USD)', 'logVolume', 'dailyHighLow']]
# tmp2_out = tmp2[['1b. open (USD)', '2b. high (USD)', '3b. low (USD)', 'logVolume', 'dailyHighLow', 'Coin']]
# print(tmp2_out.tail())
# print(tmp['Date'].max())
# print(mytmp2_use.head(), mytmp2_use.tail())
###################################
# compute changes
###################################
data_ret = (mytmp2_use).diff(1).dropna()
# data_ret = data_ret.set_index(['Date'])
# print(data_ret.head())
# print(data_ret.head(), data_ret.tail())
# print(data_ret.info(verbose=True))
###################################
# construct model
###################################
model = VAR(data_ret, freq='D')
# print(data_ret)
###################################
# Fit model using 8 lags
###################################
results = model.fit(numLags)
# print(results.summary())
###########################
# append forecast
###########################
mytmp2_use_fcst = pd.DataFrame(results.forecast(data_ret.values[-numLags:], forecastDays))
mytmp2_use_fcst.columns = mytmp2_use.columns
mytmp2_use_fcst['Coin'] = coin
mytmp2_use_fcst['Date'] = (fcst_date).strftime('%Y-%m-%d')
# print((datetime.datetime.now() + datetime.timedelta(day)).strftime('%Y-%m-%d'))
mytmp2_use_fcst = mytmp2_use_fcst.set_index(['Date'])
# tmp2_out = data_ret.append(tmp2_use_fcst, ignore_index = False)
# print(tmp2_use_fcst)
resultsOut = resultsOut.append(mytmp2_use_fcst, ignore_index = False)
# # print(resultsOut)
# resultsOut.columns = ['TS', 'P', 'Variable', 'Coin', 'prev30dayAvg', 'prior30dayAvg']
return resultsOut
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 -myfcasts[myfcasts['Coin'] == 'ADA'].sort_values(by=['Coin', 'Date']).tail()
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 -dfOut.set_index('Date', inplace=True)
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 -dfOut[dfOut['Coin'] == 'ADA'].tail()
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 -myfcasts[myfcasts['Coin'] == 'ADA'].head()
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 - 2 - 3 -# data_in_long['nextDate'] =
# myfcasts.reset_index(inplace=True)
# (myfcasts['Date'] - timedelta(days = (-1)))
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 -myfcasts.reset_index(inplace=True)
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 -myfcasts['Date'] = pd.to_datetime(myfcasts['Date'])
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 -myfcasts.head()
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 - 2 - 3 - 4 - 5 - 6 - 7 - 8 -##############################
# IMPORTANT!!
# reset Forecast Date to Today
##############################
myfcasts['priorDate'] = (myfcasts['Date'] - timedelta(days = (1)))
## myfcasts['priorDate'] = (myfcasts['Date'] - timedelta(days = (0)))
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 -myfcasts.head()
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 - 2 -myfcasts.rename(columns={"Date": "Forecast Date", "priorDate": "Date"}, inplace=True)
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 - 2 - 3 - 4 - 5 - 6 -##############################
# the fcasts data output
##############################
myfcasts.head()
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 - 2 - 3 - 4 - 5 - 6 -##############################
# go back to original data.
##############################
dfOut.tail()
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 - 2 -dfOut.dtypes
-
-
-
-
- Open float64 -High float64 -Low float64 -Close float64 -Adj Close float64 -Volume int64 -Coin object -dtype: object
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 - 2 -dfOut.info(verbose=True)
-
-
-
-
- <class 'pandas.core.frame.DataFrame'> -DatetimeIndex: 10577 entries, 2021-05-09 to 2021-11-04 -Data columns (total 7 columns): - # Column Non-Null Count Dtype ---- ------ -------------- ----- - 0 Open 10577 non-null float64 - 1 High 10577 non-null float64 - 2 Low 10577 non-null float64 - 3 Close 10577 non-null float64 - 4 Adj Close 10577 non-null float64 - 5 Volume 10577 non-null int64 - 6 Coin 10577 non-null object -dtypes: float64(5), int64(1), object(1) -memory usage: 661.1+ KB -
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 - 2 -myfcasts.info(verbose=True)
-
-
-
-
- <class 'pandas.core.frame.DataFrame'> -DatetimeIndex: 1530 entries, 2021-11-04 to 2021-10-06 -Data columns (total 10 columns): - # Column Non-Null Count Dtype ---- ------ -------------- ----- - 0 Forecast Date 1530 non-null datetime64[ns] - 1 Open 1530 non-null float64 - 2 High 1530 non-null float64 - 3 Low 1530 non-null float64 - 4 Close 1530 non-null float64 - 5 logVolume 1530 non-null float64 - 6 dailyHighLow 1530 non-null float64 - 7 BTC-Close 1530 non-null float64 - 8 BTC-logVolume 1530 non-null float64 - 9 Coin 1530 non-null object -dtypes: datetime64[ns](1), float64(8), object(1) -memory usage: 131.5+ KB -
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 -# dfOut['Coin']
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 -dfOut.columns = ('Actual: ' + dfOut.columns)
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 -dfOut.rename(columns={"Actual: Coin": "Coin"}, inplace=True)
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 -dfOut[dfOut['Coin'] == 'ADA'].tail()
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 - 2 -final_output = myfcasts.merge(dfOut, how='left', on=('Date', 'Coin'), )
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 - 2 - 3 - 4 - 5 - 6 -###############################
# QA this mudder
###############################
final_output[final_output['Coin'] == 'ADA'].head(10)
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 - 2 -final_output.reset_index(inplace=True)
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 - 2 -final_output.info(verbose=True)
-
-
-
-
- <class 'pandas.core.frame.DataFrame'> -RangeIndex: 1530 entries, 0 to 1529 -Data columns (total 17 columns): - # Column Non-Null Count Dtype ---- ------ -------------- ----- - 0 Date 1530 non-null datetime64[ns] - 1 Forecast Date 1530 non-null datetime64[ns] - 2 Open 1530 non-null float64 - 3 High 1530 non-null float64 - 4 Low 1530 non-null float64 - 5 Close 1530 non-null float64 - 6 logVolume 1530 non-null float64 - 7 dailyHighLow 1530 non-null float64 - 8 BTC-Close 1530 non-null float64 - 9 BTC-logVolume 1530 non-null float64 - 10 Coin 1530 non-null object - 11 Actual: Open 1530 non-null float64 - 12 Actual: High 1530 non-null float64 - 13 Actual: Low 1530 non-null float64 - 14 Actual: Close 1530 non-null float64 - 15 Actual: Adj Close 1530 non-null float64 - 16 Actual: Volume 1530 non-null int64 -dtypes: datetime64[ns](2), float64(13), int64(1), object(1) -memory usage: 203.3+ KB -
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 - 2 - 3 - 4 - 5 - 6 -####################
# format column names
####################
final_output.columns = final_output.columns.str.replace('[#,@,&,:, ,-]', '_')
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 -final_output.columns
-
-
-
-
- Index(['Date', 'Forecast_Date', 'Open', 'High', 'Low', 'Close', 'logVolume', - 'dailyHighLow', 'BTC_Close', 'BTC_logVolume', 'Coin', 'Actual__Open', - 'Actual__High', 'Actual__Low', 'Actual__Close', 'Actual__Adj_Close', - 'Actual__Volume'], - dtype='object')
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 - 2 - 3 - 4 - 5 - 6 -##############################
# paste previous day price
##############################
dfOut[dfOut['Coin'] == 'ADA'].sort_values(by = 'Date', ascending = False).head(10)
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 -tmpdfOut = dfOut.reset_index()
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 - 2 -tmpdfOut_lag1.rename(columns={"priorDate": "Date", "Actual: Close": "Tomorrow_Close", }, inplace=True)
-
-
-
-
- /usr/local/lib/python3.7/dist-packages/pandas/core/frame.py:4308: SettingWithCopyWarning: -A value is trying to be set on a copy of a slice from a DataFrame - -See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy - errors=errors, -
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 -tmpdfOut_lag1[tmpdfOut_lag1['Coin'] == 'ADA'].tail()
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 -final_output[final_output['Coin'] == 'ADA'].head()
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 - 2 - 3 - 4 - 5 - 6 -##############################
# extract only necessary data
##############################
tmpfcast = final_output[['Coin', 'Date', 'Actual__Close', 'Forecast_Date', 'Close']]
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 -tmpfcast[tmpfcast['Coin'] == 'ADA'].head()
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 - 2 - 3 -
tmpfcast['Close Price'] = tmpfcast['Actual__Close'] + tmpfcast['Close']
-
-
-
-
- /usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:2: SettingWithCopyWarning: -A value is trying to be set on a copy of a slice from a DataFrame. -Try using .loc[row_indexer,col_indexer] = value instead - -See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy - -
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 - 2 - 3 - 4 - 5 - 6 - 7 -# WRONG
# tmpfcast['Miss'] = tmpfcast['Close Price'] - tmpfcast['Actual__Close']
# .sort_values(['Coin', 'Date'], ascending = False).head(30)
tmpfcast.sort_values(['Coin', 'Date'], ascending = False).head(10)
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 - 2 -tmpfcast[tmpfcast['Coin'] == 'ADA'].head()
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 - 2 - 3 - 4 - 5 - 6 -#######################
# compare to actuals #
#######################
tmpdfOut_lag1[tmpdfOut_lag1['Coin'] == 'ADA'].tail()
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 - 2 - 3 -#######################################################
#######################################################
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 -tmpfcast2[tmpfcast2['Coin'] == 'ADA'].tail()
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 - 2 -tmpfcast2.rename(columns={"Actual__Close": "Today_Price", "Close": "Forecast_PM", "Close Price": "Forecast_Price", "Tomorrow_Close": "Actual_Price"}, inplace=True)
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 -tmpfcast2.head()
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 - 2 -tmpfcast2['Actual_PM'] = tmpfcast2['Actual_Price'] - tmpfcast2['Today_Price']
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 - 2 - 3 -tmpfcast2 = tmpfcast2[['Coin', 'Date', 'Today_Price', 'Forecast_Date', 'Forecast_PM',
'Forecast_Price', 'Actual_PM', 'Actual_Price']]
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 -tmpfcast2[tmpfcast2['Coin'] == 'ADA'].tail()
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 - 2 - 3 -tmpfcast2['Miss'] = tmpfcast2['Forecast_Price'] - tmpfcast2['Actual_Price']
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 - 2 -tmpfcast2['absMiss'] = abs(tmpfcast2['Miss'])
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 -tmpfcast2[tmpfcast2['Coin'] == 'ADA'].tail()
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 - 2 - 3 - 4 - 5 - 6 -###################
# calc avg abs miss
# calc avg sq miss
###################
(tmpfcast2.groupby('Coin')['absMiss'].mean()).sort_values(ascending = True)
-
-
-
-
- Coin -SHIB 0.000009 -MIR 0.000292 -USDC 0.000403 -USDT 0.000443 -ACH 0.007573 -IOTX 0.008994 -DNT 0.009602 -ANKR 0.010465 -CRO 0.013413 -XLM 0.014450 -DOGE 0.016835 -OXT 0.039436 -FET 0.040219 -NKN 0.040902 -CHZ 0.043270 -COTI 0.043946 -XRP 0.056233 -ZRX 0.077928 -CVC 0.082446 -BAT 0.086313 -LRC 0.100664 -MATIC 0.143131 -ENJ 0.158678 -STORJ 0.171876 -BNT 0.179655 -EOS 0.257915 -RLC 0.300727 -XTZ 0.408985 -BAND 0.419635 -SNX 0.511751 -UMA 0.697998 -OMG 0.966044 -MANA 1.007439 -BAL 1.491386 -REP 1.599674 -LINK 1.618269 -ETC 2.358549 -AVAX 3.691565 -ZEN 4.584268 -FIL 5.512985 -MLN 7.905663 -LTC 8.084823 -ZEC 9.136212 -DASH 11.012556 -QNT 18.729430 -BCH 21.631537 -COMP 22.646309 -MKR 113.445509 -ETH 174.683899 -BTC 1470.787377 -YFI 1740.134741 -Name: absMiss, dtype: float64
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 - 2 - 3 - 4 - 5 - 6 - 7 - 8 -####################
# write to DB
####################
table_id = 'my_dataset.actuals_forecasts'
tmpfcast2.to_gbq(table_id, project_id = 'crypto1-307915', if_exists = 'replace')
print(datetime.now())
-
-
-
-
- 1it [00:02, 2.89s/it]
2021-11-04 23:38:26.026163 -
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 -tmpfcast2.head()
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 - 2 - 3 - 4 - 5 - 6 - 7 - 8 -###############################################
# add section to create multi model output
###############################################
# final_output.dtypes
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 -
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 -# final_output['priorDate'] = (final_output['Date'] - timedelta(days = (1)))
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 -# final_output
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 -# final_output.dtypes
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 - 2 - 3 - 4 -
# final_output[['Coin', 'priorDate', 'Date', 'Actual__Close', 'Forecast_Date', 'Close']].sort_values(['Coin', 'Date'], ascending = False).head(30)
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 -
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 -
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 -
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 - 2 - 3 - 4 - 5 - 6 - 7 - 8 - 9 - 10 -######################
# pull a single ticker
######################
# mytmp = yf.download(tickers='DOGE-USD', period = '48h', interval = '5m')
mytmp = yf.download(tickers='ADA-USD', period = '90d', interval = '1d', tz='CST')
# myBTC = yf.download(tickers='BTC-USD', period = '90d', interval = '1d')
-
-
-
-
- [*********************100%***********************] 1 of 1 completed -
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 -# mytmp = mytmp[mytmp['Volume'] != 0].head()
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 -# mytmp.shape
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 -mytmp.tail()
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 -# myBTC.tail()
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 -# myBTC.columns = ('BTC-' + myBTC.columns)
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 -# myBTC.head()
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 -# mytmp2 = mytmp.merge(myBTC, how = 'left', on='Date')
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 -# mytmp2.head()
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 - 2 -# mytmp2['BTC-Volume'].plot()
# plt.grid()
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 - 2 - 3 -# mytmp2['logVolume'] = np.log(mytmp2.loc[:, 'Volume'])
# mytmp2['BTC-logVolume'] = np.log(mytmp2.loc[:, 'BTC-Volume'])
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 -# mytmp2.head()
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 - 2 -# mytmp2_use = mytmp2[['Open', 'High', 'Low', 'Adj Close', 'logVolume', 'BTC-Open', 'BTC-High', 'BTC-Low', 'BTC-Adj Close', 'BTC-logVolume']]
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 - 2 -# data_ret = (mytmp2_use).diff(1).dropna()
# data_ret = (mytmp).diff(1).dropna()
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 -# data_ret.head()
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 -# data_ret.shape
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 -#data_ret.reset_index(inplace=True)
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 -#data_ret.head()
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 -# data_ret.dtypes
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 -#data_ret.set_index(['Date'], inplace=True)
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 -# data_ret['Adj Close'].plot()
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 - 2 - 3 - 4 - 5 - 6 - 7 - 8 -# model = VAR(data_ret)
###################################
# Fit model using 8 lags
###################################
# results = model.fit(10)
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 -# results.params
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 - 2 - 3 - 4 - 5 - 6 -##################################
# create 1 day head forecast
##################################
# pd.DataFrame(results.forecast(data_ret.values[-10:],1))
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 - 2 - 3 -# tmp.pivot_table(index=['Variable', 'Date'],
# columns='Coin',
# values='Value').reset_index().round(3)
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 -# mytmp[mytmp['Volume'] != 0]
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 -# dfOut
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 -# dfOut2 = pd.concat(dfOut)
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 -# dfOut.shape
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 -# coinbase_coins.iloc[1,0]
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 - 2 - 3 -yf.download(tickers=(str(coinbase_coins.iloc[1,0]) + '-' + str('USD')), period = '2h', interval = '15m', tz = 'CST').tail(5)
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 -# yf.download(tickers=(str('ETC') + '-' + str('USD')), period = '90d', interval = '1d')
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 -
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 -
-
-
-
-
-
-
-
-
-
-
-
-
-
-
[ ]
-
-
-
- 1 -
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-