added more feats

app.py

@@ -162,6 +162,8 @@ with st.form("choose_model"):
             new_pred['OHLC4_VIX'] = new_pred['OHLC4_VIX'].astype(float)
             new_pred['OHLC4_VIX_n1'] = new_pred['OHLC4_VIX_n1'].astype(float)
             new_pred['OHLC4_VIX_n2'] = new_pred['OHLC4_VIX_n2'].astype(float)
+            new_pred['VIXOpen'] = new_pred['VIXOpen'].astype(bool)
+            new_pred['VVIXOpen'] = new_pred['VVIXOpen'].astype(bool)
             # new_pred['OHLC4_Current_Trend'] = new_pred['OHLC4_Current_Trend'].astype(bool)
             # new_pred['OHLC4_Trend'] = new_pred['OHLC4_Trend'].astype(bool)
             new_pred['OpenL1'] = new_pred['OpenL1'].astype(float)
@@ -194,7 +196,7 @@ with st.form("choose_model"):
 
             my_bar.progress(0.66, "Training models...")
             def train_models():
-                res1, xgbr = walk_forward_validation(df_final.dropna(), 'Target_clf', 120, 1)
+                res1, xgbr = walk_forward_validation(df_final.dropna(), 'Target_clf', 1)
                 return res1, xgbr
             res1, xgbr = train_models()
             # st.success("✅ Models trained")
@@ -230,6 +232,10 @@ with st.form("choose_model"):
             new_pred['OHLC4_VIX'] = new_pred['OHLC4_VIX'].astype(float)
             new_pred['OHLC4_VIX_n1'] = new_pred['OHLC4_VIX_n1'].astype(float)
             new_pred['OHLC4_VIX_n2'] = new_pred['OHLC4_VIX_n2'].astype(float)
+            new_pred['CurrentVIXTrend'] = new_pred['CurrentVIXTrend'].astype(bool)
+            new_pred['SPX30IntraPerf'] = new_pred['SPX30IntraPerf'].astype(float)
+            new_pred['VIX30IntraPerf'] = new_pred['VIX30IntraPerf'].astype(float)
+            new_pred['VVIX30IntraPerf'] = new_pred['VVIX30IntraPerf'].astype(float)
             # new_pred['OpenL1'] = new_pred['OpenL1'].astype(float)
             # new_pred['OpenL2'] = new_pred['OpenL2'].astype(float)
             # new_pred['OpenH1'] = new_pred['OpenH1'].astype(float)

model_day.py

@@ -62,6 +62,8 @@ model_cols = [
     'OHLC4_VIX',
     'OHLC4_VIX_n1',
     'OHLC4_VIX_n2',
+    'VIXOpen',
+    'VVIXOpen',
     'OpenL1',
     'OpenL2',
     'OpenH1',
@@ -251,10 +253,13 @@ def get_data():
         # releases[rid]['df'] = releases[rid]['df'].set_index('ds')
 
     vix = yf.Ticker('^VIX')
+    vvix = yf.Ticker('^VVIX')
     spx = yf.Ticker('^GSPC')
 
     prices_vix = vix.history(start='2018-07-01', interval='1d')
     prices_spx = spx.history(start='2018-07-01', interval='1d')
+    prices_vvix = vvix.history(start='2018-07-01', interval='1d')
+
     prices_spx['index'] = [str(x).split()[0] for x in prices_spx.index]
     prices_spx['index'] = pd.to_datetime(prices_spx['index']).dt.date
     prices_spx.index = prices_spx['index']
@@ -265,46 +270,44 @@ def get_data():
     prices_vix.index = prices_vix['index']
     prices_vix = prices_vix.drop(columns='index')
 
+    prices_vvix['index'] = [str(x).split()[0] for x in prices_vvix.index]
+    prices_vvix['index'] = pd.to_datetime(prices_vvix['index']).dt.date
+    prices_vvix.index = prices_vvix['index']
+    prices_vvix = prices_vvix.drop(columns='index')
+
     data = prices_spx.merge(prices_vix[['Open','High','Low','Close']], left_index=True, right_index=True, suffixes=['','_VIX'])
+    data = data.merge(prices_vvix[['Open','High','Low','Close']], left_index=True, right_index=True, suffixes=['','_VVIX'])
     data.index = pd.DatetimeIndex(data.index)
 
     # Features
     data['PrevClose'] = data['Close'].shift(1)
     data['Perf5Day'] = data['Close'] > data['Close'].shift(5)
-    data['Perf5Day_n1'] = data['Perf5Day'].shift(1)
-    data['Perf5Day_n1'] = data['Perf5Day_n1'].astype(bool)
+    data['Perf5Day_n1'] = data['Perf5Day'].shift(1).astype(bool)
     data['GreenDay'] = (data['Close'] > data['PrevClose']) * 1
     data['RedDay'] = (data['Close'] <= data['PrevClose']) * 1
-
     data['VIX5Day'] = data['Close_VIX'] > data['Close_VIX'].shift(5)
-    data['VIX5Day_n1'] = data['VIX5Day'].astype(bool)
-
-    data['Range'] = data[['Open','High']].max(axis=1) - data[['Low','Open']].min(axis=1) # Current day range in points
+    data['VIX5Day_n1'] = data['VIX5Day'].shift(1).astype(bool)
+    data['VIXOpen'] = data['Open_VIX'] > data['Close_VIX'].shift(1)
+    data['VVIXOpen'] = data['Open_VVIX'] > data['Close_VVIX'].shift(1)
+    data['VIXOpen'] = data['VIXOpen'].astype(bool)
+    data['VVIXOpen'] = data['VVIXOpen'].astype(bool)
+    data['Range'] = data[['Open','High']].max(axis=1) - data[['Low','Open']].min(axis=1)
     data['RangePct'] = data['Range'] / data['Close']
     data['VIXLevel'] = pd.qcut(data['Close_VIX'], 4)
     data['OHLC4_VIX'] = data[['Open_VIX','High_VIX','Low_VIX','Close_VIX']].mean(axis=1)
     data['OHLC4'] = data[['Open','High','Low','Close']].mean(axis=1)
     data['OHLC4_Trend'] = data['OHLC4'] > data['OHLC4'].shift(1)
-    data['OHLC4_Trend_n1'] = data['OHLC4_Trend'].shift(1)
-    data['OHLC4_Trend_n1'] = data['OHLC4_Trend_n1'].astype(float)
-    data['OHLC4_Trend_n2'] = data['OHLC4_Trend'].shift(1)
-    data['OHLC4_Trend_n2'] = data['OHLC4_Trend_n2'].astype(float)
+    data['OHLC4_Trend_n1'] = data['OHLC4_Trend'].shift(1).astype(float)
+    data['OHLC4_Trend_n2'] = data['OHLC4_Trend'].shift(2).astype(float)
     data['RangePct_n1'] = data['RangePct'].shift(1)
     data['RangePct_n2'] = data['RangePct'].shift(2)
     data['OHLC4_VIX_n1'] = data['OHLC4_VIX'].shift(1)
     data['OHLC4_VIX_n2'] = data['OHLC4_VIX'].shift(2)
-    data['CurrentGap'] = (data['Open'] - data['PrevClose']) / data['PrevClose']
-    data['CurrentGap'] = data['CurrentGap'].shift(-1)
+    data['CurrentGap'] = ((data['Open'] - data['PrevClose']) / data['PrevClose']).shift(-1)
     data['DayOfWeek'] = pd.to_datetime(data.index)
     data['DayOfWeek'] = data['DayOfWeek'].dt.day
-
-    # Calculate up
     data['up'] = 100 * (data['High'].shift(1) - data['Open'].shift(1)) / data['Close'].shift(1)
-
-    # Calculate upSD
     data['upSD'] = data['up'].rolling(30).std(ddof=0)
-
-    # Calculate aveUp
     data['aveUp'] = data['up'].rolling(30).mean()
     data['H1'] = data['Open'] + (data['aveUp'] / 100) * data['Open']
     data['H2'] = data['Open'] + ((data['aveUp'] + data['upSD']) / 100) * data['Open']
@@ -313,21 +316,18 @@ def get_data():
     data['aveDown'] = data['down'].rolling(30).mean()
     data['L1'] = data['Open'] - (data['aveDown'] / 100) * data['Open']
     data['L2'] = data['Open'] - ((data['aveDown'] + data['upSD']) / 100) * data['Open']
-
-    data = data.assign(
-        L1Touch = lambda x: x['Low'] < x['L1'],
-        L2Touch = lambda x: x['Low'] < x['L2'],
-        H1Touch = lambda x: x['High'] > x['H1'],
-        H2Touch = lambda x: x['High'] > x['H2'],
-        L1Break = lambda x: x['Close'] < x['L1'],
-        L2Break = lambda x: x['Close'] < x['L2'],
-        H1Break = lambda x: x['Close'] > x['H1'],
-        H2Break = lambda x: x['Close'] > x['H2'],
-        OpenL1 = lambda x: x['Open'] / x['L1'],
-        OpenL2 = lambda x: x['Open'] / x['L2'],
-        OpenH1 = lambda x: x['Open'] / x['H1'],
-        OpenH2 = lambda x: x['Open'] / x['H2']
-    )
+    data['L1Touch'] = data['Low'] < data['L1']
+    data['L2Touch'] = data['Low'] < data['L2']
+    data['H1Touch'] = data['High'] > data['H1']
+    data['H2Touch'] = data['High'] > data['H2']
+    data['L1Break'] = data['Close'] < data['L1']
+    data['L2Break'] = data['Close'] < data['L2']
+    data['H1Break'] = data['Close'] > data['H1']
+    data['H2Break'] = data['Close'] > data['H2']
+    data['OpenL1'] = data['Open'] / data['L1']
+    data['OpenL2'] = data['Open'] / data['L2']
+    data['OpenH1'] = data['Open'] / data['H1']
+    data['OpenH2'] = data['Open'] / data['H2']
 
     level_cols = [
         'L1Touch',
@@ -409,6 +409,8 @@ def get_data():
         'OHLC4_VIX',
         'OHLC4_VIX_n1',
         'OHLC4_VIX_n2',
+        'VIXOpen',
+        'VVIXOpen',
         'OpenL1',
         'OpenL2',
         'OpenH1',

model_intra.py

@@ -13,6 +13,7 @@ from pandas.tseries.offsets import BDay
 from datasets import load_dataset
 import lightgbm as lgb
 from sklearn.model_selection import TimeSeriesSplit
+import json
 
 data_start_date = '2018-07-01'
 
@@ -37,6 +38,10 @@ model_cols = [
     'OHLC4_VIX_n2',
     'OHLC4_Current_Trend',
     'OHLC4_Trend',
+    'CurrentVIXTrend',
+    'SPX30IntraPerf',
+    'VIX30IntraPerf',
+    'VVIX30IntraPerf',
     # 'OpenL1',
     # 'OpenL2',
     # 'OpenH1',
@@ -62,7 +67,7 @@ model_cols = [
 ]
 
 # If the dataset is gated/private, make sure you have run huggingface-cli login
-def walk_forward_validation(df, target_column, num_training_rows, num_periods):
+def walk_forward_validation(df, target_column, num_periods):
     
     df = df[model_cols + [target_column]]
     df[target_column] = df[target_column].astype(bool)
@@ -81,13 +86,13 @@ def walk_forward_validation(df, target_column, num_training_rows, num_periods):
         y_train = df[target_column].iloc[train_index]
         X_test = df.drop(target_column, axis=1).iloc[test_index]
         y_test = df[target_column].iloc[test_index]
-
-        # Fit the model to the training data
+    
+        y_train = y_train.astype(bool)
         model = lgb.LGBMClassifier(n_estimators=10, random_state=42, verbosity=-1)
         model.fit(X_train, y_train)
         # Make a prediction on the test data
         predictions = model.predict_proba(X_test)[:,-1]
-        
+            
         # Create a DataFrame to store the true and predicted values
         result_df = pd.DataFrame({'True': y_test, 'Predicted': predictions}, index=y_test.index)
         overall_results.append(result_df)
@@ -189,62 +194,102 @@ def get_data(periods_30m = 1):
         releases[rid]['df'].index = pd.DatetimeIndex(releases[rid]['df'].index)
 
     vix = yf.Ticker('^VIX')
+    vvix = yf.Ticker('^VVIX')
     spx = yf.Ticker('^GSPC')
 
     # Pull in data
-    data = load_dataset("boomsss/spx_intra", split='train')
-
-    rows = [d['text'] for d in data]
-    rows = [x.split(',') for x in rows]
-
-    fr = pd.DataFrame(columns=[
-        'Datetime','Open','High','Low','Close'
-    ], data = rows)
-
-    fr['Datetime'] = pd.to_datetime(fr['Datetime'])
-    fr['Datetime'] = fr['Datetime'].dt.tz_localize('America/New_York')
-    fr = fr.set_index('Datetime')
-    fr['Open'] = pd.to_numeric(fr['Open'])
-    fr['High'] = pd.to_numeric(fr['High'])
-    fr['Low'] = pd.to_numeric(fr['Low'])
-    fr['Close'] = pd.to_numeric(fr['Close'])
-    
+    data_files = {"spx": "SPX_full_30min.txt", "vix": "VIX_full_30min.txt", "vvix":'VVIX_full_30min.txt'}
+    data = load_dataset("boomsss/spx_intra", data_files=data_files)
+    dfs = []
+    for ticker in data.keys():
+        rows = [d['text'] for d in data[ticker]]
+        rows = [x.split(',') for x in rows]
+
+        fr = pd.DataFrame(columns=[
+            'Datetime','Open','High','Low','Close'
+        ], data = rows)
+
+        fr['Datetime'] = pd.to_datetime(fr['Datetime'])
+        fr['Datetime'] = fr['Datetime'].dt.tz_localize('America/New_York')
+        fr = fr.set_index('Datetime')
+        fr['Open'] = pd.to_numeric(fr['Open'])
+        fr['High'] = pd.to_numeric(fr['High'])
+        fr['Low'] = pd.to_numeric(fr['Low'])
+        fr['Close'] = pd.to_numeric(fr['Close'])
+        dfs.append(fr)
+
+    df_30m = pd.concat(dfs, axis=1)
+
+    df_30m.columns = [
+        'Open30',
+        'High30',
+        'Low30',
+        'Close30',
+        'Open_VIX30',
+        'High_VIX30',
+        'Low_VIX30',
+        'Close_VIX30',
+        'Open_VVIX30',
+        'High_VVIX30',
+        'Low_VVIX30',
+        'Close_VVIX30'
+    ]
+
     # Get incremental date
-    last_date = fr.index.date[-1]
+    last_date = df_30m.index.date[-1]
     last_date = last_date + datetime.timedelta(days=1)
-    # Get incremental data
+
+    # Get incremental data for each index
     spx1 = yf.Ticker('^GSPC')
+    vix1 = yf.Ticker('^VIX')
+    vvix1 = yf.Ticker('^VVIX')    
     yfp = spx1.history(start=last_date, interval='30m')
+    yf_vix = vix1.history(start=last_date, interval='30m')
+    yf_vvix = vvix1.history(start=last_date, interval='30m')
 
     if len(yfp) > 0:
-        # Concat current and incremental
-        df_30m = pd.concat([fr, yfp])
+        # Convert indexes to EST if not already
+        for _df in [yfp, yf_vix, yf_vvix]:
+            if _df.index.tz.zone != 'America/New_York':
+                _df['Datetime'] = pd.to_datetime(_df.index)
+                _df['Datetime'] = _df['Datetime'].dt.tz_convert('America/New_York')
+                _df.set_index('Datetime', inplace=True)
+        # Concat them 
+        df_inc = pd.concat([yfp, yf_vix, yf_vvix], axis=1)
+        df_inc = df_inc.loc[
+            (df_inc.index.time >= datetime.time(9,30)) & (df_inc.index.time < datetime.time(16,00))
+        ]
+        df_30m = pd.concat([df_30m, df_inc])
     else:
-        df_30m = fr.copy()
-
-    # Get the first 30 minute bar
-    df_30m = df_30m.reset_index()
-    df_30m['Datetime'] = df_30m['Datetime'].dt.date
-    df_30m = df_30m.groupby('Datetime').head(periods_30m)
-    df_30m = df_30m.set_index('Datetime',drop=True)
-    # Rename the columns
-    df_30m = df_30m[['Open','High','Low','Close']]
-
-    opens_intra = df_30m.groupby('Datetime')['Open'].head(1)
-    highs_intra = df_30m.groupby('Datetime')['High'].max()
-    lows_intra = df_30m.groupby('Datetime')['Low'].min()
-    closes_intra = df_30m.groupby('Datetime')['Close'].tail(1)
-    
-    df_intra = pd.DataFrame(index=df_30m.index.unique())
-    df_intra['Open'] = opens_intra
-    df_intra['High'] = highs_intra
-    df_intra['Low'] = lows_intra
-    df_intra['Close'] = closes_intra
+        df_30m = df_30m.copy()
+
+    df_30m = df_30m.loc[
+                (df_30m.index.time >= datetime.time(9,30)) & (df_30m.index.time < datetime.time(16,00))
+            ]
+    df_30m['dt'] = df_30m.index.date
+    df_30m = df_30m.groupby('dt').head(periods_30m)
+    df_30m = df_30m.set_index('dt',drop=True)
+    df_30m.index.name = 'Datetime'
+
+    df_30m['SPX30IntraPerf'] = (df_30m['Close30'] / df_30m['Close30'].shift(1)) - 1
+    df_30m['VIX30IntraPerf'] = (df_30m['Close_VIX30'] / df_30m['Close_VIX30'].shift(1)) - 1
+    df_30m['VVIX30IntraPerf'] = (df_30m['Close_VVIX30'] / df_30m['Close_VVIX30'].shift(1)) - 1
+
+    opens_intra = df_30m.groupby('Datetime')[[c for c in df_30m.columns if 'Open' in c]].head(1)
+    highs_intra = df_30m.groupby('Datetime')[[c for c in df_30m.columns if 'High' in c]].max()
+    lows_intra = df_30m.groupby('Datetime')[[c for c in df_30m.columns if 'Low' in c]].min()
+    closes_intra = df_30m.groupby('Datetime')[[c for c in df_30m.columns if 'Close' in c]].tail(1)
+    spx_intra = df_30m.groupby('Datetime')['SPX30IntraPerf'].tail(1)
+    vix_intra = df_30m.groupby('Datetime')['VIX30IntraPerf'].tail(1)
+    vvix_intra = df_30m.groupby('Datetime')['VVIX30IntraPerf'].tail(1)
+
+    df_intra = pd.concat([opens_intra, highs_intra, lows_intra, closes_intra, spx_intra, vix_intra, vvix_intra], axis=1)
 
-    df_intra.columns = ['Open30','High30','Low30','Close30']
 
     prices_vix = vix.history(start=data_start_date, interval='1d')
+    prices_vvix = vvix.history(start=data_start_date, interval='1d')
     prices_spx = spx.history(start=data_start_date, interval='1d')
+
     prices_spx['index'] = [str(x).split()[0] for x in prices_spx.index]
     prices_spx['index'] = pd.to_datetime(prices_spx['index']).dt.date
     prices_spx.index = prices_spx['index']
@@ -257,8 +302,15 @@ def get_data(periods_30m = 1):
     prices_vix = prices_vix.drop(columns='index')
     prices_vix.index = pd.DatetimeIndex(prices_vix.index)
 
+    prices_vvix['index'] = [str(x).split()[0] for x in prices_vvix.index]
+    prices_vvix['index'] = pd.to_datetime(prices_vvix['index']).dt.date
+    prices_vvix.index = prices_vvix['index']
+    prices_vvix = prices_vvix.drop(columns='index')
+    prices_vvix.index = pd.DatetimeIndex(prices_vvix.index)
+
     data = prices_spx.merge(df_intra, left_index=True, right_index=True)
     data = data.merge(prices_vix[['Open','High','Low','Close']], left_index=True, right_index=True, suffixes=['','_VIX'])
+    data = data.merge(prices_vvix[['Open','High','Low','Close']], left_index=True, right_index=True, suffixes=['','_VVIX'])
 
     # Features
     data['PrevClose'] = data['Close'].shift(1)
@@ -271,6 +323,9 @@ def get_data(periods_30m = 1):
     data['VIX5Day'] = data['Close_VIX'] > data['Close_VIX'].shift(5)
     data['VIX5Day_n1'] = data['VIX5Day'].astype(bool)
 
+    data['VVIX5Day'] = data['Close_VVIX'] > data['Close_VVIX'].shift(5)
+    data['VVIX5Day_n1'] = data['VVIX5Day'].astype(bool)
+
     data['Range'] = data[['Open','High']].max(axis=1) - data[['Low','Open']].min(axis=1) # Current day range in points
     data['RangePct'] = data['Range'] / data['Close']
     data['VIXLevel'] = pd.qcut(data['Close_VIX'], 4)
@@ -302,6 +357,10 @@ def get_data(periods_30m = 1):
     data['OHLC4_Current_Trend'] = data['OHLC4_Current_Trend'].astype(bool)
     data['HistClose30toPrevClose'] = (data['Close30'] / data['PrevClose']) - 1
 
+    data['CurrentCloseVIX30'] = data['Close_VIX30'].shift(-1)
+    data['CurrentOpenVIX30'] = data['Open_VIX30'].shift(-1)
+
+    data['CurrentVIXTrend'] = data['CurrentCloseVIX30'] > data['Close_VIX']
 
     # Open to High
     data['CurrentHigh30toClose'] = (data['CurrentHigh30'] / data['Close']) - 1
@@ -399,8 +458,10 @@ def get_data(periods_30m = 1):
         return df.groupby(pd.qcut(df[col_name], q))['GreenDay'].mean()
 
     probas = []
+    # Given the current price level
     for i, pct in enumerate(data['CurrentClose30toClose']):
         try:
+            # Split
             df_q = get_quintiles(data.iloc[:i], 'HistClose30toPrevClose', 10)
             for q in df_q.index:
                 if q.left <= pct <= q.right: