tgs part 4

app.py

@@ -164,9 +164,9 @@ with st.form("choose_model"):
 
             with st.spinner("Training models..."):
                 def train_models():
-                    res1, xgbr, seq2 = walk_forward_validation_seq(df_final.dropna(), 'Target_clf', 'Target', 100, 1)
-                    return res1, xgbr, seq2
-                res1, xgbr, seq2 = train_models()
+                    res1, xgbr = walk_forward_validation(df_final.dropna(), 'Target_clf', 100, 1)
+                    return res1, xgbr
+                res1, xgbr = train_models()
             # st.success("✅ Models trained")
 
             with st.spinner("Getting new prediction..."):
@@ -212,7 +212,7 @@ with st.form("choose_model"):
                 new_pred['H2BreakPct'] = new_pred['H2BreakPct'].astype(float)
                 new_pred['GreenProbas'] = new_pred['GreenProbas'].astype(float)
 
-                seq_proba = seq_predict_proba(new_pred, xgbr, seq2)
+                seq_proba = seq_predict_proba(new_pred, xgbr)
 
         st.info(f'as of {option} on {curr_date} 👇🏽', icon="🔮")
 

model_intra.py

@@ -3,53 +3,16 @@ import pandas as pd
 import pandas_datareader as pdr
 import numpy as np
 import yfinance as yf
-import json
 import requests
 from bs4 import BeautifulSoup
 from typing import List
-import xgboost as xgb
 from tqdm import tqdm
-from sklearn import linear_model
-import joblib
 import os
-from sklearn.metrics import roc_auc_score, precision_score, recall_score
 import datetime
 from pandas.tseries.offsets import BDay
 from datasets import load_dataset
 import lightgbm as lgb
 
-# If the dataset is gated/private, make sure you have run huggingface-cli login
-def walk_forward_validation(df, target_column, num_training_rows, num_periods):
-    
-    # Create an XGBRegressor model
-    # model = xgb.XGBRegressor(n_estimators=100, objective='reg:squarederror', random_state = 42)
-    model = linear_model.LinearRegression()
-
-    overall_results = []
-    # Iterate over the rows in the DataFrame, one step at a time
-    for i in tqdm(range(num_training_rows, df.shape[0] - num_periods + 1),desc='LR Model'):
-        # Split the data into training and test sets
-        X_train = df.drop(target_column, axis=1).iloc[:i]
-        y_train = df[target_column].iloc[:i]
-        X_test = df.drop(target_column, axis=1).iloc[i:i+num_periods]
-        y_test = df[target_column].iloc[i:i+num_periods]
-        
-        # Fit the model to the training data
-        model.fit(X_train, y_train)
-        
-        # Make a prediction on the test data
-        predictions = model.predict(X_test)
-        
-        # Create a DataFrame to store the true and predicted values
-        result_df = pd.DataFrame({'True': y_test, 'Predicted': predictions}, index=y_test.index)
-        
-        overall_results.append(result_df)
-
-    df_results = pd.concat(overall_results)
-    # model.save_model('model_lr.bin')
-    # Return the true and predicted values, and fitted model
-    return df_results, model
-
 model_cols = [
     'BigNewsDay',
     'Quarter',
@@ -85,46 +48,32 @@ model_cols = [
     # 'GapFillGreenProba'
 ]
 
-def walk_forward_validation_seq(df, target_column_clf, target_column_regr, num_training_rows, num_periods):
-
-    # Create run the regression model to get its target
-    res, model1 = walk_forward_validation(df.drop(columns=[target_column_clf]).dropna(), target_column_regr, num_training_rows, num_periods)
-    # joblib.dump(model1, 'model1.bin')
-
-    # Merge the result df back on the df for feeding into the classifier
-    for_merge = res[['Predicted']]
-    for_merge.columns = ['RegrModelOut']
-    for_merge['RegrModelOut'] = for_merge['RegrModelOut'] > 0
-    df = df.merge(for_merge, left_index=True, right_index=True)
-    df = df.drop(columns=[target_column_regr])
-    df = df[model_cols + ['RegrModelOut', target_column_clf]]
+# If the dataset is gated/private, make sure you have run huggingface-cli login
+def walk_forward_validation(df, target_column, num_training_rows, num_periods):
     
-    df[target_column_clf] = df[target_column_clf].astype(bool)
-    df['RegrModelOut'] = df['RegrModelOut'].astype(bool)
+    df = df[model_cols + [target_column]]
+    df[target_column] = df[target_column].astype(bool)
+
+    # Model
+    model = lgb.LGBMClassifier(n_estimators=10, random_state=42, verbosity=-1)
 
-    # Create an XGBRegressor model
-    # model2 = xgb.XGBClassifier(n_estimators=10, random_state = 42)
-    model2 = lgb.LGBMClassifier(n_estimators=10, random_state=42, verbosity=-1)
-    # model = linear_model.LogisticRegression(max_iter=1500)
-    
     overall_results = []
     # Iterate over the rows in the DataFrame, one step at a time
-    for i in tqdm(range(num_training_rows, df.shape[0] - num_periods + 1),'CLF Model'):
+    for i in tqdm(range(num_training_rows, df.shape[0] - num_periods + 1),desc='LGB Model'):
         # Split the data into training and test sets
-        X_train = df.drop(target_column_clf, axis=1).iloc[:i]
-        y_train = df[target_column_clf].iloc[:i]
-        X_test = df.drop(target_column_clf, axis=1).iloc[i:i+num_periods]
-        y_test = df[target_column_clf].iloc[i:i+num_periods]
+        X_train = df.drop(target_column, axis=1).iloc[:i]
+        y_train = df[target_column].iloc[:i]
+        X_test = df.drop(target_column, axis=1).iloc[i:i+num_periods]
+        y_test = df[target_column].iloc[i:i+num_periods]
         
         # Fit the model to the training data
-        model2.fit(X_train, y_train)
+        model.fit(X_train, y_train)
         
         # Make a prediction on the test data
-        predictions = model2.predict_proba(X_test)[:,-1]
+        predictions = model.predict_proba(X_test)[:,-1]
         
         # Create a DataFrame to store the true and predicted values
         result_df = pd.DataFrame({'True': y_test, 'Predicted': predictions}, index=y_test.index)
-        
         overall_results.append(result_df)
 
     df_results = pd.concat(overall_results)
@@ -134,32 +83,23 @@ def walk_forward_validation_seq(df, target_column_clf, target_column_regr, num_t
         return df.groupby(pd.cut(df[col_name], q))['True'].mean()
 
     greenprobas = []
-    meanprobas = []
     for i, pct in tqdm(enumerate(df_results['Predicted']), desc='Calibrating Probas'):
         try:
             df_q = get_quantiles(df_results.iloc[:i], 'Predicted', 7)
             for q in df_q.index:
                 if q.left <= pct <= q.right:
                     p = df_q[q]
-                    c = (q.left + q.right) / 2
         except:
             p = None
-            c = None
 
         greenprobas.append(p)
-        meanprobas.append(c)
 
     df_results['CalibPredicted'] = greenprobas
 
-    return df_results, model1, model2
-
+    return df_results, model
 
-def seq_predict_proba(df, trained_reg_model, trained_clf_model):
-    regr_pred = trained_reg_model.predict(df)
-    regr_pred = regr_pred > 0
-    new_df = df.copy()
-    new_df['RegrModelOut'] = regr_pred
-    clf_pred_proba = trained_clf_model.predict_proba(new_df[model_cols + ['RegrModelOut']])[:,-1]
+def seq_predict_proba(df, trained_clf_model):
+    clf_pred_proba = trained_clf_model.predict_proba(df[model_cols])[:,-1]
     return clf_pred_proba
 
 def get_data(periods_30m = 1):
@@ -298,18 +238,18 @@ def get_data(periods_30m = 1):
     # Rename the columns
     df_30m = df_30m[['Open','High','Low','Close']]
 
-    opens_1h = df_30m.groupby('Datetime')['Open'].head(1)
-    highs_1h = df_30m.groupby('Datetime')['High'].max()
-    lows_1h = df_30m.groupby('Datetime')['Low'].min()
-    closes_1h = df_30m.groupby('Datetime')['Close'].tail(1)
+    opens_intra = df_30m.groupby('Datetime')['Open'].head(1)
+    highs_intra = df_30m.groupby('Datetime')['High'].max()
+    lows_intra = df_30m.groupby('Datetime')['Low'].min()
+    closes_intra = df_30m.groupby('Datetime')['Close'].tail(1)
     
-    df_1h = pd.DataFrame(index=df_30m.index.unique())
-    df_1h['Open'] = opens_1h
-    df_1h['High'] = highs_1h
-    df_1h['Low'] = lows_1h
-    df_1h['Close'] = closes_1h
+    df_intra = pd.DataFrame(index=df_30m.index.unique())
+    df_intra['Open'] = opens_intra
+    df_intra['High'] = highs_intra
+    df_intra['Low'] = lows_intra
+    df_intra['Close'] = closes_intra
 
-    df_1h.columns = ['Open30','High30','Low30','Close30']
+    df_intra.columns = ['Open30','High30','Low30','Close30']
 
     prices_vix = vix.history(start='2018-07-01', interval='1d')
     prices_spx = spx.history(start='2018-07-01', interval='1d')
@@ -327,7 +267,7 @@ def get_data(periods_30m = 1):
     prices_vix.index = pd.DatetimeIndex(prices_vix.index)
 
 
-    data = prices_spx.merge(df_1h, left_index=True, right_index=True)
+    data = prices_spx.merge(df_intra, left_index=True, right_index=True)
     data = data.merge(prices_vix[['Open','High','Low','Close']], left_index=True, right_index=True, suffixes=['','_VIX'])
 
     # Features
@@ -417,10 +357,10 @@ def get_data(periods_30m = 1):
         OpenL2 = lambda x: np.where(x['Open'] < x['L2'], 1, 0),
         OpenH1 = lambda x: np.where(x['Open'] > x['H1'], 1, 0),
         OpenH2 = lambda x: np.where(x['Open'] > x['H2'], 1, 0),
-        CloseL1 = lambda x: np.where(x['Close'] < x['L1'], 1, 0),
-        CloseL2 = lambda x: np.where(x['Close'] < x['L2'], 1, 0),
-        CloseH1 = lambda x: np.where(x['Close'] > x['H1'], 1, 0),
-        CloseH2 = lambda x: np.where(x['Close'] > x['H2'], 1, 0)
+        CloseL1 = lambda x: np.where(x['Close30'] < x['L1'], 1, 0),
+        CloseL2 = lambda x: np.where(x['Close30'] < x['L2'], 1, 0),
+        CloseH1 = lambda x: np.where(x['Close30'] > x['H1'], 1, 0),
+        CloseH2 = lambda x: np.where(x['Close30'] > x['H2'], 1, 0)
     )
 
     data['OpenL1'] = data['OpenL1'].shift(-1)
@@ -445,7 +385,7 @@ def get_data(periods_30m = 1):
 
     for col in level_cols:
         data[col+'Pct'] = data[col].rolling(100).mean()
-        data[col+'Pct'] = data[col+'Pct'].shift(-1)
+        # data[col+'Pct'] = data[col+'Pct'].shift(-1)
 
 
     def get_quintiles(df, col_name, q):