modify refs for intra models

model_1h.py

@@ -237,6 +237,13 @@ def get_data():
     # Get incremental data
     spx1 = yf.Ticker('^GSPC')
     yfp = spx1.history(start=last_date, interval='30m')
+
+    if len(yfp) > 0:
+        # Concat current and incremental
+        df_30m = pd.concat([fr, yfp])
+    else:
+        df_30m = fr.copy()
+        
     # Concat current and incremental
     df_30m = pd.concat([fr, yfp])
     # Get the first 30 minute bar

model_30m.py

@@ -226,9 +226,6 @@ def get_data():
     fr['High'] = pd.to_numeric(fr['High'])
     fr['Low'] = pd.to_numeric(fr['Low'])
     fr['Close'] = pd.to_numeric(fr['Close'])
-
-    # Set index for ready to concat
-
     
     # Get incremental date
     last_date = fr.index.date[-1]
@@ -236,8 +233,13 @@ def get_data():
     # Get incremental data
     spx1 = yf.Ticker('^GSPC')
     yfp = spx1.history(start=last_date, interval='30m')
-    # Concat current and incremental
-    df_30m = pd.concat([fr, yfp])
+    
+    if len(yfp) > 0:
+        # Concat current and incremental
+        df_30m = pd.concat([fr, yfp])
+    else:
+        df_30m = fr.copy()
+
     # Get the first 30 minute bar
     df_30m = df_30m.reset_index()
     df_30m['Datetime'] = df_30m['Datetime'].dt.date

model_90m.py

@@ -237,6 +237,13 @@ def get_data():
     # Get incremental data
     spx1 = yf.Ticker('^GSPC')
     yfp = spx1.history(start=last_date, interval='30m')
+
+    if len(yfp) > 0:
+        # Concat current and incremental
+        df_30m = pd.concat([fr, yfp])
+    else:
+        df_30m = fr.copy()
+        
     # Concat current and incremental
     df_30m = pd.concat([fr, yfp])
     # Get the first 30 minute bar

model_day.py

@@ -15,10 +15,6 @@ import os
 from sklearn.metrics import roc_auc_score, precision_score, recall_score
 import datetime
 from pandas.tseries.offsets import BDay
-from datasets import load_dataset
-
-# If the dataset is gated/private, make sure you have run huggingface-cli login
-dataset = load_dataset("boomsss/spx_intra", split="train")
 
 def walk_forward_validation(df, target_column, num_training_rows, num_periods):