Finish integration

app.py

@@ -10,6 +10,7 @@ from utils import myOptimizer
 import torch
 from collections import OrderedDict
 from tqdm import tqdm
+import statistics
 
 
 import datetime
@@ -17,57 +18,53 @@ def get_time():
     return datetime.datetime.now().time()
 
 
-def init_rl_agent(train, test):
-    date_split = '01.09.2022 00:00:00.000 GMT-0500'
+def pretrain_rl_agent():
+    global equity
+    observations = env_train.reset()
 
-    learning_rate = 0.001
-    first_momentum = 0.0
-    second_momentum = 0.0001
-    transaction_cost = 0.0001
-    adaptation_rate = 0.01
-    state_size = 15
-    equity = 1.0
+    for _ in tqdm(range(state_size, len(train))):
+        observations = torch.as_tensor(observations).float()
+        action = agent(observations)
+        observations, reward, _ = env_train.step(action.data.to("cpu").numpy())
 
-    agent = Policy(input_channels=state_size)
-    optimizer = myOptimizer(learning_rate, first_momentum, second_momentum, adaptation_rate, transaction_cost)
+        action.backward()
 
-    history = []
-    for i in range(1, state_size):
-        c = train.iloc[i, :]['Close'] - train.iloc[i - 1, :]['Close']
-        history.append(c)
+        for name, param in agent.named_parameters():
+            grad_n = param.grad
+            param = param + optimizer.step(grad_n, reward, observations[-1], model_gradients_history[name])
+            checkpoint[name] = param
+            model_gradients_history.update({name: grad_n})
 
-    env = Environment(train, history=history, state_size=state_size)
-    observation = env.reset()
+        # equity += env_train.profit
+        optimizer.after_step(reward)
+        agent.load_state_dict(checkpoint)
 
-    return env, agent, optimizer, state_size, observation, date_split, equity
 
+def make_prediction(observations):
+    # observations: 0-14
+    action = agent(torch.as_tensor(observations).float())
+    # returned observation: 1- 15
+    observations, reward, _ = env_test.step(action.data.to("cpu").numpy())
 
-def make_prediction(env, agent, optimizer, state_size, observation, data, date_split, equity):
-    model_gradients_history = dict()
-    checkpoint = OrderedDict()
+    action.backward()
 
     for name, param in agent.named_parameters():
-        model_gradients_history.update({name: torch.zeros_like(param)})
+        grad_n = param.grad
+        param = param + optimizer.step(grad_n, reward, observations[-1], model_gradients_history[name])
+        checkpoint[name] = param
+        model_gradients_history.update({name: grad_n})
 
-    for i in tqdm(range(state_size, len(data[:date_split]))):
-        observation = torch.as_tensor(observation).float()
-        action = agent(observation)
-        observation, reward, _ = env.step(action.data.to("cpu").numpy())
-
-        action.backward()
-
-        for name, param in agent.named_parameters():
-            grad_n = param.grad
-            param = param + optimizer.step(grad_n, reward, observation[-1], model_gradients_history[name])
-            checkpoint[name] = param
-            model_gradients_history.update({name: grad_n})
+    # equity += env_test.profit
+    optimizer.after_step(reward)
+    agent.load_state_dict(checkpoint)
 
-        if i > 10000:
-            equity += env.profit
+    return action, observations   # [-1.0, 1.0] * leverage
 
-        optimizer.after_step(reward)
-        agent.load_state_dict(checkpoint)
 
+# ----------------------------------------------------------------------------------------------------------------------
+#   For visualization
+# ----------------------------------------------------------------------------------------------------------------------
+profit = 0.0
 
 counter = 0
 start_year, test_year = 2021, 2023
@@ -83,18 +80,71 @@ dt_obs = set([d.strftime("%Y-%m-%d") for d in pd.to_datetime(df_data[datetime_co
 # Define dates with missing values
 dt_breaks = [d for d in dt_all.strftime("%Y-%m-%d").tolist() if not d in list(dt_obs)]
 
-
 df_data_test = df_data[df_data['Date'].dt.year == test_year]
 df_data_train = df_data[df_data['Date'].dt.year != test_year]
+# ----------------------------------------------------------------------------------------------------------------------
+
+# ----------------------------------------------------------------------------------------------------------------------
+#   For RL Agent
+# ----------------------------------------------------------------------------------------------------------------------
+data = pd.read_csv(f'./data/EURUSD_Candlestick_1_M_BID_01.01.{start_year}-04.02.2023.csv')
+data = data.tail(50000)
+data = data.set_index('Local time')
+date_split = '01.01.2023 16:04:00.000 GMT-0600'
+
+learning_rate = 0.001
+first_momentum = 0.0
+second_momentum = 0.0001
+transaction_cost = 0.0001
+adaptation_rate = 0.01
+state_size = 15
+equity = 1.0
+
+train = data[:date_split]
+test = train.iloc[-(state_size - 1)] + data[date_split:]
+
+# Initialize agent and optimizer
+agent = Policy(input_channels=state_size)
+optimizer = myOptimizer(learning_rate, first_momentum, second_momentum, adaptation_rate, transaction_cost)
+
+history = []
+for i in range(1, state_size):
+    c = train.iloc[i, :]['Close'] - train.iloc[i - 1, :]['Close']
+    history.append(c)
+
+# Initialize train and test environments
+env_train = Environment(train, history=history, state_size=state_size)
+env_test = Environment(test, history=env_train.history, state_size=state_size)
+
+model_gradients_history = dict()
+checkpoint = OrderedDict()
+
+for name, param in agent.named_parameters():
+    model_gradients_history.update({name: torch.zeros_like(param)})
+
+pretrain_rl_agent()
+observations = env_test.reset()
+# ----------------------------------------------------------------------------------------------------------------------
 
 
 def trading_plot():
     global counter
     global df_data_train
+    global observations
+    global profit
+    actions = []
 
     if counter < len(df_data_test):
         df_data_train = df_data_train.append(df_data_test.iloc[counter])
         counter += 1
+
+        last_observation = observations[-1]
+        for i in range(1440):
+            action, observations = make_prediction(observations)
+            actions.append(action.item())
+        position = statistics.mean(actions)
+        profit += -1.0 * (last_observation - observations[-1]) * position
+
     else:
         df_data_train = df_data
 
@@ -133,6 +183,7 @@ def trading_plot():
 # The UI of the demo defines here.
 with gr.Blocks() as demo:
     gr.Markdown("Auto trade bot.")
+    gr.Markdown(f"Profit: {profit}")
 
     # dt = gr.Textbox(label="Current time")
     # demo.queue().load(get_time, inputs=None, outputs=dt, every=1)