Updates

DPAgent.py

@@ -4,20 +4,25 @@ from gymnasium.envs.toy_text.frozen_lake import generate_random_map
 from matplotlib import pyplot as plt
 from tqdm import trange
 from Shared import Shared
+import warnings
 
 
 class DPAgent(Shared):
-    def __init__(self, theta=1e-10, **kwargs):
+    def __init__(self,/,**kwargs):
         super().__init__(**kwargs)
-        self.theta = theta
+        self.theta = kwargs.get('theta', 1e-10)
+        print(self.theta)
         self.V = np.zeros(self.env.observation_space.n)
-        self.Pi = None
+        self.Pi = np.zeros(self.env.observation_space.n, self.env.action_space.n)
+        if self.gamma >= 1.0:
+            warnings.warn("DP will never converge with a gamma value =1.0. Try 0.99?", UserWarning)
 
     def policy(self, state):
         return self.Pi[state]
 
-    def train(self, **kwargs):
+    def train(self, *args, **kwargs):
         i = 0
+        print(self.gamma)
         while True:
             delta = 0
             V_prev = np.copy(self.V)
@@ -38,10 +43,10 @@ class DPAgent(Shared):
             if delta < self.theta:
                 break
             i += 1
-            self.test()
+            # if i % 100 == 0 and i != 0:
+            #     self.test()
             print(f"Iteration {i}: delta={delta}")
             # break
-
         # policy = [self.policy(state, return_value=True)[0] for state in range(self.env.observation_space.n)]
         self.Pi = np.empty((self.env.observation_space.n, self.env.action_space.n))
         for s in range(self.env.observation_space.n):
@@ -51,23 +56,18 @@ class DPAgent(Shared):
                     # if state == self.env.observation_space.n-1: reward = 1
                     expected_value += probability * (reward + self.gamma * self.V[next_state])
                 self.Pi[s,a] = expected_value
-        self.Pi = np.argmax(self.Pi, axis=1)
-        print(self.Pi)
+        idxs = np.argmax(self.Pi, axis=1)
+        print(idxs)
+        self.Pi = np.zeros((self.env.observation_space.n,self.env.action_space.n))
+        self.Pi[np.arange(self.env.observation_space.n),idxs] = 1
+
+        # print(self.Pi)
         # return self.V, self.Pi
 
 
 if __name__ == "__main__":
     # env = gym.make('FrozenLake-v1', render_mode='human')
-    dp = DPAgent("FrozenLake-v1", is_slippery=False, desc=[
-        "SFFFFFFF",
-        "FFFFFFFH",
-        "FFFHFFFF",
-        "FFFFFHFF",
-        "FFFHFFFF",
-        "FHHFFFHF",
-        "FHFFHFHF",
-        "FFFHFFFG",
-    ])
+    dp = DPAgent(env_name="FrozenLake-v1")
     dp.train()
     dp.save_policy('dp_policy.npy')
     env = gym.make('FrozenLake-v1', render_mode='human', is_slippery=False, desc=[
@@ -84,8 +84,7 @@ if __name__ == "__main__":
     state, _ = env.reset()
     done = False
     while not done:
-        action = dp.policy(state)
-        action = np.argmax(action)
+        action = dp.choose_action(state)
         state, reward, done, _, _ = env.step(action)
         env.render()
 

MCAgent.py

@@ -8,6 +8,7 @@ class MCAgent(Shared):
     
     def __init__(
         self,
+        /,
         env_name="CliffWalking-v0",
         gamma=0.99,
         epsilon=0.1,

Shared.py

@@ -8,7 +8,7 @@ import wandb
 class Shared:
     
     def __init__(
-        self,
+        self,/,
         env_name="CliffWalking-v0",
         gamma=0.99,
         epsilon=0.1,
@@ -24,7 +24,7 @@ class Shared:
         self.env_name = env_name
         self.epsilon, self.gamma = epsilon, gamma
 
-        self.env_kwargs = kwargs
+        self.env_kwargs = {k:v for k,v in kwargs.items() if k in ['render_mode']}
         if self.env_name == "FrozenLake-v1":
             # Can use defaults by defining map_name (4x4 or 8x8) or custom map by defining desc
             # self.env_kwargs["map_name"] = "8x8"

run.py

@@ -5,7 +5,6 @@ from agents import AGENTS_MAP
 
 def main():
     parser = argparse.ArgumentParser()
-
     ### Train/Test parameters
     parser.add_argument(
         "--train",
@@ -77,7 +76,7 @@ def main():
     parser.add_argument(
         "--gamma",
         type=float,
-        default=1.0,
+        default=0.99,
         help="The value for the discount factor to use. (default: 1.0)",
     )
     parser.add_argument(
@@ -128,13 +127,8 @@ def main():
     )
 
     args = parser.parse_args()
-    
-    agent = AGENTS_MAP[args.agent](
-        args.env,
-        gamma=args.gamma,
-        epsilon=args.epsilon,
-        render_mode=args.render_mode,
-    )
+    print(vars(args))
+    agent = AGENTS_MAP[args.agent](**dict(args._get_kwargs()))
 
     run_name = f"{agent.__class__.__name__}_{args.env}_e{args.n_train_episodes}_s{args.max_steps}_g{args.gamma}_e{args.epsilon}_{args.update_type}"
     if args.wandb_run_name_suffix is not None: