Updates

AgentBase.py

@@ -91,7 +91,12 @@ class AgentBase:
             p=[1.0 - self.epsilon_override, self.epsilon_override],
         )
 
-    def generate_episode(self, policy, max_steps=500, render=False, **kwargs):
+    def generate_episode(self, policy, max_steps=None, render=False, **kwargs):
+        if max_steps is None:
+            # If max_steps is not specified, we use a rough estimate of
+            # the maximum number of steps it should take to solve the environment
+            max_steps = self.n_states * self.n_actions
+
         state, _ = self.env.reset()
         episode_hist, solved, done = [], False, False
         rgb_array = self.env.render() if render else None
@@ -139,7 +144,7 @@ class AgentBase:
         rgb_array = self.env.render() if render else None
         yield episode_hist, solved, rgb_array
 
-    def run_episode(self, policy, max_steps=500, render=False, **kwargs):
+    def run_episode(self, policy, max_steps=None, render=False, **kwargs):
         # Run the generator until the end
         episode_hist, solved, rgb_array = list(
             self.generate_episode(policy, max_steps, render, **kwargs)

run.py

@@ -39,8 +39,8 @@ def main():
     parser.add_argument(
         "--max_steps",
         type=int,
-        default=200,
-        help="The maximum number of steps per episode before the episode is forced to end. (default: 200)",
+        default=None,
+        help="The maximum number of steps per episode before the episode is forced to end. If not provided, defaults to the number of states in the environment. (default: None)",
     )
 
     ### Agent parameters

test_params.py

@@ -31,18 +31,21 @@ env, num_tests, wandb_project = args.env, args.num_tests, args.wandb_project
 agent = "MCAgent"
 
 vals_update_type = [
-    "first_visit"
+    # "on_policy",
+    "off_policy",
 ]  # Note: Every visit takes too long due to these environment's reward structure
-vals_gamma = [1.0, 0.98, 0.96, 0.94]
+# vals_gamma = [1.0, 0.98, 0.96, 0.94]
 vals_epsilon = [0.1, 0.2, 0.3, 0.4, 0.5]
-# vals_gamma = [1.0]
+vals_gamma = [1.0]
 # vals_epsilon = [0.5]
 
+vals_size = [8, 16, 32, 64]
+
 if env == "CliffWalking-v0":
     n_train_episodes = 2500
     max_steps = 200
 elif env == "FrozenLake-v1":
-    n_train_episodes = 5000
+    n_train_episodes = 25000
     max_steps = 200
 elif env == "Taxi-v3":
     n_train_episodes = 10000
@@ -53,9 +56,10 @@ else:
 
 def run_test(args):
     command = f"python3 run.py --train --agent {agent} --env {env}"
-    command += f" --n_train_episodes {n_train_episodes} --max_steps {max_steps}"
-    command += f" --gamma {args[0]} --epsilon {args[1]} --update_type {args[2]}"
-    command += f" --run_name_suffix {args[3]}"
+    # command += f" --n_train_episodes {n_train_episodes} --max_steps {max_steps}"
+    command += f" --n_train_episodes {n_train_episodes}"
+    for k, v in args.items():
+        command += f" --{k} {v}"
     if wandb_project is not None:
         command += f" --wandb_project {wandb_project}"
     command += " --no_save"
@@ -67,7 +71,28 @@ with multiprocessing.Pool(8) as p:
     for update_type in vals_update_type:
         for gamma in vals_gamma:
             for eps in vals_epsilon:
-                tests.extend((gamma, eps, update_type, i) for i in range(num_tests))
+                if env == "FrozenLake-v1":
+                    for size in vals_size:
+                        tests.extend(
+                            {
+                                "gamma": gamma,
+                                "epsilon": eps,
+                                "update_type": update_type,
+                                "size": size,
+                                "run_name_suffix": i,
+                            }
+                            for i in range(num_tests)
+                        )
+                else:
+                    tests.extend(
+                        {
+                            "gamma": gamma,
+                            "epsilon": eps,
+                            "update_type": update_type,
+                            "run_name_suffix": i,
+                        }
+                        for i in range(num_tests)
+                    )
     random.shuffle(tests)
 
     p.map(run_test, tests)