Updates

MonteCarloAgent.py

@@ -27,9 +27,18 @@ class MonteCarloAgent:
 
         self.env_kwargs = kwargs
         if self.env_name == "FrozenLake-v1":
-            self.env_kwargs["desc"] = None
-            self.env_kwargs["map_name"] = "4x4"
-            self.env_kwargs["is_slippery"] = "False"
+            self.env_kwargs["desc"] = [
+                "SFFFFFFF",
+                "FFFFFFFH",
+                "FFFHFFFF",
+                "FFFFFHFF",
+                "FFFHFFFF",
+                "FHHFFFHF",
+                "FHFFHFHF",
+                "FFFHFFFG",
+            ]
+            # self.env_kwargs["map_name"] = "8x8"
+            self.env_kwargs["is_slippery"] = False
 
         self.env = gym.make(self.env_name, **self.env_kwargs)
 
@@ -67,7 +76,7 @@ class MonteCarloAgent:
         # The ability to override was mostly added for testing purposes and for the demo.
         greedy_action = np.argmax(self.Pi[state])
 
-        if greedy:
+        if greedy or epsilon_override == 0:
             return greedy_action
 
         if epsilon_override is None:
@@ -80,21 +89,30 @@ class MonteCarloAgent:
 
     def generate_episode(self, max_steps=500, render=False, **kwargs):
         state, _ = self.env.reset()
-        episode_hist, solved, rgb_array = [], False, None
+        episode_hist, solved, rgb_array = (
+            [],
+            False,
+            self.env.render() if render else None,
+        )
 
         # Generate an episode following the current policy
-        while len(episode_hist) < max_steps:
-            rgb_array = self.env.render() if render else None
-
+        for _ in range(max_steps):
             # Sample an action from the policy
             action = self.choose_action(state, **kwargs)
             # Take the action and observe the reward and next state
-            next_state, reward, done, truncated, _ = self.env.step(action)
+            next_state, reward, done, _, _ = self.env.step(action)
+
+            if self.env_name == "FrozenLake-v1":
+                if done:
+                    reward = 100 if reward == 1 else -10
+                else:
+                    reward = -1
 
             # Keeping track of the trajectory
             episode_hist.append((state, action, reward))
             yield episode_hist, solved, rgb_array
 
+            rgb_array = self.env.render() if render else None
             # For CliffWalking-v0 and Taxi-v3, the episode is solved when it terminates
             if done and (
                 self.env_name == "CliffWalking-v0" or self.env_name == "Taxi-v3"
@@ -103,12 +121,17 @@ class MonteCarloAgent:
                 break
 
             # For FrozenLake-v1, the episode terminates when the agent moves into a hole or reaches the goal
-            # We consider the episode solved when the agent reaches the goal (done == True and reward == 1)
-            if done and self.env_name == "FrozenLake-v1" and reward == 1:
-                solved = True
-                break
-
-            if done or truncated:
+            # We consider the episode solved when the agent reaches the goal
+            if done and self.env_name == "FrozenLake-v1":
+                if next_state == self.env.nrow * self.env.ncol - 1:
+                    solved = True
+                    # print("Solved!")
+                    break
+                else:
+                    done = False
+                    next_state, _ = self.env.reset()
+
+            if solved or done:
                 break
 
             state = next_state

demo.py

@@ -134,6 +134,8 @@ def run(policy_fname, n_test_episodes, max_steps, render_fps, epsilon):
         return f"{step + 1}"
 
     for episode in range(n_test_episodes):
+        time.sleep(1.0)
+
         for step, (episode_hist, solved, rgb_array) in enumerate(
             agent.generate_episode(
                 max_steps=max_steps, render=True, epsilon_override=live_epsilon
@@ -145,7 +147,7 @@ def run(policy_fname, n_test_episodes, max_steps, render_fps, epsilon):
             state, action, reward = episode_hist[-1]
             curr_policy = agent.Pi[state]
 
-            rgb_array_height, rgb_array_width = 128, 512
+            rgb_array_height, rgb_array_width = 150, 512
             rgb_array = cv2.resize(
                 rgb_array,
                 (
@@ -202,7 +204,7 @@ def run(policy_fname, n_test_episodes, max_steps, render_fps, epsilon):
             )
 
             if env_action_map:
-                action_name = env_action_map.get(action, action)
+                action_name = env_action_map.get(action, "")
 
                 cv2.putText(
                     policy_viz,
@@ -222,7 +224,7 @@ def run(policy_fname, n_test_episodes, max_steps, render_fps, epsilon):
                 )
 
             print(
-                f"Episode: {ep_str(episode + 1)} - step: {step_str(step)} - state: {state} - action: {action} - reward: {reward} (epsilon: {live_epsilon:.2f}) (frame time: {1 / render_fps:.2f}s)"
+                f"Episode: {ep_str(episode + 1)} - step: {step_str(step)} - state: {state} - action: {action} - reward: {reward} (epsilon: {live_epsilon:.2f}) (frame time: {1 / live_render_fps:.2f}s)"
             )
 
             yield agent_type, env_name, rgb_array, policy_viz, ep_str(
@@ -396,5 +398,5 @@ with gr.Blocks(title="CS581 Demo") as demo:
         ],
     )
 
-demo.queue(concurrency_count=3)
+demo.queue(concurrency_count=2)
 demo.launch()