Merge branch 'master' into gradio

README.md

@@ -116,6 +116,13 @@ If everything runs smoothly, you have successfully imported the Atari ROMs and s
 
 Reference: [StackOverflow answer](https://stackoverflow.com/a/68143504/38626)
 
+
+### support new env
+We also support other new env using Gym format, for new env you need to 
+1. Translate your Gym env to TextGym env, make `<your_env>_translator.py, <your_env>policies.py`, put them into `./envs/`, and add your env in `./envs/__init__.py`.
+2. Add the PPO performance (best or expert) of your env in `./record_reflexion.csv`
+3. Test it using shell command (recommend using COT, SPP, self-reflexion, and exe under L1&L3 level). Testing examples can be found in `./shell`.
+
 ### Visulization with Gradio
 
 > Gradio is an open-source Python package that allows you to quickly build a demo or web application for your machine learning model, API, or any arbitary Python function. You can then share a link to your demo or web application in just a few seconds using Gradio’s built-in sharing features. No JavaScript, CSS, or web hosting experience needed! [from https://www.gradio.app/guides/quickstart]
@@ -134,4 +141,4 @@ And then run the following Python file in the root directory:
 python gradio_reflexion.py
 ```
 
-The visulization web application will open in a browser on http://server-ip-address:7860 if running from a file. If you are running within a notebook, the demo will appear embedded within the notebook.
+The visulization web application will open in a browser on http://server-ip-address:7860 if running from a file. If you are running within a notebook, the demo will appear embedded within the notebook.

envs/__init__.py

@@ -18,7 +18,6 @@ from .atari import mspacman_policies, mspacman_translator
 from .atari import montezumarevenge_policies, montezumarevenge_translator
 register_environments()
 
-from .mujoco import ant_translator, ant_policies
 
 REGISTRY = {}
 REGISTRY["sampling_wrapper"] = SettableStateEnv
@@ -139,6 +138,99 @@ REGISTRY["RepresentedMontezumaRevenge_basic_policies"] = [
     montezumarevenge_policies.dedicated_18_policy,
 ]
 
+REGISTRY["RepresentedMsPacman_init_translator"] = mspacman_translator.GameDescriber
+REGISTRY["RepresentedMsPacman_basic_translator"] = mspacman_translator.BasicStateSequenceTranslator
+REGISTRY["RepresentedMsPacman_basic_policies"] = [
+    mspacman_policies.real_random_policy,
+    mspacman_policies.pseudo_random_policy,
+    mspacman_policies.dedicated_1_policy,
+    mspacman_policies.dedicated_2_policy,
+    mspacman_policies.dedicated_3_policy,
+    mspacman_policies.dedicated_4_policy,
+    mspacman_policies.dedicated_5_policy,
+    mspacman_policies.dedicated_6_policy,
+    mspacman_policies.dedicated_7_policy,
+    mspacman_policies.dedicated_8_policy,
+    mspacman_policies.dedicated_9_policy,
+]
+
+REGISTRY["RepresentedMontezumaRevenge_init_translator"] = montezumarevenge_translator.GameDescriber
+REGISTRY["RepresentedMontezumaRevenge_basic_translator"] = montezumarevenge_translator.BasicStateSequenceTranslator
+REGISTRY["RepresentedMontezumaRevenge_basic_policies"] = [
+    montezumarevenge_policies.real_random_policy,
+    montezumarevenge_policies.pseudo_random_policy,
+    montezumarevenge_policies.dedicated_1_policy,
+    montezumarevenge_policies.dedicated_2_policy,
+    montezumarevenge_policies.dedicated_3_policy,
+    montezumarevenge_policies.dedicated_4_policy,
+    montezumarevenge_policies.dedicated_5_policy,
+    montezumarevenge_policies.dedicated_6_policy,
+    montezumarevenge_policies.dedicated_7_policy,
+    montezumarevenge_policies.dedicated_8_policy,
+    montezumarevenge_policies.dedicated_9_policy,
+    montezumarevenge_policies.dedicated_10_policy,
+    montezumarevenge_policies.dedicated_11_policy,
+    montezumarevenge_policies.dedicated_12_policy,
+    montezumarevenge_policies.dedicated_13_policy,
+    montezumarevenge_policies.dedicated_14_policy,
+    montezumarevenge_policies.dedicated_15_policy,
+    montezumarevenge_policies.dedicated_16_policy,
+    montezumarevenge_policies.dedicated_17_policy,
+    montezumarevenge_policies.dedicated_18_policy,
+]
+
+## For mujoco env
+
+
+from .mujoco import invertedPendulum_translator, invertedPendulum_policies
+from .mujoco import invertedDoublePendulum_translator, invertedDoublePendulum_policies
+
+from .mujoco import swimmer_translator, swimmer_policies
+
+from .mujoco import reacher_translator, reacher_policies
+
+from .mujoco import hopper_translator, hopper_policies
+from .mujoco import walker2d_translator, walker2d_policies
+
+
+
+
+
+REGISTRY["invertedPendulum_init_translator"] = invertedPendulum_translator.GameDescriber
+REGISTRY["invertedPendulum_basic_translator"] = invertedPendulum_translator.BasicStateSequenceTranslator
+REGISTRY["invertedPendulum_policies"] = [invertedPendulum_policies.pseudo_random_policy, invertedPendulum_policies.real_random_policy]
+REGISTRY["invertedDoublePendulum_init_translator"] = invertedDoublePendulum_translator.GameDescriber
+REGISTRY["invertedDoublePendulum_basic_translator"] = invertedDoublePendulum_translator.BasicStateSequenceTranslator
+REGISTRY["invertedDoublePendulum_policies"] = [invertedDoublePendulum_policies.pseudo_random_policy, invertedDoublePendulum_policies.real_random_policy]
+
+
+REGISTRY["swimmer_init_translator"] = swimmer_translator.GameDescriber
+REGISTRY["swimmer_basic_translator"] = swimmer_translator.BasicStateSequenceTranslator
+REGISTRY["swimmer_policies"] = [swimmer_policies.pseudo_random_policy, swimmer_policies.real_random_policy]
+
+REGISTRY["reacher_init_translator"] = reacher_translator.GameDescriber
+REGISTRY["reacher_basic_translator"] = reacher_translator.BasicStateSequenceTranslator
+REGISTRY["reacher_policies"] = [reacher_policies.pseudo_random_policy, reacher_policies.real_random_policy]
+
+REGISTRY["hopper_init_translator"] = hopper_translator.GameDescriber
+REGISTRY["hopper_basic_translator"] = hopper_translator.BasicStateSequenceTranslator
+REGISTRY["hopper_policies"] = [hopper_policies.pseudo_random_policy, hopper_policies.real_random_policy]
+REGISTRY["walker2d_init_translator"] = walker2d_translator.GameDescriber
+REGISTRY["walker2d_basic_translator"] = walker2d_translator.BasicStateSequenceTranslator
+REGISTRY["walker2d_policies"] = [walker2d_policies.pseudo_random_policy, walker2d_policies.real_random_policy]
+
+
+from .mujoco import halfcheetah_translator, halfcheetah_policies
+REGISTRY["halfcheetah_init_translator"] = halfcheetah_translator.GameDescriber
+REGISTRY["halfcheetah_basic_translator"] = halfcheetah_translator.BasicStateSequenceTranslator
+REGISTRY["halfcheetah_policies"] = [halfcheetah_policies.pseudo_random_policy, halfcheetah_policies.real_random_policy]
+
+from .mujoco import pusher_translator, pusher_policies
+REGISTRY["pusher_init_translator"] = pusher_translator.GameDescriber
+REGISTRY["pusher_basic_translator"] = pusher_translator.BasicStateSequenceTranslator
+REGISTRY["pusher_policies"] = [pusher_policies.pseudo_random_policy, pusher_policies.real_random_policy]
+
+from .mujoco import ant_translator, ant_policies
 REGISTRY["ant_init_translator"] = ant_translator.GameDescriber
 REGISTRY["ant_basic_translator"] = ant_translator.BasicStateSequenceTranslator
-REGISTRY["ant_policies"] = [ant_policies.pseudo_random_policy, ant_policies.real_random_policy]
+REGISTRY["ant_policies"] = [ant_policies.pseudo_random_policy, ant_policies.real_random_policy]

envs/mujoco/ant_translator.py

@@ -1,3 +1,8 @@
+'''Ant
+Action Space Box(-1.0, 1.0, (8,), float32)
+Observation Space Box(-inf, inf, (27,), float64)
+'''
+
 class BasicLevelTranslator:
     def __init__(self):
         pass

envs/mujoco/halfcheetah_policies.py

@@ -0,0 +1,15 @@
+import numpy as np
+import random
+
+def pseudo_random_policy(state, pre_action):
+    def get_description():
+        return "Select action randomly"
+    pseudo_random_policy.description = get_description()
+    return [2 * random.random() - 1 for i in range(6)]
+
+
+def real_random_policy(state, pre_action=1):
+    def get_description():
+        return "Select action with a random policy"
+    real_random_policy.description = get_description()
+    return [2 * random.random() - 1 for i in range(6)]

envs/mujoco/halfcheetah_translator.py

@@ -0,0 +1,95 @@
+class BasicLevelTranslator:
+    def __init__(self):
+        pass
+    
+    def translate(self, state):
+        (front_tip_z, front_tip_angle, back_thigh_angle_1, back_shin_angle_1, 
+         tip_velocity_x, tip_velocity_y, front_tip_angular_velocity, 
+         back_thigh_angular_velocity_1, front_tip_x, front_tip_y, front_tip_angle_2, 
+         back_thigh_angle_2, back_shin_angle_2, tip_velocity_angular_x, 
+         tip_velocity_angular_y, front_tip_angular_velocity_2, 
+         back_thigh_angular_velocity_2) = state[:17]
+
+        res = (
+            f"The front tip is at a z-coordinate of {front_tip_z:.2f} meters. "
+            f"The angle of the front tip is {front_tip_angle:.2f} radians. "
+            f"The angles of the back thigh are {back_thigh_angle_1:.2f} and {back_thigh_angle_2:.2f} radians. "
+            f"The angles of the back shin are {back_shin_angle_1:.2f} and {back_shin_angle_2:.2f} radians. "
+            f"The tip has velocity along the x-axis of {tip_velocity_x:.2f} m/s. "
+            f"The tip has velocity along the y-axis of {tip_velocity_y:.2f} m/s. "
+            f"The angular velocity of the front tip is {front_tip_angular_velocity:.2f} radians/s. "
+            f"The angular velocities of the back thigh are {back_thigh_angular_velocity_1:.2f} and {back_thigh_angular_velocity_2:.2f} radians/s. "
+            f"The x-coordinate of the front tip is {front_tip_x:.2f} meters. "
+            f"The y-coordinate of the front tip is {front_tip_y:.2f} meters. "
+            f"The angle of the front tip is {front_tip_angle_2:.2f} radians. "
+            f"The angular velocity of the tip along the x-axis is {tip_velocity_angular_x:.2f} radians/s. "
+            f"The angular velocity of the tip along the y-axis is {tip_velocity_angular_y:.2f} radians/s. "
+            f"The angular velocity of the back shin is {front_tip_angular_velocity_2:.2f} radians/s."
+        )
+        return res
+
+class GameDescriber:
+    def __init__(self, args):
+        self.is_only_local_obs = args.is_only_local_obs == 1
+        self.max_episode_len = args.max_episode_len
+        self.action_desc_dict = {
+        }
+        self.reward_desc_dict = {
+        }
+    
+    def translate_terminate_state(self, state, episode_len, max_episode_len): 
+        return ""
+        
+    def translate_potential_next_state(self, state, action):
+        return ""
+
+    def describe_goal(self):
+        return "The goal is to make the Half-Cheetah run forward (right) as fast as possible."
+
+    def describe_game(self):
+        return (
+            "In the Half-Cheetah game, you control a 2-dimensional robot with 9 links and 8 joints. "
+            "The goal is to apply torque to the joints to make the cheetah run forward (right) as fast as possible. "
+            "You can control the back thigh, back shin, and back foot rotors for the back legs, and the front thigh, "
+            "front shin, and front foot rotors for the front legs. The episode ends after 1000 timesteps. "
+            "Your reward is based on how much forward progress you make and how much control effort you apply."
+        )
+
+    def describe_action(self):
+        return (
+            "Your next move: \n"
+            "Please select six numerical values, each one within the range of [-1,1], "
+            "which represents the torque being applied to the back thigh rotor, "
+            "back shin rotor, back foot rotor, front thigh rotor, front shin rotor, "
+            "and front foot rotor respectively."
+        )
+
+class BasicStateSequenceTranslator(BasicLevelTranslator):
+    def translate(self, infos, is_current=False):
+        descriptions = []
+        if is_current:
+            state_desc = BasicLevelTranslator().translate(infos[-1]['state'])
+            return state_desc
+        for i, info in enumerate(infos):
+            assert 'state' in info, "info should contain state information"
+
+            state_desc = BasicLevelTranslator().translate(info['state'])
+            action_desc = (
+                "Take Action: "
+                "Apply Back Thigh Torque: {:.2f}, "
+                "Apply Back Shin Torque: {:.2f}, "
+                "Apply Back Foot Torque: {:.2f}, "
+                "Apply Front Thigh Torque: {:.2f}, "
+                "Apply Front Shin Torque: {:.2f}, "
+                "Apply Front Foot Torque: {:.2f}"
+            ).format(
+                info['action'][0], info['action'][1], info['action'][2],
+                info['action'][3], info['action'][4], info['action'][5]
+            )
+
+            reward_desc = f"Result: Forward Reward of {info['forward_reward']:.2f}, "
+            ctrl_cost_desc = f"Control Cost of {info['ctrl_cost']:.2f}, "
+            total_reward_desc = f"Total Reward of {info['reward']:.2f}, "
+            next_state_desc = BasicLevelTranslator().translate(info['next_state'])
+            descriptions.append(f"{state_desc}.\\n {action_desc} \\n {reward_desc} {ctrl_cost_desc} {total_reward_desc} \\n Transit to {next_state_desc}")
+        return descriptions

envs/mujoco/hopper_policies.py

@@ -0,0 +1,15 @@
+import numpy as np
+import random
+
+def pseudo_random_policy(state, pre_action):
+    def get_description():
+        return "Select action randomly"
+    pseudo_random_policy.description = get_description()
+    return [2 * random.random() - 1 for i in range(3)]
+
+
+def real_random_policy(state, pre_action=1):
+    def get_description():
+        return "Select action with a random policy"
+    real_random_policy.description = get_description()
+    return [2 * random.random() - 1 for i in range(3)]

envs/mujoco/hopper_translator.py

@@ -0,0 +1,84 @@
+'''
+Action Space Box(-1.0, 1.0, (3,), float32)
+Observation Space Box(-inf, inf, (11,), float64)
+'''
+
+class BasicLevelTranslator:
+    def __init__(self):
+        pass
+
+    def translate(self, state):
+        (top_z, top_angle, thigh_angle, leg_angle, foot_angle, 
+         top_x_velocity, top_z_velocity, top_angular_velocity, 
+         thigh_angular_velocity, leg_angular_velocity, foot_angular_velocity) = state[:11]
+
+        res = (
+            f"The top is at a z-coordinate of {top_z:.2f} meters. "
+            f"The angle of the top is {top_angle:.2f} radians. "
+            f"The angle of the thigh joint is {thigh_angle:.2f} radians. "
+            f"The angle of the leg joint is {leg_angle:.2f} radians. "
+            f"The angle of the foot joint is {foot_angle:.2f} radians. "
+            f"The x-coordinate velocity of the top is {top_x_velocity:.2f} m/s. "
+            f"The z-coordinate (height) velocity of the top is {top_z_velocity:.2f} m/s. "
+            f"The angular velocity of the top is {top_angular_velocity:.2f} radians/s. "
+            f"The angular velocity of the thigh hinge is {thigh_angular_velocity:.2f} radians/s. "
+            f"The angular velocity of the leg hinge is {leg_angular_velocity:.2f} radians/s. "
+            f"The angular velocity of the foot hinge is {foot_angular_velocity:.2f} radians/s."
+        )
+        return res
+
+class GameDescriber:
+    def __init__(self, args):
+        self.is_only_local_obs = args.is_only_local_obs == 1
+        self.max_episode_len = args.max_episode_len
+        self.action_desc_dict = {}
+        self.reward_desc_dict = {}
+    
+    def translate_terminate_state(self, state, episode_len, max_episode_len):
+        return ""
+        
+    def translate_potential_next_state(self, state, action):
+        return ""
+    
+    def describe_goal(self):
+        return (
+            "The goal in the Hopper environment is to make the one-legged hopper move forward (right) "
+            "by applying torques to the thigh, leg, and foot joints."
+        )
+
+    def describe_game(self):
+        return (
+            "In the Hopper environment, you control a one-legged hopper consisting of a torso, thigh, leg, "
+            "and a foot on which it rests. Your objective is to apply torques to the thigh, leg, and foot joints "
+            "to make the hopper perform hops in the positive x-direction. The environment provides observations "
+            "of the hopper's body parts and velocities, including the height, angles of joints, and angular velocities. "
+            "The episode ends when certain termination conditions are met."
+        )
+
+    def describe_action(self):
+        return (
+            "Your next move: \n Please provide a list of three numerical values, each within the range of [-1,1], "
+            "representing the torques to be applied at the thigh, leg, and foot joints of the hopper."
+        )
+
+    
+class BasicStateSequenceTranslator(BasicLevelTranslator):
+    def translate(self, infos, is_current=False):
+        descriptions = []
+        if is_current:
+            state_desc = BasicLevelTranslator().translate(infos[-1]['state'])
+            return state_desc
+        for i, info in enumerate(infos):
+            assert 'state' in info, "info should contain state information"
+
+            state_desc = BasicLevelTranslator().translate(info['state'])
+            action_desc = (
+                f"Take Action: Apply Thigh Torque: {info['action'][0]:.2f}, "
+                f"Leg Torque: {info['action'][1]:.2f}, Foot Torque: {info['action'][2]:.2f}"
+            )
+
+            reward_desc = f"Result: Reward of {info['reward']:.2f}, "
+            next_state_desc = BasicLevelTranslator().translate(info['next_state'])
+            descriptions.append(f"{state_desc}.\\n {action_desc} \\n {reward_desc} \\n Transit to {next_state_desc}")
+        return descriptions
+

envs/mujoco/invertedDoublePendulum_policies.py

@@ -0,0 +1,15 @@
+import numpy as np
+import random
+
+def pseudo_random_policy(state, pre_action):
+    def get_description():
+        return "Select action randomly"
+    pseudo_random_policy.description = get_description()
+    return [6 * random.random() - 3 for i in range(1)]
+
+
+def real_random_policy(state, pre_action=1):
+    def get_description():
+        return "Select action with a random policy"
+    real_random_policy.description = get_description()
+    return [6 * random.random() - 3 for i in range(1)]

envs/mujoco/invertedDoublePendulum_translator.py

@@ -0,0 +1,68 @@
+'''InvertedDoublePendulum-v4
+Action Space Box(-1.0, 1.0, (1,), float32)
+Observation Space Box(-inf, inf, (11,), float64)
+'''
+
+class BasicLevelTranslator:
+    def translate(self, state):
+        res = (
+            f"Position of the cart: {state[0]:.2f} m\n"
+            f"Vertical angle of the pole: {state[1]:.2f} rad\n"
+            f"Linear velocity of the cart: {state[2]:.2f} m/s\n"
+            f"Angular velocity of the pole: {state[3]:.2f} rad/s"
+        )
+        return res
+
+class GameDescriber:
+    def __init__(self, args):
+        self.is_only_local_obs = args.is_only_local_obs == 1
+        self.max_episode_len = args.max_episode_len
+        self.action_desc_dict = {
+            0: "Apply a force in the range [-1, 1] to the cart to control its motion.",
+        }
+        self.reward_desc_dict = {}
+    
+    def translate_terminate_state(self, state, episode_len, max_episode_len):
+        return ""
+        
+    def translate_potential_next_state(self, state, action):
+        return ""
+    
+    def describe_goal(self):
+        return (
+            "The goal in the Inverted Pendulum environment is to balance the pole on top of the cart "\
+            "by applying continuous forces to the cart, keeping it upright."
+        )
+
+    def describe_game(self):
+        return (
+            "In the Inverted Pendulum environment, you control a cart that can move linearly with a pole "\
+            "attached to it. Your objective is to balance the pole on top of the cart by applying forces "\
+            "to the cart in a way that keeps the pole upright. "\
+            "The environment provides observations of the cart's position, pole angle, velocities, "\
+            "and angular velocities. The goal is to maintain balance as long as possible."
+        )
+
+    def describe_action(self):
+        return (
+            "Your next move: \n Please provide a numerical value for the force to be applied to the cart. "\
+            "This value should be within the range of [-3, 3], where a positive value indicates applying force "\
+            "in the right direction, and a negative value indicates applying force in the left direction."
+        )
+
+class BasicStateSequenceTranslator(BasicLevelTranslator):
+    def translate(self, infos, is_current=False):
+        descriptions = []
+        if is_current:
+            state_desc = BasicLevelTranslator().translate(infos[-1]['state'])
+            return state_desc
+        for i, info in enumerate(infos):
+            assert 'state' in info, "info should contain state information"
+            state_desc = BasicLevelTranslator().translate(info['state'])
+            action_desc = f"Applied Force on Cart: {info['action'][0]:.2f}"
+            reward_desc = f"Result: Reward of {info['reward']:.2f}"
+            next_state_desc = BasicLevelTranslator().translate(info['next_state'])
+            descriptions.append(
+                f"{state_desc}\n{action_desc}\n{reward_desc}\nTransit to\n{next_state_desc}"
+            )
+        return descriptions

envs/mujoco/invertedPendulum_policies.py

@@ -0,0 +1,15 @@
+import numpy as np
+import random
+
+def pseudo_random_policy(state, pre_action):
+    def get_description():
+        return "Select action randomly"
+    pseudo_random_policy.description = get_description()
+    return [2 * random.random() - 1 for i in range(1)]
+
+
+def real_random_policy(state, pre_action=1):
+    def get_description():
+        return "Select action with a random policy"
+    real_random_policy.description = get_description()
+    return [2 * random.random() - 1 for i in range(1)]

envs/mujoco/invertedPendulum_translator.py

@@ -0,0 +1,73 @@
+'''InvertedPendulum-v4
+Action Space Box(-3.0, 3.0, (1,), float32)
+Observation Space Box(-inf, inf, (4,), float64)
+'''
+
+class BasicLevelTranslator:
+    def translate(self, state):
+        res = (
+            f"Position of the cart: {state[0]:.2f} m\n"
+            f"Sine of the angle between cart and first pole: {state[1]:.2f}\n"
+            f"Sine of the angle between two poles: {state[2]:.2f}\n"
+            f"Cosine of the angle between cart and first pole: {state[3]:.2f}\n"
+            f"Cosine of the angle between two poles: {state[4]:.2f}\n"
+            f"Velocity of the cart: {state[5]:.2f} m/s\n"
+            f"Angular velocity of angle between cart and first pole: {state[6]:.2f} rad/s\n"
+            f"Angular velocity of angle between two poles: {state[7]:.2f} rad/s\n"
+            f"Constraint Force 1: {state[8]:.2f} N\n"
+            f"Constraint Force 2: {state[9]:.2f} N\n"
+            f"Constraint Force 3: {state[10]:.2f} N"
+        )
+        return res
+
+class GameDescriber:
+    def __init__(self, args):
+        self.is_only_local_obs = args.is_only_local_obs == 1
+        self.max_episode_len = args.max_episode_len
+        self.action_desc_dict = {
+            0: "Apply a force in the range [-3, 3] to the cart to control its motion.",
+        }
+        self.reward_desc_dict = {}
+    
+    def translate_terminate_state(self, state, episode_len, max_episode_len):
+        return ""
+        
+    def translate_potential_next_state(self, state, action):
+        return ""
+    
+    def describe_goal(self):
+        return (
+            "The goal in the InvertedDoublePendulum environment is to balance the two poles "\
+            "on top of the cart by applying continuous forces on the cart."
+        )
+
+    def describe_game(self):
+        return (
+            "In the InvertedDoublePendulum environment, you control a system with a cart and two poles. "\
+            "Your objective is to balance the two poles on top of the cart by applying continuous forces "\
+            "to the cart. The environment provides observations of the cart's position, angles of the poles, "\
+            "and their angular velocities. The episode ends when certain termination conditions are met."
+        )
+
+    def describe_action(self):
+        return (
+            "Your next move: \n Please provide a numerical value within the range of [-3,3], "\
+            "representing the force to be applied to the cart."
+        )
+
+class BasicStateSequenceTranslator(BasicLevelTranslator):
+    def translate(self, infos, is_current=False):
+        descriptions = []
+        if is_current:
+            state_desc = BasicLevelTranslator().translate(infos[-1]['state'])
+            return state_desc
+        for i, info in enumerate(infos):
+            assert 'state' in info, "info should contain state information"
+            state_desc = BasicLevelTranslator().translate(info['state'])
+            action_desc = f"Applied Force on Cart: {info['action'][0]:.2f}"
+            reward_desc = f"Result: Reward of {info['reward']:.2f}"
+            next_state_desc = BasicLevelTranslator().translate(info['next_state'])
+            descriptions.append(
+                f"{state_desc}\n{action_desc}\n{reward_desc}\nTransit to\n{next_state_desc}"
+            )
+        return descriptions

envs/mujoco/pusher_policies.py

@@ -0,0 +1,15 @@
+import numpy as np
+import random
+
+def pseudo_random_policy(state, pre_action):
+    def get_description():
+        return "Select action randomly"
+    pseudo_random_policy.description = get_description()
+    return [4 * random.random() - 2 for i in range(7)]
+
+
+def real_random_policy(state, pre_action=1):
+    def get_description():
+        return "Select action with a random policy"
+    real_random_policy.description = get_description()
+    return [4 * random.random() - 2 for i in range(7)]

envs/mujoco/pusher_translator.py

@@ -0,0 +1,93 @@
+'''Pusher
+Action Space Box(-2.0, 2.0, (7,), float32)
+Observation Space Box(-inf, inf, (23,), float64)
+'''
+import math
+
+class BasicLevelTranslator:
+    def __init__(self):
+        pass
+
+    def translate(self, state):
+
+        joint_angles = state[:7]
+        joint_velocities = state[7:14]
+        fingertip_coords = state[14:17]
+        object_coords = state[17:20]
+        goal_coords = state[20:]
+
+        joint_angle_degrees = [math.degrees(angle) for angle in joint_angles]
+        joint_velocity_degrees = [math.degrees(velocity) for velocity in joint_velocities]
+
+        res = (f"Rotation of the panning shoulder: {joint_angle_degrees[0]:.2f} degrees, "
+               f"Rotation of the shoulder lifting joint: {joint_angle_degrees[1]:.2f} degrees, "
+               f"Rotation of the shoulder rolling joint: {joint_angle_degrees[2]:.2f} degrees, "
+               f"Rotation of the elbow joint: {joint_angle_degrees[3]:.2f} degrees, "
+               f"Rotation of the forearm rolling joint: {joint_angle_degrees[4]:.2f} degrees, "
+               f"Rotation of the wrist flexing joint: {joint_angle_degrees[5]:.2f} degrees, "
+               f"Rotation of the wrist rolling joint: {joint_angle_degrees[6]:.2f} degrees, "
+               f"Rotational velocity of the panning shoulder: {joint_velocity_degrees[0]:.2f} degrees/s, "
+               f"Rotational velocity of the shoulder lifting joint: {joint_velocity_degrees[1]:.2f} degrees/s, "
+               f"Rotational velocity of the shoulder rolling joint: {joint_velocity_degrees[2]:.2f} degrees/s, "
+               f"Rotational velocity of the elbow joint: {joint_velocity_degrees[3]:.2f} degrees/s, "
+               f"Rotational velocity of the forearm rolling joint: {joint_velocity_degrees[4]:.2f} degrees/s, "
+               f"Rotational velocity of the wrist flexing joint: {joint_velocity_degrees[5]:.2f} degrees/s, "
+               f"Rotational velocity of the wrist rolling joint: {joint_velocity_degrees[6]:.2f} degrees/s, "
+               f"Fingertip coordinates (x, y, z): ({fingertip_coords[0]:.2f}, {fingertip_coords[1]:.2f}, {fingertip_coords[2]:.2f}), "
+               f"Object coordinates (x, y, z): ({object_coords[0]:.2f}, {object_coords[1]:.2f}, {object_coords[2]:.2f}), "
+               f"Goal coordinates (x, y, z): ({goal_coords[0]:.2f}, {goal_coords[1]:.2f}, {goal_coords[2]:.2f}).")
+        return res
+
+
+class GameDescriber:
+    def __init__(self, args):
+        self.is_only_local_obs = args.is_only_local_obs == 1
+        self.max_episode_len = args.max_episode_len
+        self.action_desc_dict = {
+        }
+        self.reward_desc_dict = {
+        }
+    
+    def translate_terminate_state(self, state, episode_len, max_episode_len): 
+        return ""
+        
+    def translate_potential_next_state(self, state, action):
+        return ""
+
+    def describe_goal(self):
+        return "The goal is to move the target cylinder (object) to the goal position using the robot's end effector (fingertip)."
+
+    def describe_game(self):
+        return ("In the Pusher game, you control a multi-jointed robot arm to manipulate a target cylinder (object) "
+                "and place it in a goal position using the robot's fingertip (end effector). The robot has shoulder, elbow, "
+                "forearm, and wrist joints that you can control with torque values. The observation space includes joint angles, "
+                "angular velocities of joints, fingertip coordinates, object coordinates, and goal coordinates. The reward is "
+                "based on the distance between the fingertip and the object, the distance between the object and the goal, "
+                "and control penalties for large actions.")
+
+    def describe_action(self):
+        return ("Your next move: \n Please provide a list of 7 numerical values within the range [-2, 2], "
+                "representing the torques applied to the robot's joints (shoulder, elbow, forearm, and wrist).")
+
+
+class BasicStateSequenceTranslator(BasicLevelTranslator):
+    def translate(self, infos, is_current=False):
+        descriptions = []
+        if is_current:
+            state_desc = BasicLevelTranslator().translate(infos[-1]['state'])
+            return state_desc
+        for info in infos:
+            assert 'state' in info, "info should contain state information"
+
+            state_desc = BasicLevelTranslator().translate(info['state'])
+            action_desc = ("Take Action: Apply Torques - "
+                           "Shoulder Pan: {:.2f}, Shoulder Lift: {:.2f}, Shoulder Roll: {:.2f}, "
+                           "Elbow Flex: {:.2f}, Forearm Roll: {:.2f}, Wrist Flex: {:.2f}, Wrist Roll: {:.2f}"
+                           ).format(info['action'][0], info['action'][1], info['action'][2], info['action'][3],
+                                    info['action'][4], info['action'][5], info['action'][6])
+
+            reward_desc = f"Result: Reward of {info['reward']:.2f}"
+            next_state_desc = BasicLevelTranslator().translate(info['next_state'])
+            descriptions.append(f"{state_desc}\n{action_desc}\n{reward_desc}\nTransit to {next_state_desc}")
+        return descriptions
+

envs/mujoco/reacher_policies.py

@@ -0,0 +1,15 @@
+import numpy as np
+import random
+
+def pseudo_random_policy(state, pre_action):
+    def get_description():
+        return "Select action randomly"
+    pseudo_random_policy.description = get_description()
+    return [2 * random.random() - 1 for i in range(2)]
+
+
+def real_random_policy(state, pre_action=1):
+    def get_description():
+        return "Select action with a random policy"
+    real_random_policy.description = get_description()
+    return [2 * random.random() - 1 for i in range(2)]

envs/mujoco/reacher_translator.py

@@ -0,0 +1,67 @@
+'''Reacher
+Action Space Box(-1.0, 1.0, (2,), float32)
+
+Observation Space Box(-inf, inf, (11,), float64)
+'''
+class BasicLevelTranslator:
+    def __init__(self):
+        pass
+
+    def translate(self, state):
+        (cos_angle_arm1, cos_angle_arm2, sin_angle_arm1, sin_angle_arm2, 
+         target_x, target_y, angular_vel_arm1, angular_vel_arm2, 
+         diff_x, diff_y, diff_z) = state
+
+        res = (f"Arm1 has a cosine angle of {cos_angle_arm1:.2f} and a sine angle of {sin_angle_arm1:.2f}. "\
+               f"Arm2 has a cosine angle of {cos_angle_arm2:.2f} and a sine angle of {sin_angle_arm2:.2f}. "\
+               f"Target position is at ({target_x:.2f}, {target_y:.2f}). "\
+               f"Arm1's angular velocity is {angular_vel_arm1:.2f} rad/s, and Arm2's is {angular_vel_arm2:.2f} rad/s. "\
+               f"Vector difference between fingertip and target is ({diff_x:.2f}, {diff_y:.2f}, {diff_z:.2f}).")
+        return res
+
+class GameDescriber:
+    def __init__(self, args):
+        self.is_only_local_obs = args.is_only_local_obs == 1
+        self.max_episode_len = args.max_episode_len
+        self.action_desc_dict = {
+        }
+        self.reward_desc_dict = {
+        }
+    
+    def translate_terminate_state(self, state, episode_len, max_episode_len): 
+        return ""
+        
+    def translate_potential_next_state(self, state, action):
+        return ""
+
+    def describe_goal(self):
+        return "The goal is to control a two-jointed robot arm to move its end effector (fingertip) close to a randomly spawned target."
+
+    def describe_game(self):
+        return ("In the Reacher game, you control a two-jointed robot arm. The objective is to maneuver the arm's fingertip close to a target. "\
+                "The observation space includes the cosine and sine of the arm angles, coordinates of the target, angular velocities of the arms, "\
+                "and the vector from the fingertip to the target. The episode ends after 50 timesteps or if any state space value becomes non-finite. "\
+                "Rewards are given based on the distance of the fingertip from the target and the magnitude of actions applied.")
+
+    def describe_action(self):
+        return ("Your next move: \n Please provide two numerical values representing the torques applied at the two hinge joints. "\
+                "Each value should be within the range of [-1, 1].")
+
+class BasicStateSequenceTranslator(BasicLevelTranslator):
+    def translate(self, infos, is_current=False):
+        descriptions = []
+        if is_current:
+            state_desc = BasicLevelTranslator().translate(infos[-1]['state'])
+            return state_desc
+        for i, info in enumerate(infos):
+            assert 'state' in info, "info should contain state information"
+
+            state_desc = BasicLevelTranslator().translate(info['state'])
+            action_desc = ("Take Action: Apply Torque at Joint 1: {:.2f}, "
+                           "Joint 2 Torque: {:.2f}"
+                          ).format(info['action'][0], info['action'][1])
+
+            reward_desc = f"Result: Reward of {info['reward']:.2f}, "
+            next_state_desc = BasicLevelTranslator().translate(info['next_state'])
+            descriptions.append(f"{state_desc}.\\n {action_desc} \\n {reward_desc} \\n Transit to {next_state_desc}")
+        return descriptions

envs/mujoco/swimmer_policies.py

@@ -0,0 +1,15 @@
+import numpy as np
+import random
+
+def pseudo_random_policy(state, pre_action):
+    def get_description():
+        return "Select action randomly"
+    pseudo_random_policy.description = get_description()
+    return [2 * random.random() - 1 for i in range(2)]
+
+
+def real_random_policy(state, pre_action=1):
+    def get_description():
+        return "Select action with a random policy"
+    real_random_policy.description = get_description()
+    return [2 * random.random() - 1 for i in range(2)]

envs/mujoco/swimmer_translator.py

@@ -0,0 +1,80 @@
+'''Swimmer
+Action Space Box(-1.0, 1.0, (2,), float32)
+
+Observation Space Box(-inf, inf, (8,), float64)
+'''
+
+class BasicLevelTranslator:
+    def translate(self, state):
+        res = (
+            f"Angle of the front tip: {state[0]:.2f} rad\n"
+            f"Angle of the first rotor: {state[1]:.2f} rad\n"
+            f"Angle of the second rotor: {state[2]:.2f} rad\n"
+            f"Velocity of the tip along the x-axis: {state[3]:.2f} m/s\n"
+            f"Velocity of the tip along the y-axis: {state[4]:.2f} m/s\n"
+            f"Angular velocity of front tip: {state[5]:.2f} rad/s\n"
+            f"Angular velocity of the first rotor: {state[6]:.2f} rad/s\n"
+            f"Angular velocity of the second rotor: {state[7]:.2f} rad/s"
+        )
+        return res
+
+class GameDescriber:
+
+    def __init__(self, args):
+        self.is_only_local_obs = args.is_only_local_obs == 1
+        self.max_episode_len = args.max_episode_len
+        self.action_desc_dict = {
+        }
+        self.reward_desc_dict = {
+        }
+    
+    def translate_terminate_state(self, state, episode_len, max_episode_len): 
+        return ""
+        
+    def translate_potential_next_state(self, state, action):
+        return ""
+
+    def describe_goal(self):
+        return (
+            "The goal in the Swimmer environment is to move as fast as possible towards the right "\
+            "by applying torque to the rotors and utilizing fluid friction. The swimmer consists of "\
+            "three or more segments connected by rotors, and the objective is to achieve efficient "\
+            "swimming motion."
+        )
+
+    def describe_game(self):
+        return (
+            "In the Swimmer environment, you control a swimmer consisting of three or more segments "\
+            "connected by rotors. Your goal is to make the swimmer move as fast as possible to the right "\
+            "in a two-dimensional pool. You can achieve this by applying torques to the rotors and utilizing "\
+            "fluid friction. The environment provides observations of the swimmer's angles, velocities, "\
+            "and angular velocities."
+        )
+
+    def describe_action(self):
+        return (
+            "Your next move: \nPlease provide a list of two numerical values, each within the range of [-1, 1], "\
+            "representing the torques to be applied to the two rotors of the swimmer. These torques will help "\
+            "control the swimmer's movement and achieve efficient swimming."
+        )
+
+
+class BasicStateSequenceTranslator(BasicLevelTranslator):
+    def translate(self, infos, is_current=False):
+        descriptions = []
+        if is_current:
+            state_desc = BasicLevelTranslator().translate(infos[-1]['state'])
+            return state_desc
+        for i, info in enumerate(infos):
+            assert 'state' in info, "info should contain state information"
+            state_desc = BasicLevelTranslator().translate(info['state'])
+            action_desc = (
+                "Torques Applied: "
+                f"Rotor 1: {info['action'][0]:.2f}, Rotor 2: {info['action'][1]:.2f}"
+            )
+            reward_desc = f"Result: Reward of {info['reward']:.2f}"
+            next_state_desc = BasicLevelTranslator().translate(info['next_state'])
+            descriptions.append(
+                f"{state_desc}\n{action_desc}\n{reward_desc}\nTransit to\n{next_state_desc}"
+            )
+        return descriptions

envs/mujoco/walker2d_policies.py

@@ -0,0 +1,15 @@
+import numpy as np
+import random
+
+def pseudo_random_policy(state, pre_action):
+    def get_description():
+        return "Select action randomly"
+    pseudo_random_policy.description = get_description()
+    return [2 * random.random() - 1 for i in range(6)]
+
+
+def real_random_policy(state, pre_action=1):
+    def get_description():
+        return "Select action with a random policy"
+    real_random_policy.description = get_description()
+    return [2 * random.random() - 1 for i in range(6)]

envs/mujoco/walker2d_translator.py

@@ -0,0 +1,86 @@
+
+'''Walker2d
+Action Space Box(-1.0, 1.0, (6,), float32)
+Observation Space Box(-inf, inf, (17,), float64)
+'''
+class BasicLevelTranslator:
+    def translate(self, state):
+        res = (
+            f"Z-coordinate of the top (height of walker): {state[0]:.2f} m\n"
+            f"Angle of the top: {state[1]:.2f} rad\n"
+            f"Angle of the thigh joint: {state[2]:.2f} rad\n"
+            f"Angle of the leg joint: {state[3]:.2f} rad\n"
+            f"Angle of the foot joint: {state[4]:.2f} rad\n"
+            f"Angle of the left thigh joint: {state[5]:.2f} rad\n"
+            f"Angle of the left leg joint: {state[6]:.2f} rad\n"
+            f"Angle of the left foot joint: {state[7]:.2f} rad\n"
+            f"Velocity of the x-coordinate of the top: {state[8]:.2f} m/s\n"
+            f"Velocity of the z-coordinate (height) of the top: {state[9]:.2f} m/s\n"
+            f"Angular velocity of the angle of the top: {state[10]:.2f} rad/s\n"
+            f"Angular velocity of the thigh hinge: {state[11]:.2f} rad/s\n"
+            f"Angular velocity of the leg hinge: {state[12]:.2f} rad/s\n"
+            f"Angular velocity of the foot hinge: {state[13]:.2f} rad/s\n"
+            f"Angular velocity of the thigh hinge (left): {state[14]:.2f} rad/s\n"
+            f"Angular velocity of the leg hinge (left): {state[15]:.2f} rad/s\n"
+            f"Angular velocity of the foot hinge (left): {state[16]:.2f} rad/s"
+        )
+        return res
+
+class GameDescriber:
+    def __init__(self, args):
+        self.is_only_local_obs = args.is_only_local_obs == 1
+        self.max_episode_len = args.max_episode_len
+        self.action_desc_dict = {
+        }
+        self.reward_desc_dict = {
+        }
+    
+    def translate_terminate_state(self, state, episode_len, max_episode_len): 
+        return ""
+        
+    def translate_potential_next_state(self, state, action):
+        return ""
+
+    def describe_goal(self):
+        return (
+            "The goal in the Walker2D environment is to coordinate both sets of feet, legs, and thighs "
+            "to move in the forward (right) direction by applying torques to the six hinges connecting "
+            "the six body parts. The objective is to make the robot walk forward."
+        )
+
+    def describe_game(self):
+        return (
+            "In the Walker2D environment, you control a two-dimensional two-legged walker with four main body parts. "
+            "Your objective is to make the walker move forward by coordinating the torques applied to the six hinges "
+            "connecting the body parts. The environment provides observations of the walker's body parts and velocities, "
+            "including the torso, leg, and thigh angles, orientations, and velocities. The goal is to make the walker walk "
+            "forward in the positive x-direction."
+        )
+
+    def describe_action(self):
+        return (
+            "Your next move: \nPlease provide a list of six numerical values, each within the range of [-1, 1], "
+            "representing the torques to be applied at the six hinge joints of the walker. These torques will help "
+            "coordinate the walker's movements and make it walk in the desired direction."
+        )
+
+class BasicStateSequenceTranslator(BasicLevelTranslator):
+    def translate(self, infos, is_current=False):
+        descriptions = []
+        if is_current:
+            state_desc = BasicLevelTranslator().translate(infos[-1]['state'])
+            return state_desc
+        for i, info in enumerate(infos):
+            assert 'state' in info, "info should contain state information"
+            state_desc = BasicLevelTranslator().translate(info['state'])
+            action_desc = (
+                "Torques Applied: "
+                f"Thigh: {info['action'][0]:.2f}, Leg: {info['action'][1]:.2f}, Foot: {info['action'][2]:.2f}, "
+                f"Left Thigh: {info['action'][3]:.2f}, Left Leg: {info['action'][4]:.2f}, Left Foot: {info['action'][5]:.2f}"
+            )
+            reward_desc = f"Result: Reward of {info['reward']:.2f}"
+            next_state_desc = BasicLevelTranslator().translate(info['next_state'])
+            descriptions.append(
+                f"{state_desc}\n{action_desc}\n{reward_desc}\nTransit to\n{next_state_desc}"
+            )
+        return descriptions

main_reflexion.py

@@ -292,7 +292,7 @@ if __name__ == "__main__":
     parser.add_argument(
         "--api_type",
         type=str,
-        default="azure",
+        default="openai",
         choices=["azure", "openai"],
         help="choose api type, now support azure and openai"
     )

record_reflexion.csv

@@ -12,4 +12,11 @@ RepresentedBoxing-v0,1,expert,200.0
 RepresentedPong-v0,1,expert,200.0
 RepresentedMsPacman-v0,1,expert,10000.0
 RepresentedMontezumaRevenge-v0,1,expert,10000.0
-Ant-v4,1,expert,5000
+Ant-v4,1,expert,5000.2
+HalfCheetah-v4,1,expert,12138.8
+Hopper-v4,1,expert,3542.2
+Walker2d-v4,1,expert,5000.0
+Swimmer-v4,1,expert,44.4
+Reacher-v4,1,expert,-2.6
+Pusher-v4,1,expert,-52.3
+

shell/test_mujoco_ant.sh

@@ -1,6 +1,29 @@
+
+# Ant-v4
+
+# COT
+python main_reflexion.py --env_name Ant-v4 --init_summarizer ant_init_translator --curr_summarizer ant_basic_translator --decider cot_actor --prompt_level 1 --num_trails 1
+
+python main_reflexion.py --env_name Ant-v4 --init_summarizer ant_init_translator --curr_summarizer ant_basic_translator --decider cot_actor --prompt_level 3 --num_trails 5 --distiller traj_distiller 
+
+# SPP
+python main_reflexion.py --env_name Ant-v4 --init_summarizer ant_init_translator --curr_summarizer ant_basic_translator --decider spp_actor --prompt_level 1 --num_trails 1
+
+python main_reflexion.py --env_name Ant-v4 --init_summarizer ant_init_translator --curr_summarizer ant_basic_translator --decider spp_actor --prompt_level 3 --num_trails 5 --distiller traj_distiller 
+
+
+
+# REFLEXION
+python main_reflexion.py --env_name Ant-v4 --init_summarizer ant_init_translator --curr_summarizer ant_basic_translator --decider reflexion_actor --prompt_level 1 --num_trails 1 --distiller reflect_distiller 
+
+python main_reflexion.py --env_name Ant-v4 --init_summarizer ant_init_translator --curr_summarizer ant_basic_translator --decider reflexion_actor --prompt_level 3 --num_trails 5 --distiller reflect_distiller 
+
+python main_reflexion.py --env_name Ant-v4 --init_summarizer ant_init_translator --curr_summarizer ant_basic_translator --decider reflexion_actor --prompt_level 5 --num_trails 1 --distiller reflect_distiller 
+
+
 # exe
 python main_reflexion.py --env_name Ant-v4 --init_summarizer ant_init_translator --curr_summarizer ant_basic_translator --decider exe_actor --prompt_level 1 --num_trails 1 --distiller guide_generator --api_type openai
-python main_reflexion.py --env_name CliffWalking-v0 --init_summarizer cliffwalking_init_translator --curr_summarizer cliffwalking_basic_translator --decider exe_actor --prompt_level 2 --num_trails 1 --distiller guide_generator --prompt_path "envs/toy_text/few_shot_examples/cliffwalking"
-python main_reflexion.py --env_name CliffWalking-v0 --init_summarizer cliffwalking_init_translator --curr_summarizer cliffwalking_basic_translator --decider exe_actor --prompt_level 3 --num_trails 5 --distiller guide_generator
-python main_reflexion.py --env_name CliffWalking-v0 --init_summarizer cliffwalking_init_translator --curr_summarizer cliffwalking_basic_translator --decider exe_actor --prompt_level 4 --num_trails 1 --distiller guide_generator --prompt_path "envs/toy_text/few_shot_examples/cliffwalking"
-python main_reflexion.py --env_name CliffWalking-v0 --init_summarizer cliffwalking_init_translator --curr_summarizer cliffwalking_basic_translator --decider exe_actor --prompt_level 5 --num_trails 1 --distiller guide_generator
+
+python main_reflexion.py --env_name Ant-v4 --init_summarizer ant_init_translator --curr_summarizer ant_basic_translator --decider exe_actor --prompt_level 3 --num_trails 5 --distiller guide_generator
+
+python main_reflexion.py --env_name Ant-v4 --init_summarizer ant_init_translator --curr_summarizer ant_basic_translator --decider exe_actor --prompt_level 5 --num_trails 1 --distiller guide_generator

shell/test_mujoco_halfcheetah.sh

@@ -0,0 +1,51 @@
+
+# HalfCheetah-v4
+# Naive Actor
+python main_reflexion.py --env_name HalfCheetah-v4 --init_summarizer halfcheetah_init_translator --curr_summarizer halfcheetah_basic_translator --decider naive_actor --prompt_level 1 --num_trails 1 
+python main_reflexion.py --env_name HalfCheetah-v4 --init_summarizer halfcheetah_init_translator --curr_summarizer halfcheetah_basic_translator --decider naive_actor --prompt_level 2 --num_trails 1 --distiller traj_distiller --prompt_path "envs/classic_control/few_shot_examples/halfcheetahpole" 
+python main_reflexion.py --env_name HalfCheetah-v4 --init_summarizer halfcheetah_init_translator --curr_summarizer halfcheetah_basic_translator --decider naive_actor --prompt_level 3 --num_trails 5 --distiller traj_distiller  
+python main_reflexion.py --env_name HalfCheetah-v4 --init_summarizer halfcheetah_init_translator --curr_summarizer halfcheetah_basic_translator --decider naive_actor --prompt_level 4 --num_trails 1 --distiller traj_distiller --prompt_path "envs/classic_control/few_shot_examples/halfcheetahpole" 
+python main_reflexion.py --env_name HalfCheetah-v4 --init_summarizer halfcheetah_init_translator --curr_summarizer halfcheetah_basic_translator --decider naive_actor --prompt_level 5 --num_trails 1 
+
+# COT
+python main_reflexion.py --env_name HalfCheetah-v4 --init_summarizer halfcheetah_init_translator --curr_summarizer halfcheetah_basic_translator --decider cot_actor --prompt_level 1 --num_trails 1 
+python main_reflexion.py --env_name HalfCheetah-v4 --init_summarizer halfcheetah_init_translator --curr_summarizer halfcheetah_basic_translator --decider cot_actor --prompt_level 2 --num_trails 1 --distiller traj_distiller --prompt_path "envs/classic_control/few_shot_examples/halfcheetahpole" 
+python main_reflexion.py --env_name HalfCheetah-v4 --init_summarizer halfcheetah_init_translator --curr_summarizer halfcheetah_basic_translator --decider cot_actor --prompt_level 3 --num_trails 5 --distiller traj_distiller  
+python main_reflexion.py --env_name HalfCheetah-v4 --init_summarizer halfcheetah_init_translator --curr_summarizer halfcheetah_basic_translator --decider cot_actor --prompt_level 4 --num_trails 1 --distiller traj_distiller --prompt_path "envs/classic_control/few_shot_examples/halfcheetahpole"
+python main_reflexion.py --env_name HalfCheetah-v4 --init_summarizer halfcheetah_init_translator --curr_summarizer halfcheetah_basic_translator --decider cot_actor --prompt_level 5 --num_trails 1 
+
+# self consistency
+python main_reflexion.py --env_name HalfCheetah-v4 --init_summarizer halfcheetah_init_translator --curr_summarizer halfcheetah_basic_translator --decider self_consistency_actor --prompt_level 1 --num_trails 1 
+python main_reflexion.py --env_name HalfCheetah-v4 --init_summarizer halfcheetah_init_translator --curr_summarizer halfcheetah_basic_translator --decider self_consistency_actor --prompt_level 2 --num_trails 1 --distiller traj_distiller --prompt_path "envs/classic_control/few_shot_examples/halfcheetahpole" 
+python main_reflexion.py --env_name HalfCheetah-v4 --init_summarizer halfcheetah_init_translator --curr_summarizer halfcheetah_basic_translator --decider self_consistency_actor --prompt_level 3 --num_trails 5 --distiller traj_distiller  
+python main_reflexion.py --env_name HalfCheetah-v4 --init_summarizer halfcheetah_init_translator --curr_summarizer halfcheetah_basic_translator --decider self_consistency_actor --prompt_level 4 --num_trails 1 --distiller traj_distiller --prompt_path "envs/classic_control/few_shot_examples/halfcheetahpole" 
+python main_reflexion.py --env_name HalfCheetah-v4 --init_summarizer halfcheetah_init_translator --curr_summarizer halfcheetah_basic_translator --decider self_consistency_actor --prompt_level 5 --num_trails 1 
+
+# self-ask
+python main_reflexion.py --env_name HalfCheetah-v4 --init_summarizer halfcheetah_init_translator --curr_summarizer halfcheetah_basic_translator --decider selfask_actor --prompt_level 1 --num_trails 1 
+python main_reflexion.py --env_name HalfCheetah-v4 --init_summarizer halfcheetah_init_translator --curr_summarizer halfcheetah_basic_translator --decider selfask_actor --prompt_level 2 --num_trails 1 --distiller traj_distiller --prompt_path "envs/classic_control/few_shot_examples/halfcheetahpole" 
+python main_reflexion.py --env_name HalfCheetah-v4 --init_summarizer halfcheetah_init_translator --curr_summarizer halfcheetah_basic_translator --decider selfask_actor --prompt_level 3 --num_trails 5 --distiller traj_distiller  
+python main_reflexion.py --env_name HalfCheetah-v4 --init_summarizer halfcheetah_init_translator --curr_summarizer halfcheetah_basic_translator --decider selfask_actor --prompt_level 4 --num_trails 1 --distiller traj_distiller --prompt_path "envs/classic_control/few_shot_examples/halfcheetahpole" 
+python main_reflexion.py --env_name HalfCheetah-v4 --init_summarizer halfcheetah_init_translator --curr_summarizer halfcheetah_basic_translator --decider selfask_actor --prompt_level 5 --num_trails 1 
+
+# SPP
+python main_reflexion.py --env_name HalfCheetah-v4 --init_summarizer halfcheetah_init_translator --curr_summarizer halfcheetah_basic_translator --decider spp_actor --prompt_level 1 --num_trails 1 
+python main_reflexion.py --env_name HalfCheetah-v4 --init_summarizer halfcheetah_init_translator --curr_summarizer halfcheetah_basic_translator --decider spp_actor --prompt_level 2 --num_trails 1 --distiller traj_distiller --prompt_path "envs/classic_control/few_shot_examples/halfcheetahpole" 
+python main_reflexion.py --env_name HalfCheetah-v4 --init_summarizer halfcheetah_init_translator --curr_summarizer halfcheetah_basic_translator --decider spp_actor --prompt_level 3 --num_trails 5 --distiller traj_distiller  
+python main_reflexion.py --env_name HalfCheetah-v4 --init_summarizer halfcheetah_init_translator --curr_summarizer halfcheetah_basic_translator --decider spp_actor --prompt_level 4 --num_trails 1 --distiller traj_distiller --prompt_path "envs/classic_control/few_shot_examples/halfcheetahpole" 
+python main_reflexion.py --env_name HalfCheetah-v4 --init_summarizer halfcheetah_init_translator --curr_summarizer halfcheetah_basic_translator --decider spp_actor --prompt_level 5 --num_trails 1 
+
+# REFLEXION
+python main_reflexion.py --env_name HalfCheetah-v4 --init_summarizer halfcheetah_init_translator --curr_summarizer halfcheetah_basic_translator --decider reflexion_actor --prompt_level 1 --num_trails 1 --distiller reflect_distiller 
+python main_reflexion.py --env_name HalfCheetah-v4 --init_summarizer halfcheetah_init_translator --curr_summarizer halfcheetah_basic_translator --decider reflexion_actor --prompt_level 2 --num_trails 1 --distiller reflect_distiller --prompt_path "envs/classic_control/few_shot_examples/halfcheetahpole" 
+python main_reflexion.py --env_name HalfCheetah-v4 --init_summarizer halfcheetah_init_translator --curr_summarizer halfcheetah_basic_translator --decider reflexion_actor --prompt_level 3 --num_trails 5 --distiller reflect_distiller 
+python main_reflexion.py --env_name HalfCheetah-v4 --init_summarizer halfcheetah_init_translator --curr_summarizer halfcheetah_basic_translator --decider reflexion_actor --prompt_level 4 --num_trails 1 --distiller reflect_distiller --prompt_path "envs/classic_control/few_shot_examples/halfcheetahpole" 
+python main_reflexion.py --env_name HalfCheetah-v4 --init_summarizer halfcheetah_init_translator --curr_summarizer halfcheetah_basic_translator --decider reflexion_actor --prompt_level 5 --num_trails 1 --distiller reflect_distiller 
+
+
+# exe
+python main_reflexion.py --env_name HalfCheetah-v4 --init_summarizer halfcheetah_init_translator --curr_summarizer halfcheetah_basic_translator --decider exe_actor --prompt_level 1 --num_trails 1 --distiller guide_generator --api_type openai
+python main_reflexion.py --env_name HalfCheetah-v4 --init_summarizer halfcheetah_init_translator --curr_summarizer halfcheetah_basic_translator --decider exe_actor --prompt_level 2 --num_trails 1 --distiller guide_generator --prompt_path "envs/toy_text/few_shot_examples/cliffwalking"
+python main_reflexion.py --env_name HalfCheetah-v4 --init_summarizer halfcheetah_init_translator --curr_summarizer halfcheetah_basic_translator --decider exe_actor --prompt_level 3 --num_trails 5 --distiller guide_generator
+python main_reflexion.py --env_name HalfCheetah-v4 --init_summarizer halfcheetah_init_translator --curr_summarizer halfcheetah_basic_translator --decider exe_actor --prompt_level 4 --num_trails 1 --distiller guide_generator --prompt_path "envs/toy_text/few_shot_examples/cliffwalking"
+python main_reflexion.py --env_name HalfCheetah-v4 --init_summarizer halfcheetah_init_translator --curr_summarizer halfcheetah_basic_translator --decider exe_actor --prompt_level 5 --num_trails 1 --distiller guide_generator

shell/test_mujoco_hopper.sh

@@ -0,0 +1,28 @@
+# Hopper-v4
+
+# COT
+python main_reflexion.py --env_name Hopper-v4 --init_summarizer hopper_init_translator --curr_summarizer hopper_basic_translator --decider cot_actor --prompt_level 1 --num_trails 1
+
+python main_reflexion.py --env_name Hopper-v4 --init_summarizer hopper_init_translator --curr_summarizer hopper_basic_translator --decider cot_actor --prompt_level 3 --num_trails 5 --distiller traj_distiller 
+
+# SPP
+python main_reflexion.py --env_name Hopper-v4 --init_summarizer hopper_init_translator --curr_summarizer hopper_basic_translator --decider spp_actor --prompt_level 1 --num_trails 1
+
+python main_reflexion.py --env_name Hopper-v4 --init_summarizer hopper_init_translator --curr_summarizer hopper_basic_translator --decider spp_actor --prompt_level 3 --num_trails 5 --distiller traj_distiller 
+
+
+
+# REFLEXION
+python main_reflexion.py --env_name Hopper-v4 --init_summarizer hopper_init_translator --curr_summarizer hopper_basic_translator --decider reflexion_actor --prompt_level 1 --num_trails 1 --distiller reflect_distiller 
+
+python main_reflexion.py --env_name Hopper-v4 --init_summarizer hopper_init_translator --curr_summarizer hopper_basic_translator --decider reflexion_actor --prompt_level 3 --num_trails 5 --distiller reflect_distiller 
+
+python main_reflexion.py --env_name Hopper-v4 --init_summarizer hopper_init_translator --curr_summarizer hopper_basic_translator --decider reflexion_actor --prompt_level 5 --num_trails 1 --distiller reflect_distiller 
+
+
+# exe
+python main_reflexion.py --env_name Hopper-v4 --init_summarizer hopper_init_translator --curr_summarizer hopper_basic_translator --decider exe_actor --prompt_level 1 --num_trails 1 --distiller guide_generator --api_type openai
+
+python main_reflexion.py --env_name Hopper-v4 --init_summarizer hopper_init_translator --curr_summarizer hopper_basic_translator --decider exe_actor --prompt_level 3 --num_trails 5 --distiller guide_generator
+
+python main_reflexion.py --env_name Hopper-v4 --init_summarizer hopper_init_translator --curr_summarizer hopper_basic_translator --decider exe_actor --prompt_level 5 --num_trails 1 --distiller guide_generator

shell/test_mujoco_invertedDoublePendulum.sh

@@ -0,0 +1,27 @@
+# InvertedDoublePendulum-v4
+
+# COT
+python main_reflexion.py --env_name InvertedDoublePendulum-v4 --init_summarizer invertedDoublePendulum_init_translator --curr_summarizer invertedDoublePendulum_basic_translator --decider cot_actor --prompt_level 1 --num_trails 1
+
+python main_reflexion.py --env_name InvertedDoublePendulum-v4 --init_summarizer invertedDoublePendulum_init_translator --curr_summarizer invertedDoublePendulum_basic_translator --decider cot_actor --prompt_level 3 --num_trails 5 --distiller traj_distiller 
+
+# SPP
+python main_reflexion.py --env_name InvertedDoublePendulum-v4 --init_summarizer invertedDoublePendulum_init_translator --curr_summarizer invertedDoublePendulum_basic_translator --decider spp_actor --prompt_level 1 --num_trails 1
+
+python main_reflexion.py --env_name InvertedDoublePendulum-v4 --init_summarizer invertedDoublePendulum_init_translator --curr_summarizer invertedDoublePendulum_basic_translator --decider spp_actor --prompt_level 3 --num_trails 5 --distiller traj_distiller 
+
+
+# REFLEXION
+python main_reflexion.py --env_name InvertedDoublePendulum-v4 --init_summarizer invertedDoublePendulum_init_translator --curr_summarizer invertedDoublePendulum_basic_translator --decider reflexion_actor --prompt_level 1 --num_trails 1 --distiller reflect_distiller 
+
+python main_reflexion.py --env_name InvertedDoublePendulum-v4 --init_summarizer invertedDoublePendulum_init_translator --curr_summarizer invertedDoublePendulum_basic_translator --decider reflexion_actor --prompt_level 3 --num_trails 5 --distiller reflect_distiller 
+
+python main_reflexion.py --env_name InvertedDoublePendulum-v4 --init_summarizer invertedDoublePendulum_init_translator --curr_summarizer invertedDoublePendulum_basic_translator --decider reflexion_actor --prompt_level 5 --num_trails 1 --distiller reflect_distiller 
+
+
+# exe
+python main_reflexion.py --env_name InvertedDoublePendulum-v4 --init_summarizer invertedDoublePendulum_init_translator --curr_summarizer invertedDoublePendulum_basic_translator --decider exe_actor --prompt_level 1 --num_trails 1 --distiller guide_generator --api_type openai
+
+python main_reflexion.py --env_name InvertedDoublePendulum-v4 --init_summarizer invertedDoublePendulum_init_translator --curr_summarizer invertedDoublePendulum_basic_translator --decider exe_actor --prompt_level 3 --num_trails 5 --distiller guide_generator
+
+python main_reflexion.py --env_name InvertedDoublePendulum-v4 --init_summarizer invertedDoublePendulum_init_translator --curr_summarizer invertedDoublePendulum_basic_translator --decider exe_actor --prompt_level 5 --num_trails 1 --distiller guide_generator

shell/test_mujoco_invertedPendulum.sh

@@ -0,0 +1,25 @@
+# InvertedPendulum-v4
+
+# COT
+python main_reflexion.py --env_name InvertedPendulum-v4 --init_summarizer invertedPendulum_init_translator --curr_summarizer invertedPendulum_basic_translator --decider cot_actor --prompt_level 1 --num_trails 1
+
+python main_reflexion.py --env_name InvertedPendulum-v4 --init_summarizer invertedPendulum_init_translator --curr_summarizer invertedPendulum_basic_translator --decider cot_actor --prompt_level 3 --num_trails 5 --distiller traj_distiller 
+
+# SPP
+python main_reflexion.py --env_name InvertedPendulum-v4 --init_summarizer invertedPendulum_init_translator --curr_summarizer invertedPendulum_basic_translator --decider spp_actor --prompt_level 1 --num_trails 1
+
+python main_reflexion.py --env_name InvertedPendulum-v4 --init_summarizer invertedPendulum_init_translator --curr_summarizer invertedPendulum_basic_translator --decider spp_actor --prompt_level 3 --num_trails 5 --distiller traj_distiller 
+
+
+
+# REFLEXION
+python main_reflexion.py --env_name InvertedPendulum-v4 --init_summarizer invertedPendulum_init_translator --curr_summarizer invertedPendulum_basic_translator --decider reflexion_actor --prompt_level 1 --num_trails 1 --distiller reflect_distiller 
+
+python main_reflexion.py --env_name InvertedPendulum-v4 --init_summarizer invertedPendulum_init_translator --curr_summarizer invertedPendulum_basic_translator --decider reflexion_actor --prompt_level 3 --num_trails 5 --distiller reflect_distiller 
+
+
+
+# exe
+python main_reflexion.py --env_name InvertedPendulum-v4 --init_summarizer invertedPendulum_init_translator --curr_summarizer invertedPendulum_basic_translator --decider exe_actor --prompt_level 1 --num_trails 1 --distiller guide_generator --api_type openai
+
+python main_reflexion.py --env_name InvertedPendulum-v4 --init_summarizer invertedPendulum_init_translator --curr_summarizer invertedPendulum_basic_translator --decider exe_actor --prompt_level 3 --num_trails 5 --distiller guide_generator

shell/test_mujoco_pusher.sh

@@ -0,0 +1,27 @@
+# Pusher-v4
+
+# COT
+python main_reflexion.py --env_name Pusher-v4 --init_summarizer pusher_init_translator --curr_summarizer pusher_basic_translator --decider cot_actor --prompt_level 1 --num_trails 1
+
+python main_reflexion.py --env_name Pusher-v4 --init_summarizer pusher_init_translator --curr_summarizer pusher_basic_translator --decider cot_actor --prompt_level 3 --num_trails 5 --distiller traj_distiller 
+
+# SPP
+python main_reflexion.py --env_name Pusher-v4 --init_summarizer pusher_init_translator --curr_summarizer pusher_basic_translator --decider spp_actor --prompt_level 1 --num_trails 1
+
+python main_reflexion.py --env_name Pusher-v4 --init_summarizer pusher_init_translator --curr_summarizer pusher_basic_translator --decider spp_actor --prompt_level 3 --num_trails 5 --distiller traj_distiller 
+
+
+# REFLEXION
+python main_reflexion.py --env_name Pusher-v4 --init_summarizer pusher_init_translator --curr_summarizer pusher_basic_translator --decider reflexion_actor --prompt_level 1 --num_trails 1 --distiller reflect_distiller 
+
+python main_reflexion.py --env_name Pusher-v4 --init_summarizer pusher_init_translator --curr_summarizer pusher_basic_translator --decider reflexion_actor --prompt_level 3 --num_trails 5 --distiller reflect_distiller 
+
+python main_reflexion.py --env_name Pusher-v4 --init_summarizer pusher_init_translator --curr_summarizer pusher_basic_translator --decider reflexion_actor --prompt_level 5 --num_trails 1 --distiller reflect_distiller 
+
+
+# exe
+python main_reflexion.py --env_name Pusher-v4 --init_summarizer pusher_init_translator --curr_summarizer pusher_basic_translator --decider exe_actor --prompt_level 1 --num_trails 1 --distiller guide_generator --api_type openai
+
+python main_reflexion.py --env_name Pusher-v4 --init_summarizer pusher_init_translator --curr_summarizer pusher_basic_translator --decider exe_actor --prompt_level 3 --num_trails 5 --distiller guide_generator
+
+python main_reflexion.py --env_name Pusher-v4 --init_summarizer pusher_init_translator --curr_summarizer pusher_basic_translator --decider exe_actor --prompt_level 5 --num_trails 1 --distiller guide_generator

shell/test_mujoco_reacher.sh

@@ -0,0 +1,27 @@
+# Reacher-v4
+
+# COT
+python main_reflexion.py --env_name Reacher-v4 --init_summarizer reacher_init_translator --curr_summarizer reacher_basic_translator --decider cot_actor --prompt_level 1 --num_trails 1
+
+python main_reflexion.py --env_name Reacher-v4 --init_summarizer reacher_init_translator --curr_summarizer reacher_basic_translator --decider cot_actor --prompt_level 3 --num_trails 5 --distiller traj_distiller 
+
+# SPP
+python main_reflexion.py --env_name Reacher-v4 --init_summarizer reacher_init_translator --curr_summarizer reacher_basic_translator --decider spp_actor --prompt_level 1 --num_trails 1
+
+python main_reflexion.py --env_name Reacher-v4 --init_summarizer reacher_init_translator --curr_summarizer reacher_basic_translator --decider spp_actor --prompt_level 3 --num_trails 5 --distiller traj_distiller 
+
+
+# REFLEXION
+python main_reflexion.py --env_name Reacher-v4 --init_summarizer reacher_init_translator --curr_summarizer reacher_basic_translator --decider reflexion_actor --prompt_level 1 --num_trails 1 --distiller reflect_distiller 
+
+python main_reflexion.py --env_name Reacher-v4 --init_summarizer reacher_init_translator --curr_summarizer reacher_basic_translator --decider reflexion_actor --prompt_level 3 --num_trails 5 --distiller reflect_distiller 
+
+python main_reflexion.py --env_name Reacher-v4 --init_summarizer reacher_init_translator --curr_summarizer reacher_basic_translator --decider reflexion_actor --prompt_level 5 --num_trails 1 --distiller reflect_distiller 
+
+
+# exe
+python main_reflexion.py --env_name Reacher-v4 --init_summarizer reacher_init_translator --curr_summarizer reacher_basic_translator --decider exe_actor --prompt_level 1 --num_trails 1 --distiller guide_generator --api_type openai
+
+python main_reflexion.py --env_name Reacher-v4 --init_summarizer reacher_init_translator --curr_summarizer reacher_basic_translator --decider exe_actor --prompt_level 3 --num_trails 5 --distiller guide_generator
+
+python main_reflexion.py --env_name Reacher-v4 --init_summarizer reacher_init_translator --curr_summarizer reacher_basic_translator --decider exe_actor --prompt_level 5 --num_trails 1 --distiller guide_generator

shell/test_mujoco_swimmer.sh

@@ -0,0 +1,27 @@
+# Swimmer-v4
+
+# COT
+python main_reflexion.py --env_name Swimmer-v4 --init_summarizer swimmer_init_translator --curr_summarizer swimmer_basic_translator --decider cot_actor --prompt_level 1 --num_trails 1
+
+python main_reflexion.py --env_name Swimmer-v4 --init_summarizer swimmer_init_translator --curr_summarizer swimmer_basic_translator --decider cot_actor --prompt_level 3 --num_trails 5 --distiller traj_distiller 
+
+# SPP
+python main_reflexion.py --env_name Swimmer-v4 --init_summarizer swimmer_init_translator --curr_summarizer swimmer_basic_translator --decider spp_actor --prompt_level 1 --num_trails 1
+
+python main_reflexion.py --env_name Swimmer-v4 --init_summarizer swimmer_init_translator --curr_summarizer swimmer_basic_translator --decider spp_actor --prompt_level 3 --num_trails 5 --distiller traj_distiller 
+
+
+# REFLEXION
+python main_reflexion.py --env_name Swimmer-v4 --init_summarizer swimmer_init_translator --curr_summarizer swimmer_basic_translator --decider reflexion_actor --prompt_level 1 --num_trails 1 --distiller reflect_distiller 
+
+python main_reflexion.py --env_name Swimmer-v4 --init_summarizer swimmer_init_translator --curr_summarizer swimmer_basic_translator --decider reflexion_actor --prompt_level 3 --num_trails 5 --distiller reflect_distiller 
+
+python main_reflexion.py --env_name Swimmer-v4 --init_summarizer swimmer_init_translator --curr_summarizer swimmer_basic_translator --decider reflexion_actor --prompt_level 5 --num_trails 1 --distiller reflect_distiller 
+
+
+# exe
+python main_reflexion.py --env_name Swimmer-v4 --init_summarizer swimmer_init_translator --curr_summarizer swimmer_basic_translator --decider exe_actor --prompt_level 1 --num_trails 1 --distiller guide_generator --api_type openai
+
+python main_reflexion.py --env_name Swimmer-v4 --init_summarizer swimmer_init_translator --curr_summarizer swimmer_basic_translator --decider exe_actor --prompt_level 3 --num_trails 5 --distiller guide_generator
+
+python main_reflexion.py --env_name Swimmer-v4 --init_summarizer swimmer_init_translator --curr_summarizer swimmer_basic_translator --decider exe_actor --prompt_level 5 --num_trails 1 --distiller guide_generator

shell/test_mujoco_walker2d.sh

@@ -0,0 +1,28 @@
+# Walker2d-v4
+
+# COT
+python main_reflexion.py --env_name Walker2d-v4 --init_summarizer walker2d_init_translator --curr_summarizer walker2d_basic_translator --decider cot_actor --prompt_level 1 --num_trails 1
+
+python main_reflexion.py --env_name Walker2d-v4 --init_summarizer walker2d_init_translator --curr_summarizer walker2d_basic_translator --decider cot_actor --prompt_level 3 --num_trails 5 --distiller traj_distiller 
+
+# SPP
+python main_reflexion.py --env_name Walker2d-v4 --init_summarizer walker2d_init_translator --curr_summarizer walker2d_basic_translator --decider spp_actor --prompt_level 1 --num_trails 1
+
+python main_reflexion.py --env_name Walker2d-v4 --init_summarizer walker2d_init_translator --curr_summarizer walker2d_basic_translator --decider spp_actor --prompt_level 3 --num_trails 5 --distiller traj_distiller 
+
+
+
+# REFLEXION
+python main_reflexion.py --env_name Walker2d-v4 --init_summarizer walker2d_init_translator --curr_summarizer walker2d_basic_translator --decider reflexion_actor --prompt_level 1 --num_trails 1 --distiller reflect_distiller 
+
+python main_reflexion.py --env_name Walker2d-v4 --init_summarizer walker2d_init_translator --curr_summarizer walker2d_basic_translator --decider reflexion_actor --prompt_level 3 --num_trails 5 --distiller reflect_distiller 
+
+python main_reflexion.py --env_name Walker2d-v4 --init_summarizer walker2d_init_translator --curr_summarizer walker2d_basic_translator --decider reflexion_actor --prompt_level 5 --num_trails 1 --distiller reflect_distiller 
+
+
+# exe
+python main_reflexion.py --env_name Walker2d-v4 --init_summarizer walker2d_init_translator --curr_summarizer walker2d_basic_translator --decider exe_actor --prompt_level 1 --num_trails 1 --distiller guide_generator --api_type openai
+
+python main_reflexion.py --env_name Walker2d-v4 --init_summarizer walker2d_init_translator --curr_summarizer walker2d_basic_translator --decider exe_actor --prompt_level 3 --num_trails 5 --distiller guide_generator
+
+python main_reflexion.py --env_name Walker2d-v4 --init_summarizer walker2d_init_translator --curr_summarizer walker2d_basic_translator --decider exe_actor --prompt_level 5 --num_trails 1 --distiller guide_generator