first commit

.gitignore

@@ -156,7 +156,7 @@ dmypy.json
 # Cython debug symbols
 cython_debug/
 images/ 
-gpt.py
+# gpt.py
 test.ipynb
 results
 wandb/
@@ -189,4 +189,7 @@ test_
 *.ipynb
 
 # gradio
-flagged
+flagged
+
+# hf
+policy.pth

app.py

@@ -0,0 +1,400 @@
+import envs
+import deciders
+import distillers
+import prompts as task_prompts
+import datetime
+import time
+from envs.translator import InitSummarizer, CurrSummarizer, FutureSummarizer, Translator
+import gym
+import pandas as pd
+import random
+import datetime
+from loguru import logger
+from argparse import Namespace
+import gradio as gr
+import subprocess
+import openai
+import os
+import shutil
+import subprocess
+from pathlib import Path
+from urllib.request import urlretrieve
+
+
+def set_seed(seed):
+    random.seed(seed)         
+
+def main_progress(
+        api_type, openai_key, env_name, decider_name, 
+        prompt_level, num_trails, seed
+    ):
+    init_summarizer = env_name.split("-")[0] + '_init_translator'
+    curr_summarizer = env_name.split("-")[0] + '_basic_translator'
+    if "Represented" not in init_summarizer:
+        init_summarizer = init_summarizer.lower()
+        curr_summarizer = curr_summarizer.lower()
+    args = Namespace(
+        env_name=env_name,
+        init_summarizer=init_summarizer,
+        curr_summarizer=curr_summarizer,
+        decider=decider_name,
+        prompt_level=prompt_level,
+        num_trails=num_trails,
+        seed=seed,
+        future_summarizer=None,
+        env="base_env",
+        gpt_version="gpt-3.5-turbo",
+        render="rgb_array",
+        max_episode_len=200,
+        max_query_tokens=5000,
+        max_tokens=2000,
+        distiller="traj_distiller",
+        prompt_path=None,
+        use_short_mem=1,
+        short_mem_num=10,
+        is_only_local_obs=1,
+        api_type=api_type,
+    )
+
+    if args.api_type != "azure" and args.api_type != "openai":
+        raise ValueError(f"The {args.api_type} is not supported, please use 'azure' or 'openai' !")
+    
+    # Please note when using "azure", the model name is gpt-35-turbo while using "openai", the model name is "gpt-3.5-turbo"
+    if args.api_type == "azure":
+        if args.gpt_version == "gpt-3.5-turbo":
+            args.gpt_version = 'gpt-35-turbo'
+    elif args.api_type == "openai":
+        if args.gpt_version == "gpt-35-turbo":
+            args.gpt_version = 'gpt-3.5-turbo'
+
+    # Get the specified translator, environment, and ChatGPT model
+    env_class = envs.REGISTRY[args.env]
+    init_summarizer = InitSummarizer(envs.REGISTRY[args.init_summarizer], args)
+    curr_summarizer = CurrSummarizer(envs.REGISTRY[args.curr_summarizer])
+    
+    if args.future_summarizer:
+        future_summarizer = FutureSummarizer(
+            envs.REGISTRY[args.future_summarizer],
+            envs.REGISTRY["cart_policies"],
+            future_horizon=args.future_horizon,
+        )
+    else:
+        future_summarizer = None
+
+    decider_class = deciders.REGISTRY[args.decider]
+    distiller_class = distillers.REGISTRY[args.distiller]
+    sampling_env = envs.REGISTRY["sampling_wrapper"](gym.make(args.env_name))
+    if args.prompt_level == 5:
+        prompts_class = task_prompts.REGISTRY[(args.env_name,args.decider)]()
+    else:
+        prompts_class = task_prompts.REGISTRY[(args.decider)]()
+    translator = Translator(
+        init_summarizer, curr_summarizer, future_summarizer, env=sampling_env
+    )
+    environment = env_class(
+        gym.make(args.env_name, render_mode=args.render), translator
+    )
+
+    logfile = (
+        f"llm.log/output-{args.env_name}-{args.decider}-{args.gpt_version}-l{args.prompt_level}"
+        f"-{datetime.datetime.now().timestamp()}.log"
+    )
+
+    logfile_reflexion = (
+        f"llm.log/memory-{args.env_name}-{args.decider}-{args.gpt_version}-l{args.prompt_level}"
+        f"-{datetime.datetime.now().timestamp()}.log"
+    )
+    my_distiller = distiller_class(logfile=logfile_reflexion,args=args)
+
+    args.game_description = environment.game_description
+    args.goal_description = environment.goal_description
+    args.action_description = environment.action_description
+    args.action_desc_dict = environment.action_desc_dict
+    args.reward_desc_dict = environment.reward_desc_dict
+
+    logger.add(logfile, colorize=True, enqueue=True, filter=lambda x: '[Reflexion Memory]' not in x['message'])
+
+    decider = decider_class(openai_key, environment.env.action_space, args, prompts_class, my_distiller, temperature=0.0, logger=logger, max_tokens=args.max_tokens)
+    
+    # Evaluate the translator
+    utilities = []
+    df = pd.read_csv('record_reflexion.csv', sep=',')
+    filtered_df = df[(df['env'] == args.env_name) & (df['decider'] == 'expert') & (df['level'] == 1)]
+    expert_score = filtered_df['avg_score'].item()
+    seeds = [i for i in range(1000)]
+    # prompt_file = "prompt.txt"
+    # f = open(prompt_file,"w+")
+    num_trails = args.num_trails
+    if not "Blackjack" in args.env_name:
+        curriculums = 1
+    else:
+        curriculums = 20
+    for curriculum in range(curriculums):
+        for trail in range(num_trails): 
+            if "Blackjack" in args.env_name:
+                seed = seeds[curriculum*curriculums + num_trails - trail - 1]
+            else:
+                seed = args.seed
+
+            # single run
+            # Reset the environment
+            if not "Blackjack" in args.env_name:
+                set_seed(args.seed)
+                seed = args.seed
+                # Reset the environment
+                state_description, env_info = environment.reset(seed=args.seed)
+            else:
+                set_seed(seed)
+                # Reset the environment
+                state_description, env_info = environment.reset(seed=seed)
+            game_description = environment.get_game_description()
+            goal_description = environment.get_goal_description()
+            action_description = environment.get_action_description()
+
+            # Initialize the statistics
+            frames = []
+            utility = 0
+            current_total_tokens = 0
+            current_total_cost = 0
+            # state_description, prompt, response, action = None, None, None, None
+            start_time = datetime.datetime.now()
+            # Run the game for a maximum number of steps
+            for round in range(args.max_episode_len):
+                # Keep asking ChatGPT for an action until it provides a valid one
+                error_flag = True
+                retry_num = 1
+                for error_i in range(retry_num):
+                    try:
+                        action, prompt, response, tokens, cost = decider.act(
+                            state_description,
+                            action_description,
+                            env_info,
+                            game_description,
+                            goal_description,
+                            logfile
+                        )
+
+                        state_description, reward, termination, truncation, env_info = environment.step_llm(
+                            action
+                        )
+                        if "Cliff" in args.env_name or "Frozen" in args.env_name:
+                            decider.env_history.add('reward', env_info['potential_state'] + environment.reward_desc_dict[reward])
+                        else:
+                            decider.env_history.add('reward', f"The player get rewards {reward}.")
+                            
+                        utility += reward
+
+                        # Update the statistics
+                        current_total_tokens += tokens
+                        current_total_cost += cost
+                        error_flag = False
+                        break
+                    except Exception as e:
+                        print(e)
+                        raise e
+                        if error_i < retry_num-1:
+                            if "Cliff" in args.env_name or "Frozen" in args.env_name:
+                                decider.env_history.remove_invalid_state()
+                            decider.env_history.remove_invalid_state()
+                        if logger:
+                            logger.debug(f"Error: {e}, Retry! ({error_i+1}/{retry_num})")
+                        continue
+                if error_flag:
+                    action = decider.default_action
+                    state_description, reward, termination, truncation, env_info = environment.step_llm(
+                            action
+                        )
+
+                    decider.env_history.add('action', decider.default_action)
+
+                    if "Cliff" in args.env_name or "Frozen" in args.env_name:
+                        # decider.env_history.add('reward', reward)
+                        decider.env_history.add('reward', env_info['potential_state'] + environment.reward_desc_dict[reward])
+                    utility += reward
+
+                    
+                    logger.info(f"Seed: {seed}")
+                    logger.info(f'The optimal action is: {decider.default_action}.')
+                    logger.info(f"Now it is round {round}.")
+                else:
+                    current_total_tokens += tokens
+                    current_total_cost += cost
+                    logger.info(f"Seed: {seed}")
+                    logger.info(f"current_total_tokens: {current_total_tokens}")
+                    logger.info(f"current_total_cost: {current_total_cost}")
+                    logger.info(f"Now it is round {round}.")
+
+                # return results
+                yield environment.render(), state_description, prompt, response, action
+
+                if termination or truncation:
+                    if logger:
+                        logger.info(f"Terminated!")
+                    break
+                time.sleep(5)
+            decider.env_history.add(
+                'terminate_state', environment.get_terminate_state(round+1, args.max_episode_len))
+            decider.env_history.add("cummulative_reward", str(utility))
+            # Record the final reward
+            if logger:
+                logger.info(f"Cummulative reward: {utility}.")
+                end_time = datetime.datetime.now()
+                time_diff = end_time - start_time
+                logger.info(f"Time consumer: {time_diff.total_seconds()} s")
+            
+            utilities.append(utility)
+            # TODO: set env sucess utility threshold
+            if trail < num_trails -1:
+                if args.decider in ['reflexion']:
+                    if utility < expert_score: 
+                        decider.update_mem() 
+                else:
+                    decider.update_mem() 
+        decider.clear_mem()
+    return utilities
+
+# def pause():
+#     for i in range(31415926):
+#         time.sleep(0.1)
+#         yield i            
+
+if __name__ == "__main__":
+
+
+    # install Atari ROMs
+    subprocess.run(['AutoROM', '--accept-license'])
+
+    # install mujoco
+
+    # Step 1: Download and set up MuJoCo
+    MUJOCO_URL = "https://github.com/google-deepmind/mujoco/releases/download/2.1.0/mujoco210-linux-x86_64.tar.gz"
+    MUJOCO_FILENAME = "mujoco210-linux-x86_64.tar.gz"
+
+    # Download MuJoCo
+    print("Downloading MuJoCo...")
+    urlretrieve(MUJOCO_URL, MUJOCO_FILENAME)
+
+    # Create and move to ~/.mujoco directory
+    mujoco_dir = Path.home() / ".mujoco"
+    mujoco_dir.mkdir(exist_ok=True)
+    shutil.move(MUJOCO_FILENAME, str(mujoco_dir / MUJOCO_FILENAME))
+
+    # Extract the file
+    print("Extracting MuJoCo...")
+    subprocess.run(["tar", "-zxvf", str(mujoco_dir / MUJOCO_FILENAME)], cwd=mujoco_dir)
+
+    # Edit .bashrc
+    bashrc_path = Path.home() / ".bashrc"
+    mujoco_path = mujoco_dir / "mujoco210" / "bin"
+    export_line = f"export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:{mujoco_path}\n"
+
+    with open(bashrc_path, "a") as bashrc_file:
+        bashrc_file.write(export_line)
+
+    # Set LD_LIBRARY_PATH for the current process
+    ld_lib_path = os.environ.get("LD_LIBRARY_PATH", "")
+    new_ld_lib_path = f"{ld_lib_path}{mujoco_path}"
+    os.environ["LD_LIBRARY_PATH"] = new_ld_lib_path
+
+    # Step 2: Install gym[mujoco]
+    print("Installing gym[MuJoCo]...")
+    subprocess.run(["pip", "install", "gym[mujoco]"])
+
+    # # Set render
+    os.environ["MUJOCO_GL"] = "egl"
+    # os.environ["DISPLAY"] = ":0"
+    # print(f'LD_LIBRARY_PATH: {os.environ["LD_LIBRARY_PATH"]}')
+    # assert os.path.exists(str(mujoco_path))
+    # subprocess.run("cp -r /home/user/.mujoco/mujoco210/bin/* /usr/lib/", shell=True)
+    # import mujoco_py
+    # flag = 'gpu' in str(mujoco_py.cymj).split('/')[-1]
+    # print(f'flag: {flag}')
+    # if not flag:
+    #     ld_lib_path = os.environ.get("LD_LIBRARY_PATH", "")
+    #     new_ld_lib_path = f"{ld_lib_path}:/usr/lib/nvidia-000"
+    #     os.environ["LD_LIBRARY_PATH"] = new_ld_lib_path
+    #     subprocess.run(["sudo", "mkdir", "-p", "/usr/lib/nvidia-000"])
+    #     assert 'gpu' in str(mujoco_py.cymj).split('/')[-1]
+
+
+    custom_css = """
+        #render {
+            flex-grow: 1;
+        }
+        #input_text .tabs {
+            display: flex;
+            flex-direction: column;
+            flex-grow: 1;
+        }
+        #input_text .tabitem[style="display: block;"] {
+            flex-grow: 1;
+            display: flex !important;
+        }
+        #input_text .gap {
+            flex-grow: 1;
+        }
+        #input_text .form {
+            flex-grow: 1 !important;
+        }
+        #input_text .form > :last-child{
+            flex-grow: 1;
+        }
+    """
+
+    with gr.Blocks(theme=gr.themes.Monochrome(), css=custom_css) as demo:
+        with gr.Row():
+            api_type = gr.Dropdown(["azure", "openai"], label="API Type", scale=1)
+            openai_key = gr.Textbox(label="OpenAI API Key", type="password", scale=3)
+        with gr.Row():
+            env_name = gr.Dropdown(
+                ["CartPole-v0",
+                 "LunarLander-v2",
+                 "Acrobot-v1",
+                 "MountainCar-v0",
+                 "Blackjack-v1",
+                 "Taxi-v3",
+                 "CliffWalking-v0",
+                 "FrozenLake-v1",
+                 "MountainCarContinuous-v0",
+                 "Ant-v4",
+                 "RepresentedBoxing-v0", 
+                 "RepresentedPong-v0", 
+                 "RepresentedMsPacman-v0", 
+                 "RepresentedMontezumaRevenge-v0"], 
+                label="Environment Name")
+            decider_name = gr.Dropdown(
+                ["naive_actor", 
+                 "cot_actor", 
+                 "spp_actor", 
+                 "reflexion_actor"], 
+                 label="Decider")
+            # prompt_level = gr.Dropdown([1, 2, 3, 4, 5], label="Prompt Level")
+            # TODO: support more prompt levels
+            prompt_level = gr.Dropdown([1, 3], label="Prompt Level")
+        with gr.Row():
+            num_trails = gr.Slider(1, 100, 1, label="Number of Trails", scale=2)
+            seed = gr.Slider(1, 1000, 1, label="Seed", scale=2)
+            run = gr.Button("Run", scale=1)
+            # pause_ = gr.Button("Pause")
+            # resume = gr.Button("Resume")
+            stop = gr.Button("Stop", scale=1)
+        with gr.Row():
+            with gr.Column():
+                render = gr.Image(label="render", elem_id="render")
+            with gr.Column(elem_id="input_text"):
+                state = gr.Textbox(label="translated state")
+                prompt = gr.Textbox(label="prompt", max_lines=20)
+        with gr.Row():
+            response = gr.Textbox(label="response")
+            action = gr.Textbox(label="parsed action")
+        run_event = run.click(
+            fn=main_progress, 
+            inputs=[
+                api_type, openai_key, env_name, 
+                decider_name, prompt_level, num_trails, seed], 
+            outputs=[render, state, prompt, response, action])
+        stop.click(fn=None, inputs=None, outputs=None, cancels=[run_event])
+        # pause_event = pause_.click(fn=pause, inputs=None, outputs=None)
+        # resume.click(fn=None, inputs=None, outputs=None, cancels=[pause_event])
+
+    demo.launch()

deciders/act.py

@@ -26,7 +26,7 @@ class RandomAct():
         return action, '', '', '', 0, 0
 
 class NaiveAct(gpt):
-    def __init__(self, action_space, args, prompts, distiller, temperature=0.0, max_tokens=2048, logger=None):
+    def __init__(self, openai_key, action_space, args, prompts, distiller, temperature=0.0, max_tokens=2048, logger=None):
         self.action_space = action_space
         self.temperature = temperature
         self.action_desc_dict = args.action_desc_dict
@@ -39,7 +39,7 @@ class NaiveAct(gpt):
         else:
             model = args.gpt_version
         self.encoding = tiktoken.encoding_for_model(model)
-        super().__init__(args)
+        super().__init__(args, openai_key)
         self.distiller = distiller
         self.fewshot_example_initialization(args.prompt_level, args.prompt_path, distiller = self.distiller)
         if isinstance(self.action_space, Discrete):

deciders/cot.py

@@ -17,8 +17,8 @@ from .utils import run_chain
 
 
 class ChainOfThought(NaiveAct):
-    def __init__(self, action_space, args, prompts, distiller, temperature=0.1, max_tokens=None, logger=None):
-        super().__init__(action_space, args, prompts, distiller, temperature, max_tokens,logger)
+    def __init__(self, openai_key, action_space, args, prompts, distiller, temperature=0.1, max_tokens=None, logger=None):
+        super().__init__(openai_key, action_space, args, prompts, distiller, temperature, max_tokens,logger)
 
     def act(
         self,

deciders/exe.py

@@ -20,8 +20,8 @@ from loguru import logger
 
 
 class EXE(NaiveAct):
-    def __init__(self, action_space, args, prompts, distiller, temperature=0., max_tokens=None, logger=None, fixed_suggestion=None, fixed_insight=None):
-        super().__init__(action_space, args, prompts, distiller, temperature, max_tokens, logger)
+    def __init__(self, openai_key, action_space, args, prompts, distiller, temperature=0., max_tokens=None, logger=None, fixed_suggestion=None, fixed_insight=None):
+        super().__init__(openai_key, action_space, args, prompts, distiller, temperature, max_tokens, logger)
         self.pre_memory = []
         self.post_memory = []
         self.is_first = True

deciders/gpt.py

@@ -0,0 +1,11 @@
+import openai
+class gpt: 
+    def __init__(self, args, api_key=None):
+        if args.api_type == "azure":
+            openai.api_type = "azure"
+            openai.api_version = "2023-05-15"
+            # Your Azure OpenAI resource's endpoint value.
+            openai.api_base = "https://midivi-main-scu1.openai.azure.com/"
+            openai.api_key = api_key
+        else:
+            openai.api_key = api_key

deciders/reflexion.py

@@ -19,8 +19,8 @@ from .utils import run_chain
 
 
 class Reflexion(NaiveAct):
-    def __init__(self, action_space, args, prompts, distiller, temperature=0.1, max_tokens=None, logger=None):
-        super().__init__(action_space, args, prompts, distiller, temperature, max_tokens, logger)
+    def __init__(self, openai_key, action_space, args, prompts, distiller, temperature=0.1, max_tokens=None, logger=None):
+        super().__init__(openai_key, action_space, args, prompts, distiller, temperature, max_tokens, logger)
     
     def num_tokens_from_string(self,string: str) -> int:
         """Returns the number of tokens in a text string."""

deciders/self_consistency.py

@@ -17,9 +17,9 @@ from .utils import run_chain
 
 
 class SelfConsistency(NaiveAct):
-    def __init__(self, action_space, args, prompts, distiller, temperature=0.1, max_tokens=None, logger=None):
+    def __init__(self, openai_key, action_space, args, prompts, distiller, temperature=0.1, max_tokens=None, logger=None):
         temperature = 0.7
-        super().__init__(action_space, args, prompts, distiller, temperature, max_tokens, logger)
+        super().__init__(openai_key, action_space, args, prompts, distiller, temperature, max_tokens, logger)
         self.temperature = temperature
 
     def act(

deciders/selfask.py

@@ -17,8 +17,8 @@ from .utils import run_chain
 
 
 class SelfAskAct(NaiveAct):
-    def __init__(self, action_space, args, prompts, distiller, temperature=0.1, max_tokens=None, logger=None):
-        super().__init__(action_space, args, prompts, distiller, temperature, max_tokens,logger)
+    def __init__(self, openai_key, action_space, args, prompts, distiller, temperature=0.1, max_tokens=None, logger=None):
+        super().__init__(openai_key, action_space, args, prompts, distiller, temperature, max_tokens,logger)
     
     def act(
         self,

deciders/spp.py

@@ -16,8 +16,8 @@ from .act import NaiveAct
 from .utils import run_chain
 
 class SPP(NaiveAct):
-    def __init__(self, action_space, args, prompts, distiller, temperature=0.1, max_tokens=None, logger=None):
-        super().__init__(action_space, args, prompts, distiller, temperature, max_tokens, logger)
+    def __init__(self, openai_key, action_space, args, prompts, distiller, temperature=0.1, max_tokens=None, logger=None):
+        super().__init__(openai_key, action_space, args, prompts, distiller, temperature, max_tokens, logger)
 
     def act(
         self,

deciders/utils.py

@@ -19,8 +19,30 @@ Model = Literal["gpt-4", "gpt-35-turbo", "text-davinci-003"]
 # from .gpt import gpt
 # gpt().__init__()
 
-import timeout_decorator
-@timeout_decorator.timeout(30)
+# import timeout_decorator
+# @timeout_decorator.timeout(30)
+# def run_chain(chain, *args, **kwargs):
+#     return chain.run(*args, **kwargs)
+import concurrent.futures
+
+def timeout_decorator(timeout):
+    def decorator(function):
+        def wrapper(*args, **kwargs):
+            with concurrent.futures.ThreadPoolExecutor(max_workers=1) as executor:
+                future = executor.submit(function, *args, **kwargs)
+                try:
+                    return future.result(timeout)
+                except concurrent.futures.TimeoutError:
+                    raise RuntimeError(
+                        f"Function '{function.__name__}' timed out after {timeout} seconds"
+                    )
+                except Exception as e:
+                    raise e
+        return wrapper
+    return decorator
+
+
+@timeout_decorator(30)
 def run_chain(chain, *args, **kwargs):
     return chain.run(*args, **kwargs)
 
@@ -86,5 +108,4 @@ def get_chat(prompt: str, api_type: str = "azure", model: str = "gpt-35-turbo",
             temperature=temperature,
             # request_timeout = 1
         )
-        return response.choices[0]["message"]["content"]
-
+        return response.choices[0]["message"]["content"]

envs/__init__.py

@@ -18,24 +18,25 @@ from .atari import mspacman_policies, mspacman_translator
 from .atari import montezumarevenge_policies, montezumarevenge_translator
 register_environments()
 
+from .mujoco import ant_translator, ant_policies
 
 REGISTRY = {}
 REGISTRY["sampling_wrapper"] = SettableStateEnv
 REGISTRY["base_env"] = BaseEnv
-REGISTRY["cart_init_translator"] = cartpole_translator.GameDescriber
-REGISTRY["cart_basic_translator"] = cartpole_translator.BasicStateSequenceTranslator
+REGISTRY["cartpole_init_translator"] = cartpole_translator.GameDescriber
+REGISTRY["cartpole_basic_translator"] = cartpole_translator.BasicStateSequenceTranslator
 REGISTRY["acrobot_init_translator"] = acrobot_translator.GameDescriber
 REGISTRY["acrobot_basic_translator"] = acrobot_translator.BasicStateSequenceTranslator
 REGISTRY["mountaincar_init_translator"] = mountaincar_translator.GameDescriber
 REGISTRY["mountaincar_basic_translator"] = mountaincar_translator.BasicStateSequenceTranslator
 
-REGISTRY["cart_policies"] = [cartpole_policies.dedicated_1_policy, cartpole_policies.dedicated_2_policy, cartpole_policies.pseudo_random_policy, cartpole_policies.real_random_policy]
+REGISTRY["cartpole_policies"] = [cartpole_policies.dedicated_1_policy, cartpole_policies.dedicated_2_policy, cartpole_policies.pseudo_random_policy, cartpole_policies.real_random_policy]
 REGISTRY["acrobot_policies"] = [acrobot_policies.dedicated_1_policy, acrobot_policies.dedicated_2_policy, acrobot_policies.dedicated_3_policy, acrobot_policies.pseudo_random_policy, acrobot_policies.real_random_policy]
 REGISTRY["mountaincar_policies"] = [mountaincar_policies.dedicated_1_policy, mountaincar_policies.dedicated_2_policy, mountaincar_policies.dedicated_3_policy, mountaincar_policies.pseudo_random_policy, mountaincar_policies.real_random_policy]
 
-REGISTRY["lunarLander_init_translator"] = LunarLander_translator.GameDescriber
-REGISTRY["lunarLander_basic_translator"] = LunarLander_translator.BasicStateSequenceTranslator
-REGISTRY["lunarLander_policies"] = [LunarLander_policies.dedicated_1_policy, LunarLander_policies.dedicated_2_policy, LunarLander_policies.dedicated_3_policy,LunarLander_policies.dedicated_4_policy, LunarLander_policies.pseudo_random_policy, LunarLander_policies.real_random_policy]
+REGISTRY["lunarlander_init_translator"] = LunarLander_translator.GameDescriber
+REGISTRY["lunarlander_basic_translator"] = LunarLander_translator.BasicStateSequenceTranslator
+REGISTRY["lunarlander_policies"] = [LunarLander_policies.dedicated_1_policy, LunarLander_policies.dedicated_2_policy, LunarLander_policies.dedicated_3_policy,LunarLander_policies.dedicated_4_policy, LunarLander_policies.pseudo_random_policy, LunarLander_policies.real_random_policy]
 
 REGISTRY["blackjack_init_translator"] = blackjack_translator.GameDescriber
 REGISTRY["blackjack_basic_translator"] = blackjack_translator.BasicStateSequenceTranslator
@@ -54,9 +55,9 @@ REGISTRY["frozenlake_basic_translator"] = frozenlake_translator.BasicStateSequen
 REGISTRY["frozenlake_policies"] = [frozenlake_policies.dedicated_1_policy, frozenlake_policies.dedicated_2_policy, frozenlake_policies.dedicated_3_policy, frozenlake_policies.dedicated_4_policy, frozenlake_policies.pseudo_random_policy, frozenlake_policies.real_random_policy]
 
 
-REGISTRY["mountaincarContinuous_init_translator"] = mountaincarContinuous_translator.GameDescriber
-REGISTRY["mountaincarContinuous_basic_translator"] = mountaincarContinuous_translator.BasicStateSequenceTranslator
-REGISTRY["mountaincarContinuous_policies"] = [mountaincarContinuous_policies.pseudo_random_policy, mountaincarContinuous_policies.real_random_policy]
+REGISTRY["mountaincarcontinuous_init_translator"] = mountaincarContinuous_translator.GameDescriber
+REGISTRY["mountaincarcontinuous_basic_translator"] = mountaincarContinuous_translator.BasicStateSequenceTranslator
+REGISTRY["mountaincarcontinuous_policies"] = [mountaincarContinuous_policies.pseudo_random_policy, mountaincarContinuous_policies.real_random_policy]
 
 
 REGISTRY["RepresentedBoxing_init_translator"] = Boxing_translator.GameDescriber
@@ -138,47 +139,6 @@ REGISTRY["RepresentedMontezumaRevenge_basic_policies"] = [
     montezumarevenge_policies.dedicated_18_policy,
 ]
 
-REGISTRY["RepresentedMsPacman_init_translator"] = mspacman_translator.GameDescriber
-REGISTRY["RepresentedMsPacman_basic_translator"] = mspacman_translator.BasicStateSequenceTranslator
-REGISTRY["RepresentedMsPacman_basic_policies"] = [
-    mspacman_policies.real_random_policy,
-    mspacman_policies.pseudo_random_policy,
-    mspacman_policies.dedicated_1_policy,
-    mspacman_policies.dedicated_2_policy,
-    mspacman_policies.dedicated_3_policy,
-    mspacman_policies.dedicated_4_policy,
-    mspacman_policies.dedicated_5_policy,
-    mspacman_policies.dedicated_6_policy,
-    mspacman_policies.dedicated_7_policy,
-    mspacman_policies.dedicated_8_policy,
-    mspacman_policies.dedicated_9_policy,
-]
-
-REGISTRY["RepresentedMontezumaRevenge_init_translator"] = montezumarevenge_translator.GameDescriber
-REGISTRY["RepresentedMontezumaRevenge_basic_translator"] = montezumarevenge_translator.BasicStateSequenceTranslator
-REGISTRY["RepresentedMontezumaRevenge_basic_policies"] = [
-    montezumarevenge_policies.real_random_policy,
-    montezumarevenge_policies.pseudo_random_policy,
-    montezumarevenge_policies.dedicated_1_policy,
-    montezumarevenge_policies.dedicated_2_policy,
-    montezumarevenge_policies.dedicated_3_policy,
-    montezumarevenge_policies.dedicated_4_policy,
-    montezumarevenge_policies.dedicated_5_policy,
-    montezumarevenge_policies.dedicated_6_policy,
-    montezumarevenge_policies.dedicated_7_policy,
-    montezumarevenge_policies.dedicated_8_policy,
-    montezumarevenge_policies.dedicated_9_policy,
-    montezumarevenge_policies.dedicated_10_policy,
-    montezumarevenge_policies.dedicated_11_policy,
-    montezumarevenge_policies.dedicated_12_policy,
-    montezumarevenge_policies.dedicated_13_policy,
-    montezumarevenge_policies.dedicated_14_policy,
-    montezumarevenge_policies.dedicated_15_policy,
-    montezumarevenge_policies.dedicated_16_policy,
-    montezumarevenge_policies.dedicated_17_policy,
-    montezumarevenge_policies.dedicated_18_policy,
-]
-
 ## For mujoco env
 
 
@@ -196,12 +156,12 @@ from .mujoco import walker2d_translator, walker2d_policies
 
 
 
-REGISTRY["invertedPendulum_init_translator"] = invertedPendulum_translator.GameDescriber
-REGISTRY["invertedPendulum_basic_translator"] = invertedPendulum_translator.BasicStateSequenceTranslator
-REGISTRY["invertedPendulum_policies"] = [invertedPendulum_policies.pseudo_random_policy, invertedPendulum_policies.real_random_policy]
-REGISTRY["invertedDoublePendulum_init_translator"] = invertedDoublePendulum_translator.GameDescriber
-REGISTRY["invertedDoublePendulum_basic_translator"] = invertedDoublePendulum_translator.BasicStateSequenceTranslator
-REGISTRY["invertedDoublePendulum_policies"] = [invertedDoublePendulum_policies.pseudo_random_policy, invertedDoublePendulum_policies.real_random_policy]
+REGISTRY["invertedpendulum_init_translator"] = invertedPendulum_translator.GameDescriber
+REGISTRY["invertedpendulum_basic_translator"] = invertedPendulum_translator.BasicStateSequenceTranslator
+REGISTRY["invertedpendulum_policies"] = [invertedPendulum_policies.pseudo_random_policy, invertedPendulum_policies.real_random_policy]
+REGISTRY["inverteddoublependulum_init_translator"] = invertedDoublePendulum_translator.GameDescriber
+REGISTRY["inverteddoublependulum_basic_translator"] = invertedDoublePendulum_translator.BasicStateSequenceTranslator
+REGISTRY["inverteddoublependulum_policies"] = [invertedDoublePendulum_policies.pseudo_random_policy, invertedDoublePendulum_policies.real_random_policy]
 
 
 REGISTRY["swimmer_init_translator"] = swimmer_translator.GameDescriber

packages.txt

@@ -0,0 +1,10 @@
+swig
+libosmesa6-dev 
+libgl1-mesa-glx 
+libglfw3
+libglew-dev
+patchelf
+libxrender1 
+libgl1-mesa-dev
+xpra
+libglfw3-dev

requirements.txt

@@ -0,0 +1,188 @@
+# # _libgcc_mutex==0.1
+# # _openmp_mutex==5.1
+# asttokens==2.0.5
+# async-timeout==4.0.2
+# backcall==0.2.0
+# # blas==1.0
+# brotlipy==0.7.0
+# # ca-certificates==2023.01.10
+# cached-property==1.5.2
+# cffi==1.15.1
+# chardet==4.0.0
+# comm==0.1.2
+# cryptography==39.0.1
+# # cudatoolkit==11.3.1
+# debugpy==1.5.1
+# decorator==5.1.1
+# executing==0.8.3
+# frozenlist==1.3.3
+# # hdf5==1.10.6
+# idna==3.4
+# importlib_metadata==6.0.0
+# intel-openmp==2023.1.0
+# ipykernel==6.19.2
+# ipython==8.12.0
+# jedi==0.18.1
+# jupyter_client==8.1.0
+# jupyter_core==5.3.0
+# # ld_impl_linux-64==2.38
+# # libffi==3.4.4
+# # libgcc-ng==11.2.0
+# # libgfortran-ng==11.2.0
+# # libgfortran5==11.2.0
+# # libgomp==11.2.0
+# # libllvm14==14.0.6
+# # libprotobuf==3.20.3
+# # libsodium==1.0.18
+# # libstdcxx-ng==11.2.0
+# matplotlib-inline==0.1.6
+# mkl==2023.1.0
+# mkl-service==2.4.0
+# mkl_fft==1.3.6
+# mkl_random==1.2.2
+# # ncurses==6.4
+# nest-asyncio==1.5.6
+# numpy==1.24.3
+# # numpy-base==1.24.3
+# # openssl==3.0.10
+# packaging==23.1
+# parso==0.8.3
+# # pcre==8.45
+# pexpect==4.8.0
+# pickleshare==0.7.5
+# pip==23.1.2
+# platformdirs==2.5.2
+# prompt-toolkit==3.0.36
+# ptyprocess==0.7.0
+# pure_eval==0.2.2
+# pycparser==2.21
+# pygments==2.15.1
+# pyopenssl==23.0.0
+# pysocks==1.7.1
+# # python==3.8.16
+# python-dateutil==2.8.2
+# # python_abi==3.8
+# pyzmq==25.1.0
+# # readline==8.2
+# setuptools==67.8.0
+# six==1.16.0
+# # sqlite==3.41.2
+# stack_data==0.2.0
+# tbb==2021.8.0
+# # tk==8.6.12
+# tornado==6.2
+# traitlets==5.7.1
+# typing_extensions==4.7.1
+# wcwidth==0.2.5
+# wheel==0.38.4
+# # xz==5.4.2
+# # yaml==0.2.5
+# # zeromq==4.3.4
+# # zlib==1.2.13
+ale-py==0.8.1
+absl-py==1.4.0
+aiohttp==3.8.4
+aiosignal==1.3.1
+annotated-types==0.5.0
+anyio==3.7.1
+appdirs==1.4.4
+aquarel==0.0.5
+attrs==23.1.0
+box2d-py==2.3.5
+cachetools==5.3.1
+certifi==2023.5.7
+charset-normalizer==3.1.0
+click==8.1.6
+cloudpickle==2.2.1
+colorama==0.4.6
+contourpy==1.1.0
+cycler==0.11.0
+dataclasses-json==0.5.14
+distro==1.8.0
+docker-pycreds==0.4.0
+exceptiongroup==1.2.0
+filelock==3.12.3
+fonttools==4.40.0
+fsspec==2023.6.0
+gitdb==4.0.10
+gitpython==3.1.32
+google-auth==2.22.0
+google-auth-oauthlib==1.0.0
+greenlet==2.0.2
+grpcio==1.57.0
+gym==0.26.2
+gym-notices==0.0.8
+gym[accept-rom-license]
+h11==0.14.0
+h5py==3.9.0
+httpcore==1.0.2
+httpx==0.25.2
+# huggingface-hub==0.16.4
+importlib-metadata==6.6.0
+importlib-resources==5.12.0
+joblib==1.3.2
+kiwisolver==1.4.4
+langchain==0.0.270
+langsmith==0.0.25
+llvmlite==0.40.1
+logger==1.4
+loguru==0.7.0
+markdown==3.4.4
+markupsafe==2.1.3
+marshmallow==3.20.1
+matplotlib==3.7.1
+multidict==6.0.4
+mypy-extensions==1.0.0
+numba==0.57.1
+numexpr==2.8.5
+oauthlib==3.2.2
+openai==0.27.8
+pandas==2.0.3
+pathtools==0.1.2
+pillow==9.5.0
+protobuf==3.19.6
+psutil==5.9.5
+pyasn1==0.5.0
+pyasn1-modules==0.3.0
+# pydantic==1.10.11
+# pydantic-core==2.6.1
+pygame==2.1.0
+pyparsing==3.0.9
+pytz==2023.3.post1
+pyyaml==6.0.1
+regex==2023.8.8
+requests==2.31.0
+requests-oauthlib==1.3.1
+rsa==4.9
+safetensors==0.3.3
+seaborn==0.13.0
+sentry-sdk==1.28.1
+setproctitle==1.3.2
+smmap==5.0.0
+sniffio==1.3.0
+sqlalchemy==2.0.20
+swig==4.1.1
+tenacity==8.2.3
+tensorboard==2.14.0
+tensorboard-data-server==0.7.1
+threadpoolctl==3.2.0
+tiktoken==0.4.0
+timeout-decorator==0.5.0
+tokenizers==0.13.3
+tqdm==4.65.0
+transformers==4.30.2
+typing-inspect==0.9.0
+tzdata==2023.3
+urllib3==1.26.16
+v==1
+wandb==0.15.5
+werkzeug==2.3.7
+win32-setctime==1.1.0
+yarl==1.9.2
+zipp==3.15.0
+git+https://[email protected]/hyyh28/atari-representation-learning.git
+gradio
+# gradio==4.13.0
+mujoco-py==2.1.2.14
+cython==0.29.37
+ruamel.yaml==0.18.5

yaml2rep.py

@@ -0,0 +1,17 @@
+import ruamel.yaml
+
+yaml = ruamel.yaml.YAML()
+data = yaml.load(open('environment.yaml'))
+
+requirements = []
+for dep in data['dependencies']:
+    if isinstance(dep, str):
+        package, package_version = dep.split('=')
+        requirements.append(package + '==' + package_version)
+    elif isinstance(dep, dict):
+        for preq in dep.get('pip', []):
+            requirements.append(preq)
+
+with open('requirements.txt', 'w') as fp:
+    for requirement in requirements:
+       print(requirement, file=fp)