and so it begins...

rllib/ppo-asm.yaml

@@ -0,0 +1,30 @@
+asm:
+    env: src.asm.asm
+    run: PPO
+    stop:
+        time_total_s: 24000
+    config:
+        lambda: 0.95
+        kl_coeff: 0.5
+        clip_param: 0.2
+        vf_clip_param: 400.0
+        entropy_coeff: 0.0001
+        rollout_fragment_length: auto
+        num_sgd_iter: 10
+        num_envs_per_worker: 24
+        min_time_s_per_iteration: 360
+        lr: 0.0003
+
+        # Run with Learner- and RLModule API (new stack).
+        _enable_learner_api: true
+        _enable_rl_module_api: true
+        # Use N Learner worker on the GPU
+        num_learner_workers: 2
+        num_gpus_per_learner_worker: 1
+        num_gpus: 0  # No GPU needed for driver.
+        # Since we are using learner workers, the driver process does not need
+        # a CPU in particular.
+        num_cpus_for_local_worker: 1
+        # Need to unset this b/c we are using the RLModule API, which
+        # provides exploration control via the RLModule's `forward_exploration` method.
+        exploration_config: {}

rllib/ppo-caribou.yaml

@@ -0,0 +1,32 @@
+caribou:
+    env: src.caribou.s3a2
+    run: PPO
+    stop:
+        time_total_s: 24000
+    config:
+        lambda: 0.95
+        kl_coeff: 0.5
+        clip_param: 0.2
+        vf_clip_param: 400.0
+        entropy_coeff: 0.0001
+#        train_batch_size: 64000
+        rollout_fragment_length: auto
+#        sgd_minibatch_size: 16000
+        num_sgd_iter: 10
+        num_envs_per_worker: 24
+        min_time_s_per_iteration: 30
+        lr: 0.0003
+
+        # Run with Learner- and RLModule API (new stack).
+        _enable_learner_api: true
+        _enable_rl_module_api: true
+        # Use N Learner worker on the GPU
+        num_learner_workers: 2
+        num_gpus_per_learner_worker: 1
+        num_gpus: 0  # No GPU needed for driver.
+        # Since we are using learner workers, the driver process does not need
+        # a CPU in particular.
+        num_cpus_for_local_worker: 1
+        # Need to unset this b/c we are using the RLModule API, which
+        # provides exploration control via the RLModule's `forward_exploration` method.
+        exploration_config: {}

rllib/rllib-environment.yml

@@ -0,0 +1,198 @@
+name: rllib
+channels:
+  - conda-forge
+dependencies:
+  - _libgcc_mutex=0.1=conda_forge
+  - _openmp_mutex=4.5=2_gnu
+  - anyio=4.0.0=pyhd8ed1ab_0
+  - argon2-cffi=23.1.0=pyhd8ed1ab_0
+  - argon2-cffi-bindings=21.2.0=py311hd4cff14_3
+  - arrow=1.2.3=pyhd8ed1ab_0
+  - asttokens=2.2.1=pyhd8ed1ab_0
+  - async-lru=2.0.4=pyhd8ed1ab_0
+  - attrs=23.1.0=pyh71513ae_1
+  - babel=2.12.1=pyhd8ed1ab_1
+  - backcall=0.2.0=pyh9f0ad1d_0
+  - backports=1.0=pyhd8ed1ab_3
+  - backports.functools_lru_cache=1.6.5=pyhd8ed1ab_0
+  - beautifulsoup4=4.12.2=pyha770c72_0
+  - bleach=6.0.0=pyhd8ed1ab_0
+  - brotli-python=1.1.0=py311hb755f60_0
+  - bzip2=1.0.8=h7f98852_4
+  - ca-certificates=2023.7.22=hbcca054_0
+  - cached-property=1.5.2=hd8ed1ab_1
+  - cached_property=1.5.2=pyha770c72_1
+  - certifi=2023.7.22=pyhd8ed1ab_0
+  - cffi=1.15.1=py311h409f033_3
+  - charset-normalizer=3.2.0=pyhd8ed1ab_0
+  - comm=0.1.4=pyhd8ed1ab_0
+  - debugpy=1.6.8=py311hb755f60_0
+  - decorator=5.1.1=pyhd8ed1ab_0
+  - defusedxml=0.7.1=pyhd8ed1ab_0
+  - entrypoints=0.4=pyhd8ed1ab_0
+  - exceptiongroup=1.1.3=pyhd8ed1ab_0
+  - executing=1.2.0=pyhd8ed1ab_0
+  - fqdn=1.5.1=pyhd8ed1ab_0
+  - idna=3.4=pyhd8ed1ab_0
+  - importlib-metadata=6.8.0=pyha770c72_0
+  - importlib_metadata=6.8.0=hd8ed1ab_0
+  - importlib_resources=6.0.1=pyhd8ed1ab_0
+  - ipykernel=6.25.1=pyh71e2992_0
+  - ipython=8.15.0=pyh0d859eb_0
+  - isoduration=20.11.0=pyhd8ed1ab_0
+  - jedi=0.19.0=pyhd8ed1ab_0
+  - jinja2=3.1.2=pyhd8ed1ab_1
+  - json5=0.9.14=pyhd8ed1ab_0
+  - jsonpointer=2.0=py_0
+  - jsonschema=4.19.0=pyhd8ed1ab_1
+  - jsonschema-specifications=2023.7.1=pyhd8ed1ab_0
+  - jsonschema-with-format-nongpl=4.19.0=pyhd8ed1ab_1
+  - jupyter-lsp=2.2.0=pyhd8ed1ab_0
+  - jupyter_client=8.3.1=pyhd8ed1ab_0
+  - jupyter_core=5.3.1=py311h38be061_0
+  - jupyter_events=0.7.0=pyhd8ed1ab_2
+  - jupyter_server=2.7.3=pyhd8ed1ab_0
+  - jupyter_server_terminals=0.4.4=pyhd8ed1ab_1
+  - jupyterlab=4.0.5=pyhd8ed1ab_0
+  - jupyterlab_pygments=0.2.2=pyhd8ed1ab_0
+  - jupyterlab_server=2.24.0=pyhd8ed1ab_0
+  - ld_impl_linux-64=2.40=h41732ed_0
+  - libexpat=2.5.0=hcb278e6_1
+  - libffi=3.4.2=h7f98852_5
+  - libgcc-ng=13.1.0=he5830b7_0
+  - libgomp=13.1.0=he5830b7_0
+  - libnsl=2.0.0=h7f98852_0
+  - libsodium=1.0.18=h36c2ea0_1
+  - libsqlite=3.43.0=h2797004_0
+  - libstdcxx-ng=13.1.0=hfd8a6a1_0
+  - libuuid=2.38.1=h0b41bf4_0
+  - libzlib=1.2.13=hd590300_5
+  - markupsafe=2.1.3=py311h459d7ec_0
+  - matplotlib-inline=0.1.6=pyhd8ed1ab_0
+  - mistune=3.0.1=pyhd8ed1ab_0
+  - nbclient=0.8.0=pyhd8ed1ab_0
+  - nbconvert-core=7.8.0=pyhd8ed1ab_0
+  - nbformat=5.9.2=pyhd8ed1ab_0
+  - ncurses=6.4=hcb278e6_0
+  - nest-asyncio=1.5.6=pyhd8ed1ab_0
+  - notebook-shim=0.2.3=pyhd8ed1ab_0
+  - openssl=3.1.2=hd590300_0
+  - overrides=7.4.0=pyhd8ed1ab_0
+  - packaging=23.1=pyhd8ed1ab_0
+  - pandocfilters=1.5.0=pyhd8ed1ab_0
+  - parso=0.8.3=pyhd8ed1ab_0
+  - pexpect=4.8.0=pyh1a96a4e_2
+  - pickleshare=0.7.5=py_1003
+  - pip=23.2.1=pyhd8ed1ab_0
+  - pkgutil-resolve-name=1.3.10=pyhd8ed1ab_0
+  - platformdirs=3.10.0=pyhd8ed1ab_0
+  - prometheus_client=0.17.1=pyhd8ed1ab_0
+  - prompt-toolkit=3.0.39=pyha770c72_0
+  - prompt_toolkit=3.0.39=hd8ed1ab_0
+  - psutil=5.9.5=py311h2582759_0
+  - ptyprocess=0.7.0=pyhd3deb0d_0
+  - pure_eval=0.2.2=pyhd8ed1ab_0
+  - pycparser=2.21=pyhd8ed1ab_0
+  - pygments=2.16.1=pyhd8ed1ab_0
+  - pysocks=1.7.1=pyha2e5f31_6
+  - python=3.11.5=hab00c5b_0_cpython
+  - python-dateutil=2.8.2=pyhd8ed1ab_0
+  - python-fastjsonschema=2.18.0=pyhd8ed1ab_0
+  - python-json-logger=2.0.7=pyhd8ed1ab_0
+  - python_abi=3.11=3_cp311
+  - pytz=2023.3=pyhd8ed1ab_0
+  - pyyaml=6.0.1=py311h459d7ec_0
+  - pyzmq=25.1.1=py311h75c88c4_0
+  - readline=8.2=h8228510_1
+  - referencing=0.30.2=pyhd8ed1ab_0
+  - requests=2.31.0=pyhd8ed1ab_0
+  - rfc3339-validator=0.1.4=pyhd8ed1ab_0
+  - rfc3986-validator=0.1.1=pyh9f0ad1d_0
+  - rpds-py=0.10.0=py311h46250e7_0
+  - send2trash=1.8.2=pyh41d4057_0
+  - setuptools=68.1.2=pyhd8ed1ab_0
+  - six=1.16.0=pyh6c4a22f_0
+  - sniffio=1.3.0=pyhd8ed1ab_0
+  - soupsieve=2.5=pyhd8ed1ab_0
+  - stack_data=0.6.2=pyhd8ed1ab_0
+  - terminado=0.17.1=pyh41d4057_0
+  - tinycss2=1.2.1=pyhd8ed1ab_0
+  - tk=8.6.12=h27826a3_0
+  - tomli=2.0.1=pyhd8ed1ab_0
+  - tornado=6.3.3=py311h459d7ec_0
+  - traitlets=5.9.0=pyhd8ed1ab_0
+  - typing-extensions=4.7.1=hd8ed1ab_0
+  - typing_extensions=4.7.1=pyha770c72_0
+  - typing_utils=0.1.0=pyhd8ed1ab_0
+  - uri-template=1.3.0=pyhd8ed1ab_0
+  - urllib3=2.0.4=pyhd8ed1ab_0
+  - wcwidth=0.2.6=pyhd8ed1ab_0
+  - webcolors=1.13=pyhd8ed1ab_0
+  - webencodings=0.5.1=py_1
+  - websocket-client=1.6.2=pyhd8ed1ab_0
+  - wheel=0.41.2=pyhd8ed1ab_0
+  - xz=5.2.6=h166bdaf_0
+  - yaml=0.2.5=h7f98852_2
+  - zeromq=4.3.4=h9c3ff4c_1
+  - zipp=3.16.2=pyhd8ed1ab_0
+  - pip:
+      - aiosignal==1.3.1
+      - click==8.1.7
+      - cloudpickle==2.2.1
+      - cmake==3.27.2
+      - contourpy==1.1.0
+      - cycler==0.11.0
+      - dm-tree==0.1.8
+      - filelock==3.12.3
+      - fonttools==4.42.1
+      - frozenlist==1.4.0
+      - gputil==1.4.0
+      - grpcio==1.57.0
+      - gymnasium==0.26.3
+      - gymnasium-notices==0.0.1
+      - imageio==2.31.3
+      - kiwisolver==1.4.5
+      - lazy-loader==0.3
+      - lit==16.0.6
+      - lz4==4.3.2
+      - markdown-it-py==3.0.0
+      - matplotlib==3.7.2
+      - mdurl==0.1.2
+      - mizani==0.9.3
+      - mpmath==1.3.0
+      - msgpack==1.0.5
+      - networkx==3.1
+      - numpy==1.25.2
+      - nvidia-cublas-cu11==11.10.3.66
+      - nvidia-cuda-cupti-cu11==11.7.101
+      - nvidia-cuda-nvrtc-cu11==11.7.99
+      - nvidia-cuda-runtime-cu11==11.7.99
+      - nvidia-cudnn-cu11==8.5.0.96
+      - nvidia-cufft-cu11==10.9.0.58
+      - nvidia-curand-cu11==10.2.10.91
+      - nvidia-cusolver-cu11==11.4.0.1
+      - nvidia-cusparse-cu11==11.7.4.91
+      - nvidia-nccl-cu11==2.14.3
+      - nvidia-nvtx-cu11==11.7.91
+      - pandas==2.1.0
+      - patsy==0.5.3
+      - pillow==10.0.0
+      - plotnine==0.12.3
+      - polars==0.19.1
+      - protobuf==4.24.2
+      - pyarrow==13.0.0
+      - pyparsing==3.0.9
+      - pywavelets==1.4.1
+      - ray==2.6.3
+      - rich==13.5.2
+      - scikit-image==0.21.0
+      - scipy==1.11.2
+      - statsmodels==0.14.0
+      - sympy==1.12
+      - tensorboardx==2.6.2.2
+      - tifffile==2023.8.30
+      - torch==2.0.1
+      - triton==2.0.0
+      - typer==0.9.0
+      - tzdata==2023.3
+prefix: /srv/conda/envs/rllib

sb3/a2c_action_is_S.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0b7fa6fd325433369b748db5442d61797146410efee2706c1a219524bac2e952
+size 91923

sb3/ars_action_is_S.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:46600c05fed8776737c3fbcd2612a168f09a0ce6feb6530fb92dda1db6a42b0e
+size 52044

sb3/ppo_action_is_S.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7a4ffd7ea0d6c1944a33d87b388552ed106b624cceac3f31acbb0f8ac70a0c14
+size 134569

sb3/sb3-environment.yml

@@ -0,0 +1,155 @@
+name: sb3
+channels:
+  - conda-forge
+dependencies:
+  - _libgcc_mutex=0.1=conda_forge
+  - _openmp_mutex=4.5=2_gnu
+  - asttokens=2.2.1=pyhd8ed1ab_0
+  - backcall=0.2.0=pyh9f0ad1d_0
+  - backports=1.0=pyhd8ed1ab_3
+  - backports.functools_lru_cache=1.6.5=pyhd8ed1ab_0
+  - bzip2=1.0.8=h7f98852_4
+  - ca-certificates=2023.7.22=hbcca054_0
+  - comm=0.1.4=pyhd8ed1ab_0
+  - debugpy=1.6.8=py311hb755f60_0
+  - decorator=5.1.1=pyhd8ed1ab_0
+  - exceptiongroup=1.1.3=pyhd8ed1ab_0
+  - executing=1.2.0=pyhd8ed1ab_0
+  - importlib-metadata=6.8.0=pyha770c72_0
+  - importlib_metadata=6.8.0=hd8ed1ab_0
+  - ipykernel=6.25.1=pyh71e2992_0
+  - ipython=8.15.0=pyh0d859eb_0
+  - jedi=0.19.0=pyhd8ed1ab_0
+  - jupyter_client=8.3.1=pyhd8ed1ab_0
+  - jupyter_core=5.3.1=py311h38be061_0
+  - ld_impl_linux-64=2.40=h41732ed_0
+  - libexpat=2.5.0=hcb278e6_1
+  - libffi=3.4.2=h7f98852_5
+  - libgcc-ng=13.1.0=he5830b7_0
+  - libgomp=13.1.0=he5830b7_0
+  - libnsl=2.0.0=h7f98852_0
+  - libsodium=1.0.18=h36c2ea0_1
+  - libsqlite=3.43.0=h2797004_0
+  - libstdcxx-ng=13.1.0=hfd8a6a1_0
+  - libuuid=2.38.1=h0b41bf4_0
+  - libzlib=1.2.13=hd590300_5
+  - matplotlib-inline=0.1.6=pyhd8ed1ab_0
+  - ncurses=6.4=hcb278e6_0
+  - nest-asyncio=1.5.6=pyhd8ed1ab_0
+  - openssl=3.1.2=hd590300_0
+  - packaging=23.1=pyhd8ed1ab_0
+  - parso=0.8.3=pyhd8ed1ab_0
+  - pexpect=4.8.0=pyh1a96a4e_2
+  - pickleshare=0.7.5=py_1003
+  - pip=23.2.1=pyhd8ed1ab_0
+  - platformdirs=3.10.0=pyhd8ed1ab_0
+  - prompt-toolkit=3.0.39=pyha770c72_0
+  - prompt_toolkit=3.0.39=hd8ed1ab_0
+  - psutil=5.9.5=py311h2582759_0
+  - ptyprocess=0.7.0=pyhd3deb0d_0
+  - pure_eval=0.2.2=pyhd8ed1ab_0
+  - pygments=2.16.1=pyhd8ed1ab_0
+  - python=3.11.5=hab00c5b_0_cpython
+  - python-dateutil=2.8.2=pyhd8ed1ab_0
+  - python_abi=3.11=3_cp311
+  - pyzmq=25.1.1=py311h75c88c4_0
+  - readline=8.2=h8228510_1
+  - setuptools=68.1.2=pyhd8ed1ab_0
+  - six=1.16.0=pyh6c4a22f_0
+  - stack_data=0.6.2=pyhd8ed1ab_0
+  - tk=8.6.12=h27826a3_0
+  - tornado=6.3.3=py311h459d7ec_0
+  - traitlets=5.9.0=pyhd8ed1ab_0
+  - typing-extensions=4.7.1=hd8ed1ab_0
+  - typing_extensions=4.7.1=pyha770c72_0
+  - wcwidth=0.2.6=pyhd8ed1ab_0
+  - wheel=0.41.2=pyhd8ed1ab_0
+  - xz=5.2.6=h166bdaf_0
+  - zeromq=4.3.4=h9c3ff4c_1
+  - zipp=3.16.2=pyhd8ed1ab_0
+  - pip:
+      - absl-py==1.4.0
+      - aiosignal==1.3.1
+      - arrow==1.2.3
+      - attrs==23.1.0
+      - cachetools==5.3.1
+      - certifi==2023.7.22
+      - charset-normalizer==3.2.0
+      - click==8.1.7
+      - cloudpickle==2.2.1
+      - cmake==3.27.2
+      - contourpy==1.1.0
+      - cycler==0.11.0
+      - defusedxml==0.7.1
+      - farama-notifications==0.0.4
+      - filelock==3.12.3
+      - fonttools==4.42.1
+      - frozenlist==1.4.0
+      - glances==3.4.0.3
+      - google-auth==2.22.0
+      - google-auth-oauthlib==1.0.0
+      - grpcio==1.57.0
+      - gymnasium==0.29.1
+      - idna==3.4
+      - ipywidgets==8.1.0
+      - jinja2==3.1.2
+      - jsonschema==4.19.0
+      - jsonschema-specifications==2023.7.1
+      - jupyterlab-widgets==3.0.8
+      - kiwisolver==1.4.5
+      - lit==16.0.6
+      - markdown==3.4.4
+      - markupsafe==2.1.3
+      - matplotlib==3.7.2
+      - mizani==0.9.3
+      - mpmath==1.3.0
+      - msgpack==1.0.5
+      - networkx==3.1
+      - numpy==1.25.2
+      - nvidia-cublas-cu11==11.10.3.66
+      - nvidia-cuda-cupti-cu11==11.7.101
+      - nvidia-cuda-nvrtc-cu11==11.7.99
+      - nvidia-cuda-runtime-cu11==11.7.99
+      - nvidia-cudnn-cu11==8.5.0.96
+      - nvidia-cufft-cu11==10.9.0.58
+      - nvidia-curand-cu11==10.2.10.91
+      - nvidia-cusolver-cu11==11.4.0.1
+      - nvidia-cusparse-cu11==11.7.4.91
+      - nvidia-nccl-cu11==2.14.3
+      - nvidia-nvtx-cu11==11.7.91
+      - oauthlib==3.2.2
+      - pandas==2.1.0
+      - patsy==0.5.3
+      - pillow==10.0.0
+      - plotnine==0.12.3
+      - polars==0.19.1
+      - protobuf==4.24.2
+      - py3nvml==0.2.7
+      - pyarrow==13.0.0
+      - pyasn1==0.5.0
+      - pyasn1-modules==0.3.0
+      - pyparsing==3.0.9
+      - pytz==2023.3
+      - pyyaml==6.0.1
+      - ray==2.6.3
+      - referencing==0.30.2
+      - requests==2.31.0
+      - requests-oauthlib==1.3.1
+      - rpds-py==0.10.0
+      - rsa==4.9
+      - sb3-contrib==2.1.0
+      - scipy==1.11.2
+      - stable-baselines3==2.1.0
+      - statsmodels==0.14.0
+      - sympy==1.12
+      - tensorboard==2.14.0
+      - tensorboard-data-server==0.7.1
+      - torch==2.0.1
+      - triton==2.0.0
+      - tzdata==2023.3
+      - ujson==5.8.0
+      - urllib3==1.26.16
+      - werkzeug==2.3.7
+      - widgetsnbextension==4.0.8
+      - xmltodict==0.13.0
+prefix: /srv/conda/envs/sb3

sb3/tqc_action_is_S.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a76826c3de94ea83437bb61fe7339447824e00e5ec806de982e11dbbc653bbf4
+size 3181288