Upload 2 files

ae.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2ac8ee0cd77a073c0fcd13a1810873acc184cc83d27f94ea91b2f59d34df4d4e
+size 3624052520

config.json

@@ -0,0 +1,24 @@
+{
+    "trainer": {
+        "trainer_class": "TrainerTopK",
+        "dict_class": "AutoEncoderTopK",
+        "lr": 0.00016329931618554522,
+        "steps": 30000,
+        "seed": 42,
+        "activation_dim": 18432,
+        "dict_size": 24576,
+        "k": 64,
+        "device": "cuda:0",
+        "wandb_name": "all_modules_topk64_dict24576_seed42_batch8196",
+        "submodule_list": "[GPT2MLP(\n  (c_fc): Conv1D()\n  (c_proj): Conv1D()\n  (act): NewGELUActivation()\n  (dropout): Dropout(p=0.1, inplace=False)\n), GPT2MLP(\n  (c_fc): Conv1D()\n  (c_proj): Conv1D()\n  (act): NewGELUActivation()\n  (dropout): Dropout(p=0.1, inplace=False)\n), GPT2MLP(\n  (c_fc): Conv1D()\n  (c_proj): Conv1D()\n  (act): NewGELUActivation()\n  (dropout): Dropout(p=0.1, inplace=False)\n), GPT2MLP(\n  (c_fc): Conv1D()\n  (c_proj): Conv1D()\n  (act): NewGELUActivation()\n  (dropout): Dropout(p=0.1, inplace=False)\n), GPT2MLP(\n  (c_fc): Conv1D()\n  (c_proj): Conv1D()\n  (act): NewGELUActivation()\n  (dropout): Dropout(p=0.1, inplace=False)\n), GPT2MLP(\n  (c_fc): Conv1D()\n  (c_proj): Conv1D()\n  (act): NewGELUActivation()\n  (dropout): Dropout(p=0.1, inplace=False)\n), GPT2MLP(\n  (c_fc): Conv1D()\n  (c_proj): Conv1D()\n  (act): NewGELUActivation()\n  (dropout): Dropout(p=0.1, inplace=False)\n), GPT2MLP(\n  (c_fc): Conv1D()\n  (c_proj): Conv1D()\n  (act): NewGELUActivation()\n  (dropout): Dropout(p=0.1, inplace=False)\n), GPT2MLP(\n  (c_fc): Conv1D()\n  (c_proj): Conv1D()\n  (act): NewGELUActivation()\n  (dropout): Dropout(p=0.1, inplace=False)\n), GPT2MLP(\n  (c_fc): Conv1D()\n  (c_proj): Conv1D()\n  (act): NewGELUActivation()\n  (dropout): Dropout(p=0.1, inplace=False)\n), GPT2MLP(\n  (c_fc): Conv1D()\n  (c_proj): Conv1D()\n  (act): NewGELUActivation()\n  (dropout): Dropout(p=0.1, inplace=False)\n), GPT2MLP(\n  (c_fc): Conv1D()\n  (c_proj): Conv1D()\n  (act): NewGELUActivation()\n  (dropout): Dropout(p=0.1, inplace=False)\n), GPT2SdpaAttention(\n  (c_attn): Conv1D()\n  (c_proj): Conv1D()\n  (attn_dropout): Dropout(p=0.1, inplace=False)\n  (resid_dropout): Dropout(p=0.1, inplace=False)\n), GPT2SdpaAttention(\n  (c_attn): Conv1D()\n  (c_proj): Conv1D()\n  (attn_dropout): Dropout(p=0.1, inplace=False)\n  (resid_dropout): Dropout(p=0.1, inplace=False)\n), GPT2SdpaAttention(\n  (c_attn): Conv1D()\n  (c_proj): Conv1D()\n  (attn_dropout): Dropout(p=0.1, inplace=False)\n  (resid_dropout): Dropout(p=0.1, inplace=False)\n), GPT2SdpaAttention(\n  (c_attn): Conv1D()\n  (c_proj): Conv1D()\n  (attn_dropout): Dropout(p=0.1, inplace=False)\n  (resid_dropout): Dropout(p=0.1, inplace=False)\n), GPT2SdpaAttention(\n  (c_attn): Conv1D()\n  (c_proj): Conv1D()\n  (attn_dropout): Dropout(p=0.1, inplace=False)\n  (resid_dropout): Dropout(p=0.1, inplace=False)\n), GPT2SdpaAttention(\n  (c_attn): Conv1D()\n  (c_proj): Conv1D()\n  (attn_dropout): Dropout(p=0.1, inplace=False)\n  (resid_dropout): Dropout(p=0.1, inplace=False)\n), GPT2SdpaAttention(\n  (c_attn): Conv1D()\n  (c_proj): Conv1D()\n  (attn_dropout): Dropout(p=0.1, inplace=False)\n  (resid_dropout): Dropout(p=0.1, inplace=False)\n), GPT2SdpaAttention(\n  (c_attn): Conv1D()\n  (c_proj): Conv1D()\n  (attn_dropout): Dropout(p=0.1, inplace=False)\n  (resid_dropout): Dropout(p=0.1, inplace=False)\n), GPT2SdpaAttention(\n  (c_attn): Conv1D()\n  (c_proj): Conv1D()\n  (attn_dropout): Dropout(p=0.1, inplace=False)\n  (resid_dropout): Dropout(p=0.1, inplace=False)\n), GPT2SdpaAttention(\n  (c_attn): Conv1D()\n  (c_proj): Conv1D()\n  (attn_dropout): Dropout(p=0.1, inplace=False)\n  (resid_dropout): Dropout(p=0.1, inplace=False)\n), GPT2SdpaAttention(\n  (c_attn): Conv1D()\n  (c_proj): Conv1D()\n  (attn_dropout): Dropout(p=0.1, inplace=False)\n  (resid_dropout): Dropout(p=0.1, inplace=False)\n), GPT2SdpaAttention(\n  (c_attn): Conv1D()\n  (c_proj): Conv1D()\n  (attn_dropout): Dropout(p=0.1, inplace=False)\n  (resid_dropout): Dropout(p=0.1, inplace=False)\n)]"
+    },
+    "buffer": {
+        "d_submodule": 768,
+        "io": "out",
+        "n_ctxs": 128,
+        "ctx_len": 128,
+        "refresh_batch_size": 256,
+        "out_batch_size": 8192,
+        "device": "cuda:0"
+    }
+}