Training in progress, step 500

config.json

@@ -1,5 +1,5 @@
 {
-  "_name_or_path": "tattabio/gLM2_150M",
+  "_name_or_path": "tattabio/gLM2_650M",
   "architectures": [
     "gLM2ForMaskedLM"
   ],
@@ -8,10 +8,10 @@
     "AutoModel": "modeling_glm2.gLM2Model",
     "AutoModelForMaskedLM": "modeling_glm2.gLM2ForMaskedLM"
   },
-  "depth": 30,
-  "dim": 640,
+  "depth": 33,
+  "dim": 1280,
   "ffn_dim_multiplier": null,
-  "heads": 10,
+  "heads": 20,
   "model_type": "gLM2",
   "norm_eps": 1e-05,
   "swiglu_multiple_of": 256,

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:47c3ea48a3d6443a0fd08b53e26ae04984ed6d6381f3928afee8fb4a155e9eaf
-size 609855088
+oid sha256:38384749b3fa8fe08d6d05430325f30ab4a6dfb4936f267fce232c98c501dc69
+size 2682482800

runs/Oct29_18-35-09_torch-flash-large-0-6/events.out.tfevents.1730226910.torch-flash-large-0-6.10810.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2d5b31cdcb3b62fe9760a97d7a398b2c01828fd4df9b4debc7f1c32fc9b04aba
-size 9001
+oid sha256:a797bb6b26c5557fec5d7d41147a52180d2a68a96881b963e0b0ac76a7bdff8d
+size 9423

runs/Oct29_21-31-10_torch-flash-large-0-6/events.out.tfevents.1730237472.torch-flash-large-0-6.11056.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e2e526f20b445afc85b084cc286cf58861cc53a6a75bf76402dd0322f55aaa89
+size 6893

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c1d0607b0ef32a7fb9356d7e9cea98c6b3f4ff9d058f42339926e1f0132b13a7
+oid sha256:eb6051eee29c31ca0092abe3bc9f1d55068e299690664ab4b9484f44a7a3cebf
 size 5240