Training in progress, step 12000, checkpoint

checkpoint-12000/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6d6594bcda7c37f46af5a7034a0a3a5e5404b400fd58503a826192b4a17a7e26
+oid sha256:1cad5ac5a5ac9ae4ebcc434ed367cd166b5c8d9198470a60a6a4734ba58b34d8
 size 51007160

checkpoint-12000/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d074f60296c3775bf8ee10baa8786f755def3ce442bd21e24fa06620063edec6
+oid sha256:af955ba6d5a07870556a93eaed58863f5b28875d0e0c95dc1b635384e7ca9c7b
 size 102078202

checkpoint-12000/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:830ab2b6136d8d2aab64443305d7325934223d7dc3a59a6214ea6709b98b0520
+oid sha256:5dee6be081b446e09b1bd96d3dd8c7d6f80a5e3fc6ada987b1d58900a7916323
 size 14308

checkpoint-12000/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5463b724c2cd6909cd79c85b289d00a369e4fb6bb042482b75620c59845c7360
+oid sha256:a300d405a31ed9a6a3aacb8615e61e340e96b3c687b201bcf6ed4ae9f299ec86
 size 1000

checkpoint-12000/trainer_state.json

@@ -1,7 +1,7 @@
 {
-  "best_metric": 4.1580729484558105,
-  "best_model_checkpoint": "/home/p318482/babyLM_controlled/models_trained/fr_clm/childes_30/checkpoint-12000",
-  "epoch": 18.867924528301888,
+  "best_metric": 4.983631134033203,
+  "best_model_checkpoint": "/home/p318482/babyLM_controlled/models_trained/de_clm/childes_30/checkpoint-12000",
+  "epoch": 12.578616352201259,
   "eval_steps": 2000,
   "global_step": 12000,
   "is_hyper_param_search": false,
@@ -9,79 +9,79 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 3.1446540880503147,
-      "eval_loss": 6.67229700088501,
-      "eval_runtime": 0.6568,
-      "eval_samples_per_second": 1386.938,
-      "eval_steps_per_second": 86.779,
+      "epoch": 2.0964360587002098,
+      "eval_loss": 7.102903366088867,
+      "eval_runtime": 0.9708,
+      "eval_samples_per_second": 1416.286,
+      "eval_steps_per_second": 88.582,
       "step": 2000
     },
     {
-      "epoch": 6.289308176100629,
-      "grad_norm": 1.4630517959594727,
+      "epoch": 4.1928721174004195,
+      "grad_norm": 1.3964662551879883,
       "learning_rate": 1e-05,
-      "loss": 6.5359,
+      "loss": 6.9987,
       "step": 4000
     },
     {
-      "epoch": 6.289308176100629,
-      "eval_loss": 5.102311611175537,
-      "eval_runtime": 0.6383,
-      "eval_samples_per_second": 1427.13,
-      "eval_steps_per_second": 89.294,
+      "epoch": 4.1928721174004195,
+      "eval_loss": 5.884151935577393,
+      "eval_runtime": 0.966,
+      "eval_samples_per_second": 1423.408,
+      "eval_steps_per_second": 89.028,
       "step": 4000
     },
     {
-      "epoch": 9.433962264150944,
-      "eval_loss": 4.675341606140137,
-      "eval_runtime": 0.6396,
-      "eval_samples_per_second": 1424.427,
-      "eval_steps_per_second": 89.124,
+      "epoch": 6.289308176100629,
+      "eval_loss": 5.54873514175415,
+      "eval_runtime": 0.9657,
+      "eval_samples_per_second": 1423.84,
+      "eval_steps_per_second": 89.055,
       "step": 6000
     },
     {
-      "epoch": 12.578616352201259,
-      "grad_norm": 2.3778915405273438,
+      "epoch": 8.385744234800839,
+      "grad_norm": 2.7172107696533203,
       "learning_rate": 2e-05,
-      "loss": 4.34,
+      "loss": 5.2204,
       "step": 8000
     },
     {
-      "epoch": 12.578616352201259,
-      "eval_loss": 4.42986536026001,
-      "eval_runtime": 0.646,
-      "eval_samples_per_second": 1410.229,
-      "eval_steps_per_second": 88.236,
+      "epoch": 8.385744234800839,
+      "eval_loss": 5.2793288230896,
+      "eval_runtime": 0.9644,
+      "eval_samples_per_second": 1425.779,
+      "eval_steps_per_second": 89.176,
       "step": 8000
     },
     {
-      "epoch": 15.723270440251572,
-      "eval_loss": 4.2789154052734375,
-      "eval_runtime": 0.6485,
-      "eval_samples_per_second": 1404.856,
-      "eval_steps_per_second": 87.9,
+      "epoch": 10.482180293501049,
+      "eval_loss": 5.10486364364624,
+      "eval_runtime": 0.9641,
+      "eval_samples_per_second": 1426.204,
+      "eval_steps_per_second": 89.203,
       "step": 10000
     },
     {
-      "epoch": 18.867924528301888,
-      "grad_norm": 2.512312889099121,
-      "learning_rate": 2.9995e-05,
-      "loss": 3.8692,
+      "epoch": 12.578616352201259,
+      "grad_norm": 2.500443458557129,
+      "learning_rate": 2.99925e-05,
+      "loss": 4.7358,
       "step": 12000
     },
     {
-      "epoch": 18.867924528301888,
-      "eval_loss": 4.1580729484558105,
-      "eval_runtime": 0.6403,
-      "eval_samples_per_second": 1422.769,
-      "eval_steps_per_second": 89.021,
+      "epoch": 12.578616352201259,
+      "eval_loss": 4.983631134033203,
+      "eval_runtime": 0.9644,
+      "eval_samples_per_second": 1425.809,
+      "eval_steps_per_second": 89.178,
       "step": 12000
     }
   ],
   "logging_steps": 4000,
   "max_steps": 100000,
   "num_input_tokens_seen": 0,
-  "num_train_epochs": 158,
+  "num_train_epochs": 105,
   "save_steps": 4000,
   "stateful_callbacks": {
     "TrainerControl": {
@@ -95,7 +95,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 3102504217214976.0,
+  "total_flos": 3102794094231552.0,
   "train_batch_size": 16,
   "trial_name": null,
   "trial_params": null

checkpoint-12000/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6af9de70859d62cbd3b61f71e8a5bc95702dbc6ddb62d2994641a31953e4ea9b
+oid sha256:0c779122676b54107edc62ae0b9293c062733193c5f82a36a2bc097bca192814
 size 5368