Training in progress, step 32000, checkpoint

checkpoint-32000/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:18a733bd99880c6cc1b137b2e231d709a7fa19ee96fb95a17f8f6aa594e02107
+oid sha256:dd4a6973dd00085448343157918fa81acf1c80e455463a4cbcb6b5891451e8dd
 size 51007160

checkpoint-32000/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:29c0f79dbcf34b655024cba486568f04b90573ef975730cb037a1680d4eddeb2
+oid sha256:9aaf27507a0ed8b0d35f33cdd6a05fe3030bb5f6b1b3b11a53fa0ef8653dd42d
 size 102078202

checkpoint-32000/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:09c3a358ee70daf1253ab986e28dd72ff92ca451da07ecd9228648f03de62c75
+oid sha256:446e0a7c49e6a10011b27f8818a4e93e9e250d631bd5f884a8b1561ca35443e1
 size 14308

checkpoint-32000/trainer_state.json

@@ -1,7 +1,7 @@
 {
-  "best_metric": 3.9348273277282715,
-  "best_model_checkpoint": "/home/p318482/babyLM_controlled/models_trained/fr_clm/childes_30/checkpoint-28000",
-  "epoch": 50.314465408805034,
+  "best_metric": 4.583011150360107,
+  "best_model_checkpoint": "/home/p318482/babyLM_controlled/models_trained/de_clm/childes_30/checkpoint-32000",
+  "epoch": 33.542976939203356,
   "eval_steps": 2000,
   "global_step": 32000,
   "is_hyper_param_search": false,
@@ -9,194 +9,194 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 3.1446540880503147,
-      "eval_loss": 6.67229700088501,
-      "eval_runtime": 0.6568,
-      "eval_samples_per_second": 1386.938,
-      "eval_steps_per_second": 86.779,
+      "epoch": 2.0964360587002098,
+      "eval_loss": 7.102903366088867,
+      "eval_runtime": 0.9708,
+      "eval_samples_per_second": 1416.286,
+      "eval_steps_per_second": 88.582,
       "step": 2000
     },
     {
-      "epoch": 6.289308176100629,
-      "grad_norm": 1.4630517959594727,
+      "epoch": 4.1928721174004195,
+      "grad_norm": 1.3964662551879883,
       "learning_rate": 1e-05,
-      "loss": 6.5359,
+      "loss": 6.9987,
       "step": 4000
     },
     {
-      "epoch": 6.289308176100629,
-      "eval_loss": 5.102311611175537,
-      "eval_runtime": 0.6383,
-      "eval_samples_per_second": 1427.13,
-      "eval_steps_per_second": 89.294,
+      "epoch": 4.1928721174004195,
+      "eval_loss": 5.884151935577393,
+      "eval_runtime": 0.966,
+      "eval_samples_per_second": 1423.408,
+      "eval_steps_per_second": 89.028,
       "step": 4000
     },
     {
-      "epoch": 9.433962264150944,
-      "eval_loss": 4.675341606140137,
-      "eval_runtime": 0.6396,
-      "eval_samples_per_second": 1424.427,
-      "eval_steps_per_second": 89.124,
+      "epoch": 6.289308176100629,
+      "eval_loss": 5.54873514175415,
+      "eval_runtime": 0.9657,
+      "eval_samples_per_second": 1423.84,
+      "eval_steps_per_second": 89.055,
       "step": 6000
     },
     {
-      "epoch": 12.578616352201259,
-      "grad_norm": 2.3778915405273438,
+      "epoch": 8.385744234800839,
+      "grad_norm": 2.7172107696533203,
       "learning_rate": 2e-05,
-      "loss": 4.34,
+      "loss": 5.2204,
       "step": 8000
     },
     {
-      "epoch": 12.578616352201259,
-      "eval_loss": 4.42986536026001,
-      "eval_runtime": 0.646,
-      "eval_samples_per_second": 1410.229,
-      "eval_steps_per_second": 88.236,
+      "epoch": 8.385744234800839,
+      "eval_loss": 5.2793288230896,
+      "eval_runtime": 0.9644,
+      "eval_samples_per_second": 1425.779,
+      "eval_steps_per_second": 89.176,
       "step": 8000
     },
     {
-      "epoch": 15.723270440251572,
-      "eval_loss": 4.2789154052734375,
-      "eval_runtime": 0.6485,
-      "eval_samples_per_second": 1404.856,
-      "eval_steps_per_second": 87.9,
+      "epoch": 10.482180293501049,
+      "eval_loss": 5.10486364364624,
+      "eval_runtime": 0.9641,
+      "eval_samples_per_second": 1426.204,
+      "eval_steps_per_second": 89.203,
       "step": 10000
     },
     {
-      "epoch": 18.867924528301888,
-      "grad_norm": 2.512312889099121,
-      "learning_rate": 2.9995e-05,
-      "loss": 3.8692,
+      "epoch": 12.578616352201259,
+      "grad_norm": 2.500443458557129,
+      "learning_rate": 2.99925e-05,
+      "loss": 4.7358,
       "step": 12000
     },
     {
-      "epoch": 18.867924528301888,
-      "eval_loss": 4.1580729484558105,
-      "eval_runtime": 0.6403,
-      "eval_samples_per_second": 1422.769,
-      "eval_steps_per_second": 89.021,
+      "epoch": 12.578616352201259,
+      "eval_loss": 4.983631134033203,
+      "eval_runtime": 0.9644,
+      "eval_samples_per_second": 1425.809,
+      "eval_steps_per_second": 89.178,
       "step": 12000
     },
     {
-      "epoch": 22.0125786163522,
-      "eval_loss": 4.082061767578125,
-      "eval_runtime": 0.6415,
-      "eval_samples_per_second": 1420.041,
-      "eval_steps_per_second": 88.85,
+      "epoch": 14.675052410901468,
+      "eval_loss": 4.882917404174805,
+      "eval_runtime": 0.9686,
+      "eval_samples_per_second": 1419.612,
+      "eval_steps_per_second": 88.79,
       "step": 14000
     },
     {
-      "epoch": 25.157232704402517,
-      "grad_norm": 2.4205257892608643,
+      "epoch": 16.771488469601678,
+      "grad_norm": 2.400749444961548,
       "learning_rate": 3.999e-05,
-      "loss": 3.5905,
+      "loss": 4.4216,
       "step": 16000
     },
     {
-      "epoch": 25.157232704402517,
-      "eval_loss": 4.033827781677246,
-      "eval_runtime": 0.6416,
-      "eval_samples_per_second": 1419.823,
-      "eval_steps_per_second": 88.836,
+      "epoch": 16.771488469601678,
+      "eval_loss": 4.802889823913574,
+      "eval_runtime": 0.9763,
+      "eval_samples_per_second": 1408.393,
+      "eval_steps_per_second": 88.089,
       "step": 16000
     },
     {
-      "epoch": 28.30188679245283,
-      "eval_loss": 3.9935402870178223,
-      "eval_runtime": 0.6614,
-      "eval_samples_per_second": 1377.343,
-      "eval_steps_per_second": 86.178,
+      "epoch": 18.867924528301888,
+      "eval_loss": 4.74226188659668,
+      "eval_runtime": 0.976,
+      "eval_samples_per_second": 1408.882,
+      "eval_steps_per_second": 88.119,
       "step": 18000
     },
     {
-      "epoch": 31.446540880503143,
-      "grad_norm": 2.5674731731414795,
+      "epoch": 20.964360587002098,
+      "grad_norm": 2.2613022327423096,
       "learning_rate": 4.9985e-05,
-      "loss": 3.3778,
+      "loss": 4.1842,
       "step": 20000
     },
     {
-      "epoch": 31.446540880503143,
-      "eval_loss": 3.9675967693328857,
-      "eval_runtime": 0.6406,
-      "eval_samples_per_second": 1422.059,
-      "eval_steps_per_second": 88.976,
+      "epoch": 20.964360587002098,
+      "eval_loss": 4.690371513366699,
+      "eval_runtime": 0.9669,
+      "eval_samples_per_second": 1422.037,
+      "eval_steps_per_second": 88.942,
       "step": 20000
     },
     {
-      "epoch": 34.59119496855346,
-      "eval_loss": 3.9494004249572754,
-      "eval_runtime": 0.6409,
-      "eval_samples_per_second": 1421.391,
-      "eval_steps_per_second": 88.934,
+      "epoch": 23.060796645702307,
+      "eval_loss": 4.645771503448486,
+      "eval_runtime": 0.966,
+      "eval_samples_per_second": 1423.404,
+      "eval_steps_per_second": 89.027,
       "step": 22000
     },
     {
-      "epoch": 37.735849056603776,
-      "grad_norm": 2.273855447769165,
+      "epoch": 25.157232704402517,
+      "grad_norm": 2.2588465213775635,
       "learning_rate": 5.9980000000000005e-05,
-      "loss": 3.1983,
+      "loss": 3.9858,
       "step": 24000
     },
     {
-      "epoch": 37.735849056603776,
-      "eval_loss": 3.9382567405700684,
-      "eval_runtime": 0.6423,
-      "eval_samples_per_second": 1418.386,
-      "eval_steps_per_second": 88.746,
+      "epoch": 25.157232704402517,
+      "eval_loss": 4.623382568359375,
+      "eval_runtime": 0.9666,
+      "eval_samples_per_second": 1422.538,
+      "eval_steps_per_second": 88.973,
       "step": 24000
     },
     {
-      "epoch": 40.880503144654085,
-      "eval_loss": 3.936068296432495,
-      "eval_runtime": 0.6434,
-      "eval_samples_per_second": 1415.972,
-      "eval_steps_per_second": 88.595,
+      "epoch": 27.253668763102727,
+      "eval_loss": 4.6056084632873535,
+      "eval_runtime": 0.9825,
+      "eval_samples_per_second": 1399.422,
+      "eval_steps_per_second": 87.527,
       "step": 26000
     },
     {
-      "epoch": 44.0251572327044,
-      "grad_norm": 2.377107858657837,
+      "epoch": 29.350104821802937,
+      "grad_norm": 2.1845760345458984,
       "learning_rate": 6.997500000000001e-05,
-      "loss": 3.0374,
+      "loss": 3.8189,
       "step": 28000
     },
     {
-      "epoch": 44.0251572327044,
-      "eval_loss": 3.9348273277282715,
-      "eval_runtime": 0.6383,
-      "eval_samples_per_second": 1427.128,
-      "eval_steps_per_second": 89.293,
+      "epoch": 29.350104821802937,
+      "eval_loss": 4.590851783752441,
+      "eval_runtime": 1.0367,
+      "eval_samples_per_second": 1326.293,
+      "eval_steps_per_second": 82.954,
       "step": 28000
     },
     {
-      "epoch": 47.16981132075472,
-      "eval_loss": 3.9625139236450195,
-      "eval_runtime": 0.6461,
-      "eval_samples_per_second": 1409.936,
-      "eval_steps_per_second": 88.218,
+      "epoch": 31.446540880503143,
+      "eval_loss": 4.586838245391846,
+      "eval_runtime": 0.9705,
+      "eval_samples_per_second": 1416.743,
+      "eval_steps_per_second": 88.611,
       "step": 30000
     },
     {
-      "epoch": 50.314465408805034,
-      "grad_norm": 2.460918426513672,
+      "epoch": 33.542976939203356,
+      "grad_norm": 2.2118289470672607,
       "learning_rate": 7.997e-05,
-      "loss": 2.887,
+      "loss": 3.6763,
       "step": 32000
     },
     {
-      "epoch": 50.314465408805034,
-      "eval_loss": 3.9899542331695557,
-      "eval_runtime": 0.6383,
-      "eval_samples_per_second": 1427.149,
-      "eval_steps_per_second": 89.295,
+      "epoch": 33.542976939203356,
+      "eval_loss": 4.583011150360107,
+      "eval_runtime": 0.9706,
+      "eval_samples_per_second": 1416.604,
+      "eval_steps_per_second": 88.602,
       "step": 32000
     }
   ],
   "logging_steps": 4000,
   "max_steps": 100000,
   "num_input_tokens_seen": 0,
-  "num_train_epochs": 158,
+  "num_train_epochs": 105,
   "save_steps": 4000,
   "stateful_callbacks": {
     "TrainerControl": {
@@ -210,7 +210,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 8273312258457600.0,
+  "total_flos": 8274118257967104.0,
   "train_batch_size": 16,
   "trial_name": null,
   "trial_params": null

checkpoint-32000/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6af9de70859d62cbd3b61f71e8a5bc95702dbc6ddb62d2994641a31953e4ea9b
+oid sha256:0c779122676b54107edc62ae0b9293c062733193c5f82a36a2bc097bca192814
 size 5368