Training in progress, step 36000, checkpoint

checkpoint-36000/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:cd4c1a9e11a89a6cde54864a17bb6d7061c8874ee48b501ae6de556b0c1bac77
+oid sha256:29419cec635968aa525f23d35c669b5bf2041e5a2cf052001b8248fe8bd905f4
 size 51007160

checkpoint-36000/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:404cf802c17b8b96d055abdbc6036be9116a61433854cd7cf252e158cc31b01e
+oid sha256:75c62d244e0db446ecb110df796bb5be95c462915a79e838601fad05e34ef375
 size 102078202

checkpoint-36000/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:eb616a971c34a6a37a5bbc0216928e42b060156bc566262f7d14068b4ad5759c
+oid sha256:2a7decb7e959750cf1f95ba0ba86cb51b96df552bbec3e4df195ce0cb31321c0
 size 14244

checkpoint-36000/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a5e3cc1389fd061e56f46ee9cf4a1474d5d3a2aa6cd16d38bf1eb706acac347a
+oid sha256:c58c8bab71cea73096e54bcd1f275388b0bbb92c06a4b92d779827f150db8a6f
 size 1000

checkpoint-36000/trainer_state.json

@@ -1,7 +1,7 @@
 {
-  "best_metric": 3.937915563583374,
-  "best_model_checkpoint": "/home/p318482/babyLM_controlled/models_trained/fr_clm/childes_42/checkpoint-24000",
-  "epoch": 56.60377358490566,
+  "best_metric": 4.579585075378418,
+  "best_model_checkpoint": "/home/p318482/babyLM_controlled/models_trained/de_clm/childes_42/checkpoint-32000",
+  "epoch": 37.735849056603776,
   "eval_steps": 2000,
   "global_step": 36000,
   "is_hyper_param_search": false,
@@ -9,217 +9,217 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 3.1446540880503147,
-      "eval_loss": 6.666613578796387,
-      "eval_runtime": 0.6599,
-      "eval_samples_per_second": 1380.413,
-      "eval_steps_per_second": 86.37,
+      "epoch": 2.0964360587002098,
+      "eval_loss": 7.0908098220825195,
+      "eval_runtime": 0.9834,
+      "eval_samples_per_second": 1398.23,
+      "eval_steps_per_second": 87.453,
       "step": 2000
     },
     {
-      "epoch": 6.289308176100629,
-      "grad_norm": 1.431768536567688,
+      "epoch": 4.1928721174004195,
+      "grad_norm": 1.4699604511260986,
       "learning_rate": 1e-05,
-      "loss": 6.5315,
+      "loss": 6.9765,
       "step": 4000
     },
     {
-      "epoch": 6.289308176100629,
-      "eval_loss": 5.101934432983398,
-      "eval_runtime": 0.6406,
-      "eval_samples_per_second": 1422.145,
-      "eval_steps_per_second": 88.982,
+      "epoch": 4.1928721174004195,
+      "eval_loss": 5.874638557434082,
+      "eval_runtime": 0.973,
+      "eval_samples_per_second": 1413.214,
+      "eval_steps_per_second": 88.39,
       "step": 4000
     },
     {
-      "epoch": 9.433962264150944,
-      "eval_loss": 4.677049160003662,
-      "eval_runtime": 0.6439,
-      "eval_samples_per_second": 1414.724,
-      "eval_steps_per_second": 88.517,
+      "epoch": 6.289308176100629,
+      "eval_loss": 5.544075965881348,
+      "eval_runtime": 0.9898,
+      "eval_samples_per_second": 1389.228,
+      "eval_steps_per_second": 86.89,
       "step": 6000
     },
     {
-      "epoch": 12.578616352201259,
-      "grad_norm": 2.6387760639190674,
-      "learning_rate": 2e-05,
-      "loss": 4.3407,
+      "epoch": 8.385744234800839,
+      "grad_norm": 2.451873779296875,
+      "learning_rate": 1.9997500000000003e-05,
+      "loss": 5.2182,
       "step": 8000
     },
     {
-      "epoch": 12.578616352201259,
-      "eval_loss": 4.4404096603393555,
-      "eval_runtime": 0.6445,
-      "eval_samples_per_second": 1413.55,
-      "eval_steps_per_second": 88.444,
+      "epoch": 8.385744234800839,
+      "eval_loss": 5.278811454772949,
+      "eval_runtime": 0.9809,
+      "eval_samples_per_second": 1401.823,
+      "eval_steps_per_second": 87.678,
       "step": 8000
     },
     {
-      "epoch": 15.723270440251572,
-      "eval_loss": 4.286661624908447,
-      "eval_runtime": 0.6446,
-      "eval_samples_per_second": 1413.35,
-      "eval_steps_per_second": 88.431,
+      "epoch": 10.482180293501049,
+      "eval_loss": 5.099233150482178,
+      "eval_runtime": 0.9694,
+      "eval_samples_per_second": 1418.396,
+      "eval_steps_per_second": 88.714,
       "step": 10000
     },
     {
-      "epoch": 18.867924528301888,
-      "grad_norm": 2.381523609161377,
+      "epoch": 12.578616352201259,
+      "grad_norm": 2.490121841430664,
       "learning_rate": 2.9995e-05,
-      "loss": 3.8725,
+      "loss": 4.7379,
       "step": 12000
     },
     {
-      "epoch": 18.867924528301888,
-      "eval_loss": 4.171908855438232,
-      "eval_runtime": 0.6441,
-      "eval_samples_per_second": 1414.321,
-      "eval_steps_per_second": 88.492,
+      "epoch": 12.578616352201259,
+      "eval_loss": 4.970981597900391,
+      "eval_runtime": 0.9729,
+      "eval_samples_per_second": 1413.251,
+      "eval_steps_per_second": 88.392,
       "step": 12000
     },
     {
-      "epoch": 22.0125786163522,
-      "eval_loss": 4.096552848815918,
-      "eval_runtime": 0.6506,
-      "eval_samples_per_second": 1400.196,
-      "eval_steps_per_second": 87.608,
+      "epoch": 14.675052410901468,
+      "eval_loss": 4.875853061676025,
+      "eval_runtime": 0.9902,
+      "eval_samples_per_second": 1388.55,
+      "eval_steps_per_second": 86.848,
       "step": 14000
     },
     {
-      "epoch": 25.157232704402517,
-      "grad_norm": 2.4479799270629883,
-      "learning_rate": 3.999e-05,
-      "loss": 3.5953,
+      "epoch": 16.771488469601678,
+      "grad_norm": 2.485079765319824,
+      "learning_rate": 3.99925e-05,
+      "loss": 4.4249,
       "step": 16000
     },
     {
-      "epoch": 25.157232704402517,
-      "eval_loss": 4.033241271972656,
-      "eval_runtime": 0.6465,
-      "eval_samples_per_second": 1409.06,
-      "eval_steps_per_second": 88.163,
+      "epoch": 16.771488469601678,
+      "eval_loss": 4.8005266189575195,
+      "eval_runtime": 0.9709,
+      "eval_samples_per_second": 1416.144,
+      "eval_steps_per_second": 88.573,
       "step": 16000
     },
     {
-      "epoch": 28.30188679245283,
-      "eval_loss": 3.9956283569335938,
-      "eval_runtime": 0.6443,
-      "eval_samples_per_second": 1413.837,
-      "eval_steps_per_second": 88.462,
+      "epoch": 18.867924528301888,
+      "eval_loss": 4.743557453155518,
+      "eval_runtime": 0.9764,
+      "eval_samples_per_second": 1408.167,
+      "eval_steps_per_second": 88.074,
       "step": 18000
     },
     {
-      "epoch": 31.446540880503143,
-      "grad_norm": 2.266655921936035,
-      "learning_rate": 4.9985e-05,
-      "loss": 3.3822,
+      "epoch": 20.964360587002098,
+      "grad_norm": 2.311079978942871,
+      "learning_rate": 4.99875e-05,
+      "loss": 4.1842,
       "step": 20000
     },
     {
-      "epoch": 31.446540880503143,
-      "eval_loss": 3.970871686935425,
-      "eval_runtime": 0.6424,
-      "eval_samples_per_second": 1418.161,
-      "eval_steps_per_second": 88.732,
+      "epoch": 20.964360587002098,
+      "eval_loss": 4.692162990570068,
+      "eval_runtime": 0.9728,
+      "eval_samples_per_second": 1413.396,
+      "eval_steps_per_second": 88.401,
       "step": 20000
     },
     {
-      "epoch": 34.59119496855346,
-      "eval_loss": 3.951535224914551,
-      "eval_runtime": 0.6747,
-      "eval_samples_per_second": 1350.174,
-      "eval_steps_per_second": 84.479,
+      "epoch": 23.060796645702307,
+      "eval_loss": 4.648115634918213,
+      "eval_runtime": 0.9711,
+      "eval_samples_per_second": 1415.941,
+      "eval_steps_per_second": 88.561,
       "step": 22000
     },
     {
-      "epoch": 37.735849056603776,
-      "grad_norm": 2.3022546768188477,
-      "learning_rate": 5.9980000000000005e-05,
-      "loss": 3.2038,
+      "epoch": 25.157232704402517,
+      "grad_norm": 2.3314199447631836,
+      "learning_rate": 5.998250000000001e-05,
+      "loss": 3.9843,
       "step": 24000
     },
     {
-      "epoch": 37.735849056603776,
-      "eval_loss": 3.937915563583374,
-      "eval_runtime": 0.6427,
-      "eval_samples_per_second": 1417.462,
-      "eval_steps_per_second": 88.689,
+      "epoch": 25.157232704402517,
+      "eval_loss": 4.615537166595459,
+      "eval_runtime": 0.9745,
+      "eval_samples_per_second": 1410.994,
+      "eval_steps_per_second": 88.251,
       "step": 24000
     },
     {
-      "epoch": 40.880503144654085,
-      "eval_loss": 3.935504913330078,
-      "eval_runtime": 0.6463,
-      "eval_samples_per_second": 1409.641,
-      "eval_steps_per_second": 88.199,
+      "epoch": 27.253668763102727,
+      "eval_loss": 4.59817361831665,
+      "eval_runtime": 0.9725,
+      "eval_samples_per_second": 1413.828,
+      "eval_steps_per_second": 88.429,
       "step": 26000
     },
     {
-      "epoch": 44.0251572327044,
-      "grad_norm": 2.377114772796631,
-      "learning_rate": 6.997500000000001e-05,
-      "loss": 3.0443,
+      "epoch": 29.350104821802937,
+      "grad_norm": 2.2022252082824707,
+      "learning_rate": 6.99775e-05,
+      "loss": 3.8181,
       "step": 28000
     },
     {
-      "epoch": 44.0251572327044,
-      "eval_loss": 3.951632261276245,
-      "eval_runtime": 0.6452,
-      "eval_samples_per_second": 1412.017,
-      "eval_steps_per_second": 88.348,
+      "epoch": 29.350104821802937,
+      "eval_loss": 4.584521293640137,
+      "eval_runtime": 0.9704,
+      "eval_samples_per_second": 1416.917,
+      "eval_steps_per_second": 88.622,
       "step": 28000
     },
     {
-      "epoch": 47.16981132075472,
-      "eval_loss": 3.9560022354125977,
-      "eval_runtime": 0.6457,
-      "eval_samples_per_second": 1410.953,
-      "eval_steps_per_second": 88.281,
+      "epoch": 31.446540880503143,
+      "eval_loss": 4.581137657165527,
+      "eval_runtime": 0.9781,
+      "eval_samples_per_second": 1405.736,
+      "eval_steps_per_second": 87.922,
       "step": 30000
     },
     {
-      "epoch": 50.314465408805034,
-      "grad_norm": 2.4269351959228516,
-      "learning_rate": 7.997e-05,
-      "loss": 2.8937,
+      "epoch": 33.542976939203356,
+      "grad_norm": 2.109912872314453,
+      "learning_rate": 7.997250000000001e-05,
+      "loss": 3.6751,
       "step": 32000
     },
     {
-      "epoch": 50.314465408805034,
-      "eval_loss": 3.9868452548980713,
-      "eval_runtime": 0.642,
-      "eval_samples_per_second": 1418.921,
-      "eval_steps_per_second": 88.78,
+      "epoch": 33.542976939203356,
+      "eval_loss": 4.579585075378418,
+      "eval_runtime": 0.9725,
+      "eval_samples_per_second": 1413.852,
+      "eval_steps_per_second": 88.43,
       "step": 32000
     },
     {
-      "epoch": 53.459119496855344,
-      "eval_loss": 4.025730133056641,
-      "eval_runtime": 0.6451,
-      "eval_samples_per_second": 1412.122,
-      "eval_steps_per_second": 88.355,
+      "epoch": 35.63941299790356,
+      "eval_loss": 4.582756042480469,
+      "eval_runtime": 0.9724,
+      "eval_samples_per_second": 1413.996,
+      "eval_steps_per_second": 88.439,
       "step": 34000
     },
     {
-      "epoch": 56.60377358490566,
-      "grad_norm": 2.4868056774139404,
-      "learning_rate": 8.9965e-05,
-      "loss": 2.7503,
+      "epoch": 37.735849056603776,
+      "grad_norm": 2.1550486087799072,
+      "learning_rate": 8.996750000000001e-05,
+      "loss": 3.5484,
       "step": 36000
     },
     {
-      "epoch": 56.60377358490566,
-      "eval_loss": 4.050300598144531,
-      "eval_runtime": 0.6446,
-      "eval_samples_per_second": 1413.342,
-      "eval_steps_per_second": 88.431,
+      "epoch": 37.735849056603776,
+      "eval_loss": 4.586859226226807,
+      "eval_runtime": 0.979,
+      "eval_samples_per_second": 1404.481,
+      "eval_steps_per_second": 87.844,
       "step": 36000
     }
   ],
   "logging_steps": 4000,
   "max_steps": 100000,
   "num_input_tokens_seen": 0,
-  "num_train_epochs": 158,
+  "num_train_epochs": 105,
   "save_steps": 4000,
   "stateful_callbacks": {
     "TrainerControl": {
@@ -233,7 +233,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 9307480330862592.0,
+  "total_flos": 9308384302743552.0,
   "train_batch_size": 16,
   "trial_name": null,
   "trial_params": null

checkpoint-36000/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e6d9a18dff8e8501ed8e1286b71e34a5ee0478c42b015fd65b8981e86e58a8c2
+oid sha256:7e7557b0dc8ef07c9db255e1167d98819525b043deb4934f580e21d716914111
 size 5368