Training in progress, step 1875

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:da32d3ea1bfbd0ebd3b10a65710db858a2b52840601b85c79d7dea239a7ad777
+oid sha256:55936da28ccaa45e5ddca8dcdc226991b73479c6b56eb976c62b4594f394ab98
 size 104900720

config.json

@@ -1,5 +1,5 @@
 {
-  "_name_or_path": "elinas/llama-13b-hf-transformers-4.29",
+  "_name_or_path": "TheBloke/Llama-2-13B-fp16",
   "architectures": [
     "LlamaForCausalLM"
   ],
@@ -9,14 +9,14 @@
   "hidden_size": 5120,
   "initializer_range": 0.02,
   "intermediate_size": 13824,
-  "max_position_embeddings": 2048,
+  "max_position_embeddings": 4096,
   "model_type": "llama",
   "num_attention_heads": 40,
   "num_hidden_layers": 40,
   "num_key_value_heads": 40,
   "pad_token_id": 0,
   "pretraining_tp": 1,
-  "rms_norm_eps": 1e-06,
+  "rms_norm_eps": 1e-05,
   "rope_scaling": null,
   "tie_word_embeddings": false,
   "torch_dtype": "float32",

last-checkpoint/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:da32d3ea1bfbd0ebd3b10a65710db858a2b52840601b85c79d7dea239a7ad777
+oid sha256:55936da28ccaa45e5ddca8dcdc226991b73479c6b56eb976c62b4594f394ab98
 size 104900720

last-checkpoint/{global_step615 → global_step1875}/zero_pp_rank_0_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fc7a512b9dd649ea7b8bcd0f4bfeba78db117a4d21241eb3c445f28b9b035b44
+oid sha256:b86c8546dd1666401b84d90ca0b94ef6cb6c0b73117ae6c1d687ddb1e11f648b
 size 6508524919

last-checkpoint/{global_step615 → global_step1875}/zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c6ac434744b5670e20dbf7de0f019f547687454f89ea0280eb188b83b45f13ef
+oid sha256:fee3a7fd29d3119d2b21bf40c50d75a2eebabc650ed9e0e16b2e8bcfe9449031
 size 39324734

last-checkpoint/{global_step615 → global_step1875}/zero_pp_rank_1_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0ae5fa3e3b1dcbb4a4e0fe8d8297960bf81e85883e8626388f2741f1256a7f42
+oid sha256:86353a275442ecc1818f29ee23975deedfc0c42b7dc80def1b26bff124460652
 size 6508524919

last-checkpoint/{global_step615 → global_step1875}/zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8287daec126b9d2fb8e8cd51ec9da468856db2418f4582876b55fe61757ef61a
+oid sha256:bb9c8eabf344996ecedd94f084d70e33300335e97cdc132bc047b014185c6059
 size 39324734

last-checkpoint/{global_step615 → global_step1875}/zero_pp_rank_2_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5e4f6ed520b87d564d216f089f3527986451a9a7fda56fa546fd646514bd44f9
+oid sha256:25055a4966ab33becc0bcb18d051dd85b05d08342ee8e7609d2649be7f0fe9d4
 size 6508524919

last-checkpoint/{global_step615 → global_step1875}/zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f4042166c4f941b64617579ea00f6d7f095a5be1bc6c0e8bbdf9c40550033e6b
+oid sha256:a1a1d5289fa7a41de645b8cb7890fb4ed7624b274892688cef21dfb5703cc831
 size 39324734

last-checkpoint/{global_step615 → global_step1875}/zero_pp_rank_3_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8f2f0a3215aac7e2ec10298d856832621b2db0c91916c00868411fc3298786ee
+oid sha256:da7d9d6595b20c78f8650a456533ca3ca7fd8a0d09c8b080efd314c39ab51fc7
 size 6508524919

last-checkpoint/{global_step615 → global_step1875}/zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f92054239ef1b333a36e40ed281baf5e518dbf38687fd7707a744cc060ea4bfe
+oid sha256:9bef3cc71b28be9da43756109bd915e5a8693bc2e9f945c3a2c35f77d8fd5a36
 size 39324734

last-checkpoint/{global_step615 → global_step1875}/zero_pp_rank_4_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7807b566bad78986341cee14e7d4aa53bc8a0200ca5eb2d5f029a2beaf7ebf66
+oid sha256:4dd45dd6271af5951105ff05aa28f48e34d0aa71ad8e9374c030e8495033d274
 size 6508524919

last-checkpoint/{global_step615 → global_step1875}/zero_pp_rank_4_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e0479c922106519c15b9049c6ef433091218afc9898d85ecf50d0bca0f00f1dd
+oid sha256:c9c7d905697ce3ebcb3108c7a8d9c377f03e508805123a11d78736db0628e3b0
 size 39324734

last-checkpoint/{global_step615 → global_step1875}/zero_pp_rank_5_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c1b68b41313f92d5461f5650971d0d7fc2021f831e8b209bbe7684a9fa6de036
+oid sha256:7214390febf54a55dfed233c4252b5a5e1fe4a2e5a1dce4c268ff0b8af78add1
 size 6508524919

last-checkpoint/{global_step615 → global_step1875}/zero_pp_rank_5_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e6d43038f68b214eaaf0de4696eac4ce4fbf8b203f0278eae4d39d6b03b2c8b5
+oid sha256:1d2e9a3e15eab72cb27e2ad212925c0bcc40c109704d5c12d76a00c783c0714b
 size 39324734

last-checkpoint/{global_step615 → global_step1875}/zero_pp_rank_6_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ca1774553df058481dced20adaab498c5c060713fc551b7824be8b5b899199f0
+oid sha256:a4d315b42177b90d73673c0917af534743f494256bd881f5858dd280d4432bbb
 size 6508524919

last-checkpoint/{global_step615 → global_step1875}/zero_pp_rank_6_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0d9e28784bda3c89c935090afe7e9c4f28a0319586034f42c7a6ddb62cb23145
+oid sha256:5bb2afbc9c7cb4f347271f6e4391eb9c7956b9e8fc68c0117bf80cb52c8cba7c
 size 39324734

last-checkpoint/{global_step615 → global_step1875}/zero_pp_rank_7_mp_rank_00_model_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ad420ae915a4f4dda92d5e553c7fa52197f613dd9a32042c920b5f57993795db
+oid sha256:f6a90eac5198209d06ad10d0085cb23471f4de6a307c6b2d74b5cb7067c96729
 size 6508524919

last-checkpoint/{global_step615 → global_step1875}/zero_pp_rank_7_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e0d966f44a532fe90a784275aa0ecbcca4250af5ac121375136da07434f1e132
+oid sha256:b1e2129e9ba4b98ab3bf5f0c3afc2e58206d9d375a9163c28300e85e1b38fe30
 size 39324734

last-checkpoint/latest

@@ -1 +1 @@
-global_step615
+global_step1875

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2650662faaa72b41472de1537f17ead9d812f09f4fb8cd2d674128a4712a4e7d
+oid sha256:9d914bd28016f6122c43d9f2c83724d2eb3fb30e66e779af973bb2d54cc8392b
 size 21687

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a4627a7e7a7dfe2ee147a65e0d6015baf626890fa1b156301c335b9ad9e316e6
+oid sha256:6211fb1ee2fec138ceb8c47005fc9bc2418104c532ee8dfc1b97c6ff0a4d8ca6
 size 21687

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:26052e9c4d518cdd7d9b6100278576ee5087e52a6278f6b0143dacdb8bf18fdb
+oid sha256:5e6e93d0e06b92d456cdf9ae8d458dc04fe62bcaff12dcfd66e34ec4d96610a3
 size 21687

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:979fe1e078e9bce07319930c2cf56f16954ac1c8ce9e6e4acceb06554c8cef58
+oid sha256:bd94ec78519765c3ee8dc3b4493f42b5fd0c4cd2976fd129e1f8ac18f99ae6d2
 size 21687

last-checkpoint/rng_state_4.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:dd01729cff1a8faf266bd665eeba2ab4f2ed3a9a830f734b2db4c2eae107dcff
+oid sha256:63751f591f898abcc08b990d99973e8f008e12981693a109dce30ef0cd238781
 size 21687

last-checkpoint/rng_state_5.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a1e8fe0589efc84bd5ff71928ffae22410544c7901ec04e9e97eb1a38525f7fe
+oid sha256:ee6e3fa3cef42dc11afb890036889f87f425dfc577cbf9e1d5de01bb444eb373
 size 21687

last-checkpoint/rng_state_6.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:425f4a8de6e80c8b017685beb871dcc15ab951bf996eecf512432f3e6d9c9abf
+oid sha256:61d84c5c8cd6aa95d17a38e63e5b602a1ef6cf5067ed65357c055b82af7e9406
 size 21687

last-checkpoint/rng_state_7.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:08811a4460727a46d30b4089dffa5d5fea621645158464f2a8d71c7027d8688d
+oid sha256:1182410896d797ec61be8102c6e7744e694a891c867f4e04ed79c5737df19e2c
 size 21687

last-checkpoint/trainer_state.json

@@ -1,754 +1,2266 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 1.0,
-  "global_step": 615,
+  "epoch": 0.7203227045716482,
+  "global_step": 1875,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 0.01,
-      "learning_rate": 1.3979400086720373e-06,
-      "loss": 1.8018,
+      "epoch": 0.0,
+      "learning_rate": 0.00013979400086720374,
+      "loss": 1.8118,
       "step": 5
     },
     {
-      "epoch": 0.02,
-      "learning_rate": 1.9999999999999995e-06,
-      "loss": 1.7649,
+      "epoch": 0.0,
+      "learning_rate": 0.00019999999999999998,
+      "loss": 1.6525,
       "step": 10
     },
     {
-      "epoch": 0.02,
-      "learning_rate": 1.9934426229508195e-06,
-      "loss": 1.7617,
+      "epoch": 0.01,
+      "learning_rate": 0.0001998460354118553,
+      "loss": 1.5208,
       "step": 15
     },
     {
-      "epoch": 0.03,
-      "learning_rate": 1.9852459016393443e-06,
-      "loss": 1.9239,
+      "epoch": 0.01,
+      "learning_rate": 0.00019965357967667439,
+      "loss": 1.4179,
       "step": 20
     },
     {
-      "epoch": 0.04,
-      "learning_rate": 1.9770491803278686e-06,
-      "loss": 1.7685,
+      "epoch": 0.01,
+      "learning_rate": 0.00019946112394149347,
+      "loss": 1.5179,
       "step": 25
     },
     {
-      "epoch": 0.05,
-      "learning_rate": 1.9688524590163934e-06,
-      "loss": 1.7113,
+      "epoch": 0.01,
+      "learning_rate": 0.00019926866820631255,
+      "loss": 1.4164,
       "step": 30
     },
     {
-      "epoch": 0.06,
-      "learning_rate": 1.960655737704918e-06,
-      "loss": 1.8364,
+      "epoch": 0.01,
+      "learning_rate": 0.00019907621247113163,
+      "loss": 1.3599,
       "step": 35
     },
     {
-      "epoch": 0.07,
-      "learning_rate": 1.9524590163934425e-06,
-      "loss": 1.7904,
+      "epoch": 0.02,
+      "learning_rate": 0.00019888375673595074,
+      "loss": 1.4631,
       "step": 40
     },
     {
-      "epoch": 0.07,
-      "learning_rate": 1.9442622950819673e-06,
-      "loss": 1.7484,
+      "epoch": 0.02,
+      "learning_rate": 0.00019869130100076983,
+      "loss": 1.4569,
       "step": 45
     },
     {
-      "epoch": 0.08,
-      "learning_rate": 1.9360655737704916e-06,
-      "loss": 1.694,
+      "epoch": 0.02,
+      "learning_rate": 0.00019849884526558894,
+      "loss": 1.4491,
       "step": 50
     },
     {
-      "epoch": 0.09,
-      "learning_rate": 1.9278688524590163e-06,
-      "loss": 1.7381,
+      "epoch": 0.02,
+      "learning_rate": 0.00019830638953040802,
+      "loss": 1.4852,
       "step": 55
     },
     {
-      "epoch": 0.1,
-      "learning_rate": 1.919672131147541e-06,
-      "loss": 1.7264,
+      "epoch": 0.02,
+      "learning_rate": 0.0001981139337952271,
+      "loss": 1.5018,
       "step": 60
     },
     {
-      "epoch": 0.11,
-      "learning_rate": 1.9114754098360654e-06,
-      "loss": 1.6833,
+      "epoch": 0.02,
+      "learning_rate": 0.00019792147806004618,
+      "loss": 1.4777,
       "step": 65
     },
     {
-      "epoch": 0.11,
-      "learning_rate": 1.9032786885245902e-06,
-      "loss": 1.7088,
+      "epoch": 0.03,
+      "learning_rate": 0.0001977290223248653,
+      "loss": 1.4531,
       "step": 70
     },
     {
-      "epoch": 0.12,
-      "learning_rate": 1.8950819672131145e-06,
-      "loss": 1.6512,
+      "epoch": 0.03,
+      "learning_rate": 0.00019753656658968438,
+      "loss": 1.3477,
       "step": 75
     },
     {
-      "epoch": 0.13,
-      "learning_rate": 1.8868852459016393e-06,
-      "loss": 1.5752,
+      "epoch": 0.03,
+      "learning_rate": 0.0001973441108545035,
+      "loss": 1.5621,
       "step": 80
     },
     {
-      "epoch": 0.14,
-      "learning_rate": 1.8786885245901638e-06,
-      "loss": 1.8083,
+      "epoch": 0.03,
+      "learning_rate": 0.00019715165511932257,
+      "loss": 1.3823,
       "step": 85
     },
     {
-      "epoch": 0.15,
-      "learning_rate": 1.8704918032786886e-06,
-      "loss": 1.7016,
+      "epoch": 0.03,
+      "learning_rate": 0.00019695919938414165,
+      "loss": 1.4521,
       "step": 90
     },
     {
-      "epoch": 0.15,
-      "learning_rate": 1.862295081967213e-06,
-      "loss": 1.6425,
+      "epoch": 0.04,
+      "learning_rate": 0.00019676674364896076,
+      "loss": 1.4822,
       "step": 95
     },
     {
-      "epoch": 0.16,
-      "learning_rate": 1.8540983606557377e-06,
-      "loss": 1.6506,
+      "epoch": 0.04,
+      "learning_rate": 0.00019657428791377982,
+      "loss": 1.4313,
       "step": 100
     },
     {
-      "epoch": 0.17,
-      "learning_rate": 1.845901639344262e-06,
-      "loss": 1.6875,
+      "epoch": 0.04,
+      "learning_rate": 0.00019638183217859893,
+      "loss": 1.3581,
       "step": 105
     },
     {
-      "epoch": 0.18,
-      "learning_rate": 1.8377049180327868e-06,
-      "loss": 1.667,
+      "epoch": 0.04,
+      "learning_rate": 0.000196189376443418,
+      "loss": 1.3917,
       "step": 110
     },
     {
-      "epoch": 0.19,
-      "learning_rate": 1.8295081967213113e-06,
-      "loss": 1.7864,
+      "epoch": 0.04,
+      "learning_rate": 0.00019599692070823712,
+      "loss": 1.4632,
       "step": 115
     },
     {
-      "epoch": 0.2,
-      "learning_rate": 1.8213114754098359e-06,
-      "loss": 1.7222,
+      "epoch": 0.05,
+      "learning_rate": 0.0001958044649730562,
+      "loss": 1.4535,
       "step": 120
     },
     {
-      "epoch": 0.2,
-      "learning_rate": 1.8131147540983606e-06,
-      "loss": 1.6307,
+      "epoch": 0.05,
+      "learning_rate": 0.0001956120092378753,
+      "loss": 1.487,
       "step": 125
     },
     {
-      "epoch": 0.21,
-      "learning_rate": 1.8049180327868852e-06,
-      "loss": 1.678,
+      "epoch": 0.05,
+      "learning_rate": 0.0001954195535026944,
+      "loss": 1.3851,
       "step": 130
     },
     {
-      "epoch": 0.22,
-      "learning_rate": 1.7967213114754097e-06,
-      "loss": 1.6302,
+      "epoch": 0.05,
+      "learning_rate": 0.00019522709776751348,
+      "loss": 1.3467,
       "step": 135
     },
     {
-      "epoch": 0.23,
-      "learning_rate": 1.7885245901639343e-06,
-      "loss": 1.7003,
+      "epoch": 0.05,
+      "learning_rate": 0.00019503464203233256,
+      "loss": 1.4262,
       "step": 140
     },
     {
-      "epoch": 0.24,
-      "learning_rate": 1.780327868852459e-06,
-      "loss": 1.6147,
+      "epoch": 0.06,
+      "learning_rate": 0.00019484218629715167,
+      "loss": 1.326,
       "step": 145
     },
     {
-      "epoch": 0.24,
-      "learning_rate": 1.7721311475409834e-06,
-      "loss": 1.6948,
+      "epoch": 0.06,
+      "learning_rate": 0.00019464973056197075,
+      "loss": 1.471,
       "step": 150
     },
     {
-      "epoch": 0.25,
-      "learning_rate": 1.7639344262295081e-06,
-      "loss": 1.6458,
+      "epoch": 0.06,
+      "learning_rate": 0.00019445727482678984,
+      "loss": 1.4928,
       "step": 155
     },
     {
-      "epoch": 0.26,
-      "learning_rate": 1.7557377049180327e-06,
-      "loss": 1.6142,
+      "epoch": 0.06,
+      "learning_rate": 0.00019426481909160895,
+      "loss": 1.3059,
       "step": 160
     },
     {
-      "epoch": 0.27,
-      "learning_rate": 1.7475409836065572e-06,
-      "loss": 1.5277,
+      "epoch": 0.06,
+      "learning_rate": 0.00019407236335642803,
+      "loss": 1.3448,
       "step": 165
     },
     {
-      "epoch": 0.28,
-      "learning_rate": 1.7393442622950818e-06,
-      "loss": 1.5558,
+      "epoch": 0.07,
+      "learning_rate": 0.0001938799076212471,
+      "loss": 1.4605,
       "step": 170
     },
     {
-      "epoch": 0.28,
-      "learning_rate": 1.7311475409836065e-06,
-      "loss": 1.56,
+      "epoch": 0.07,
+      "learning_rate": 0.0001936874518860662,
+      "loss": 1.3755,
       "step": 175
     },
     {
-      "epoch": 0.29,
-      "learning_rate": 1.7229508196721309e-06,
-      "loss": 1.6358,
+      "epoch": 0.07,
+      "learning_rate": 0.0001934949961508853,
+      "loss": 1.4118,
       "step": 180
     },
     {
-      "epoch": 0.3,
-      "learning_rate": 1.7147540983606556e-06,
-      "loss": 1.5567,
+      "epoch": 0.07,
+      "learning_rate": 0.0001933025404157044,
+      "loss": 1.302,
       "step": 185
     },
     {
-      "epoch": 0.31,
-      "learning_rate": 1.7065573770491804e-06,
-      "loss": 1.4662,
+      "epoch": 0.07,
+      "learning_rate": 0.0001931100846805235,
+      "loss": 1.441,
       "step": 190
     },
     {
-      "epoch": 0.32,
-      "learning_rate": 1.6983606557377047e-06,
-      "loss": 1.5566,
+      "epoch": 0.07,
+      "learning_rate": 0.00019291762894534258,
+      "loss": 1.4891,
       "step": 195
     },
     {
-      "epoch": 0.33,
-      "learning_rate": 1.6901639344262295e-06,
-      "loss": 1.5082,
+      "epoch": 0.08,
+      "learning_rate": 0.0001927251732101617,
+      "loss": 1.4324,
       "step": 200
     },
     {
-      "epoch": 0.33,
-      "learning_rate": 1.681967213114754e-06,
-      "loss": 1.5863,
+      "epoch": 0.08,
+      "learning_rate": 0.00019253271747498077,
+      "loss": 1.3829,
       "step": 205
     },
     {
-      "epoch": 0.34,
-      "learning_rate": 1.6737704918032786e-06,
-      "loss": 1.4774,
+      "epoch": 0.08,
+      "learning_rate": 0.00019234026173979986,
+      "loss": 1.3732,
       "step": 210
     },
     {
-      "epoch": 0.35,
-      "learning_rate": 1.6655737704918031e-06,
-      "loss": 1.4835,
+      "epoch": 0.08,
+      "learning_rate": 0.00019214780600461894,
+      "loss": 1.3262,
       "step": 215
     },
     {
-      "epoch": 0.36,
-      "learning_rate": 1.6573770491803279e-06,
-      "loss": 1.5662,
+      "epoch": 0.08,
+      "learning_rate": 0.00019195535026943802,
+      "loss": 1.3651,
       "step": 220
     },
     {
-      "epoch": 0.37,
-      "learning_rate": 1.6491803278688522e-06,
-      "loss": 1.5638,
+      "epoch": 0.09,
+      "learning_rate": 0.00019176289453425713,
+      "loss": 1.4975,
       "step": 225
     },
     {
-      "epoch": 0.37,
-      "learning_rate": 1.640983606557377e-06,
-      "loss": 1.5964,
+      "epoch": 0.09,
+      "learning_rate": 0.00019157043879907621,
+      "loss": 1.341,
       "step": 230
     },
     {
-      "epoch": 0.38,
-      "learning_rate": 1.6327868852459015e-06,
-      "loss": 1.4099,
+      "epoch": 0.09,
+      "learning_rate": 0.00019137798306389532,
+      "loss": 1.4237,
       "step": 235
     },
     {
-      "epoch": 0.39,
-      "learning_rate": 1.624590163934426e-06,
-      "loss": 1.4246,
+      "epoch": 0.09,
+      "learning_rate": 0.0001911855273287144,
+      "loss": 1.444,
       "step": 240
     },
     {
-      "epoch": 0.4,
-      "learning_rate": 1.6163934426229508e-06,
-      "loss": 1.4101,
+      "epoch": 0.09,
+      "learning_rate": 0.0001909930715935335,
+      "loss": 1.4071,
       "step": 245
     },
     {
-      "epoch": 0.41,
-      "learning_rate": 1.6081967213114754e-06,
-      "loss": 1.4219,
+      "epoch": 0.1,
+      "learning_rate": 0.00019080061585835257,
+      "loss": 1.3715,
       "step": 250
     },
     {
-      "epoch": 0.41,
-      "learning_rate": 1.6e-06,
-      "loss": 1.4839,
+      "epoch": 0.1,
+      "learning_rate": 0.00019060816012317168,
+      "loss": 1.3385,
       "step": 255
     },
     {
-      "epoch": 0.42,
-      "learning_rate": 1.5918032786885245e-06,
-      "loss": 1.4891,
+      "epoch": 0.1,
+      "learning_rate": 0.00019041570438799076,
+      "loss": 1.4171,
       "step": 260
     },
     {
-      "epoch": 0.43,
-      "learning_rate": 1.5836065573770492e-06,
-      "loss": 1.466,
+      "epoch": 0.1,
+      "learning_rate": 0.00019022324865280987,
+      "loss": 1.3596,
       "step": 265
     },
     {
-      "epoch": 0.44,
-      "learning_rate": 1.5754098360655736e-06,
-      "loss": 1.3792,
+      "epoch": 0.1,
+      "learning_rate": 0.00019003079291762896,
+      "loss": 1.3444,
       "step": 270
     },
     {
-      "epoch": 0.45,
-      "learning_rate": 1.5672131147540983e-06,
-      "loss": 1.4344,
+      "epoch": 0.11,
+      "learning_rate": 0.00018983833718244807,
+      "loss": 1.4197,
       "step": 275
     },
     {
-      "epoch": 0.46,
-      "learning_rate": 1.5590163934426229e-06,
-      "loss": 1.3924,
+      "epoch": 0.11,
+      "learning_rate": 0.00018964588144726715,
+      "loss": 1.3405,
       "step": 280
     },
     {
-      "epoch": 0.46,
-      "learning_rate": 1.5508196721311474e-06,
-      "loss": 1.3575,
+      "epoch": 0.11,
+      "learning_rate": 0.0001894534257120862,
+      "loss": 1.4371,
       "step": 285
     },
     {
-      "epoch": 0.47,
-      "learning_rate": 1.542622950819672e-06,
-      "loss": 1.3742,
+      "epoch": 0.11,
+      "learning_rate": 0.00018926096997690532,
+      "loss": 1.3825,
       "step": 290
     },
     {
-      "epoch": 0.48,
-      "learning_rate": 1.5344262295081967e-06,
-      "loss": 1.4584,
+      "epoch": 0.11,
+      "learning_rate": 0.0001890685142417244,
+      "loss": 1.4089,
       "step": 295
     },
     {
-      "epoch": 0.49,
-      "learning_rate": 1.526229508196721e-06,
-      "loss": 1.3866,
+      "epoch": 0.12,
+      "learning_rate": 0.0001888760585065435,
+      "loss": 1.4502,
       "step": 300
     },
     {
-      "epoch": 0.5,
-      "learning_rate": 1.5180327868852458e-06,
-      "loss": 1.4555,
+      "epoch": 0.12,
+      "learning_rate": 0.0001886836027713626,
+      "loss": 1.3097,
       "step": 305
     },
     {
-      "epoch": 0.5,
-      "learning_rate": 1.5098360655737706e-06,
-      "loss": 1.3399,
+      "epoch": 0.12,
+      "learning_rate": 0.0001884911470361817,
+      "loss": 1.3422,
       "step": 310
     },
     {
-      "epoch": 0.51,
-      "learning_rate": 1.501639344262295e-06,
-      "loss": 1.3791,
+      "epoch": 0.12,
+      "learning_rate": 0.00018829869130100078,
+      "loss": 1.4088,
       "step": 315
     },
     {
-      "epoch": 0.52,
-      "learning_rate": 1.4934426229508197e-06,
-      "loss": 1.3861,
+      "epoch": 0.12,
+      "learning_rate": 0.00018810623556581987,
+      "loss": 1.3372,
       "step": 320
     },
     {
-      "epoch": 0.53,
-      "learning_rate": 1.4852459016393442e-06,
-      "loss": 1.4207,
+      "epoch": 0.12,
+      "learning_rate": 0.00018791377983063895,
+      "loss": 1.3307,
       "step": 325
     },
     {
-      "epoch": 0.54,
-      "learning_rate": 1.4770491803278688e-06,
-      "loss": 1.424,
+      "epoch": 0.13,
+      "learning_rate": 0.00018772132409545806,
+      "loss": 1.4299,
       "step": 330
     },
     {
-      "epoch": 0.54,
-      "learning_rate": 1.4688524590163933e-06,
-      "loss": 1.408,
+      "epoch": 0.13,
+      "learning_rate": 0.00018752886836027714,
+      "loss": 1.4264,
       "step": 335
     },
     {
-      "epoch": 0.55,
-      "learning_rate": 1.460655737704918e-06,
-      "loss": 1.3347,
+      "epoch": 0.13,
+      "learning_rate": 0.00018733641262509625,
+      "loss": 1.3151,
       "step": 340
     },
     {
-      "epoch": 0.56,
-      "learning_rate": 1.4524590163934424e-06,
-      "loss": 1.4363,
+      "epoch": 0.13,
+      "learning_rate": 0.00018714395688991533,
+      "loss": 1.3675,
       "step": 345
     },
     {
-      "epoch": 0.57,
-      "learning_rate": 1.4442622950819672e-06,
-      "loss": 1.4262,
+      "epoch": 0.13,
+      "learning_rate": 0.00018695150115473442,
+      "loss": 1.3794,
       "step": 350
     },
     {
-      "epoch": 0.58,
-      "learning_rate": 1.4360655737704917e-06,
-      "loss": 1.4408,
+      "epoch": 0.14,
+      "learning_rate": 0.00018675904541955353,
+      "loss": 1.3892,
       "step": 355
     },
     {
-      "epoch": 0.59,
-      "learning_rate": 1.4278688524590163e-06,
-      "loss": 1.4739,
+      "epoch": 0.14,
+      "learning_rate": 0.00018656658968437258,
+      "loss": 1.4612,
       "step": 360
     },
     {
-      "epoch": 0.59,
-      "learning_rate": 1.419672131147541e-06,
-      "loss": 1.3784,
+      "epoch": 0.14,
+      "learning_rate": 0.0001863741339491917,
+      "loss": 1.283,
       "step": 365
     },
     {
-      "epoch": 0.6,
-      "learning_rate": 1.4114754098360656e-06,
-      "loss": 1.3516,
+      "epoch": 0.14,
+      "learning_rate": 0.00018618167821401078,
+      "loss": 1.4235,
       "step": 370
     },
     {
-      "epoch": 0.61,
-      "learning_rate": 1.4032786885245901e-06,
-      "loss": 1.2991,
+      "epoch": 0.14,
+      "learning_rate": 0.00018598922247882988,
+      "loss": 1.3036,
       "step": 375
     },
     {
-      "epoch": 0.62,
-      "learning_rate": 1.3950819672131147e-06,
-      "loss": 1.3686,
+      "epoch": 0.15,
+      "learning_rate": 0.00018579676674364897,
+      "loss": 1.3886,
       "step": 380
     },
     {
-      "epoch": 0.63,
-      "learning_rate": 1.3868852459016394e-06,
-      "loss": 1.5018,
+      "epoch": 0.15,
+      "learning_rate": 0.00018560431100846808,
+      "loss": 1.3226,
       "step": 385
     },
     {
-      "epoch": 0.63,
-      "learning_rate": 1.3786885245901638e-06,
-      "loss": 1.3487,
+      "epoch": 0.15,
+      "learning_rate": 0.00018541185527328716,
+      "loss": 1.3966,
       "step": 390
     },
     {
-      "epoch": 0.64,
-      "learning_rate": 1.3704918032786885e-06,
-      "loss": 1.4386,
+      "epoch": 0.15,
+      "learning_rate": 0.00018521939953810624,
+      "loss": 1.3835,
       "step": 395
     },
     {
-      "epoch": 0.65,
-      "learning_rate": 1.362295081967213e-06,
-      "loss": 1.4099,
+      "epoch": 0.15,
+      "learning_rate": 0.00018502694380292533,
+      "loss": 1.3544,
       "step": 400
     },
     {
-      "epoch": 0.66,
-      "learning_rate": 1.3540983606557376e-06,
-      "loss": 1.3353,
+      "epoch": 0.16,
+      "learning_rate": 0.00018483448806774444,
+      "loss": 1.5449,
       "step": 405
     },
     {
-      "epoch": 0.67,
-      "learning_rate": 1.3459016393442622e-06,
-      "loss": 1.4481,
+      "epoch": 0.16,
+      "learning_rate": 0.00018464203233256352,
+      "loss": 1.3814,
       "step": 410
     },
     {
-      "epoch": 0.67,
-      "learning_rate": 1.337704918032787e-06,
-      "loss": 1.3743,
+      "epoch": 0.16,
+      "learning_rate": 0.0001844495765973826,
+      "loss": 1.4272,
       "step": 415
     },
     {
-      "epoch": 0.68,
-      "learning_rate": 1.3295081967213113e-06,
-      "loss": 1.3592,
+      "epoch": 0.16,
+      "learning_rate": 0.0001842571208622017,
+      "loss": 1.4256,
       "step": 420
     },
     {
-      "epoch": 0.69,
-      "learning_rate": 1.321311475409836e-06,
-      "loss": 1.3307,
+      "epoch": 0.16,
+      "learning_rate": 0.0001840646651270208,
+      "loss": 1.367,
       "step": 425
     },
     {
-      "epoch": 0.7,
-      "learning_rate": 1.3131147540983608e-06,
-      "loss": 1.4015,
+      "epoch": 0.17,
+      "learning_rate": 0.00018387220939183988,
+      "loss": 1.3814,
       "step": 430
     },
     {
-      "epoch": 0.71,
-      "learning_rate": 1.3049180327868851e-06,
-      "loss": 1.4114,
+      "epoch": 0.17,
+      "learning_rate": 0.00018367975365665896,
+      "loss": 1.351,
       "step": 435
     },
     {
-      "epoch": 0.72,
-      "learning_rate": 1.2967213114754099e-06,
-      "loss": 1.4648,
+      "epoch": 0.17,
+      "learning_rate": 0.00018348729792147807,
+      "loss": 1.3966,
       "step": 440
     },
     {
-      "epoch": 0.72,
-      "learning_rate": 1.2885245901639344e-06,
-      "loss": 1.4291,
+      "epoch": 0.17,
+      "learning_rate": 0.00018329484218629715,
+      "loss": 1.4355,
       "step": 445
     },
     {
-      "epoch": 0.73,
-      "learning_rate": 1.280327868852459e-06,
-      "loss": 1.3383,
+      "epoch": 0.17,
+      "learning_rate": 0.00018310238645111626,
+      "loss": 1.3635,
       "step": 450
     },
     {
-      "epoch": 0.74,
-      "learning_rate": 1.2721311475409835e-06,
-      "loss": 1.3256,
+      "epoch": 0.17,
+      "learning_rate": 0.00018290993071593534,
+      "loss": 1.4373,
       "step": 455
     },
     {
-      "epoch": 0.75,
-      "learning_rate": 1.2639344262295083e-06,
-      "loss": 1.347,
+      "epoch": 0.18,
+      "learning_rate": 0.00018271747498075445,
+      "loss": 1.424,
       "step": 460
     },
     {
-      "epoch": 0.76,
-      "learning_rate": 1.2557377049180326e-06,
-      "loss": 1.4265,
+      "epoch": 0.18,
+      "learning_rate": 0.00018252501924557354,
+      "loss": 1.4445,
       "step": 465
     },
     {
-      "epoch": 0.76,
-      "learning_rate": 1.2475409836065574e-06,
-      "loss": 1.3616,
+      "epoch": 0.18,
+      "learning_rate": 0.00018233256351039262,
+      "loss": 1.4569,
       "step": 470
     },
     {
-      "epoch": 0.77,
-      "learning_rate": 1.239344262295082e-06,
-      "loss": 1.347,
+      "epoch": 0.18,
+      "learning_rate": 0.0001821401077752117,
+      "loss": 1.2821,
       "step": 475
     },
     {
-      "epoch": 0.78,
-      "learning_rate": 1.2311475409836065e-06,
-      "loss": 1.3326,
+      "epoch": 0.18,
+      "learning_rate": 0.00018194765204003079,
+      "loss": 1.4447,
       "step": 480
     },
     {
-      "epoch": 0.79,
-      "learning_rate": 1.222950819672131e-06,
-      "loss": 1.3019,
+      "epoch": 0.19,
+      "learning_rate": 0.0001817551963048499,
+      "loss": 1.5219,
       "step": 485
     },
     {
-      "epoch": 0.8,
-      "learning_rate": 1.2147540983606558e-06,
-      "loss": 1.4361,
+      "epoch": 0.19,
+      "learning_rate": 0.00018156274056966898,
+      "loss": 1.3272,
       "step": 490
     },
     {
-      "epoch": 0.8,
-      "learning_rate": 1.2065573770491803e-06,
-      "loss": 1.4145,
+      "epoch": 0.19,
+      "learning_rate": 0.0001813702848344881,
+      "loss": 1.3045,
       "step": 495
     },
     {
-      "epoch": 0.81,
-      "learning_rate": 1.1983606557377049e-06,
-      "loss": 1.381,
+      "epoch": 0.19,
+      "learning_rate": 0.00018117782909930717,
+      "loss": 1.4005,
       "step": 500
     },
     {
-      "epoch": 0.82,
-      "learning_rate": 1.1901639344262296e-06,
-      "loss": 1.422,
+      "epoch": 0.19,
+      "learning_rate": 0.00018098537336412625,
+      "loss": 1.277,
       "step": 505
     },
     {
-      "epoch": 0.83,
-      "learning_rate": 1.181967213114754e-06,
-      "loss": 1.3907,
+      "epoch": 0.2,
+      "learning_rate": 0.00018079291762894534,
+      "loss": 1.4171,
       "step": 510
     },
     {
-      "epoch": 0.84,
-      "learning_rate": 1.1737704918032787e-06,
-      "loss": 1.4062,
+      "epoch": 0.2,
+      "learning_rate": 0.00018060046189376445,
+      "loss": 1.3951,
       "step": 515
     },
     {
-      "epoch": 0.85,
-      "learning_rate": 1.1655737704918033e-06,
-      "loss": 1.4302,
+      "epoch": 0.2,
+      "learning_rate": 0.00018040800615858353,
+      "loss": 1.3163,
       "step": 520
     },
     {
-      "epoch": 0.85,
-      "learning_rate": 1.1573770491803278e-06,
-      "loss": 1.4916,
+      "epoch": 0.2,
+      "learning_rate": 0.00018021555042340264,
+      "loss": 1.4673,
       "step": 525
     },
     {
-      "epoch": 0.86,
-      "learning_rate": 1.1491803278688523e-06,
-      "loss": 1.3817,
+      "epoch": 0.2,
+      "learning_rate": 0.00018002309468822172,
+      "loss": 1.356,
       "step": 530
     },
     {
-      "epoch": 0.87,
-      "learning_rate": 1.1409836065573771e-06,
-      "loss": 1.3041,
+      "epoch": 0.21,
+      "learning_rate": 0.00017983063895304083,
+      "loss": 1.4882,
       "step": 535
     },
     {
-      "epoch": 0.88,
-      "learning_rate": 1.1327868852459014e-06,
-      "loss": 1.415,
+      "epoch": 0.21,
+      "learning_rate": 0.00017963818321785991,
+      "loss": 1.4118,
       "step": 540
     },
     {
-      "epoch": 0.89,
-      "learning_rate": 1.1245901639344262e-06,
-      "loss": 1.3001,
+      "epoch": 0.21,
+      "learning_rate": 0.00017944572748267897,
+      "loss": 1.4022,
       "step": 545
     },
     {
-      "epoch": 0.89,
-      "learning_rate": 1.116393442622951e-06,
-      "loss": 1.4388,
+      "epoch": 0.21,
+      "learning_rate": 0.00017925327174749808,
+      "loss": 1.4223,
       "step": 550
     },
     {
-      "epoch": 0.9,
-      "learning_rate": 1.1081967213114753e-06,
-      "loss": 1.3848,
+      "epoch": 0.21,
+      "learning_rate": 0.00017906081601231716,
+      "loss": 1.3543,
       "step": 555
     },
     {
-      "epoch": 0.91,
-      "learning_rate": 1.1e-06,
-      "loss": 1.4446,
+      "epoch": 0.22,
+      "learning_rate": 0.00017886836027713627,
+      "loss": 1.2972,
       "step": 560
     },
     {
-      "epoch": 0.92,
-      "learning_rate": 1.0918032786885246e-06,
-      "loss": 1.4944,
+      "epoch": 0.22,
+      "learning_rate": 0.00017867590454195535,
+      "loss": 1.3908,
       "step": 565
     },
     {
-      "epoch": 0.93,
-      "learning_rate": 1.0836065573770492e-06,
-      "loss": 1.4,
+      "epoch": 0.22,
+      "learning_rate": 0.00017848344880677446,
+      "loss": 1.4811,
       "step": 570
     },
     {
-      "epoch": 0.93,
-      "learning_rate": 1.0754098360655737e-06,
-      "loss": 1.3372,
+      "epoch": 0.22,
+      "learning_rate": 0.00017829099307159355,
+      "loss": 1.4248,
       "step": 575
     },
     {
-      "epoch": 0.94,
-      "learning_rate": 1.0672131147540985e-06,
-      "loss": 1.3619,
+      "epoch": 0.22,
+      "learning_rate": 0.00017809853733641263,
+      "loss": 1.4548,
       "step": 580
     },
     {
-      "epoch": 0.95,
-      "learning_rate": 1.0590163934426228e-06,
-      "loss": 1.3584,
+      "epoch": 0.22,
+      "learning_rate": 0.0001779060816012317,
+      "loss": 1.4073,
       "step": 585
     },
     {
-      "epoch": 0.96,
-      "learning_rate": 1.0508196721311476e-06,
-      "loss": 1.3189,
+      "epoch": 0.23,
+      "learning_rate": 0.00017771362586605082,
+      "loss": 1.3499,
       "step": 590
     },
     {
-      "epoch": 0.97,
-      "learning_rate": 1.042622950819672e-06,
-      "loss": 1.3524,
+      "epoch": 0.23,
+      "learning_rate": 0.0001775211701308699,
+      "loss": 1.2826,
       "step": 595
     },
     {
-      "epoch": 0.98,
-      "learning_rate": 1.0344262295081966e-06,
-      "loss": 1.3713,
+      "epoch": 0.23,
+      "learning_rate": 0.00017732871439568902,
+      "loss": 1.4861,
       "step": 600
     },
     {
-      "epoch": 0.98,
-      "learning_rate": 1.0262295081967212e-06,
-      "loss": 1.3962,
+      "epoch": 0.23,
+      "learning_rate": 0.0001771362586605081,
+      "loss": 1.4161,
       "step": 605
     },
     {
-      "epoch": 0.99,
-      "learning_rate": 1.018032786885246e-06,
-      "loss": 1.4458,
+      "epoch": 0.23,
+      "learning_rate": 0.00017694380292532718,
+      "loss": 1.3263,
       "step": 610
     },
     {
-      "epoch": 1.0,
-      "learning_rate": 1.0098360655737705e-06,
-      "loss": 1.3579,
+      "epoch": 0.24,
+      "learning_rate": 0.00017675134719014626,
+      "loss": 1.3883,
       "step": 615
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.00017655889145496535,
+      "loss": 1.3328,
+      "step": 620
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.00017636643571978446,
+      "loss": 1.3174,
+      "step": 625
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.00017617397998460354,
+      "loss": 1.3876,
+      "step": 630
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 0.00017598152424942265,
+      "loss": 1.3541,
+      "step": 635
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.00017578906851424173,
+      "loss": 1.3319,
+      "step": 640
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.00017559661277906084,
+      "loss": 1.2886,
+      "step": 645
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.00017540415704387992,
+      "loss": 1.3338,
+      "step": 650
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.000175211701308699,
+      "loss": 1.4036,
+      "step": 655
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 0.0001750192455735181,
+      "loss": 1.4543,
+      "step": 660
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.0001748267898383372,
+      "loss": 1.2902,
+      "step": 665
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.00017463433410315628,
+      "loss": 1.3255,
+      "step": 670
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.00017444187836797537,
+      "loss": 1.2875,
+      "step": 675
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.00017424942263279448,
+      "loss": 1.3105,
+      "step": 680
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 0.00017405696689761356,
+      "loss": 1.2533,
+      "step": 685
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.00017386451116243264,
+      "loss": 1.3827,
+      "step": 690
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.00017367205542725172,
+      "loss": 1.3549,
+      "step": 695
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.00017347959969207083,
+      "loss": 1.4396,
+      "step": 700
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.00017328714395688992,
+      "loss": 1.3179,
+      "step": 705
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.00017309468822170903,
+      "loss": 1.356,
+      "step": 710
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 0.0001729022324865281,
+      "loss": 1.2499,
+      "step": 715
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.00017270977675134722,
+      "loss": 1.4139,
+      "step": 720
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.0001725173210161663,
+      "loss": 1.249,
+      "step": 725
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.00017232486528098538,
+      "loss": 1.3054,
+      "step": 730
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.00017213240954580447,
+      "loss": 1.3415,
+      "step": 735
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 0.00017193995381062355,
+      "loss": 1.3323,
+      "step": 740
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.00017174749807544266,
+      "loss": 1.3559,
+      "step": 745
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.00017155504234026174,
+      "loss": 1.3771,
+      "step": 750
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.00017136258660508085,
+      "loss": 1.3811,
+      "step": 755
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.00017117013086989993,
+      "loss": 1.3644,
+      "step": 760
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 0.00017097767513471902,
+      "loss": 1.2619,
+      "step": 765
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.0001707852193995381,
+      "loss": 1.3795,
+      "step": 770
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.0001705927636643572,
+      "loss": 1.4482,
+      "step": 775
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.0001704003079291763,
+      "loss": 1.3213,
+      "step": 780
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.0001702078521939954,
+      "loss": 1.3406,
+      "step": 785
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 0.00017001539645881449,
+      "loss": 1.254,
+      "step": 790
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.0001698229407236336,
+      "loss": 1.4489,
+      "step": 795
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.00016963048498845268,
+      "loss": 1.3327,
+      "step": 800
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.00016943802925327173,
+      "loss": 1.2697,
+      "step": 805
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.00016924557351809084,
+      "loss": 1.3808,
+      "step": 810
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 0.00016905311778290993,
+      "loss": 1.3519,
+      "step": 815
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.00016886066204772904,
+      "loss": 1.3051,
+      "step": 820
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.00016866820631254812,
+      "loss": 1.3251,
+      "step": 825
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.00016847575057736723,
+      "loss": 1.3595,
+      "step": 830
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.0001682832948421863,
+      "loss": 1.366,
+      "step": 835
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.0001680908391070054,
+      "loss": 1.3157,
+      "step": 840
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 0.00016789838337182448,
+      "loss": 1.3809,
+      "step": 845
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.0001677059276366436,
+      "loss": 1.3305,
+      "step": 850
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.00016751347190146267,
+      "loss": 1.3454,
+      "step": 855
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.00016732101616628175,
+      "loss": 1.3914,
+      "step": 860
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.00016712856043110086,
+      "loss": 1.2801,
+      "step": 865
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 0.00016693610469591995,
+      "loss": 1.2451,
+      "step": 870
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.00016674364896073903,
+      "loss": 1.3802,
+      "step": 875
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.0001665511932255581,
+      "loss": 1.383,
+      "step": 880
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.00016635873749037722,
+      "loss": 1.3572,
+      "step": 885
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.0001661662817551963,
+      "loss": 1.381,
+      "step": 890
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 0.0001659738260200154,
+      "loss": 1.253,
+      "step": 895
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.0001657813702848345,
+      "loss": 1.3824,
+      "step": 900
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.0001655889145496536,
+      "loss": 1.3366,
+      "step": 905
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.0001653964588144727,
+      "loss": 1.2716,
+      "step": 910
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.00016520400307929177,
+      "loss": 1.317,
+      "step": 915
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 0.00016501154734411085,
+      "loss": 1.27,
+      "step": 920
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.00016481909160892994,
+      "loss": 1.463,
+      "step": 925
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.00016462663587374905,
+      "loss": 1.3101,
+      "step": 930
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.00016443418013856813,
+      "loss": 1.3305,
+      "step": 935
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.00016424172440338724,
+      "loss": 1.2637,
+      "step": 940
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.00016404926866820632,
+      "loss": 1.3564,
+      "step": 945
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 0.0001638568129330254,
+      "loss": 1.1473,
+      "step": 950
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.0001636643571978445,
+      "loss": 1.2369,
+      "step": 955
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.0001634719014626636,
+      "loss": 1.2508,
+      "step": 960
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.00016327944572748268,
+      "loss": 1.3019,
+      "step": 965
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.0001630869899923018,
+      "loss": 1.2893,
+      "step": 970
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 0.00016289453425712087,
+      "loss": 1.3052,
+      "step": 975
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.00016270207852193998,
+      "loss": 1.3312,
+      "step": 980
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.00016250962278675907,
+      "loss": 1.3215,
+      "step": 985
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.00016231716705157812,
+      "loss": 1.3382,
+      "step": 990
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.00016212471131639723,
+      "loss": 1.3263,
+      "step": 995
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 0.00016193225558121631,
+      "loss": 1.2434,
+      "step": 1000
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.00016173979984603542,
+      "loss": 1.3063,
+      "step": 1005
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.0001615473441108545,
+      "loss": 1.3254,
+      "step": 1010
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.00016135488837567362,
+      "loss": 1.2654,
+      "step": 1015
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.0001611624326404927,
+      "loss": 1.348,
+      "step": 1020
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 0.00016096997690531178,
+      "loss": 1.3155,
+      "step": 1025
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.00016077752117013086,
+      "loss": 1.2864,
+      "step": 1030
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.00016058506543494997,
+      "loss": 1.364,
+      "step": 1035
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.00016039260969976906,
+      "loss": 1.2698,
+      "step": 1040
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.00016020015396458817,
+      "loss": 1.3632,
+      "step": 1045
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 0.00016000769822940725,
+      "loss": 1.2741,
+      "step": 1050
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.00015981524249422633,
+      "loss": 1.3533,
+      "step": 1055
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.00015962278675904542,
+      "loss": 1.3571,
+      "step": 1060
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.0001594303310238645,
+      "loss": 1.2827,
+      "step": 1065
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.0001592378752886836,
+      "loss": 1.2547,
+      "step": 1070
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.0001590454195535027,
+      "loss": 1.4101,
+      "step": 1075
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 0.0001588529638183218,
+      "loss": 1.3149,
+      "step": 1080
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.00015866050808314088,
+      "loss": 1.3008,
+      "step": 1085
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.00015846805234796,
+      "loss": 1.2859,
+      "step": 1090
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.00015827559661277908,
+      "loss": 1.1892,
+      "step": 1095
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.00015808314087759816,
+      "loss": 1.364,
+      "step": 1100
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 0.00015789068514241724,
+      "loss": 1.2708,
+      "step": 1105
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.00015769822940723635,
+      "loss": 1.3591,
+      "step": 1110
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.00015750577367205543,
+      "loss": 1.2828,
+      "step": 1115
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.00015731331793687452,
+      "loss": 1.3861,
+      "step": 1120
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.00015712086220169363,
+      "loss": 1.3752,
+      "step": 1125
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 0.0001569284064665127,
+      "loss": 1.299,
+      "step": 1130
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.0001567359507313318,
+      "loss": 1.2744,
+      "step": 1135
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.00015654349499615087,
+      "loss": 1.3226,
+      "step": 1140
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.00015635103926096998,
+      "loss": 1.2342,
+      "step": 1145
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.00015615858352578907,
+      "loss": 1.2236,
+      "step": 1150
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 0.00015596612779060818,
+      "loss": 1.3178,
+      "step": 1155
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.00015577367205542726,
+      "loss": 1.315,
+      "step": 1160
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.00015558121632024637,
+      "loss": 1.2071,
+      "step": 1165
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.00015538876058506545,
+      "loss": 1.2485,
+      "step": 1170
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.00015519630484988454,
+      "loss": 1.3034,
+      "step": 1175
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 0.00015500384911470362,
+      "loss": 1.3113,
+      "step": 1180
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.0001548113933795227,
+      "loss": 1.4071,
+      "step": 1185
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.0001546189376443418,
+      "loss": 1.2806,
+      "step": 1190
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.0001544264819091609,
+      "loss": 1.2668,
+      "step": 1195
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.00015423402617398,
+      "loss": 1.2156,
+      "step": 1200
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.00015404157043879909,
+      "loss": 1.2713,
+      "step": 1205
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 0.00015384911470361817,
+      "loss": 1.3547,
+      "step": 1210
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.00015365665896843725,
+      "loss": 1.352,
+      "step": 1215
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.00015346420323325636,
+      "loss": 1.2166,
+      "step": 1220
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.00015327174749807544,
+      "loss": 1.3411,
+      "step": 1225
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.00015307929176289455,
+      "loss": 1.3127,
+      "step": 1230
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 0.00015288683602771364,
+      "loss": 1.2179,
+      "step": 1235
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.00015269438029253275,
+      "loss": 1.3099,
+      "step": 1240
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.00015250192455735183,
+      "loss": 1.3071,
+      "step": 1245
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.00015230946882217089,
+      "loss": 1.2789,
+      "step": 1250
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.00015211701308699,
+      "loss": 1.265,
+      "step": 1255
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 0.00015192455735180908,
+      "loss": 1.3423,
+      "step": 1260
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.0001517321016166282,
+      "loss": 1.3561,
+      "step": 1265
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.00015153964588144727,
+      "loss": 1.3766,
+      "step": 1270
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.00015134719014626638,
+      "loss": 1.1815,
+      "step": 1275
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.00015115473441108546,
+      "loss": 1.2731,
+      "step": 1280
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 0.00015096227867590455,
+      "loss": 1.4211,
+      "step": 1285
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.00015076982294072363,
+      "loss": 1.3037,
+      "step": 1290
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.00015057736720554274,
+      "loss": 1.2689,
+      "step": 1295
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.00015038491147036182,
+      "loss": 1.3821,
+      "step": 1300
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.00015019245573518093,
+      "loss": 1.3421,
+      "step": 1305
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 0.00015000000000000001,
+      "loss": 1.2462,
+      "step": 1310
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.0001498075442648191,
+      "loss": 1.3296,
+      "step": 1315
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.00014961508852963818,
+      "loss": 1.2991,
+      "step": 1320
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.00014942263279445726,
+      "loss": 1.2695,
+      "step": 1325
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.00014923017705927637,
+      "loss": 1.3681,
+      "step": 1330
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.00014903772132409545,
+      "loss": 1.3166,
+      "step": 1335
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 0.00014884526558891456,
+      "loss": 1.2906,
+      "step": 1340
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.00014865280985373365,
+      "loss": 1.2122,
+      "step": 1345
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.00014846035411855276,
+      "loss": 1.2767,
+      "step": 1350
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.00014826789838337184,
+      "loss": 1.3143,
+      "step": 1355
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.00014807544264819092,
+      "loss": 1.2427,
+      "step": 1360
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 0.00014788298691301,
+      "loss": 1.2413,
+      "step": 1365
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00014769053117782912,
+      "loss": 1.2943,
+      "step": 1370
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.0001474980754426482,
+      "loss": 1.2527,
+      "step": 1375
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00014730561970746728,
+      "loss": 1.3178,
+      "step": 1380
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.0001471131639722864,
+      "loss": 1.2924,
+      "step": 1385
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 0.00014692070823710547,
+      "loss": 1.1681,
+      "step": 1390
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00014672825250192456,
+      "loss": 1.2805,
+      "step": 1395
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00014653579676674364,
+      "loss": 1.2903,
+      "step": 1400
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00014634334103156275,
+      "loss": 1.3223,
+      "step": 1405
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00014615088529638183,
+      "loss": 1.208,
+      "step": 1410
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 0.00014595842956120094,
+      "loss": 1.2135,
+      "step": 1415
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00014576597382602002,
+      "loss": 1.291,
+      "step": 1420
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00014557351809083913,
+      "loss": 1.1809,
+      "step": 1425
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00014538106235565822,
+      "loss": 1.3176,
+      "step": 1430
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.0001451886066204773,
+      "loss": 1.2837,
+      "step": 1435
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 0.00014499615088529638,
+      "loss": 1.2517,
+      "step": 1440
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00014480369515011547,
+      "loss": 1.3029,
+      "step": 1445
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00014461123941493458,
+      "loss": 1.2432,
+      "step": 1450
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00014441878367975366,
+      "loss": 1.2917,
+      "step": 1455
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00014422632794457277,
+      "loss": 1.3542,
+      "step": 1460
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00014403387220939185,
+      "loss": 1.2598,
+      "step": 1465
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 0.00014384141647421093,
+      "loss": 1.2475,
+      "step": 1470
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00014364896073903002,
+      "loss": 1.2112,
+      "step": 1475
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00014345650500384913,
+      "loss": 1.2839,
+      "step": 1480
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.0001432640492686682,
+      "loss": 1.1871,
+      "step": 1485
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.00014307159353348732,
+      "loss": 1.1936,
+      "step": 1490
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 0.0001428791377983064,
+      "loss": 1.2619,
+      "step": 1495
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.0001426866820631255,
+      "loss": 1.243,
+      "step": 1500
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00014249422632794457,
+      "loss": 1.3483,
+      "step": 1505
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00014230177059276365,
+      "loss": 1.241,
+      "step": 1510
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00014210931485758276,
+      "loss": 1.2585,
+      "step": 1515
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 0.00014191685912240184,
+      "loss": 1.2525,
+      "step": 1520
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00014172440338722095,
+      "loss": 1.2934,
+      "step": 1525
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00014153194765204003,
+      "loss": 1.2277,
+      "step": 1530
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00014133949191685914,
+      "loss": 1.2294,
+      "step": 1535
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.00014114703618167823,
+      "loss": 1.245,
+      "step": 1540
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 0.0001409545804464973,
+      "loss": 1.1826,
+      "step": 1545
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.0001407621247113164,
+      "loss": 1.2436,
+      "step": 1550
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.0001405696689761355,
+      "loss": 1.1588,
+      "step": 1555
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00014037721324095459,
+      "loss": 1.3642,
+      "step": 1560
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.0001401847575057737,
+      "loss": 1.2621,
+      "step": 1565
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 0.00013999230177059278,
+      "loss": 1.2909,
+      "step": 1570
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00013979984603541186,
+      "loss": 1.2259,
+      "step": 1575
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00013960739030023094,
+      "loss": 1.2078,
+      "step": 1580
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00013941493456505003,
+      "loss": 1.2307,
+      "step": 1585
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00013922247882986914,
+      "loss": 1.2816,
+      "step": 1590
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00013903002309468822,
+      "loss": 1.2587,
+      "step": 1595
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 0.00013883756735950733,
+      "loss": 1.3018,
+      "step": 1600
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.0001386451116243264,
+      "loss": 1.2167,
+      "step": 1605
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00013845265588914552,
+      "loss": 1.2399,
+      "step": 1610
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.0001382602001539646,
+      "loss": 1.2355,
+      "step": 1615
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.0001380677444187837,
+      "loss": 1.2099,
+      "step": 1620
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 0.00013787528868360277,
+      "loss": 1.2794,
+      "step": 1625
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00013768283294842188,
+      "loss": 1.1232,
+      "step": 1630
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00013749037721324096,
+      "loss": 1.2563,
+      "step": 1635
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00013729792147806005,
+      "loss": 1.2364,
+      "step": 1640
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00013710546574287916,
+      "loss": 1.2036,
+      "step": 1645
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 0.00013691301000769824,
+      "loss": 1.209,
+      "step": 1650
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.00013672055427251732,
+      "loss": 1.2653,
+      "step": 1655
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.0001365280985373364,
+      "loss": 1.3329,
+      "step": 1660
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.0001363356428021555,
+      "loss": 1.2629,
+      "step": 1665
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.0001361431870669746,
+      "loss": 1.2393,
+      "step": 1670
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 0.0001359507313317937,
+      "loss": 1.1669,
+      "step": 1675
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.0001357582755966128,
+      "loss": 1.2177,
+      "step": 1680
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.0001355658198614319,
+      "loss": 1.1363,
+      "step": 1685
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00013537336412625098,
+      "loss": 1.1773,
+      "step": 1690
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00013518090839107006,
+      "loss": 1.2482,
+      "step": 1695
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 0.00013498845265588915,
+      "loss": 1.266,
+      "step": 1700
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00013479599692070823,
+      "loss": 1.2936,
+      "step": 1705
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00013460354118552734,
+      "loss": 1.2855,
+      "step": 1710
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00013441108545034642,
+      "loss": 1.2413,
+      "step": 1715
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00013421862971516553,
+      "loss": 1.2874,
+      "step": 1720
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.00013402617397998461,
+      "loss": 1.2527,
+      "step": 1725
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 0.0001338337182448037,
+      "loss": 1.2527,
+      "step": 1730
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00013364126250962278,
+      "loss": 1.2423,
+      "step": 1735
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.0001334488067744419,
+      "loss": 1.2544,
+      "step": 1740
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00013325635103926097,
+      "loss": 1.1613,
+      "step": 1745
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00013306389530408008,
+      "loss": 1.2273,
+      "step": 1750
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 0.00013287143956889917,
+      "loss": 1.2789,
+      "step": 1755
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00013267898383371828,
+      "loss": 1.2755,
+      "step": 1760
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00013248652809853733,
+      "loss": 1.2144,
+      "step": 1765
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00013229407236335641,
+      "loss": 1.1615,
+      "step": 1770
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.00013210161662817552,
+      "loss": 1.2173,
+      "step": 1775
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 0.0001319091608929946,
+      "loss": 1.2408,
+      "step": 1780
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00013171670515781372,
+      "loss": 1.1671,
+      "step": 1785
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.0001315242494226328,
+      "loss": 1.1994,
+      "step": 1790
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.0001313317936874519,
+      "loss": 1.3369,
+      "step": 1795
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.000131139337952271,
+      "loss": 1.1598,
+      "step": 1800
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 0.00013094688221709007,
+      "loss": 1.1784,
+      "step": 1805
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00013075442648190916,
+      "loss": 1.2743,
+      "step": 1810
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00013056197074672827,
+      "loss": 1.2127,
+      "step": 1815
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00013036951501154735,
+      "loss": 1.2738,
+      "step": 1820
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00013017705927636643,
+      "loss": 1.2232,
+      "step": 1825
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00012998460354118554,
+      "loss": 1.1547,
+      "step": 1830
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 0.00012979214780600463,
+      "loss": 1.2235,
+      "step": 1835
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.0001295996920708237,
+      "loss": 1.332,
+      "step": 1840
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.0001294072363356428,
+      "loss": 1.1242,
+      "step": 1845
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.0001292147806004619,
+      "loss": 1.1874,
+      "step": 1850
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.00012902232486528098,
+      "loss": 1.1394,
+      "step": 1855
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 0.0001288298691301001,
+      "loss": 1.1734,
+      "step": 1860
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00012863741339491918,
+      "loss": 1.2245,
+      "step": 1865
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00012844495765973829,
+      "loss": 1.1869,
+      "step": 1870
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 0.00012825250192455737,
+      "loss": 1.2558,
+      "step": 1875
     }
   ],
-  "max_steps": 1230,
+  "max_steps": 5206,
   "num_train_epochs": 2,
-  "total_flos": 204046325514240.0,
+  "total_flos": 19105201520640.0,
   "trial_name": null,
   "trial_params": null
 }

last-checkpoint/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3f43edfee59f374cac51ff219426a9f3a4f9ade245c05ad2c22d14f5e8e9c1a4
+oid sha256:762f87ade0fa6168b36ab9775461c0f8a3950e947fee8d4e9ec9e059b0407a8a
 size 6139

last-checkpoint/zero_to_fp32.py

@@ -5,7 +5,7 @@
 
 # DeepSpeed Team
 
-# This script extracts fp32 consolidated weights from a zero 2 and 3 DeepSpeed checkpoints. It gets
+# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets
 # copied into the top level checkpoint dir, so the user can easily do the conversion at any point in
 # the future. Once extracted, the weights don't require DeepSpeed and can be used in any
 # application.
@@ -63,7 +63,7 @@ def get_model_state_file(checkpoint_dir, zero_stage):
         raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist")
 
     # there should be only one file
-    if zero_stage == 2:
+    if zero_stage <= 2:
         file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt")
     elif zero_stage == 3:
         file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt")
@@ -143,7 +143,11 @@ def parse_optim_states(files, ds_checkpoint_dir):
     total_files = len(files)
     state_dicts = []
     for f in files:
-        state_dicts.append(torch.load(f, map_location=device))
+        state_dict = torch.load(f, map_location=device)
+        # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights
+        # and also handle the case where it was already removed by another helper script
+        state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None)
+        state_dicts.append(state_dict)
 
     if not ZERO_STAGE in state_dicts[0][OPTIMIZER_STATE_DICT]:
         raise ValueError(f"{files[0]} is not a zero checkpoint")
@@ -164,14 +168,14 @@ def parse_optim_states(files, ds_checkpoint_dir):
         )
 
     # the groups are named differently in each stage
-    if zero_stage == 2:
+    if zero_stage <= 2:
         fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS
     elif zero_stage == 3:
         fp32_groups_key = FP32_FLAT_GROUPS
     else:
         raise ValueError(f"unknown zero stage {zero_stage}")
 
-    if zero_stage == 2:
+    if zero_stage <= 2:
         fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))]
     elif zero_stage == 3:
         # if there is more than one param group, there will be multiple flattened tensors - one
@@ -206,7 +210,7 @@ def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir):
     zero_model_states = parse_model_states(model_files)
     print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}')
 
-    if zero_stage == 2:
+    if zero_stage <= 2:
         return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states)
     elif zero_stage == 3:
         return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states)

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3f43edfee59f374cac51ff219426a9f3a4f9ade245c05ad2c22d14f5e8e9c1a4
+oid sha256:762f87ade0fa6168b36ab9775461c0f8a3950e947fee8d4e9ec9e059b0407a8a
 size 6139