Training in progress, epoch 1, checkpoint

last-checkpoint/global_step313/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a9b52a7194cf8990578e139150e620dd6d54c5f70f687b440b2e254659d35c0d
+oid sha256:84b7891511d3eaeb18a9a39180391e53d1a3092f51db3ca242bccd7eae36b42e
 size 16060527788

last-checkpoint/global_step313/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:34cab496bf87556d9c668a950f9f0ed2380a012073a852085e30531a84414337
+oid sha256:14c59349bf0aaca7831e612b47ae91d823e280ca16a66217156893c3a3c5f3ca
 size 16060527788

last-checkpoint/global_step313/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a19493349ce956f338cfbc14d56cb36c3a5990dbf9a40efbbbcd67c3094ba241
+oid sha256:c4506fe6a9191a35b75a756845641d5b3245166414c1b45a1022a98ced5c8554
 size 16060527788

last-checkpoint/global_step313/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6e05bfec7981d86db64d0489761b2a09d6ede6d28ed84d5c9b8c347538e5f91f
+oid sha256:e1252a92a76a4ef4b3e917201f1523245b7cf7aaa2ebd7b0f696dfdf3a5f131f
 size 16060527788

last-checkpoint/latest

@@ -1 +1 @@
-global_step939
+global_step313

last-checkpoint/model-00001-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d7e8fb1adccd2e7fe2bbeefbafa82a4565057fab3010779fd5e5ab4c409bb496
+oid sha256:d7fe06acc865660a1210968b7bc088f6fb99c21090af754cb1949a8c7cdb4909
 size 4976698672

last-checkpoint/model-00002-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:dc8a42b274b60f2ed063dbc5a0db133fa0fba6e446796edd49203d49a11700aa
+oid sha256:e35f3442c3dbcd09ff321274b852038ddc4a3e42284034def18e47b84de2f100
 size 4999802720

last-checkpoint/model-00003-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e4395bbcc01e2ccad78f50643525aed5bf2d95e1b0f624507fdeacc3d0870011
+oid sha256:fd9fab64492ee1c6e4df053779be0fc5340804f432515c681cc98864551129d1
 size 4915916176

last-checkpoint/model-00004-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f60fed5474de507e5cf5f93d70d0b84d7bf7848f88ce747be3d6fa3998258d84
+oid sha256:7e073616b55f59724d580f9af0176d99e5001a0a8dec6aface93da15fc48c8b7
 size 1168138808

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ab7cf0d34d60dfda516cf9661904550e2e294e723edd07c25c738f05e8ba92d1
+oid sha256:f1ce770862b76ceecb3453a5d0b9ef65311eab0f6a60f52997726a89d718de08
 size 15024

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f7b183324e8227a51a9556d86b2ad893a8c4c52205ed4a737356c6611dac7353
+oid sha256:2bcae6573442578b752be3e988d4ccb38056b45c31b5c02eb579ec6cebcfa62e
 size 15024

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ac69e994090f4818cb1fa6f6cefa363178552c3c731c6507ff195bcb07fd5bef
+oid sha256:398ff45072fee2975e88b9e078f915103d75d94bc08753303cf855b915973623
 size 15024

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f68e61b63402f8afb1f69c960f7944965655dac11e3ccf29919c282f23931f86
+oid sha256:7f4b210bc832d4c648e3bee8c21dca26e5a8b365d6ec90c638062005a052e57b
 size 15024

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fe7d17fbcfa5bee1bf6cb5aedab3ff5a70436912c200d7301d173be443809d63
+oid sha256:6b9aad3a748e15719e20120f0a760f22686338e176ee3f1c011b6dab2f3ab19b
 size 1064

last-checkpoint/tokenizer_config.json

@@ -2053,11 +2053,15 @@
   "chat_template": "{% for message in messages %}\n{% if message['role'] == 'user' %}\n{{ '<|user|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'system' %}\n{{ '<|system|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'assistant' %}\n{{ '<|assistant|>\n'  + message['content'] + eos_token }}\n{% endif %}\n{% if loop.last and add_generation_prompt %}\n{{ '<|assistant|>' }}\n{% endif %}\n{% endfor %}",
   "clean_up_tokenization_spaces": true,
   "eos_token": "<|end_of_text|>",
+  "max_length": 1024,
   "model_input_names": [
     "input_ids",
     "attention_mask"
   ],
   "model_max_length": 2048,
   "pad_token": "<|end_of_text|>",
-  "tokenizer_class": "PreTrainedTokenizerFast"
+  "stride": 0,
+  "tokenizer_class": "PreTrainedTokenizerFast",
+  "truncation_side": "left",
+  "truncation_strategy": "longest_first"
 }

last-checkpoint/trainer_state.json

@@ -1,21 +1,21 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 3.0,
+  "epoch": 1.0,
   "eval_steps": 500,
-  "global_step": 939,
+  "global_step": 313,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
       "epoch": 0.003194888178913738,
-      "grad_norm": 14.15949417784876,
+      "grad_norm": 32.14257471485931,
       "learning_rate": 5.3191489361702125e-09,
-      "logits/chosen": -0.875,
-      "logits/rejected": -0.8671875,
-      "logps/chosen": -29.375,
-      "logps/rejected": -38.0,
+      "logits/chosen": -1.7265625,
+      "logits/rejected": -1.65625,
+      "logps/chosen": -249.0,
+      "logps/rejected": -242.0,
       "loss": 0.6914,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
@@ -25,1446 +25,484 @@
     },
     {
       "epoch": 0.03194888178913738,
-      "grad_norm": 12.702097036550947,
+      "grad_norm": 33.76096895998352,
       "learning_rate": 5.3191489361702123e-08,
-      "logits/chosen": -0.828125,
-      "logits/rejected": -0.91796875,
-      "logps/chosen": -26.875,
-      "logps/rejected": -27.25,
-      "loss": 0.6917,
-      "rewards/accuracies": 0.1944444477558136,
-      "rewards/chosen": 0.00121307373046875,
-      "rewards/margins": 0.0020751953125,
-      "rewards/rejected": -0.0008697509765625,
+      "logits/chosen": -1.765625,
+      "logits/rejected": -1.765625,
+      "logps/chosen": -240.0,
+      "logps/rejected": -244.0,
+      "loss": 0.6926,
+      "rewards/accuracies": 0.2083333283662796,
+      "rewards/chosen": 0.00555419921875,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.00555419921875,
       "step": 10
     },
     {
       "epoch": 0.06389776357827476,
-      "grad_norm": 14.201541608200035,
+      "grad_norm": 31.259900823289186,
       "learning_rate": 1.0638297872340425e-07,
-      "logits/chosen": -0.921875,
-      "logits/rejected": -0.93359375,
-      "logps/chosen": -30.75,
-      "logps/rejected": -24.625,
-      "loss": 0.6924,
-      "rewards/accuracies": 0.10000000149011612,
-      "rewards/chosen": 0.00141143798828125,
-      "rewards/margins": 7.724761962890625e-05,
-      "rewards/rejected": 0.0013275146484375,
+      "logits/chosen": -1.71875,
+      "logits/rejected": -1.75,
+      "logps/chosen": -237.0,
+      "logps/rejected": -241.0,
+      "loss": 0.6916,
+      "rewards/accuracies": 0.26249998807907104,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.006256103515625,
+      "rewards/rejected": -0.006256103515625,
       "step": 20
     },
     {
       "epoch": 0.09584664536741214,
-      "grad_norm": 12.074020986080455,
+      "grad_norm": 32.366987695897166,
       "learning_rate": 1.5957446808510638e-07,
-      "logits/chosen": -0.91796875,
-      "logits/rejected": -0.9453125,
-      "logps/chosen": -32.25,
-      "logps/rejected": -28.875,
-      "loss": 0.6916,
-      "rewards/accuracies": 0.2750000059604645,
-      "rewards/chosen": 0.0011749267578125,
-      "rewards/margins": 0.008056640625,
-      "rewards/rejected": -0.00689697265625,
+      "logits/chosen": -1.75,
+      "logits/rejected": -1.75,
+      "logps/chosen": -245.0,
+      "logps/rejected": -246.0,
+      "loss": 0.6946,
+      "rewards/accuracies": 0.22499999403953552,
+      "rewards/chosen": 6.109476089477539e-06,
+      "rewards/margins": -0.0012359619140625,
+      "rewards/rejected": 0.0012664794921875,
       "step": 30
     },
     {
       "epoch": 0.12779552715654952,
-      "grad_norm": 13.073953515135278,
+      "grad_norm": 30.81764382136956,
       "learning_rate": 2.127659574468085e-07,
-      "logits/chosen": -0.91796875,
-      "logits/rejected": -0.890625,
-      "logps/chosen": -36.0,
-      "logps/rejected": -26.375,
-      "loss": 0.6888,
-      "rewards/accuracies": 0.2750000059604645,
-      "rewards/chosen": -0.00186920166015625,
-      "rewards/margins": 0.002593994140625,
-      "rewards/rejected": -0.00445556640625,
+      "logits/chosen": -1.703125,
+      "logits/rejected": -1.734375,
+      "logps/chosen": -243.0,
+      "logps/rejected": -240.0,
+      "loss": 0.6818,
+      "rewards/accuracies": 0.4000000059604645,
+      "rewards/chosen": -0.003753662109375,
+      "rewards/margins": 0.032470703125,
+      "rewards/rejected": -0.036376953125,
       "step": 40
     },
     {
       "epoch": 0.1597444089456869,
-      "grad_norm": 11.709384491164828,
+      "grad_norm": 31.214992328374773,
       "learning_rate": 2.659574468085106e-07,
-      "logits/chosen": -0.90234375,
-      "logits/rejected": -0.8828125,
-      "logps/chosen": -27.75,
-      "logps/rejected": -23.5,
-      "loss": 0.6822,
-      "rewards/accuracies": 0.2750000059604645,
-      "rewards/chosen": 0.0009307861328125,
-      "rewards/margins": 0.0206298828125,
-      "rewards/rejected": -0.0196533203125,
+      "logits/chosen": -1.7109375,
+      "logits/rejected": -1.7265625,
+      "logps/chosen": -242.0,
+      "logps/rejected": -243.0,
+      "loss": 0.673,
+      "rewards/accuracies": 0.4375,
+      "rewards/chosen": -0.0400390625,
+      "rewards/margins": 0.033935546875,
+      "rewards/rejected": -0.07373046875,
       "step": 50
     },
     {
       "epoch": 0.19169329073482427,
-      "grad_norm": 11.881387515429367,
+      "grad_norm": 28.88257684001896,
       "learning_rate": 3.1914893617021275e-07,
-      "logits/chosen": -0.921875,
-      "logits/rejected": -0.90234375,
-      "logps/chosen": -26.25,
-      "logps/rejected": -29.0,
-      "loss": 0.6851,
-      "rewards/accuracies": 0.3125,
-      "rewards/chosen": 0.0028839111328125,
-      "rewards/margins": 0.01385498046875,
-      "rewards/rejected": -0.010986328125,
+      "logits/chosen": -1.7265625,
+      "logits/rejected": -1.75,
+      "logps/chosen": -240.0,
+      "logps/rejected": -243.0,
+      "loss": 0.6296,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.0625,
+      "rewards/margins": 0.111328125,
+      "rewards/rejected": -0.173828125,
       "step": 60
     },
     {
       "epoch": 0.22364217252396165,
-      "grad_norm": 15.065379921321332,
+      "grad_norm": 30.584634735099684,
       "learning_rate": 3.7234042553191484e-07,
-      "logits/chosen": -0.921875,
-      "logits/rejected": -0.921875,
-      "logps/chosen": -42.0,
-      "logps/rejected": -24.75,
-      "loss": 0.6652,
-      "rewards/accuracies": 0.38749998807907104,
-      "rewards/chosen": -0.015625,
-      "rewards/margins": 0.07958984375,
-      "rewards/rejected": -0.09521484375,
+      "logits/chosen": -1.7109375,
+      "logits/rejected": -1.71875,
+      "logps/chosen": -243.0,
+      "logps/rejected": -250.0,
+      "loss": 0.6439,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.080078125,
+      "rewards/margins": 0.19140625,
+      "rewards/rejected": -0.271484375,
       "step": 70
     },
     {
       "epoch": 0.25559105431309903,
-      "grad_norm": 9.160023538736992,
+      "grad_norm": 29.751500863088314,
       "learning_rate": 4.25531914893617e-07,
-      "logits/chosen": -0.94921875,
-      "logits/rejected": -0.9140625,
-      "logps/chosen": -37.5,
-      "logps/rejected": -30.875,
-      "loss": 0.6678,
-      "rewards/accuracies": 0.4000000059604645,
-      "rewards/chosen": -0.08447265625,
-      "rewards/margins": 0.0703125,
-      "rewards/rejected": -0.154296875,
+      "logits/chosen": -1.703125,
+      "logits/rejected": -1.75,
+      "logps/chosen": -240.0,
+      "logps/rejected": -251.0,
+      "loss": 0.6266,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.2060546875,
+      "rewards/margins": 0.2099609375,
+      "rewards/rejected": -0.41796875,
       "step": 80
     },
     {
       "epoch": 0.28753993610223644,
-      "grad_norm": 11.744213372479827,
+      "grad_norm": 32.267981389042255,
       "learning_rate": 4.787234042553192e-07,
-      "logits/chosen": -0.953125,
-      "logits/rejected": -0.95703125,
-      "logps/chosen": -35.0,
-      "logps/rejected": -25.125,
-      "loss": 0.6325,
-      "rewards/accuracies": 0.375,
-      "rewards/chosen": 0.047607421875,
-      "rewards/margins": 0.1240234375,
-      "rewards/rejected": -0.076171875,
+      "logits/chosen": -1.7109375,
+      "logits/rejected": -1.765625,
+      "logps/chosen": -245.0,
+      "logps/rejected": -247.0,
+      "loss": 0.6458,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.349609375,
+      "rewards/margins": 0.2060546875,
+      "rewards/rejected": -0.5546875,
       "step": 90
     },
     {
       "epoch": 0.3194888178913738,
-      "grad_norm": 12.175021032470806,
+      "grad_norm": 31.916118030442767,
       "learning_rate": 4.964497041420119e-07,
-      "logits/chosen": -0.96875,
-      "logits/rejected": -0.90625,
-      "logps/chosen": -29.0,
-      "logps/rejected": -27.375,
-      "loss": 0.6503,
-      "rewards/accuracies": 0.3375000059604645,
-      "rewards/chosen": 0.015380859375,
-      "rewards/margins": 0.09716796875,
-      "rewards/rejected": -0.08203125,
+      "logits/chosen": -1.6953125,
+      "logits/rejected": -1.7109375,
+      "logps/chosen": -246.0,
+      "logps/rejected": -243.0,
+      "loss": 0.6301,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -0.29296875,
+      "rewards/margins": 0.1767578125,
+      "rewards/rejected": -0.470703125,
       "step": 100
     },
     {
       "epoch": 0.3514376996805112,
-      "grad_norm": 15.178658782857307,
+      "grad_norm": 30.563502769937987,
       "learning_rate": 4.905325443786982e-07,
-      "logits/chosen": -0.99609375,
-      "logits/rejected": -0.8984375,
-      "logps/chosen": -31.75,
-      "logps/rejected": -34.75,
-      "loss": 0.6204,
-      "rewards/accuracies": 0.4000000059604645,
-      "rewards/chosen": 0.046142578125,
-      "rewards/margins": 0.1982421875,
-      "rewards/rejected": -0.15234375,
+      "logits/chosen": -1.6796875,
+      "logits/rejected": -1.734375,
+      "logps/chosen": -247.0,
+      "logps/rejected": -250.0,
+      "loss": 0.6481,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.23046875,
+      "rewards/margins": 0.2421875,
+      "rewards/rejected": -0.47265625,
       "step": 110
     },
     {
       "epoch": 0.38338658146964855,
-      "grad_norm": 16.16675788193662,
+      "grad_norm": 26.847013088556583,
       "learning_rate": 4.846153846153846e-07,
-      "logits/chosen": -0.96875,
-      "logits/rejected": -0.93359375,
-      "logps/chosen": -42.75,
-      "logps/rejected": -36.25,
-      "loss": 0.6309,
-      "rewards/accuracies": 0.3375000059604645,
-      "rewards/chosen": -0.2294921875,
-      "rewards/margins": 0.2216796875,
-      "rewards/rejected": -0.451171875,
+      "logits/chosen": -1.7109375,
+      "logits/rejected": -1.7109375,
+      "logps/chosen": -243.0,
+      "logps/rejected": -251.0,
+      "loss": 0.6152,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.2734375,
+      "rewards/margins": 0.306640625,
+      "rewards/rejected": -0.578125,
       "step": 120
     },
     {
       "epoch": 0.41533546325878595,
-      "grad_norm": 10.193866441849577,
+      "grad_norm": 31.074353820130803,
       "learning_rate": 4.78698224852071e-07,
-      "logits/chosen": -0.9453125,
-      "logits/rejected": -0.9375,
-      "logps/chosen": -30.25,
-      "logps/rejected": -33.0,
-      "loss": 0.6548,
-      "rewards/accuracies": 0.3125,
-      "rewards/chosen": -0.1484375,
-      "rewards/margins": 0.228515625,
-      "rewards/rejected": -0.376953125,
+      "logits/chosen": -1.703125,
+      "logits/rejected": -1.71875,
+      "logps/chosen": -240.0,
+      "logps/rejected": -260.0,
+      "loss": 0.604,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -0.47265625,
+      "rewards/margins": 0.291015625,
+      "rewards/rejected": -0.76171875,
       "step": 130
     },
     {
       "epoch": 0.4472843450479233,
-      "grad_norm": 12.44670055298775,
+      "grad_norm": 32.192668717979046,
       "learning_rate": 4.727810650887574e-07,
-      "logits/chosen": -0.94140625,
-      "logits/rejected": -0.9375,
-      "logps/chosen": -34.0,
-      "logps/rejected": -33.75,
-      "loss": 0.6025,
-      "rewards/accuracies": 0.4000000059604645,
-      "rewards/chosen": -0.1123046875,
-      "rewards/margins": 0.373046875,
-      "rewards/rejected": -0.486328125,
+      "logits/chosen": -1.6796875,
+      "logits/rejected": -1.6796875,
+      "logps/chosen": -243.0,
+      "logps/rejected": -254.0,
+      "loss": 0.5609,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.416015625,
+      "rewards/margins": 0.5234375,
+      "rewards/rejected": -0.9375,
       "step": 140
     },
     {
       "epoch": 0.4792332268370607,
-      "grad_norm": 9.718117808971781,
+      "grad_norm": 32.155266134791546,
       "learning_rate": 4.668639053254438e-07,
-      "logits/chosen": -0.97265625,
-      "logits/rejected": -0.9375,
-      "logps/chosen": -35.25,
-      "logps/rejected": -34.75,
-      "loss": 0.6049,
-      "rewards/accuracies": 0.4124999940395355,
-      "rewards/chosen": -0.333984375,
-      "rewards/margins": 0.365234375,
-      "rewards/rejected": -0.69921875,
+      "logits/chosen": -1.6953125,
+      "logits/rejected": -1.671875,
+      "logps/chosen": -246.0,
+      "logps/rejected": -251.0,
+      "loss": 0.5987,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.45703125,
+      "rewards/margins": 0.453125,
+      "rewards/rejected": -0.91015625,
       "step": 150
     },
     {
       "epoch": 0.5111821086261981,
-      "grad_norm": 12.959238451666906,
+      "grad_norm": 30.837637394035802,
       "learning_rate": 4.6094674556213014e-07,
-      "logits/chosen": -0.9296875,
-      "logits/rejected": -0.859375,
-      "logps/chosen": -34.0,
-      "logps/rejected": -32.75,
-      "loss": 0.5895,
-      "rewards/accuracies": 0.42500001192092896,
-      "rewards/chosen": -0.3359375,
-      "rewards/margins": 0.4140625,
-      "rewards/rejected": -0.75,
+      "logits/chosen": -1.6875,
+      "logits/rejected": -1.703125,
+      "logps/chosen": -248.0,
+      "logps/rejected": -252.0,
+      "loss": 0.6033,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.333984375,
+      "rewards/margins": 0.373046875,
+      "rewards/rejected": -0.70703125,
       "step": 160
     },
     {
       "epoch": 0.5431309904153354,
-      "grad_norm": 10.449440815665023,
+      "grad_norm": 25.59680166981086,
       "learning_rate": 4.5502958579881655e-07,
-      "logits/chosen": -0.8984375,
-      "logits/rejected": -0.91015625,
-      "logps/chosen": -30.75,
-      "logps/rejected": -31.75,
-      "loss": 0.6003,
-      "rewards/accuracies": 0.375,
-      "rewards/chosen": -0.34375,
-      "rewards/margins": 0.306640625,
-      "rewards/rejected": -0.6484375,
+      "logits/chosen": -1.671875,
+      "logits/rejected": -1.6796875,
+      "logps/chosen": -238.0,
+      "logps/rejected": -248.0,
+      "loss": 0.5817,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.322265625,
+      "rewards/margins": 0.30078125,
+      "rewards/rejected": -0.62109375,
       "step": 170
     },
     {
       "epoch": 0.5750798722044729,
-      "grad_norm": 21.831604570880106,
+      "grad_norm": 29.648466125367857,
       "learning_rate": 4.491124260355029e-07,
-      "logits/chosen": -0.8984375,
-      "logits/rejected": -0.90625,
-      "logps/chosen": -39.0,
-      "logps/rejected": -35.75,
-      "loss": 0.5721,
-      "rewards/accuracies": 0.42500001192092896,
-      "rewards/chosen": -0.515625,
-      "rewards/margins": 0.388671875,
-      "rewards/rejected": -0.90625,
+      "logits/chosen": -1.6875,
+      "logits/rejected": -1.6875,
+      "logps/chosen": -245.0,
+      "logps/rejected": -251.0,
+      "loss": 0.5874,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.38671875,
+      "rewards/margins": 0.50390625,
+      "rewards/rejected": -0.890625,
       "step": 180
     },
     {
       "epoch": 0.6070287539936102,
-      "grad_norm": 10.29045072439074,
+      "grad_norm": 28.442764104750204,
       "learning_rate": 4.4319526627218936e-07,
-      "logits/chosen": -0.9296875,
-      "logits/rejected": -0.87109375,
-      "logps/chosen": -43.0,
-      "logps/rejected": -30.25,
-      "loss": 0.609,
-      "rewards/accuracies": 0.4000000059604645,
-      "rewards/chosen": -0.470703125,
-      "rewards/margins": 0.419921875,
-      "rewards/rejected": -0.890625,
+      "logits/chosen": -1.6875,
+      "logits/rejected": -1.6875,
+      "logps/chosen": -246.0,
+      "logps/rejected": -249.0,
+      "loss": 0.5737,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.294921875,
+      "rewards/margins": 0.5078125,
+      "rewards/rejected": -0.80078125,
       "step": 190
     },
     {
       "epoch": 0.6389776357827476,
-      "grad_norm": 20.548507017891822,
+      "grad_norm": 26.370679072337964,
       "learning_rate": 4.372781065088757e-07,
-      "logits/chosen": -0.9375,
-      "logits/rejected": -0.83984375,
-      "logps/chosen": -36.75,
-      "logps/rejected": -35.0,
-      "loss": 0.6401,
-      "rewards/accuracies": 0.2874999940395355,
-      "rewards/chosen": -0.58984375,
-      "rewards/margins": 0.251953125,
-      "rewards/rejected": -0.83984375,
+      "logits/chosen": -1.71875,
+      "logits/rejected": -1.703125,
+      "logps/chosen": -240.0,
+      "logps/rejected": -256.0,
+      "loss": 0.5867,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.3828125,
+      "rewards/margins": 0.3671875,
+      "rewards/rejected": -0.75,
       "step": 200
     },
     {
       "epoch": 0.670926517571885,
-      "grad_norm": 9.119895639662692,
+      "grad_norm": 24.556255560376357,
       "learning_rate": 4.313609467455621e-07,
-      "logits/chosen": -0.9140625,
-      "logits/rejected": -0.8828125,
-      "logps/chosen": -32.0,
-      "logps/rejected": -32.5,
-      "loss": 0.5835,
-      "rewards/accuracies": 0.38749998807907104,
-      "rewards/chosen": -0.404296875,
-      "rewards/margins": 0.390625,
-      "rewards/rejected": -0.796875,
+      "logits/chosen": -1.703125,
+      "logits/rejected": -1.6953125,
+      "logps/chosen": -246.0,
+      "logps/rejected": -252.0,
+      "loss": 0.5853,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.349609375,
+      "rewards/margins": 0.3671875,
+      "rewards/rejected": -0.71875,
       "step": 210
     },
     {
       "epoch": 0.7028753993610224,
-      "grad_norm": 10.016002486299485,
+      "grad_norm": 29.98817140815093,
       "learning_rate": 4.2544378698224847e-07,
-      "logits/chosen": -0.94921875,
-      "logits/rejected": -0.90625,
-      "logps/chosen": -36.0,
-      "logps/rejected": -33.25,
-      "loss": 0.5812,
-      "rewards/accuracies": 0.375,
-      "rewards/chosen": -0.322265625,
-      "rewards/margins": 0.447265625,
-      "rewards/rejected": -0.76953125,
+      "logits/chosen": -1.6796875,
+      "logits/rejected": -1.71875,
+      "logps/chosen": -243.0,
+      "logps/rejected": -256.0,
+      "loss": 0.5872,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.39453125,
+      "rewards/margins": 0.55078125,
+      "rewards/rejected": -0.9453125,
       "step": 220
     },
     {
       "epoch": 0.7348242811501597,
-      "grad_norm": 8.739343921649029,
+      "grad_norm": 27.478577985922634,
       "learning_rate": 4.195266272189349e-07,
-      "logits/chosen": -0.9453125,
-      "logits/rejected": -0.8828125,
-      "logps/chosen": -40.75,
-      "logps/rejected": -36.75,
-      "loss": 0.5854,
-      "rewards/accuracies": 0.44999998807907104,
-      "rewards/chosen": -0.52734375,
-      "rewards/margins": 0.60546875,
-      "rewards/rejected": -1.1328125,
+      "logits/chosen": -1.671875,
+      "logits/rejected": -1.65625,
+      "logps/chosen": -240.0,
+      "logps/rejected": -249.0,
+      "loss": 0.5549,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.3203125,
+      "rewards/margins": 0.50390625,
+      "rewards/rejected": -0.82421875,
       "step": 230
     },
     {
       "epoch": 0.7667731629392971,
-      "grad_norm": 20.107539290726088,
+      "grad_norm": 26.416091326825903,
       "learning_rate": 4.1360946745562133e-07,
-      "logits/chosen": -0.8984375,
-      "logits/rejected": -0.87109375,
-      "logps/chosen": -33.75,
-      "logps/rejected": -30.125,
-      "loss": 0.6357,
-      "rewards/accuracies": 0.26249998807907104,
-      "rewards/chosen": -0.5546875,
-      "rewards/margins": 0.19140625,
-      "rewards/rejected": -0.74609375,
+      "logits/chosen": -1.703125,
+      "logits/rejected": -1.71875,
+      "logps/chosen": -240.0,
+      "logps/rejected": -262.0,
+      "loss": 0.538,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.32421875,
+      "rewards/margins": 0.55078125,
+      "rewards/rejected": -0.875,
       "step": 240
     },
     {
       "epoch": 0.7987220447284346,
-      "grad_norm": 13.470404552518705,
+      "grad_norm": 28.30204123280793,
       "learning_rate": 4.076923076923077e-07,
-      "logits/chosen": -0.90234375,
-      "logits/rejected": -0.92578125,
-      "logps/chosen": -39.75,
-      "logps/rejected": -39.75,
-      "loss": 0.5625,
-      "rewards/accuracies": 0.44999998807907104,
-      "rewards/chosen": -0.52734375,
-      "rewards/margins": 0.5078125,
-      "rewards/rejected": -1.03125,
+      "logits/chosen": -1.6796875,
+      "logits/rejected": -1.6875,
+      "logps/chosen": -239.0,
+      "logps/rejected": -260.0,
+      "loss": 0.5507,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.38671875,
+      "rewards/margins": 0.5546875,
+      "rewards/rejected": -0.9453125,
       "step": 250
     },
     {
       "epoch": 0.8306709265175719,
-      "grad_norm": 13.121143639536331,
+      "grad_norm": 33.17216428250153,
       "learning_rate": 4.017751479289941e-07,
-      "logits/chosen": -0.8828125,
-      "logits/rejected": -0.8359375,
-      "logps/chosen": -34.5,
-      "logps/rejected": -34.75,
-      "loss": 0.5635,
-      "rewards/accuracies": 0.38749998807907104,
-      "rewards/chosen": -0.72265625,
-      "rewards/margins": 0.484375,
-      "rewards/rejected": -1.2109375,
+      "logits/chosen": -1.6953125,
+      "logits/rejected": -1.7421875,
+      "logps/chosen": -244.0,
+      "logps/rejected": -256.0,
+      "loss": 0.5584,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.27734375,
+      "rewards/margins": 0.78125,
+      "rewards/rejected": -1.0625,
       "step": 260
     },
     {
       "epoch": 0.8626198083067093,
-      "grad_norm": 10.329929244037169,
+      "grad_norm": 27.41764842123467,
       "learning_rate": 3.9585798816568044e-07,
-      "logits/chosen": -0.90625,
-      "logits/rejected": -0.859375,
-      "logps/chosen": -37.0,
-      "logps/rejected": -41.25,
-      "loss": 0.595,
-      "rewards/accuracies": 0.44999998807907104,
-      "rewards/chosen": -0.796875,
-      "rewards/margins": 0.58203125,
-      "rewards/rejected": -1.375,
+      "logits/chosen": -1.6640625,
+      "logits/rejected": -1.65625,
+      "logps/chosen": -238.0,
+      "logps/rejected": -252.0,
+      "loss": 0.5742,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.24609375,
+      "rewards/margins": 0.6015625,
+      "rewards/rejected": -0.84765625,
       "step": 270
     },
     {
       "epoch": 0.8945686900958466,
-      "grad_norm": 16.191548100134025,
+      "grad_norm": 34.061232014315166,
       "learning_rate": 3.8994082840236685e-07,
-      "logits/chosen": -0.8515625,
-      "logits/rejected": -0.875,
-      "logps/chosen": -42.25,
-      "logps/rejected": -38.0,
-      "loss": 0.555,
-      "rewards/accuracies": 0.4375,
-      "rewards/chosen": -0.6875,
-      "rewards/margins": 0.63671875,
-      "rewards/rejected": -1.328125,
+      "logits/chosen": -1.6953125,
+      "logits/rejected": -1.703125,
+      "logps/chosen": -242.0,
+      "logps/rejected": -251.0,
+      "loss": 0.5279,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -0.3203125,
+      "rewards/margins": 0.65234375,
+      "rewards/rejected": -0.97265625,
       "step": 280
     },
     {
       "epoch": 0.9265175718849841,
-      "grad_norm": 12.02406721548161,
+      "grad_norm": 28.160658079870018,
       "learning_rate": 3.840236686390532e-07,
-      "logits/chosen": -0.94140625,
-      "logits/rejected": -0.86328125,
-      "logps/chosen": -42.5,
-      "logps/rejected": -43.5,
-      "loss": 0.5886,
-      "rewards/accuracies": 0.38749998807907104,
-      "rewards/chosen": -0.76171875,
-      "rewards/margins": 0.53125,
-      "rewards/rejected": -1.296875,
+      "logits/chosen": -1.6796875,
+      "logits/rejected": -1.6953125,
+      "logps/chosen": -248.0,
+      "logps/rejected": -252.0,
+      "loss": 0.5804,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.4296875,
+      "rewards/margins": 0.45703125,
+      "rewards/rejected": -0.88671875,
       "step": 290
     },
     {
       "epoch": 0.9584664536741214,
-      "grad_norm": 5.752074059086699,
+      "grad_norm": 29.60484241313221,
       "learning_rate": 3.7810650887573966e-07,
-      "logits/chosen": -0.87890625,
-      "logits/rejected": -0.890625,
-      "logps/chosen": -31.75,
-      "logps/rejected": -46.75,
-      "loss": 0.5734,
-      "rewards/accuracies": 0.4000000059604645,
-      "rewards/chosen": -0.66796875,
-      "rewards/margins": 0.67578125,
-      "rewards/rejected": -1.34375,
+      "logits/chosen": -1.65625,
+      "logits/rejected": -1.671875,
+      "logps/chosen": -247.0,
+      "logps/rejected": -255.0,
+      "loss": 0.5635,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.35546875,
+      "rewards/margins": 0.400390625,
+      "rewards/rejected": -0.75390625,
       "step": 300
     },
     {
       "epoch": 0.9904153354632588,
-      "grad_norm": 10.316894713261737,
+      "grad_norm": 33.368421615399974,
       "learning_rate": 3.72189349112426e-07,
-      "logits/chosen": -0.90234375,
-      "logits/rejected": -0.89453125,
-      "logps/chosen": -43.5,
-      "logps/rejected": -44.75,
-      "loss": 0.5646,
-      "rewards/accuracies": 0.42500001192092896,
-      "rewards/chosen": -0.703125,
-      "rewards/margins": 0.5859375,
-      "rewards/rejected": -1.2890625,
+      "logits/chosen": -1.7109375,
+      "logits/rejected": -1.703125,
+      "logps/chosen": -244.0,
+      "logps/rejected": -250.0,
+      "loss": 0.5843,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.2578125,
+      "rewards/margins": 0.49609375,
+      "rewards/rejected": -0.75390625,
       "step": 310
     },
     {
       "epoch": 1.0,
-      "eval_logits/chosen": -0.9296875,
-      "eval_logits/rejected": -0.921875,
-      "eval_logps/chosen": -45.5,
-      "eval_logps/rejected": -38.0,
-      "eval_loss": 0.5726562738418579,
-      "eval_rewards/accuracies": 0.4107142984867096,
-      "eval_rewards/chosen": -0.65625,
-      "eval_rewards/margins": 0.53125,
-      "eval_rewards/rejected": -1.1875,
-      "eval_runtime": 12.4256,
-      "eval_samples_per_second": 16.096,
-      "eval_steps_per_second": 0.563,
+      "eval_logits/chosen": -1.6953125,
+      "eval_logits/rejected": -1.7109375,
+      "eval_logps/chosen": -246.0,
+      "eval_logps/rejected": -248.0,
+      "eval_loss": 0.6162499785423279,
+      "eval_rewards/accuracies": 0.5714285969734192,
+      "eval_rewards/chosen": -0.380859375,
+      "eval_rewards/margins": 0.296875,
+      "eval_rewards/rejected": -0.6796875,
+      "eval_runtime": 12.6011,
+      "eval_samples_per_second": 15.872,
+      "eval_steps_per_second": 0.556,
       "step": 313
-    },
-    {
-      "epoch": 1.0223642172523961,
-      "grad_norm": 11.604345250214385,
-      "learning_rate": 3.662721893491124e-07,
-      "logits/chosen": -0.859375,
-      "logits/rejected": -0.8828125,
-      "logps/chosen": -41.75,
-      "logps/rejected": -44.25,
-      "loss": 0.4945,
-      "rewards/accuracies": 0.5375000238418579,
-      "rewards/chosen": -0.51171875,
-      "rewards/margins": 0.921875,
-      "rewards/rejected": -1.4296875,
-      "step": 320
-    },
-    {
-      "epoch": 1.0543130990415335,
-      "grad_norm": 8.164279499540298,
-      "learning_rate": 3.603550295857988e-07,
-      "logits/chosen": -0.93359375,
-      "logits/rejected": -0.8359375,
-      "logps/chosen": -41.5,
-      "logps/rejected": -47.0,
-      "loss": 0.487,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": -0.51171875,
-      "rewards/margins": 0.98046875,
-      "rewards/rejected": -1.4921875,
-      "step": 330
-    },
-    {
-      "epoch": 1.0862619808306708,
-      "grad_norm": 10.098456650224257,
-      "learning_rate": 3.544378698224852e-07,
-      "logits/chosen": -0.90234375,
-      "logits/rejected": -0.89453125,
-      "logps/chosen": -38.25,
-      "logps/rejected": -46.5,
-      "loss": 0.4921,
-      "rewards/accuracies": 0.42500001192092896,
-      "rewards/chosen": -0.74609375,
-      "rewards/margins": 0.71875,
-      "rewards/rejected": -1.4609375,
-      "step": 340
-    },
-    {
-      "epoch": 1.1182108626198084,
-      "grad_norm": 12.312245605594391,
-      "learning_rate": 3.485207100591716e-07,
-      "logits/chosen": -0.8984375,
-      "logits/rejected": -0.9140625,
-      "logps/chosen": -41.75,
-      "logps/rejected": -44.0,
-      "loss": 0.4951,
-      "rewards/accuracies": 0.42500001192092896,
-      "rewards/chosen": -0.671875,
-      "rewards/margins": 0.86328125,
-      "rewards/rejected": -1.5390625,
-      "step": 350
-    },
-    {
-      "epoch": 1.1501597444089458,
-      "grad_norm": 7.6387271402180925,
-      "learning_rate": 3.42603550295858e-07,
-      "logits/chosen": -0.95703125,
-      "logits/rejected": -0.875,
-      "logps/chosen": -39.5,
-      "logps/rejected": -43.5,
-      "loss": 0.5121,
-      "rewards/accuracies": 0.3499999940395355,
-      "rewards/chosen": -0.74609375,
-      "rewards/margins": 0.8125,
-      "rewards/rejected": -1.5625,
-      "step": 360
-    },
-    {
-      "epoch": 1.182108626198083,
-      "grad_norm": 9.066794201428493,
-      "learning_rate": 3.366863905325444e-07,
-      "logits/chosen": -0.91796875,
-      "logits/rejected": -0.890625,
-      "logps/chosen": -39.25,
-      "logps/rejected": -43.25,
-      "loss": 0.5005,
-      "rewards/accuracies": 0.375,
-      "rewards/chosen": -0.9140625,
-      "rewards/margins": 0.7890625,
-      "rewards/rejected": -1.703125,
-      "step": 370
-    },
-    {
-      "epoch": 1.2140575079872205,
-      "grad_norm": 13.248069014797954,
-      "learning_rate": 3.3076923076923075e-07,
-      "logits/chosen": -0.91796875,
-      "logits/rejected": -0.91796875,
-      "logps/chosen": -37.5,
-      "logps/rejected": -51.0,
-      "loss": 0.4653,
-      "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": -0.71484375,
-      "rewards/margins": 1.015625,
-      "rewards/rejected": -1.7265625,
-      "step": 380
-    },
-    {
-      "epoch": 1.2460063897763578,
-      "grad_norm": 10.867663199240486,
-      "learning_rate": 3.2485207100591716e-07,
-      "logits/chosen": -0.94140625,
-      "logits/rejected": -0.890625,
-      "logps/chosen": -42.25,
-      "logps/rejected": -45.75,
-      "loss": 0.4909,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": -0.5625,
-      "rewards/margins": 1.109375,
-      "rewards/rejected": -1.671875,
-      "step": 390
-    },
-    {
-      "epoch": 1.2779552715654952,
-      "grad_norm": 12.647214530771212,
-      "learning_rate": 3.189349112426035e-07,
-      "logits/chosen": -0.84765625,
-      "logits/rejected": -0.921875,
-      "logps/chosen": -29.625,
-      "logps/rejected": -60.0,
-      "loss": 0.4816,
-      "rewards/accuracies": 0.5375000238418579,
-      "rewards/chosen": -0.96484375,
-      "rewards/margins": 1.0,
-      "rewards/rejected": -1.96875,
-      "step": 400
-    },
-    {
-      "epoch": 1.3099041533546325,
-      "grad_norm": 13.831769507648124,
-      "learning_rate": 3.130177514792899e-07,
-      "logits/chosen": -0.89453125,
-      "logits/rejected": -0.85546875,
-      "logps/chosen": -32.5,
-      "logps/rejected": -43.75,
-      "loss": 0.5379,
-      "rewards/accuracies": 0.3375000059604645,
-      "rewards/chosen": -1.078125,
-      "rewards/margins": 0.50390625,
-      "rewards/rejected": -1.578125,
-      "step": 410
-    },
-    {
-      "epoch": 1.34185303514377,
-      "grad_norm": 12.24140890124011,
-      "learning_rate": 3.071005917159763e-07,
-      "logits/chosen": -0.9375,
-      "logits/rejected": -0.9375,
-      "logps/chosen": -41.0,
-      "logps/rejected": -53.25,
-      "loss": 0.4786,
-      "rewards/accuracies": 0.48750001192092896,
-      "rewards/chosen": -1.265625,
-      "rewards/margins": 1.046875,
-      "rewards/rejected": -2.3125,
-      "step": 420
-    },
-    {
-      "epoch": 1.3738019169329074,
-      "grad_norm": 8.001226654192951,
-      "learning_rate": 3.011834319526627e-07,
-      "logits/chosen": -0.95703125,
-      "logits/rejected": -0.8984375,
-      "logps/chosen": -44.5,
-      "logps/rejected": -50.0,
-      "loss": 0.4892,
-      "rewards/accuracies": 0.4749999940395355,
-      "rewards/chosen": -1.0703125,
-      "rewards/margins": 0.984375,
-      "rewards/rejected": -2.046875,
-      "step": 430
-    },
-    {
-      "epoch": 1.4057507987220448,
-      "grad_norm": 13.90707171430186,
-      "learning_rate": 2.952662721893491e-07,
-      "logits/chosen": -0.95703125,
-      "logits/rejected": -0.87890625,
-      "logps/chosen": -44.0,
-      "logps/rejected": -47.25,
-      "loss": 0.4805,
-      "rewards/accuracies": 0.42500001192092896,
-      "rewards/chosen": -1.03125,
-      "rewards/margins": 1.0546875,
-      "rewards/rejected": -2.09375,
-      "step": 440
-    },
-    {
-      "epoch": 1.4376996805111821,
-      "grad_norm": 11.71134112335707,
-      "learning_rate": 2.893491124260355e-07,
-      "logits/chosen": -0.91796875,
-      "logits/rejected": -0.86328125,
-      "logps/chosen": -46.5,
-      "logps/rejected": -45.25,
-      "loss": 0.4582,
-      "rewards/accuracies": 0.42500001192092896,
-      "rewards/chosen": -1.171875,
-      "rewards/margins": 0.83984375,
-      "rewards/rejected": -2.015625,
-      "step": 450
-    },
-    {
-      "epoch": 1.4696485623003195,
-      "grad_norm": 10.206570733769754,
-      "learning_rate": 2.834319526627219e-07,
-      "logits/chosen": -1.0,
-      "logits/rejected": -0.921875,
-      "logps/chosen": -44.25,
-      "logps/rejected": -46.75,
-      "loss": 0.4662,
-      "rewards/accuracies": 0.44999998807907104,
-      "rewards/chosen": -1.1015625,
-      "rewards/margins": 0.8984375,
-      "rewards/rejected": -2.015625,
-      "step": 460
-    },
-    {
-      "epoch": 1.5015974440894568,
-      "grad_norm": 8.481816160880287,
-      "learning_rate": 2.7751479289940824e-07,
-      "logits/chosen": -0.97265625,
-      "logits/rejected": -0.8671875,
-      "logps/chosen": -43.0,
-      "logps/rejected": -54.0,
-      "loss": 0.4951,
-      "rewards/accuracies": 0.44999998807907104,
-      "rewards/chosen": -1.2421875,
-      "rewards/margins": 1.078125,
-      "rewards/rejected": -2.3125,
-      "step": 470
-    },
-    {
-      "epoch": 1.5335463258785942,
-      "grad_norm": 9.450961138318995,
-      "learning_rate": 2.715976331360947e-07,
-      "logits/chosen": -0.96484375,
-      "logits/rejected": -0.859375,
-      "logps/chosen": -45.0,
-      "logps/rejected": -50.5,
-      "loss": 0.4609,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": -0.8359375,
-      "rewards/margins": 1.359375,
-      "rewards/rejected": -2.1875,
-      "step": 480
-    },
-    {
-      "epoch": 1.5654952076677318,
-      "grad_norm": 10.079665777936816,
-      "learning_rate": 2.6568047337278106e-07,
-      "logits/chosen": -0.9140625,
-      "logits/rejected": -0.8671875,
-      "logps/chosen": -40.5,
-      "logps/rejected": -46.25,
-      "loss": 0.484,
-      "rewards/accuracies": 0.4000000059604645,
-      "rewards/chosen": -1.171875,
-      "rewards/margins": 0.91796875,
-      "rewards/rejected": -2.09375,
-      "step": 490
-    },
-    {
-      "epoch": 1.5974440894568689,
-      "grad_norm": 5.892858649998084,
-      "learning_rate": 2.5976331360946746e-07,
-      "logits/chosen": -0.90234375,
-      "logits/rejected": -0.85546875,
-      "logps/chosen": -43.5,
-      "logps/rejected": -44.0,
-      "loss": 0.4725,
-      "rewards/accuracies": 0.42500001192092896,
-      "rewards/chosen": -0.96875,
-      "rewards/margins": 0.8984375,
-      "rewards/rejected": -1.8671875,
-      "step": 500
-    },
-    {
-      "epoch": 1.6293929712460065,
-      "grad_norm": 19.866826656455657,
-      "learning_rate": 2.538461538461538e-07,
-      "logits/chosen": -0.88671875,
-      "logits/rejected": -0.875,
-      "logps/chosen": -37.0,
-      "logps/rejected": -49.0,
-      "loss": 0.4713,
-      "rewards/accuracies": 0.44999998807907104,
-      "rewards/chosen": -1.015625,
-      "rewards/margins": 1.09375,
-      "rewards/rejected": -2.109375,
-      "step": 510
-    },
-    {
-      "epoch": 1.6613418530351438,
-      "grad_norm": 11.462164104029984,
-      "learning_rate": 2.479289940828402e-07,
-      "logits/chosen": -1.0,
-      "logits/rejected": -0.86328125,
-      "logps/chosen": -48.25,
-      "logps/rejected": -46.5,
-      "loss": 0.4437,
-      "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": -0.69140625,
-      "rewards/margins": 1.28125,
-      "rewards/rejected": -1.9765625,
-      "step": 520
-    },
-    {
-      "epoch": 1.6932907348242812,
-      "grad_norm": 4.754132390092114,
-      "learning_rate": 2.420118343195266e-07,
-      "logits/chosen": -0.984375,
-      "logits/rejected": -0.89453125,
-      "logps/chosen": -43.5,
-      "logps/rejected": -44.5,
-      "loss": 0.4539,
-      "rewards/accuracies": 0.44999998807907104,
-      "rewards/chosen": -1.0234375,
-      "rewards/margins": 1.0234375,
-      "rewards/rejected": -2.046875,
-      "step": 530
-    },
-    {
-      "epoch": 1.7252396166134185,
-      "grad_norm": 13.855605681924747,
-      "learning_rate": 2.36094674556213e-07,
-      "logits/chosen": -1.0234375,
-      "logits/rejected": -0.828125,
-      "logps/chosen": -50.0,
-      "logps/rejected": -45.75,
-      "loss": 0.4537,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": -0.81640625,
-      "rewards/margins": 1.203125,
-      "rewards/rejected": -2.015625,
-      "step": 540
-    },
-    {
-      "epoch": 1.7571884984025559,
-      "grad_norm": 13.694710524562387,
-      "learning_rate": 2.301775147928994e-07,
-      "logits/chosen": -0.98828125,
-      "logits/rejected": -0.91796875,
-      "logps/chosen": -41.75,
-      "logps/rejected": -52.5,
-      "loss": 0.4505,
-      "rewards/accuracies": 0.4749999940395355,
-      "rewards/chosen": -1.09375,
-      "rewards/margins": 1.265625,
-      "rewards/rejected": -2.359375,
-      "step": 550
-    },
-    {
-      "epoch": 1.7891373801916934,
-      "grad_norm": 12.265823828649788,
-      "learning_rate": 2.242603550295858e-07,
-      "logits/chosen": -0.98828125,
-      "logits/rejected": -0.92578125,
-      "logps/chosen": -39.75,
-      "logps/rejected": -59.0,
-      "loss": 0.4626,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": -0.86328125,
-      "rewards/margins": 1.3828125,
-      "rewards/rejected": -2.25,
-      "step": 560
-    },
-    {
-      "epoch": 1.8210862619808306,
-      "grad_norm": 15.811792160256678,
-      "learning_rate": 2.1834319526627217e-07,
-      "logits/chosen": -0.95703125,
-      "logits/rejected": -0.8984375,
-      "logps/chosen": -40.5,
-      "logps/rejected": -50.5,
-      "loss": 0.4507,
-      "rewards/accuracies": 0.4749999940395355,
-      "rewards/chosen": -0.609375,
-      "rewards/margins": 1.28125,
-      "rewards/rejected": -1.8828125,
-      "step": 570
-    },
-    {
-      "epoch": 1.8530351437699681,
-      "grad_norm": 12.03300574763422,
-      "learning_rate": 2.1242603550295858e-07,
-      "logits/chosen": -1.046875,
-      "logits/rejected": -0.91015625,
-      "logps/chosen": -42.25,
-      "logps/rejected": -49.25,
-      "loss": 0.45,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": -0.62890625,
-      "rewards/margins": 1.3984375,
-      "rewards/rejected": -2.03125,
-      "step": 580
-    },
-    {
-      "epoch": 1.8849840255591053,
-      "grad_norm": 6.498105360999796,
-      "learning_rate": 2.0650887573964496e-07,
-      "logits/chosen": -0.9921875,
-      "logits/rejected": -0.8828125,
-      "logps/chosen": -45.75,
-      "logps/rejected": -52.75,
-      "loss": 0.4347,
-      "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": -0.94921875,
-      "rewards/margins": 1.546875,
-      "rewards/rejected": -2.5,
-      "step": 590
-    },
-    {
-      "epoch": 1.9169329073482428,
-      "grad_norm": 17.196261475524192,
-      "learning_rate": 2.0059171597633133e-07,
-      "logits/chosen": -0.91796875,
-      "logits/rejected": -0.859375,
-      "logps/chosen": -39.25,
-      "logps/rejected": -50.75,
-      "loss": 0.4452,
-      "rewards/accuracies": 0.4375,
-      "rewards/chosen": -1.1875,
-      "rewards/margins": 1.171875,
-      "rewards/rejected": -2.359375,
-      "step": 600
-    },
-    {
-      "epoch": 1.9488817891373802,
-      "grad_norm": 6.682211779881807,
-      "learning_rate": 1.9467455621301774e-07,
-      "logits/chosen": -0.81640625,
-      "logits/rejected": -0.93359375,
-      "logps/chosen": -42.25,
-      "logps/rejected": -57.25,
-      "loss": 0.4483,
-      "rewards/accuracies": 0.4749999940395355,
-      "rewards/chosen": -1.2890625,
-      "rewards/margins": 1.265625,
-      "rewards/rejected": -2.546875,
-      "step": 610
-    },
-    {
-      "epoch": 1.9808306709265175,
-      "grad_norm": 8.358366031471173,
-      "learning_rate": 1.8875739644970412e-07,
-      "logits/chosen": -0.9921875,
-      "logits/rejected": -0.90234375,
-      "logps/chosen": -45.25,
-      "logps/rejected": -57.75,
-      "loss": 0.4365,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": -1.296875,
-      "rewards/margins": 1.40625,
-      "rewards/rejected": -2.703125,
-      "step": 620
-    },
-    {
-      "epoch": 2.0,
-      "eval_logits/chosen": -1.03125,
-      "eval_logits/rejected": -0.9375,
-      "eval_logps/chosen": -51.75,
-      "eval_logps/rejected": -49.75,
-      "eval_loss": 0.5380077958106995,
-      "eval_rewards/accuracies": 0.5,
-      "eval_rewards/chosen": -1.2890625,
-      "eval_rewards/margins": 1.078125,
-      "eval_rewards/rejected": -2.375,
-      "eval_runtime": 12.4486,
-      "eval_samples_per_second": 16.066,
-      "eval_steps_per_second": 0.562,
-      "step": 626
-    },
-    {
-      "epoch": 2.012779552715655,
-      "grad_norm": 4.74652717171804,
-      "learning_rate": 1.8284023668639053e-07,
-      "logits/chosen": -0.875,
-      "logits/rejected": -0.96484375,
-      "logps/chosen": -40.75,
-      "logps/rejected": -56.0,
-      "loss": 0.4227,
-      "rewards/accuracies": 0.4625000059604645,
-      "rewards/chosen": -1.25,
-      "rewards/margins": 1.078125,
-      "rewards/rejected": -2.328125,
-      "step": 630
-    },
-    {
-      "epoch": 2.0447284345047922,
-      "grad_norm": 9.093805196751525,
-      "learning_rate": 1.7692307692307693e-07,
-      "logits/chosen": -0.95703125,
-      "logits/rejected": -0.96484375,
-      "logps/chosen": -46.25,
-      "logps/rejected": -56.75,
-      "loss": 0.4062,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": -1.234375,
-      "rewards/margins": 1.6640625,
-      "rewards/rejected": -2.90625,
-      "step": 640
-    },
-    {
-      "epoch": 2.07667731629393,
-      "grad_norm": 7.904913836224486,
-      "learning_rate": 1.710059171597633e-07,
-      "logits/chosen": -0.94140625,
-      "logits/rejected": -0.84765625,
-      "logps/chosen": -44.75,
-      "logps/rejected": -53.75,
-      "loss": 0.403,
-      "rewards/accuracies": 0.4625000059604645,
-      "rewards/chosen": -1.265625,
-      "rewards/margins": 1.34375,
-      "rewards/rejected": -2.609375,
-      "step": 650
-    },
-    {
-      "epoch": 2.108626198083067,
-      "grad_norm": 9.216092905835959,
-      "learning_rate": 1.650887573964497e-07,
-      "logits/chosen": -0.85546875,
-      "logits/rejected": -0.81640625,
-      "logps/chosen": -42.5,
-      "logps/rejected": -47.0,
-      "loss": 0.4476,
-      "rewards/accuracies": 0.4124999940395355,
-      "rewards/chosen": -1.15625,
-      "rewards/margins": 1.1796875,
-      "rewards/rejected": -2.34375,
-      "step": 660
-    },
-    {
-      "epoch": 2.1405750798722045,
-      "grad_norm": 8.710740799549797,
-      "learning_rate": 1.591715976331361e-07,
-      "logits/chosen": -1.0234375,
-      "logits/rejected": -0.875,
-      "logps/chosen": -44.5,
-      "logps/rejected": -54.25,
-      "loss": 0.4187,
-      "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": -0.92578125,
-      "rewards/margins": 1.625,
-      "rewards/rejected": -2.546875,
-      "step": 670
-    },
-    {
-      "epoch": 2.1725239616613417,
-      "grad_norm": 8.094002644119932,
-      "learning_rate": 1.5325443786982248e-07,
-      "logits/chosen": -1.0703125,
-      "logits/rejected": -0.8984375,
-      "logps/chosen": -43.5,
-      "logps/rejected": -59.0,
-      "loss": 0.3973,
-      "rewards/accuracies": 0.5375000238418579,
-      "rewards/chosen": -0.7578125,
-      "rewards/margins": 2.0,
-      "rewards/rejected": -2.765625,
-      "step": 680
-    },
-    {
-      "epoch": 2.2044728434504792,
-      "grad_norm": 8.249215092288534,
-      "learning_rate": 1.4733727810650885e-07,
-      "logits/chosen": -0.96875,
-      "logits/rejected": -0.9765625,
-      "logps/chosen": -41.75,
-      "logps/rejected": -59.75,
-      "loss": 0.3862,
-      "rewards/accuracies": 0.5375000238418579,
-      "rewards/chosen": -1.2109375,
-      "rewards/margins": 1.59375,
-      "rewards/rejected": -2.8125,
-      "step": 690
-    },
-    {
-      "epoch": 2.236421725239617,
-      "grad_norm": 8.713919175156146,
-      "learning_rate": 1.4142011834319526e-07,
-      "logits/chosen": -1.015625,
-      "logits/rejected": -0.81640625,
-      "logps/chosen": -44.25,
-      "logps/rejected": -52.25,
-      "loss": 0.3898,
-      "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": -1.109375,
-      "rewards/margins": 1.6875,
-      "rewards/rejected": -2.796875,
-      "step": 700
-    },
-    {
-      "epoch": 2.268370607028754,
-      "grad_norm": 2.4789298662571055,
-      "learning_rate": 1.3550295857988164e-07,
-      "logits/chosen": -1.0703125,
-      "logits/rejected": -0.75,
-      "logps/chosen": -49.0,
-      "logps/rejected": -55.0,
-      "loss": 0.387,
-      "rewards/accuracies": 0.4625000059604645,
-      "rewards/chosen": -1.421875,
-      "rewards/margins": 1.6015625,
-      "rewards/rejected": -3.03125,
-      "step": 710
-    },
-    {
-      "epoch": 2.3003194888178915,
-      "grad_norm": 13.408171707422571,
-      "learning_rate": 1.2958579881656802e-07,
-      "logits/chosen": -0.81640625,
-      "logits/rejected": -0.859375,
-      "logps/chosen": -42.0,
-      "logps/rejected": -54.5,
-      "loss": 0.4224,
-      "rewards/accuracies": 0.4625000059604645,
-      "rewards/chosen": -1.546875,
-      "rewards/margins": 1.3984375,
-      "rewards/rejected": -2.953125,
-      "step": 720
-    },
-    {
-      "epoch": 2.3322683706070286,
-      "grad_norm": 5.024401475671983,
-      "learning_rate": 1.2366863905325443e-07,
-      "logits/chosen": -0.95703125,
-      "logits/rejected": -0.89453125,
-      "logps/chosen": -47.0,
-      "logps/rejected": -57.0,
-      "loss": 0.393,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": -1.2421875,
-      "rewards/margins": 1.671875,
-      "rewards/rejected": -2.921875,
-      "step": 730
-    },
-    {
-      "epoch": 2.364217252396166,
-      "grad_norm": 27.208339044008937,
-      "learning_rate": 1.1775147928994082e-07,
-      "logits/chosen": -0.97265625,
-      "logits/rejected": -0.921875,
-      "logps/chosen": -47.0,
-      "logps/rejected": -63.0,
-      "loss": 0.4028,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": -1.4375,
-      "rewards/margins": 1.640625,
-      "rewards/rejected": -3.078125,
-      "step": 740
-    },
-    {
-      "epoch": 2.3961661341853033,
-      "grad_norm": 10.51991490071309,
-      "learning_rate": 1.1183431952662721e-07,
-      "logits/chosen": -0.98046875,
-      "logits/rejected": -1.03125,
-      "logps/chosen": -43.5,
-      "logps/rejected": -69.0,
-      "loss": 0.4102,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": -1.5546875,
-      "rewards/margins": 2.125,
-      "rewards/rejected": -3.671875,
-      "step": 750
-    },
-    {
-      "epoch": 2.428115015974441,
-      "grad_norm": 11.046462734712232,
-      "learning_rate": 1.059171597633136e-07,
-      "logits/chosen": -0.9375,
-      "logits/rejected": -0.77734375,
-      "logps/chosen": -46.5,
-      "logps/rejected": -53.5,
-      "loss": 0.374,
-      "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": -1.265625,
-      "rewards/margins": 1.78125,
-      "rewards/rejected": -3.03125,
-      "step": 760
-    },
-    {
-      "epoch": 2.460063897763578,
-      "grad_norm": 7.941784467499254,
-      "learning_rate": 1e-07,
-      "logits/chosen": -0.79296875,
-      "logits/rejected": -0.921875,
-      "logps/chosen": -40.5,
-      "logps/rejected": -67.5,
-      "loss": 0.3808,
-      "rewards/accuracies": 0.5375000238418579,
-      "rewards/chosen": -1.6328125,
-      "rewards/margins": 1.96875,
-      "rewards/rejected": -3.609375,
-      "step": 770
-    },
-    {
-      "epoch": 2.4920127795527156,
-      "grad_norm": 18.383843736884476,
-      "learning_rate": 9.408284023668639e-08,
-      "logits/chosen": -0.953125,
-      "logits/rejected": -0.953125,
-      "logps/chosen": -44.5,
-      "logps/rejected": -63.0,
-      "loss": 0.3853,
-      "rewards/accuracies": 0.44999998807907104,
-      "rewards/chosen": -1.6015625,
-      "rewards/margins": 1.8046875,
-      "rewards/rejected": -3.390625,
-      "step": 780
-    },
-    {
-      "epoch": 2.523961661341853,
-      "grad_norm": 12.735279104095845,
-      "learning_rate": 8.816568047337278e-08,
-      "logits/chosen": -0.95703125,
-      "logits/rejected": -0.859375,
-      "logps/chosen": -46.5,
-      "logps/rejected": -53.75,
-      "loss": 0.4527,
-      "rewards/accuracies": 0.375,
-      "rewards/chosen": -1.7421875,
-      "rewards/margins": 1.2421875,
-      "rewards/rejected": -2.984375,
-      "step": 790
-    },
-    {
-      "epoch": 2.5559105431309903,
-      "grad_norm": 25.31092617144284,
-      "learning_rate": 8.224852071005916e-08,
-      "logits/chosen": -0.75,
-      "logits/rejected": -1.03125,
-      "logps/chosen": -44.25,
-      "logps/rejected": -72.0,
-      "loss": 0.3664,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": -1.8671875,
-      "rewards/margins": 1.890625,
-      "rewards/rejected": -3.765625,
-      "step": 800
-    },
-    {
-      "epoch": 2.587859424920128,
-      "grad_norm": 28.35128868207696,
-      "learning_rate": 7.633136094674555e-08,
-      "logits/chosen": -0.953125,
-      "logits/rejected": -0.83203125,
-      "logps/chosen": -50.5,
-      "logps/rejected": -55.0,
-      "loss": 0.4429,
-      "rewards/accuracies": 0.42500001192092896,
-      "rewards/chosen": -1.6796875,
-      "rewards/margins": 1.5703125,
-      "rewards/rejected": -3.25,
-      "step": 810
-    },
-    {
-      "epoch": 2.619808306709265,
-      "grad_norm": 8.972843974408116,
-      "learning_rate": 7.041420118343195e-08,
-      "logits/chosen": -1.03125,
-      "logits/rejected": -0.94140625,
-      "logps/chosen": -51.0,
-      "logps/rejected": -70.0,
-      "loss": 0.3984,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": -1.46875,
-      "rewards/margins": 2.421875,
-      "rewards/rejected": -3.890625,
-      "step": 820
-    },
-    {
-      "epoch": 2.6517571884984026,
-      "grad_norm": 8.613351315510789,
-      "learning_rate": 6.449704142011835e-08,
-      "logits/chosen": -0.96875,
-      "logits/rejected": -1.0546875,
-      "logps/chosen": -50.25,
-      "logps/rejected": -69.0,
-      "loss": 0.4115,
-      "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": -1.46875,
-      "rewards/margins": 2.03125,
-      "rewards/rejected": -3.5,
-      "step": 830
-    },
-    {
-      "epoch": 2.68370607028754,
-      "grad_norm": 9.70223823689821,
-      "learning_rate": 5.857988165680473e-08,
-      "logits/chosen": -1.03125,
-      "logits/rejected": -0.75,
-      "logps/chosen": -49.0,
-      "logps/rejected": -53.5,
-      "loss": 0.3931,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": -1.2890625,
-      "rewards/margins": 1.890625,
-      "rewards/rejected": -3.171875,
-      "step": 840
-    },
-    {
-      "epoch": 2.7156549520766773,
-      "grad_norm": 14.809661385875646,
-      "learning_rate": 5.266272189349112e-08,
-      "logits/chosen": -0.96875,
-      "logits/rejected": -0.953125,
-      "logps/chosen": -47.25,
-      "logps/rejected": -62.75,
-      "loss": 0.4115,
-      "rewards/accuracies": 0.4749999940395355,
-      "rewards/chosen": -1.6484375,
-      "rewards/margins": 1.7734375,
-      "rewards/rejected": -3.421875,
-      "step": 850
-    },
-    {
-      "epoch": 2.747603833865815,
-      "grad_norm": 5.380820798764086,
-      "learning_rate": 4.674556213017751e-08,
-      "logits/chosen": -0.98046875,
-      "logits/rejected": -0.8828125,
-      "logps/chosen": -47.0,
-      "logps/rejected": -61.25,
-      "loss": 0.3984,
-      "rewards/accuracies": 0.4625000059604645,
-      "rewards/chosen": -1.625,
-      "rewards/margins": 1.703125,
-      "rewards/rejected": -3.328125,
-      "step": 860
-    },
-    {
-      "epoch": 2.779552715654952,
-      "grad_norm": 19.658075353925057,
-      "learning_rate": 4.082840236686391e-08,
-      "logits/chosen": -0.90625,
-      "logits/rejected": -0.7578125,
-      "logps/chosen": -49.25,
-      "logps/rejected": -57.5,
-      "loss": 0.3908,
-      "rewards/accuracies": 0.4749999940395355,
-      "rewards/chosen": -1.8984375,
-      "rewards/margins": 1.578125,
-      "rewards/rejected": -3.46875,
-      "step": 870
-    },
-    {
-      "epoch": 2.8115015974440896,
-      "grad_norm": 10.442407160730754,
-      "learning_rate": 3.4911242603550294e-08,
-      "logits/chosen": -0.84375,
-      "logits/rejected": -0.91796875,
-      "logps/chosen": -41.75,
-      "logps/rejected": -63.75,
-      "loss": 0.4039,
-      "rewards/accuracies": 0.48750001192092896,
-      "rewards/chosen": -1.8125,
-      "rewards/margins": 1.53125,
-      "rewards/rejected": -3.34375,
-      "step": 880
-    },
-    {
-      "epoch": 2.8434504792332267,
-      "grad_norm": 29.6132532828452,
-      "learning_rate": 2.8994082840236687e-08,
-      "logits/chosen": -0.95703125,
-      "logits/rejected": -0.82421875,
-      "logps/chosen": -46.75,
-      "logps/rejected": -62.5,
-      "loss": 0.3814,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": -1.5859375,
-      "rewards/margins": 2.0625,
-      "rewards/rejected": -3.640625,
-      "step": 890
-    },
-    {
-      "epoch": 2.8753993610223643,
-      "grad_norm": 5.3338488869709035,
-      "learning_rate": 2.3076923076923076e-08,
-      "logits/chosen": -0.7578125,
-      "logits/rejected": -0.81640625,
-      "logps/chosen": -37.5,
-      "logps/rejected": -55.0,
-      "loss": 0.4095,
-      "rewards/accuracies": 0.4124999940395355,
-      "rewards/chosen": -1.671875,
-      "rewards/margins": 1.34375,
-      "rewards/rejected": -3.015625,
-      "step": 900
-    },
-    {
-      "epoch": 2.9073482428115014,
-      "grad_norm": 8.959126987075347,
-      "learning_rate": 1.7159763313609465e-08,
-      "logits/chosen": -1.0078125,
-      "logits/rejected": -0.84375,
-      "logps/chosen": -45.75,
-      "logps/rejected": -55.5,
-      "loss": 0.3631,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": -1.3984375,
-      "rewards/margins": 1.6015625,
-      "rewards/rejected": -3.0,
-      "step": 910
-    },
-    {
-      "epoch": 2.939297124600639,
-      "grad_norm": 12.28972695659583,
-      "learning_rate": 1.1242603550295858e-08,
-      "logits/chosen": -1.015625,
-      "logits/rejected": -0.9765625,
-      "logps/chosen": -49.75,
-      "logps/rejected": -69.0,
-      "loss": 0.4118,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": -1.703125,
-      "rewards/margins": 2.09375,
-      "rewards/rejected": -3.796875,
-      "step": 920
-    },
-    {
-      "epoch": 2.9712460063897765,
-      "grad_norm": 7.913983074444077,
-      "learning_rate": 5.325443786982248e-09,
-      "logits/chosen": -0.984375,
-      "logits/rejected": -0.7734375,
-      "logps/chosen": -46.5,
-      "logps/rejected": -49.0,
-      "loss": 0.4176,
-      "rewards/accuracies": 0.42500001192092896,
-      "rewards/chosen": -1.6484375,
-      "rewards/margins": 1.28125,
-      "rewards/rejected": -2.921875,
-      "step": 930
-    },
-    {
-      "epoch": 3.0,
-      "eval_logits/chosen": -1.078125,
-      "eval_logits/rejected": -0.91796875,
-      "eval_logps/chosen": -57.0,
-      "eval_logps/rejected": -56.75,
-      "eval_loss": 0.5467187762260437,
-      "eval_rewards/accuracies": 0.4642857015132904,
-      "eval_rewards/chosen": -1.828125,
-      "eval_rewards/margins": 1.234375,
-      "eval_rewards/rejected": -3.0625,
-      "eval_runtime": 14.7769,
-      "eval_samples_per_second": 13.535,
-      "eval_steps_per_second": 0.474,
-      "step": 939
     }
   ],
   "logging_steps": 10,
@@ -1479,7 +517,7 @@
         "should_evaluate": false,
         "should_log": false,
         "should_save": true,
-        "should_training_stop": true
+        "should_training_stop": false
       },
       "attributes": {}
     }

last-checkpoint/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:49d0ee10924680b0dbb8c60876dca1244bf6be04c6336947d4ba61389da32f15
-size 7800
+oid sha256:79dfe2d41c4493c5292614c95a7bd868fe596379a37a92ef6ab1c6a427fc531a
+size 7864