diff --git "a/checkpoint-200/trainer_state.json" "b/checkpoint-200/trainer_state.json"
new file mode 100644--- /dev/null
+++ "b/checkpoint-200/trainer_state.json"
@@ -0,0 +1,3634 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "episode": 3200,
+  "epoch": 0.019172927825909816,
+  "eval_steps": 500,
+  "global_step": 200,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "episode": 16,
+      "epoch": 9.586463912954908e-05,
+      "loss/policy_avg": 0.0339290015399456,
+      "lr": 1e-05,
+      "objective/entropy": 78.48619842529297,
+      "objective/kl": 5.6675214767456055,
+      "objective/non_score_reward": -0.2833760380744934,
+      "objective/rlhf_reward": 3.2664958328008655,
+      "objective/scores": 1.1,
+      "policy/approxkl_avg": 56.270538330078125,
+      "policy/clipfrac_avg": 1.25,
+      "policy/entropy_avg": 0.5703125,
+      "step": 0,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 3,
+      "val/ratio": 2.000828266143799
+    },
+    {
+      "episode": 32,
+      "epoch": 0.00019172927825909816,
+      "loss/policy_avg": 0.032509539276361465,
+      "lr": 9.999360940695298e-06,
+      "objective/entropy": 39.34157943725586,
+      "objective/kl": 8.134885787963867,
+      "objective/non_score_reward": -0.40674424171447754,
+      "objective/rlhf_reward": -1.6269769463688135,
+      "objective/scores": 0.0,
+      "policy/approxkl_avg": 125.53129577636719,
+      "policy/clipfrac_avg": 1.25,
+      "policy/entropy_avg": 0.46875,
+      "step": 1,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 2,
+      "val/ratio": 1.9994192123413086
+    },
+    {
+      "episode": 48,
+      "epoch": 0.00028759391738864725,
+      "loss/policy_avg": 0.2574540972709656,
+      "lr": 9.998721881390595e-06,
+      "objective/entropy": 35.90438461303711,
+      "objective/kl": 10.056818008422852,
+      "objective/non_score_reward": -0.5028409957885742,
+      "objective/rlhf_reward": -4.011363983154297,
+      "objective/scores": -0.5,
+      "policy/approxkl_avg": 197.81790161132812,
+      "policy/clipfrac_avg": 0.75,
+      "policy/entropy_avg": 0.73828125,
+      "step": 2,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 3,
+      "val/ratio": 1.99604332447052
+    },
+    {
+      "episode": 64,
+      "epoch": 0.0003834585565181963,
+      "loss/policy_avg": 0.1315518617630005,
+      "lr": 9.99808282208589e-06,
+      "objective/entropy": 163.52642822265625,
+      "objective/kl": 12.497467041015625,
+      "objective/non_score_reward": -0.6248733997344971,
+      "objective/rlhf_reward": -2.499493680894375,
+      "objective/scores": 0.0,
+      "policy/approxkl_avg": 280.7725830078125,
+      "policy/clipfrac_avg": 1.5,
+      "policy/entropy_avg": 0.716796875,
+      "step": 3,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 1,
+      "val/ratio": 1.999171257019043
+    },
+    {
+      "episode": 80,
+      "epoch": 0.0004793231956477454,
+      "loss/policy_avg": 0.024046147242188454,
+      "lr": 9.997443762781187e-06,
+      "objective/entropy": 118.5094223022461,
+      "objective/kl": 5.982309818267822,
+      "objective/non_score_reward": -0.29911553859710693,
+      "objective/rlhf_reward": -1.196462158113718,
+      "objective/scores": 0.0,
+      "policy/approxkl_avg": 52.543487548828125,
+      "policy/clipfrac_avg": 1.25,
+      "policy/entropy_avg": 0.552734375,
+      "step": 4,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 1,
+      "val/ratio": 2.001847743988037
+    },
+    {
+      "episode": 96,
+      "epoch": 0.0005751878347772945,
+      "loss/policy_avg": 0.10632362961769104,
+      "lr": 9.996804703476484e-06,
+      "objective/entropy": 152.1885986328125,
+      "objective/kl": 7.815367698669434,
+      "objective/non_score_reward": -0.3907684087753296,
+      "objective/rlhf_reward": 0.0987858943933384,
+      "objective/scores": 0.41546487678572874,
+      "policy/approxkl_avg": 88.51527404785156,
+      "policy/clipfrac_avg": 1.0,
+      "policy/entropy_avg": 0.5859375,
+      "step": 5,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 1,
+      "val/ratio": 1.9992973804473877
+    },
+    {
+      "episode": 112,
+      "epoch": 0.0006710524739068436,
+      "loss/policy_avg": 0.08422186970710754,
+      "lr": 9.99616564417178e-06,
+      "objective/entropy": 75.0154037475586,
+      "objective/kl": 17.52770233154297,
+      "objective/non_score_reward": -0.8763852119445801,
+      "objective/rlhf_reward": -3.5055407360196114,
+      "objective/scores": 0.0,
+      "policy/approxkl_avg": 443.602294921875,
+      "policy/clipfrac_avg": 1.0,
+      "policy/entropy_avg": 0.505859375,
+      "step": 6,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 2,
+      "val/ratio": 1.9972598552703857
+    },
+    {
+      "episode": 128,
+      "epoch": 0.0007669171130363926,
+      "loss/policy_avg": 0.007405903190374374,
+      "lr": 9.995526584867077e-06,
+      "objective/entropy": 51.67172622680664,
+      "objective/kl": 11.563663482666016,
+      "objective/non_score_reward": -0.5781831741333008,
+      "objective/rlhf_reward": -4.312732696533203,
+      "objective/scores": -0.5,
+      "policy/approxkl_avg": 126.90079498291016,
+      "policy/clipfrac_avg": 1.25,
+      "policy/entropy_avg": 0.546875,
+      "step": 7,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 1,
+      "val/ratio": 2.0016322135925293
+    },
+    {
+      "episode": 144,
+      "epoch": 0.0008627817521659417,
+      "loss/policy_avg": 0.13771404325962067,
+      "lr": 9.994887525562374e-06,
+      "objective/entropy": 240.35464477539062,
+      "objective/kl": 18.096904754638672,
+      "objective/non_score_reward": -0.9048453569412231,
+      "objective/rlhf_reward": -1.6719702733325317,
+      "objective/scores": 0.4868528072345416,
+      "policy/approxkl_avg": 460.8926696777344,
+      "policy/clipfrac_avg": 1.0,
+      "policy/entropy_avg": 0.90625,
+      "step": 8,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 1,
+      "val/ratio": 1.9999654293060303
+    },
+    {
+      "episode": 160,
+      "epoch": 0.0009586463912954908,
+      "loss/policy_avg": 0.41069674491882324,
+      "lr": 9.99424846625767e-06,
+      "objective/entropy": 224.78262329101562,
+      "objective/kl": 11.231921195983887,
+      "objective/non_score_reward": -0.5615960955619812,
+      "objective/rlhf_reward": -0.8225522383051791,
+      "objective/scores": 0.35595802480981553,
+      "policy/approxkl_avg": 167.4181671142578,
+      "policy/clipfrac_avg": 1.0,
+      "policy/entropy_avg": 0.7578125,
+      "step": 9,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 2,
+      "val/ratio": 1.9952213764190674
+    },
+    {
+      "episode": 176,
+      "epoch": 0.0010545110304250398,
+      "loss/policy_avg": 0.2340843677520752,
+      "lr": 9.993609406952966e-06,
+      "objective/entropy": 77.48204040527344,
+      "objective/kl": 13.726895332336426,
+      "objective/non_score_reward": -0.6863448619842529,
+      "objective/rlhf_reward": -0.34537934362888345,
+      "objective/scores": 0.6,
+      "policy/approxkl_avg": 270.8516845703125,
+      "policy/clipfrac_avg": 1.25,
+      "policy/entropy_avg": 0.876953125,
+      "step": 10,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 6,
+      "val/ratio": 1.9955778121948242
+    },
+    {
+      "episode": 192,
+      "epoch": 0.001150375669554589,
+      "loss/policy_avg": 0.1845349222421646,
+      "lr": 9.992970347648263e-06,
+      "objective/entropy": -45.138362884521484,
+      "objective/kl": 14.76271915435791,
+      "objective/non_score_reward": -0.7381359338760376,
+      "objective/rlhf_reward": -1.2192103425661722,
+      "objective/scores": 0.43333333333333335,
+      "policy/approxkl_avg": 207.85874938964844,
+      "policy/clipfrac_avg": 0.75,
+      "policy/entropy_avg": 0.703125,
+      "step": 11,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 4,
+      "val/ratio": 2.000040292739868
+    },
+    {
+      "episode": 208,
+      "epoch": 0.001246240308684138,
+      "loss/policy_avg": 0.6059431433677673,
+      "lr": 9.992331288343558e-06,
+      "objective/entropy": 40.190372467041016,
+      "objective/kl": 19.720378875732422,
+      "objective/non_score_reward": -0.9860190749168396,
+      "objective/rlhf_reward": -1.821370030120883,
+      "objective/scores": 0.5306765580733931,
+      "policy/approxkl_avg": 268.6492919921875,
+      "policy/clipfrac_avg": 0.75,
+      "policy/entropy_avg": 0.654296875,
+      "step": 12,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 4,
+      "val/ratio": 1.9966726303100586
+    },
+    {
+      "episode": 224,
+      "epoch": 0.0013421049478136871,
+      "loss/policy_avg": -0.0064672790467739105,
+      "lr": 9.991692229038855e-06,
+      "objective/entropy": 108.48332214355469,
+      "objective/kl": 5.689068794250488,
+      "objective/non_score_reward": -0.28445348143577576,
+      "objective/rlhf_reward": -1.1378139406442642,
+      "objective/scores": 0.0,
+      "policy/approxkl_avg": 15.834894180297852,
+      "policy/clipfrac_avg": 1.75,
+      "policy/entropy_avg": 0.4716796875,
+      "step": 13,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 2,
+      "val/ratio": 2.0026183128356934
+    },
+    {
+      "episode": 240,
+      "epoch": 0.001437969586943236,
+      "loss/policy_avg": 0.6670212745666504,
+      "lr": 9.991053169734152e-06,
+      "objective/entropy": 0.18174362182617188,
+      "objective/kl": 12.982845306396484,
+      "objective/non_score_reward": -0.6491422653198242,
+      "objective/rlhf_reward": -2.596569076180458,
+      "objective/scores": 0.0,
+      "policy/approxkl_avg": 330.118896484375,
+      "policy/clipfrac_avg": 1.25,
+      "policy/entropy_avg": 0.759765625,
+      "step": 14,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 3,
+      "val/ratio": 1.9997687339782715
+    },
+    {
+      "episode": 256,
+      "epoch": 0.0015338342260727853,
+      "loss/policy_avg": 0.21263472735881805,
+      "lr": 9.990414110429449e-06,
+      "objective/entropy": 249.88232421875,
+      "objective/kl": 9.040252685546875,
+      "objective/non_score_reward": -0.45201271772384644,
+      "objective/rlhf_reward": -1.808050960302353,
+      "objective/scores": 0.0,
+      "policy/approxkl_avg": 102.57914733886719,
+      "policy/clipfrac_avg": 1.0,
+      "policy/entropy_avg": 0.744140625,
+      "step": 15,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 0,
+      "val/ratio": 2.000203847885132
+    },
+    {
+      "episode": 272,
+      "epoch": 0.0016296988652023342,
+      "loss/policy_avg": 0.01660698838531971,
+      "lr": 9.989775051124744e-06,
+      "objective/entropy": 106.64703369140625,
+      "objective/kl": 11.038640022277832,
+      "objective/non_score_reward": -0.5519319772720337,
+      "objective/rlhf_reward": -2.2077280431985855,
+      "objective/scores": 0.0,
+      "policy/approxkl_avg": 164.40890502929688,
+      "policy/clipfrac_avg": 1.5,
+      "policy/entropy_avg": 0.654296875,
+      "step": 16,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 2,
+      "val/ratio": 2.000194549560547
+    },
+    {
+      "episode": 288,
+      "epoch": 0.0017255635043318834,
+      "loss/policy_avg": 0.17964985966682434,
+      "lr": 9.989135991820041e-06,
+      "objective/entropy": 29.59412956237793,
+      "objective/kl": 11.429637908935547,
+      "objective/non_score_reward": -0.5714819431304932,
+      "objective/rlhf_reward": -2.2859277576208115,
+      "objective/scores": 0.0,
+      "policy/approxkl_avg": 113.22151184082031,
+      "policy/clipfrac_avg": 1.0,
+      "policy/entropy_avg": 0.642578125,
+      "step": 17,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 3,
+      "val/ratio": 1.9986356496810913
+    },
+    {
+      "episode": 304,
+      "epoch": 0.0018214281434614326,
+      "loss/policy_avg": 0.1845007836818695,
+      "lr": 9.988496932515338e-06,
+      "objective/entropy": -2.3180160522460938,
+      "objective/kl": 15.66268539428711,
+      "objective/non_score_reward": -0.7831343412399292,
+      "objective/rlhf_reward": -1.773287498687191,
+      "objective/scores": 0.33981246656813147,
+      "policy/approxkl_avg": 162.00823974609375,
+      "policy/clipfrac_avg": 1.0,
+      "policy/entropy_avg": 0.8046875,
+      "step": 18,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 3,
+      "val/ratio": 2.0002946853637695
+    },
+    {
+      "episode": 320,
+      "epoch": 0.0019172927825909815,
+      "loss/policy_avg": 0.14623276889324188,
+      "lr": 9.987857873210635e-06,
+      "objective/entropy": 26.79373550415039,
+      "objective/kl": 16.199674606323242,
+      "objective/non_score_reward": -0.8099837303161621,
+      "objective/rlhf_reward": -3.2399348318576813,
+      "objective/scores": 0.0,
+      "policy/approxkl_avg": 345.71685791015625,
+      "policy/clipfrac_avg": 1.0,
+      "policy/entropy_avg": 0.7109375,
+      "step": 19,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 2,
+      "val/ratio": 1.9992010593414307
+    },
+    {
+      "episode": 336,
+      "epoch": 0.0020131574217205307,
+      "loss/policy_avg": 0.061316944658756256,
+      "lr": 9.987218813905932e-06,
+      "objective/entropy": 30.27604866027832,
+      "objective/kl": 13.349930763244629,
+      "objective/non_score_reward": -0.6674965620040894,
+      "objective/rlhf_reward": -4.669986248016357,
+      "objective/scores": -0.5,
+      "policy/approxkl_avg": 182.816650390625,
+      "policy/clipfrac_avg": 1.0,
+      "policy/entropy_avg": 0.56640625,
+      "step": 20,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 3,
+      "val/ratio": 1.996535301208496
+    },
+    {
+      "episode": 352,
+      "epoch": 0.0021090220608500796,
+      "loss/policy_avg": -0.08272892981767654,
+      "lr": 9.986579754601228e-06,
+      "objective/entropy": 198.63003540039062,
+      "objective/kl": 11.5382719039917,
+      "objective/non_score_reward": -0.5769136548042297,
+      "objective/rlhf_reward": -0.9076545149087907,
+      "objective/scores": 0.35,
+      "policy/approxkl_avg": 159.97686767578125,
+      "policy/clipfrac_avg": 1.5,
+      "policy/entropy_avg": 0.974609375,
+      "step": 21,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 2,
+      "val/ratio": 1.998477578163147
+    },
+    {
+      "episode": 368,
+      "epoch": 0.0022048866999796286,
+      "loss/policy_avg": 0.23813551664352417,
+      "lr": 9.985940695296524e-06,
+      "objective/entropy": 181.51829528808594,
+      "objective/kl": 13.08276653289795,
+      "objective/non_score_reward": -0.6541383266448975,
+      "objective/rlhf_reward": -4.61655330657959,
+      "objective/scores": -0.5,
+      "policy/approxkl_avg": 108.47281646728516,
+      "policy/clipfrac_avg": 1.25,
+      "policy/entropy_avg": 0.6484375,
+      "step": 22,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 1,
+      "val/ratio": 1.9991295337677002
+    },
+    {
+      "episode": 384,
+      "epoch": 0.002300751339109178,
+      "loss/policy_avg": 0.36420387029647827,
+      "lr": 9.98530163599182e-06,
+      "objective/entropy": 257.93609619140625,
+      "objective/kl": 14.696407318115234,
+      "objective/non_score_reward": -0.7348203063011169,
+      "objective/rlhf_reward": -1.3351611531415755,
+      "objective/scores": 0.40102999566398123,
+      "policy/approxkl_avg": 150.4597625732422,
+      "policy/clipfrac_avg": 1.25,
+      "policy/entropy_avg": 0.849609375,
+      "step": 23,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 1,
+      "val/ratio": 1.9980016946792603
+    },
+    {
+      "episode": 400,
+      "epoch": 0.002396615978238727,
+      "loss/policy_avg": 0.20780539512634277,
+      "lr": 9.984662576687117e-06,
+      "objective/entropy": -139.27951049804688,
+      "objective/kl": 15.462644577026367,
+      "objective/non_score_reward": -0.77313232421875,
+      "objective/rlhf_reward": -5.092529296875,
+      "objective/scores": -0.5,
+      "policy/approxkl_avg": 237.78317260742188,
+      "policy/clipfrac_avg": 1.0,
+      "policy/entropy_avg": 0.6484375,
+      "step": 24,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 7,
+      "val/ratio": 1.999894618988037
+    },
+    {
+      "episode": 416,
+      "epoch": 0.002492480617368276,
+      "loss/policy_avg": 0.2547074556350708,
+      "lr": 9.984023517382414e-06,
+      "objective/entropy": 103.24639892578125,
+      "objective/kl": 17.307334899902344,
+      "objective/non_score_reward": -0.8653668165206909,
+      "objective/rlhf_reward": -5.461467266082764,
+      "objective/scores": -0.5,
+      "policy/approxkl_avg": 167.2418212890625,
+      "policy/clipfrac_avg": 1.25,
+      "policy/entropy_avg": 0.65625,
+      "step": 25,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 4,
+      "val/ratio": 2.000932216644287
+    },
+    {
+      "episode": 432,
+      "epoch": 0.002588345256497825,
+      "loss/policy_avg": 0.3379603624343872,
+      "lr": 9.983384458077711e-06,
+      "objective/entropy": 120.86388397216797,
+      "objective/kl": 14.275808334350586,
+      "objective/non_score_reward": -0.7137903571128845,
+      "objective/rlhf_reward": -2.8551614582538605,
+      "objective/scores": 0.0,
+      "policy/approxkl_avg": 99.34181213378906,
+      "policy/clipfrac_avg": 0.75,
+      "policy/entropy_avg": 0.73828125,
+      "step": 26,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 2,
+      "val/ratio": 1.9994611740112305
+    },
+    {
+      "episode": 448,
+      "epoch": 0.0026842098956273742,
+      "loss/policy_avg": 0.1328231394290924,
+      "lr": 9.982745398773006e-06,
+      "objective/entropy": 154.86619567871094,
+      "objective/kl": 14.35202693939209,
+      "objective/non_score_reward": -0.7176014184951782,
+      "objective/rlhf_reward": -4.870405673980713,
+      "objective/scores": -0.5,
+      "policy/approxkl_avg": 91.33482360839844,
+      "policy/clipfrac_avg": 1.0,
+      "policy/entropy_avg": 0.724609375,
+      "step": 27,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 2,
+      "val/ratio": 1.9957685470581055
+    },
+    {
+      "episode": 464,
+      "epoch": 0.002780074534756923,
+      "loss/policy_avg": 0.2314174473285675,
+      "lr": 9.982106339468303e-06,
+      "objective/entropy": 60.19127655029297,
+      "objective/kl": 12.188166618347168,
+      "objective/non_score_reward": -0.6094082593917847,
+      "objective/rlhf_reward": -2.4376331865787506,
+      "objective/scores": 0.0,
+      "policy/approxkl_avg": 205.1094970703125,
+      "policy/clipfrac_avg": 1.25,
+      "policy/entropy_avg": 0.810546875,
+      "step": 28,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 2,
+      "val/ratio": 1.994727611541748
+    },
+    {
+      "episode": 480,
+      "epoch": 0.002875939173886472,
+      "loss/policy_avg": 0.1261996328830719,
+      "lr": 9.9814672801636e-06,
+      "objective/entropy": 67.08200073242188,
+      "objective/kl": 16.607372283935547,
+      "objective/non_score_reward": -0.8303685784339905,
+      "objective/rlhf_reward": -3.3214742839336395,
+      "objective/scores": 0.0,
+      "policy/approxkl_avg": 226.6929168701172,
+      "policy/clipfrac_avg": 0.75,
+      "policy/entropy_avg": 0.53125,
+      "step": 29,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 2,
+      "val/ratio": 1.9976527690887451
+    },
+    {
+      "episode": 496,
+      "epoch": 0.0029718038130160216,
+      "loss/policy_avg": 0.35239556431770325,
+      "lr": 9.980828220858897e-06,
+      "objective/entropy": 204.22116088867188,
+      "objective/kl": 14.144770622253418,
+      "objective/non_score_reward": -0.7072385549545288,
+      "objective/rlhf_reward": 1.5710457801818851,
+      "objective/scores": 1.1,
+      "policy/approxkl_avg": 124.6588363647461,
+      "policy/clipfrac_avg": 1.25,
+      "policy/entropy_avg": 0.7578125,
+      "step": 30,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 2,
+      "val/ratio": 1.9977011680603027
+    },
+    {
+      "episode": 512,
+      "epoch": 0.0030676684521455705,
+      "loss/policy_avg": 0.26766547560691833,
+      "lr": 9.980189161554194e-06,
+      "objective/entropy": 107.69725036621094,
+      "objective/kl": 12.877479553222656,
+      "objective/non_score_reward": -0.643873929977417,
+      "objective/rlhf_reward": -2.5754958018660545,
+      "objective/scores": 0.0,
+      "policy/approxkl_avg": 186.40504455566406,
+      "policy/clipfrac_avg": 1.0,
+      "policy/entropy_avg": 0.701171875,
+      "step": 31,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 1,
+      "val/ratio": 1.9972684383392334
+    },
+    {
+      "episode": 528,
+      "epoch": 0.0031635330912751195,
+      "loss/policy_avg": 0.017455143854022026,
+      "lr": 9.97955010224949e-06,
+      "objective/entropy": 69.81261444091797,
+      "objective/kl": 15.58060073852539,
+      "objective/non_score_reward": -0.7790300250053406,
+      "objective/rlhf_reward": -0.19240116024133824,
+      "objective/scores": 0.7309297535714575,
+      "policy/approxkl_avg": 326.45733642578125,
+      "policy/clipfrac_avg": 1.0,
+      "policy/entropy_avg": 0.671875,
+      "step": 32,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 2,
+      "val/ratio": 1.9994511604309082
+    },
+    {
+      "episode": 544,
+      "epoch": 0.0032593977304046684,
+      "loss/policy_avg": 0.1626880019903183,
+      "lr": 9.978911042944786e-06,
+      "objective/entropy": 49.244285583496094,
+      "objective/kl": 11.25068473815918,
+      "objective/non_score_reward": -0.5625342130661011,
+      "objective/rlhf_reward": -0.871534817901951,
+      "objective/scores": 0.34465054211822604,
+      "policy/approxkl_avg": 50.66204071044922,
+      "policy/clipfrac_avg": 1.0,
+      "policy/entropy_avg": 0.7109375,
+      "step": 33,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 3,
+      "val/ratio": 1.9966607093811035
+    },
+    {
+      "episode": 560,
+      "epoch": 0.003355262369534218,
+      "loss/policy_avg": 0.18031546473503113,
+      "lr": 9.978271983640083e-06,
+      "objective/entropy": 147.95474243164062,
+      "objective/kl": 15.950370788574219,
+      "objective/non_score_reward": -0.7975186109542847,
+      "objective/rlhf_reward": -3.190074533224106,
+      "objective/scores": 0.0,
+      "policy/approxkl_avg": 201.51080322265625,
+      "policy/clipfrac_avg": 1.5,
+      "policy/entropy_avg": 0.515625,
+      "step": 34,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 0,
+      "val/ratio": 2.0006275177001953
+    },
+    {
+      "episode": 576,
+      "epoch": 0.0034511270086637668,
+      "loss/policy_avg": -0.05689749866724014,
+      "lr": 9.977632924335378e-06,
+      "objective/entropy": 4.240108489990234,
+      "objective/kl": 13.38272762298584,
+      "objective/non_score_reward": -0.6691364049911499,
+      "objective/rlhf_reward": -4.6765456199646,
+      "objective/scores": -0.5,
+      "policy/approxkl_avg": 247.30409240722656,
+      "policy/clipfrac_avg": 1.5,
+      "policy/entropy_avg": 0.796875,
+      "step": 35,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 3,
+      "val/ratio": 1.9973469972610474
+    },
+    {
+      "episode": 592,
+      "epoch": 0.0035469916477933157,
+      "loss/policy_avg": 0.16461774706840515,
+      "lr": 9.976993865030675e-06,
+      "objective/entropy": 123.00151824951172,
+      "objective/kl": 11.21810531616211,
+      "objective/non_score_reward": -0.5609052181243896,
+      "objective/rlhf_reward": -4.243620872497559,
+      "objective/scores": -0.5,
+      "policy/approxkl_avg": 64.79019927978516,
+      "policy/clipfrac_avg": 1.25,
+      "policy/entropy_avg": 0.712890625,
+      "step": 36,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 2,
+      "val/ratio": 2.0026960372924805
+    },
+    {
+      "episode": 608,
+      "epoch": 0.003642856286922865,
+      "loss/policy_avg": 0.15750062465667725,
+      "lr": 9.976354805725972e-06,
+      "objective/entropy": 30.60162925720215,
+      "objective/kl": 17.013538360595703,
+      "objective/non_score_reward": -0.850676953792572,
+      "objective/rlhf_reward": -3.4027078449726105,
+      "objective/scores": 0.0,
+      "policy/approxkl_avg": 234.95870971679688,
+      "policy/clipfrac_avg": 0.75,
+      "policy/entropy_avg": 0.7421875,
+      "step": 37,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 4,
+      "val/ratio": 1.998824119567871
+    },
+    {
+      "episode": 624,
+      "epoch": 0.003738720926052414,
+      "loss/policy_avg": 0.03158241882920265,
+      "lr": 9.975715746421269e-06,
+      "objective/entropy": 115.71566772460938,
+      "objective/kl": 15.653677940368652,
+      "objective/non_score_reward": -0.7826838493347168,
+      "objective/rlhf_reward": -3.1307354420423508,
+      "objective/scores": 0.0,
+      "policy/approxkl_avg": 233.744873046875,
+      "policy/clipfrac_avg": 0.75,
+      "policy/entropy_avg": 0.765625,
+      "step": 38,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 2,
+      "val/ratio": 1.997577428817749
+    },
+    {
+      "episode": 640,
+      "epoch": 0.003834585565181963,
+      "loss/policy_avg": -0.031586866825819016,
+      "lr": 9.975076687116566e-06,
+      "objective/entropy": 70.69473266601562,
+      "objective/kl": 13.526529312133789,
+      "objective/non_score_reward": -0.6763265132904053,
+      "objective/rlhf_reward": 1.6946939915418628,
+      "objective/scores": 1.1,
+      "policy/approxkl_avg": 101.47872924804688,
+      "policy/clipfrac_avg": 1.25,
+      "policy/entropy_avg": 0.677734375,
+      "step": 39,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 2,
+      "val/ratio": 2.0043745040893555
+    },
+    {
+      "episode": 656,
+      "epoch": 0.003930450204311512,
+      "loss/policy_avg": 0.12032957375049591,
+      "lr": 9.97443762781186e-06,
+      "objective/entropy": 172.61669921875,
+      "objective/kl": 16.604652404785156,
+      "objective/non_score_reward": -0.8302326798439026,
+      "objective/rlhf_reward": -5.320930480957031,
+      "objective/scores": -0.5,
+      "policy/approxkl_avg": 220.98178100585938,
+      "policy/clipfrac_avg": 1.0,
+      "policy/entropy_avg": 0.642578125,
+      "step": 40,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 1,
+      "val/ratio": 1.9957454204559326
+    },
+    {
+      "episode": 672,
+      "epoch": 0.004026314843441061,
+      "loss/policy_avg": 0.0876859575510025,
+      "lr": 9.973798568507158e-06,
+      "objective/entropy": 12.816411972045898,
+      "objective/kl": 13.908916473388672,
+      "objective/non_score_reward": -0.6954457759857178,
+      "objective/rlhf_reward": -4.781783103942871,
+      "objective/scores": -0.5,
+      "policy/approxkl_avg": 74.01300048828125,
+      "policy/clipfrac_avg": 1.25,
+      "policy/entropy_avg": 0.640625,
+      "step": 41,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 3,
+      "val/ratio": 2.0018563270568848
+    },
+    {
+      "episode": 688,
+      "epoch": 0.00412217948257061,
+      "loss/policy_avg": 0.1008758619427681,
+      "lr": 9.973159509202454e-06,
+      "objective/entropy": 257.1292724609375,
+      "objective/kl": 11.528783798217773,
+      "objective/non_score_reward": -0.5764391422271729,
+      "objective/rlhf_reward": -2.3057566583156586,
+      "objective/scores": 0.0,
+      "policy/approxkl_avg": 84.59580993652344,
+      "policy/clipfrac_avg": 0.75,
+      "policy/entropy_avg": 0.75390625,
+      "step": 42,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 0,
+      "val/ratio": 1.9929132461547852
+    },
+    {
+      "episode": 704,
+      "epoch": 0.004218044121700159,
+      "loss/policy_avg": 0.44639891386032104,
+      "lr": 9.972520449897751e-06,
+      "objective/entropy": 62.445350646972656,
+      "objective/kl": 13.397602081298828,
+      "objective/non_score_reward": -0.6698801517486572,
+      "objective/rlhf_reward": -4.679520606994629,
+      "objective/scores": -0.5,
+      "policy/approxkl_avg": 185.67079162597656,
+      "policy/clipfrac_avg": 1.0,
+      "policy/entropy_avg": 0.533203125,
+      "step": 43,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 2,
+      "val/ratio": 1.9992148876190186
+    },
+    {
+      "episode": 720,
+      "epoch": 0.004313908760829708,
+      "loss/policy_avg": 0.09568839520215988,
+      "lr": 9.971881390593048e-06,
+      "objective/entropy": 129.84619140625,
+      "objective/kl": 14.350381851196289,
+      "objective/non_score_reward": -0.7175191640853882,
+      "objective/rlhf_reward": -0.9226653081940968,
+      "objective/scores": 0.4868528072345416,
+      "policy/approxkl_avg": 253.10037231445312,
+      "policy/clipfrac_avg": 1.0,
+      "policy/entropy_avg": 0.7421875,
+      "step": 44,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 1,
+      "val/ratio": 1.9957365989685059
+    },
+    {
+      "episode": 736,
+      "epoch": 0.004409773399959257,
+      "loss/policy_avg": 0.22084593772888184,
+      "lr": 9.971242331288345e-06,
+      "objective/entropy": -42.89992904663086,
+      "objective/kl": 15.893115997314453,
+      "objective/non_score_reward": -0.7946557998657227,
+      "objective/rlhf_reward": -1.44528977672259,
+      "objective/scores": 0.43333333333333335,
+      "policy/approxkl_avg": 121.7098388671875,
+      "policy/clipfrac_avg": 1.0,
+      "policy/entropy_avg": 0.751953125,
+      "step": 45,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 3,
+      "val/ratio": 2.000582218170166
+    },
+    {
+      "episode": 752,
+      "epoch": 0.004505638039088807,
+      "loss/policy_avg": 0.22875869274139404,
+      "lr": 9.97060327198364e-06,
+      "objective/entropy": 9.025165557861328,
+      "objective/kl": 22.01996612548828,
+      "objective/non_score_reward": -1.1009982824325562,
+      "objective/rlhf_reward": -1.48027405583975,
+      "objective/scores": 0.7309297535714575,
+      "policy/approxkl_avg": 321.845703125,
+      "policy/clipfrac_avg": 1.0,
+      "policy/entropy_avg": 0.5859375,
+      "step": 46,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 3,
+      "val/ratio": 1.9953069686889648
+    },
+    {
+      "episode": 768,
+      "epoch": 0.004601502678218356,
+      "loss/policy_avg": 0.07661572843790054,
+      "lr": 9.969964212678937e-06,
+      "objective/entropy": 10.382087707519531,
+      "objective/kl": 13.358439445495605,
+      "objective/non_score_reward": -0.6679220199584961,
+      "objective/rlhf_reward": 1.7283119499683384,
+      "objective/scores": 1.1,
+      "policy/approxkl_avg": 29.50304412841797,
+      "policy/clipfrac_avg": 1.25,
+      "policy/entropy_avg": 0.638671875,
+      "step": 47,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 4,
+      "val/ratio": 1.999579668045044
+    },
+    {
+      "episode": 784,
+      "epoch": 0.004697367317347905,
+      "loss/policy_avg": 0.19636262953281403,
+      "lr": 9.969325153374234e-06,
+      "objective/entropy": 0.6832618713378906,
+      "objective/kl": 8.836541175842285,
+      "objective/non_score_reward": -0.4418269991874695,
+      "objective/rlhf_reward": -1.767308071255684,
+      "objective/scores": 0.0,
+      "policy/approxkl_avg": 94.3209228515625,
+      "policy/clipfrac_avg": 1.0,
+      "policy/entropy_avg": 0.712890625,
+      "step": 48,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 3,
+      "val/ratio": 1.998420000076294
+    },
+    {
+      "episode": 800,
+      "epoch": 0.004793231956477454,
+      "loss/policy_avg": -0.02652953751385212,
+      "lr": 9.968686094069531e-06,
+      "objective/entropy": 125.6042709350586,
+      "objective/kl": 15.016199111938477,
+      "objective/non_score_reward": -0.7508100271224976,
+      "objective/rlhf_reward": -5.00324010848999,
+      "objective/scores": -0.5,
+      "policy/approxkl_avg": 207.12213134765625,
+      "policy/clipfrac_avg": 1.5,
+      "policy/entropy_avg": 0.7421875,
+      "step": 49,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 2,
+      "val/ratio": 2.002742052078247
+    },
+    {
+      "episode": 816,
+      "epoch": 0.004889096595607003,
+      "loss/policy_avg": 0.10039197653532028,
+      "lr": 9.968047034764828e-06,
+      "objective/entropy": -24.506595611572266,
+      "objective/kl": 18.726213455200195,
+      "objective/non_score_reward": -0.9363107085227966,
+      "objective/rlhf_reward": -5.745243072509766,
+      "objective/scores": -0.5,
+      "policy/approxkl_avg": 300.677490234375,
+      "policy/clipfrac_avg": 0.75,
+      "policy/entropy_avg": 0.6953125,
+      "step": 50,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 5,
+      "val/ratio": 1.9997317790985107
+    },
+    {
+      "episode": 832,
+      "epoch": 0.004984961234736552,
+      "loss/policy_avg": 0.18666991591453552,
+      "lr": 9.967407975460123e-06,
+      "objective/entropy": -47.62429428100586,
+      "objective/kl": 13.258740425109863,
+      "objective/non_score_reward": -0.6629370450973511,
+      "objective/rlhf_reward": -1.0954889049201753,
+      "objective/scores": 0.38906482631788786,
+      "policy/approxkl_avg": 110.42059326171875,
+      "policy/clipfrac_avg": 1.0,
+      "policy/entropy_avg": 0.685546875,
+      "step": 51,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 5,
+      "val/ratio": 1.9965816736221313
+    },
+    {
+      "episode": 848,
+      "epoch": 0.005080825873866101,
+      "loss/policy_avg": 0.195734903216362,
+      "lr": 9.96676891615542e-06,
+      "objective/entropy": 57.396114349365234,
+      "objective/kl": 15.980720520019531,
+      "objective/non_score_reward": -0.7990360856056213,
+      "objective/rlhf_reward": -5.196144104003906,
+      "objective/scores": -0.5,
+      "policy/approxkl_avg": 125.00595092773438,
+      "policy/clipfrac_avg": 1.25,
+      "policy/entropy_avg": 0.775390625,
+      "step": 52,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 4,
+      "val/ratio": 1.995786428451538
+    },
+    {
+      "episode": 864,
+      "epoch": 0.00517669051299565,
+      "loss/policy_avg": 0.24296848475933075,
+      "lr": 9.966129856850717e-06,
+      "objective/entropy": 99.57502746582031,
+      "objective/kl": 17.62392807006836,
+      "objective/non_score_reward": -0.8811964988708496,
+      "objective/rlhf_reward": -2.0090143916928134,
+      "objective/scores": 0.37894294565112985,
+      "policy/approxkl_avg": 99.32807922363281,
+      "policy/clipfrac_avg": 1.0,
+      "policy/entropy_avg": 0.662109375,
+      "step": 53,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 4,
+      "val/ratio": 2.0016443729400635
+    },
+    {
+      "episode": 880,
+      "epoch": 0.0052725551521251995,
+      "loss/policy_avg": 0.5130509734153748,
+      "lr": 9.965490797546014e-06,
+      "objective/entropy": 34.0892219543457,
+      "objective/kl": 14.999124526977539,
+      "objective/non_score_reward": -0.7499562501907349,
+      "objective/rlhf_reward": -1.4840530840479695,
+      "objective/scores": 0.37894294565112985,
+      "policy/approxkl_avg": 130.593017578125,
+      "policy/clipfrac_avg": 0.75,
+      "policy/entropy_avg": 0.75390625,
+      "step": 54,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 4,
+      "val/ratio": 2.000711679458618
+    },
+    {
+      "episode": 896,
+      "epoch": 0.0053684197912547485,
+      "loss/policy_avg": 0.3232521116733551,
+      "lr": 9.96485173824131e-06,
+      "objective/entropy": 69.26298522949219,
+      "objective/kl": 21.724315643310547,
+      "objective/non_score_reward": -1.086215853691101,
+      "objective/rlhf_reward": -2.788603871074274,
+      "objective/scores": 0.38906482631788786,
+      "policy/approxkl_avg": 234.98104858398438,
+      "policy/clipfrac_avg": 1.0,
+      "policy/entropy_avg": 0.68359375,
+      "step": 55,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 3,
+      "val/ratio": 1.9952917098999023
+    },
+    {
+      "episode": 912,
+      "epoch": 0.0054642844303842975,
+      "loss/policy_avg": 0.10791392624378204,
+      "lr": 9.964212678936606e-06,
+      "objective/entropy": 32.22584533691406,
+      "objective/kl": 15.846414566040039,
+      "objective/non_score_reward": -0.7923207879066467,
+      "objective/rlhf_reward": 1.230716893076897,
+      "objective/scores": 1.1,
+      "policy/approxkl_avg": 256.9724426269531,
+      "policy/clipfrac_avg": 0.75,
+      "policy/entropy_avg": 0.4736328125,
+      "step": 56,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 1,
+      "val/ratio": 1.9984806776046753
+    },
+    {
+      "episode": 928,
+      "epoch": 0.005560149069513846,
+      "loss/policy_avg": 0.37206730246543884,
+      "lr": 9.963573619631903e-06,
+      "objective/entropy": -6.195688247680664,
+      "objective/kl": 12.801559448242188,
+      "objective/non_score_reward": -0.6400780081748962,
+      "objective/rlhf_reward": -2.5603120028972626,
+      "objective/scores": 0.0,
+      "policy/approxkl_avg": 40.83631896972656,
+      "policy/clipfrac_avg": 1.0,
+      "policy/entropy_avg": 0.744140625,
+      "step": 57,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 3,
+      "val/ratio": 2.0007452964782715
+    },
+    {
+      "episode": 944,
+      "epoch": 0.005656013708643395,
+      "loss/policy_avg": 0.05091024935245514,
+      "lr": 9.9629345603272e-06,
+      "objective/entropy": -18.476280212402344,
+      "objective/kl": 18.95052719116211,
+      "objective/non_score_reward": -0.9475262761116028,
+      "objective/rlhf_reward": -3.790105164051056,
+      "objective/scores": 0.0,
+      "policy/approxkl_avg": 144.4001007080078,
+      "policy/clipfrac_avg": 1.0,
+      "policy/entropy_avg": 0.580078125,
+      "step": 58,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 2,
+      "val/ratio": 1.9988869428634644
+    },
+    {
+      "episode": 960,
+      "epoch": 0.005751878347772944,
+      "loss/policy_avg": -0.008312445133924484,
+      "lr": 9.962295501022495e-06,
+      "objective/entropy": 217.08169555664062,
+      "objective/kl": 14.908738136291504,
+      "objective/non_score_reward": -0.7454369068145752,
+      "objective/rlhf_reward": -2.9817477762699127,
+      "objective/scores": 0.0,
+      "policy/approxkl_avg": 93.395751953125,
+      "policy/clipfrac_avg": 1.0,
+      "policy/entropy_avg": 0.77734375,
+      "step": 59,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 1,
+      "val/ratio": 1.9973504543304443
+    },
+    {
+      "episode": 976,
+      "epoch": 0.005847742986902493,
+      "loss/policy_avg": 0.03407389298081398,
+      "lr": 9.961656441717792e-06,
+      "objective/entropy": 140.58189392089844,
+      "objective/kl": 20.377920150756836,
+      "objective/non_score_reward": -1.0188961029052734,
+      "objective/rlhf_reward": -6.075584411621094,
+      "objective/scores": -0.5,
+      "policy/approxkl_avg": 66.34793090820312,
+      "policy/clipfrac_avg": 1.25,
+      "policy/entropy_avg": 0.673828125,
+      "step": 60,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 2,
+      "val/ratio": 1.9996695518493652
+    },
+    {
+      "episode": 992,
+      "epoch": 0.005943607626032043,
+      "loss/policy_avg": 0.08645053207874298,
+      "lr": 9.961017382413088e-06,
+      "objective/entropy": 19.108230590820312,
+      "objective/kl": 13.644828796386719,
+      "objective/non_score_reward": -0.6822414994239807,
+      "objective/rlhf_reward": -2.7289658784866333,
+      "objective/scores": 0.0,
+      "policy/approxkl_avg": 35.71690368652344,
+      "policy/clipfrac_avg": 1.0,
+      "policy/entropy_avg": 0.564453125,
+      "step": 61,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 2,
+      "val/ratio": 1.9962527751922607
+    },
+    {
+      "episode": 1008,
+      "epoch": 0.006039472265161592,
+      "loss/policy_avg": -0.17965860664844513,
+      "lr": 9.960378323108385e-06,
+      "objective/entropy": 168.96075439453125,
+      "objective/kl": 11.691057205200195,
+      "objective/non_score_reward": -0.5845528841018677,
+      "objective/rlhf_reward": -4.338212013244629,
+      "objective/scores": -0.5,
+      "policy/approxkl_avg": 65.8020248413086,
+      "policy/clipfrac_avg": 1.25,
+      "policy/entropy_avg": 0.775390625,
+      "step": 62,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 1,
+      "val/ratio": 2.0171313285827637
+    },
+    {
+      "episode": 1024,
+      "epoch": 0.006135336904291141,
+      "loss/policy_avg": 0.061459362506866455,
+      "lr": 9.959739263803682e-06,
+      "objective/entropy": 117.6607437133789,
+      "objective/kl": 15.35727310180664,
+      "objective/non_score_reward": -0.7678636312484741,
+      "objective/rlhf_reward": -3.071454644203186,
+      "objective/scores": 0.0,
+      "policy/approxkl_avg": 187.7376708984375,
+      "policy/clipfrac_avg": 1.0,
+      "policy/entropy_avg": 0.9375,
+      "step": 63,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 2,
+      "val/ratio": 1.9990172386169434
+    },
+    {
+      "episode": 1040,
+      "epoch": 0.00623120154342069,
+      "loss/policy_avg": 0.07200516015291214,
+      "lr": 9.959100204498979e-06,
+      "objective/entropy": 20.476089477539062,
+      "objective/kl": 13.475000381469727,
+      "objective/non_score_reward": -0.6737500429153442,
+      "objective/rlhf_reward": 1.7049996197223667,
+      "objective/scores": 1.1,
+      "policy/approxkl_avg": 68.92333984375,
+      "policy/clipfrac_avg": 1.25,
+      "policy/entropy_avg": 0.6796875,
+      "step": 64,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 6,
+      "val/ratio": 1.9985809326171875
+    },
+    {
+      "episode": 1056,
+      "epoch": 0.006327066182550239,
+      "loss/policy_avg": 0.08365275710821152,
+      "lr": 9.958461145194274e-06,
+      "objective/entropy": -127.62371826171875,
+      "objective/kl": 22.050678253173828,
+      "objective/non_score_reward": -1.1025339365005493,
+      "objective/rlhf_reward": -4.410135626792908,
+      "objective/scores": 0.0,
+      "policy/approxkl_avg": 301.7841491699219,
+      "policy/clipfrac_avg": 0.5,
+      "policy/entropy_avg": 0.720703125,
+      "step": 65,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 5,
+      "val/ratio": 1.9990687370300293
+    },
+    {
+      "episode": 1072,
+      "epoch": 0.006422930821679788,
+      "loss/policy_avg": 0.040758199989795685,
+      "lr": 9.957822085889571e-06,
+      "objective/entropy": 70.77458190917969,
+      "objective/kl": 18.2130069732666,
+      "objective/non_score_reward": -0.9106502532958984,
+      "objective/rlhf_reward": -1.9807416550522907,
+      "objective/scores": 0.41546487678572874,
+      "policy/approxkl_avg": 190.14797973632812,
+      "policy/clipfrac_avg": 1.25,
+      "policy/entropy_avg": 0.439453125,
+      "step": 66,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 2,
+      "val/ratio": 1.9984623193740845
+    },
+    {
+      "episode": 1088,
+      "epoch": 0.006518795460809337,
+      "loss/policy_avg": 0.05295582860708237,
+      "lr": 9.957183026584868e-06,
+      "objective/entropy": 97.35667419433594,
+      "objective/kl": 24.64842987060547,
+      "objective/non_score_reward": -1.2324215173721313,
+      "objective/rlhf_reward": -4.929685860872269,
+      "objective/scores": 0.0,
+      "policy/approxkl_avg": 394.2406921386719,
+      "policy/clipfrac_avg": 0.75,
+      "policy/entropy_avg": 0.67578125,
+      "step": 67,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 1,
+      "val/ratio": 2.0007405281066895
+    },
+    {
+      "episode": 1104,
+      "epoch": 0.006614660099938887,
+      "loss/policy_avg": 0.14266067743301392,
+      "lr": 9.956543967280165e-06,
+      "objective/entropy": 85.57185363769531,
+      "objective/kl": 14.76464557647705,
+      "objective/non_score_reward": -0.7382322549819946,
+      "objective/rlhf_reward": -0.029210095049115647,
+      "objective/scores": 0.7309297535714575,
+      "policy/approxkl_avg": 171.19406127929688,
+      "policy/clipfrac_avg": 1.0,
+      "policy/entropy_avg": 0.6015625,
+      "step": 68,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 4,
+      "val/ratio": 1.9964725971221924
+    },
+    {
+      "episode": 1120,
+      "epoch": 0.006710524739068436,
+      "loss/policy_avg": 0.11469551920890808,
+      "lr": 9.955904907975462e-06,
+      "objective/entropy": 21.974023818969727,
+      "objective/kl": 16.630640029907227,
+      "objective/non_score_reward": -0.8315319418907166,
+      "objective/rlhf_reward": -3.326127827167511,
+      "objective/scores": 0.0,
+      "policy/approxkl_avg": 63.985679626464844,
+      "policy/clipfrac_avg": 1.25,
+      "policy/entropy_avg": 0.72265625,
+      "step": 69,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 4,
+      "val/ratio": 1.999800682067871
+    },
+    {
+      "episode": 1136,
+      "epoch": 0.006806389378197985,
+      "loss/policy_avg": 0.10287429392337799,
+      "lr": 9.955265848670757e-06,
+      "objective/entropy": 43.38239288330078,
+      "objective/kl": 22.02418327331543,
+      "objective/non_score_reward": -1.101209282875061,
+      "objective/rlhf_reward": -4.404837071895599,
+      "objective/scores": 0.0,
+      "policy/approxkl_avg": 135.89984130859375,
+      "policy/clipfrac_avg": 0.75,
+      "policy/entropy_avg": 0.69140625,
+      "step": 70,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 3,
+      "val/ratio": 1.9973747730255127
+    },
+    {
+      "episode": 1152,
+      "epoch": 0.0069022540173275335,
+      "loss/policy_avg": 2.0731570720672607,
+      "lr": 9.954626789366054e-06,
+      "objective/entropy": -7.300925254821777,
+      "objective/kl": 17.635089874267578,
+      "objective/non_score_reward": -0.8817545175552368,
+      "objective/rlhf_reward": -3.5270181000232697,
+      "objective/scores": 0.0,
+      "policy/approxkl_avg": 183.6417236328125,
+      "policy/clipfrac_avg": 0.75,
+      "policy/entropy_avg": 0.65625,
+      "step": 71,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 5,
+      "val/ratio": 1.9976426362991333
+    },
+    {
+      "episode": 1168,
+      "epoch": 0.0069981186564570825,
+      "loss/policy_avg": 0.24466943740844727,
+      "lr": 9.95398773006135e-06,
+      "objective/entropy": -26.054595947265625,
+      "objective/kl": 15.905699729919434,
+      "objective/non_score_reward": -0.7952849864959717,
+      "objective/rlhf_reward": -5.181139945983887,
+      "objective/scores": -0.5,
+      "policy/approxkl_avg": 46.05584716796875,
+      "policy/clipfrac_avg": 1.0,
+      "policy/entropy_avg": 0.625,
+      "step": 72,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 3,
+      "val/ratio": 1.9955050945281982
+    },
+    {
+      "episode": 1184,
+      "epoch": 0.0070939832955866314,
+      "loss/policy_avg": 0.4031391739845276,
+      "lr": 9.953348670756648e-06,
+      "objective/entropy": -107.55976867675781,
+      "objective/kl": 19.68102264404297,
+      "objective/non_score_reward": -0.9840512275695801,
+      "objective/rlhf_reward": -3.936204746365547,
+      "objective/scores": 0.0,
+      "policy/approxkl_avg": 232.6634521484375,
+      "policy/clipfrac_avg": 1.25,
+      "policy/entropy_avg": 0.751953125,
+      "step": 73,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 6,
+      "val/ratio": 2.00066876411438
+    },
+    {
+      "episode": 1200,
+      "epoch": 0.00718984793471618,
+      "loss/policy_avg": 0.1890699565410614,
+      "lr": 9.952709611451944e-06,
+      "objective/entropy": 118.900146484375,
+      "objective/kl": 21.680133819580078,
+      "objective/non_score_reward": -1.084006667137146,
+      "objective/rlhf_reward": -6.336027145385742,
+      "objective/scores": -0.5,
+      "policy/approxkl_avg": 266.20465087890625,
+      "policy/clipfrac_avg": 1.0,
+      "policy/entropy_avg": 0.720703125,
+      "step": 74,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 2,
+      "val/ratio": 1.997557282447815
+    },
+    {
+      "episode": 1216,
+      "epoch": 0.00728571257384573,
+      "loss/policy_avg": 0.383888304233551,
+      "lr": 9.952070552147241e-06,
+      "objective/entropy": 124.33120727539062,
+      "objective/kl": 21.27002716064453,
+      "objective/non_score_reward": -1.0635013580322266,
+      "objective/rlhf_reward": -4.254005193710327,
+      "objective/scores": 0.0,
+      "policy/approxkl_avg": 86.44483184814453,
+      "policy/clipfrac_avg": 0.75,
+      "policy/entropy_avg": 0.611328125,
+      "step": 75,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 2,
+      "val/ratio": 1.9962941408157349
+    },
+    {
+      "episode": 1232,
+      "epoch": 0.007381577212975279,
+      "loss/policy_avg": 0.23960661888122559,
+      "lr": 9.951431492842536e-06,
+      "objective/entropy": 40.334468841552734,
+      "objective/kl": 17.827497482299805,
+      "objective/non_score_reward": -0.891374945640564,
+      "objective/rlhf_reward": -3.5654996633529663,
+      "objective/scores": 0.0,
+      "policy/approxkl_avg": 94.15713500976562,
+      "policy/clipfrac_avg": 1.0,
+      "policy/entropy_avg": 0.548828125,
+      "step": 76,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 1,
+      "val/ratio": 1.9984164237976074
+    },
+    {
+      "episode": 1248,
+      "epoch": 0.007477441852104828,
+      "loss/policy_avg": 0.4706483781337738,
+      "lr": 9.950792433537833e-06,
+      "objective/entropy": 106.07322692871094,
+      "objective/kl": 19.12630844116211,
+      "objective/non_score_reward": -0.9563154578208923,
+      "objective/rlhf_reward": -3.8252618312835693,
+      "objective/scores": 0.0,
+      "policy/approxkl_avg": 185.7378387451172,
+      "policy/clipfrac_avg": 0.75,
+      "policy/entropy_avg": 0.87890625,
+      "step": 77,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 5,
+      "val/ratio": 1.9966304302215576
+    },
+    {
+      "episode": 1264,
+      "epoch": 0.007573306491234377,
+      "loss/policy_avg": 0.0665474385023117,
+      "lr": 9.950153374233129e-06,
+      "objective/entropy": 34.984527587890625,
+      "objective/kl": 23.865880966186523,
+      "objective/non_score_reward": -1.1932940483093262,
+      "objective/rlhf_reward": -3.373176074028015,
+      "objective/scores": 0.35,
+      "policy/approxkl_avg": 336.36712646484375,
+      "policy/clipfrac_avg": 1.0,
+      "policy/entropy_avg": 0.53515625,
+      "step": 78,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 2,
+      "val/ratio": 1.997634768486023
+    },
+    {
+      "episode": 1280,
+      "epoch": 0.007669171130363926,
+      "loss/policy_avg": 0.3842596113681793,
+      "lr": 9.949514314928425e-06,
+      "objective/entropy": 229.88047790527344,
+      "objective/kl": 27.72378921508789,
+      "objective/non_score_reward": -1.386189579963684,
+      "objective/rlhf_reward": -1.144758558273315,
+      "objective/scores": 1.1,
+      "policy/approxkl_avg": 524.0328369140625,
+      "policy/clipfrac_avg": 1.0,
+      "policy/entropy_avg": 0.720703125,
+      "step": 79,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 0,
+      "val/ratio": 1.997296690940857
+    },
+    {
+      "episode": 1296,
+      "epoch": 0.007765035769493475,
+      "loss/policy_avg": 0.4347228705883026,
+      "lr": 9.948875255623722e-06,
+      "objective/entropy": -43.351566314697266,
+      "objective/kl": 18.37939453125,
+      "objective/non_score_reward": -0.9189697504043579,
+      "objective/rlhf_reward": -3.6758789718151093,
+      "objective/scores": 0.0,
+      "policy/approxkl_avg": 87.53759002685547,
+      "policy/clipfrac_avg": 0.75,
+      "policy/entropy_avg": 0.521484375,
+      "step": 80,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 3,
+      "val/ratio": 1.9968631267547607
+    },
+    {
+      "episode": 1312,
+      "epoch": 0.007860900408623025,
+      "loss/policy_avg": 0.5703809261322021,
+      "lr": 9.94823619631902e-06,
+      "objective/entropy": 182.94879150390625,
+      "objective/kl": 24.6871337890625,
+      "objective/non_score_reward": -1.2343567609786987,
+      "objective/rlhf_reward": -6.937427043914795,
+      "objective/scores": -0.5,
+      "policy/approxkl_avg": 274.44744873046875,
+      "policy/clipfrac_avg": 0.75,
+      "policy/entropy_avg": 0.71875,
+      "step": 81,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 1,
+      "val/ratio": 1.9995486736297607
+    },
+    {
+      "episode": 1328,
+      "epoch": 0.007956765047752574,
+      "loss/policy_avg": 0.10641711950302124,
+      "lr": 9.947597137014316e-06,
+      "objective/entropy": 73.29893493652344,
+      "objective/kl": 17.603548049926758,
+      "objective/non_score_reward": -0.88017737865448,
+      "objective/rlhf_reward": -3.52070951461792,
+      "objective/scores": 0.0,
+      "policy/approxkl_avg": 123.0771255493164,
+      "policy/clipfrac_avg": 1.0,
+      "policy/entropy_avg": 0.505859375,
+      "step": 82,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 1,
+      "val/ratio": 1.998863935470581
+    },
+    {
+      "episode": 1344,
+      "epoch": 0.008052629686882123,
+      "loss/policy_avg": 0.12928956747055054,
+      "lr": 9.946958077709611e-06,
+      "objective/entropy": 108.6548080444336,
+      "objective/kl": 18.59684944152832,
+      "objective/non_score_reward": -0.929842472076416,
+      "objective/rlhf_reward": -2.3407678390420497,
+      "objective/scores": 0.34465054211822604,
+      "policy/approxkl_avg": 120.68421936035156,
+      "policy/clipfrac_avg": 1.0,
+      "policy/entropy_avg": 0.57421875,
+      "step": 83,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 2,
+      "val/ratio": 1.998254656791687
+    },
+    {
+      "episode": 1360,
+      "epoch": 0.008148494326011672,
+      "loss/policy_avg": 0.14865761995315552,
+      "lr": 9.946319018404908e-06,
+      "objective/entropy": 102.67412567138672,
+      "objective/kl": 23.651020050048828,
+      "objective/non_score_reward": -1.1825510263442993,
+      "objective/rlhf_reward": -4.7302040457725525,
+      "objective/scores": 0.0,
+      "policy/approxkl_avg": 73.66981506347656,
+      "policy/clipfrac_avg": 1.0,
+      "policy/entropy_avg": 0.4033203125,
+      "step": 84,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 1,
+      "val/ratio": 2.000584602355957
+    },
+    {
+      "episode": 1376,
+      "epoch": 0.00824435896514122,
+      "loss/policy_avg": 0.07400541007518768,
+      "lr": 9.945679959100205e-06,
+      "objective/entropy": 133.18292236328125,
+      "objective/kl": 11.912694931030273,
+      "objective/non_score_reward": -0.5956346988677979,
+      "objective/rlhf_reward": -0.4351277453469593,
+      "objective/scores": 0.4868528072345416,
+      "policy/approxkl_avg": 49.34624099731445,
+      "policy/clipfrac_avg": 1.0,
+      "policy/entropy_avg": 0.59375,
+      "step": 85,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 1,
+      "val/ratio": 2.000187397003174
+    },
+    {
+      "episode": 1392,
+      "epoch": 0.00834022360427077,
+      "loss/policy_avg": 0.13357847929000854,
+      "lr": 9.945040899795502e-06,
+      "objective/entropy": 112.34770202636719,
+      "objective/kl": 20.725894927978516,
+      "objective/non_score_reward": -1.0362948179244995,
+      "objective/rlhf_reward": -2.7665772224343836,
+      "objective/scores": 0.34465054211822604,
+      "policy/approxkl_avg": 157.26473999023438,
+      "policy/clipfrac_avg": 0.75,
+      "policy/entropy_avg": 0.56640625,
+      "step": 86,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 2,
+      "val/ratio": 1.9963226318359375
+    },
+    {
+      "episode": 1408,
+      "epoch": 0.008436088243400319,
+      "loss/policy_avg": 0.32753437757492065,
+      "lr": 9.944401840490799e-06,
+      "objective/entropy": 43.2598762512207,
+      "objective/kl": 19.98666000366211,
+      "objective/non_score_reward": -0.9993331432342529,
+      "objective/rlhf_reward": -5.997332572937012,
+      "objective/scores": -0.5,
+      "policy/approxkl_avg": 257.4547424316406,
+      "policy/clipfrac_avg": 1.0,
+      "policy/entropy_avg": 0.599609375,
+      "step": 87,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 3,
+      "val/ratio": 1.9966068267822266
+    },
+    {
+      "episode": 1424,
+      "epoch": 0.008531952882529868,
+      "loss/policy_avg": 0.09795168787240982,
+      "lr": 9.943762781186096e-06,
+      "objective/entropy": -59.364646911621094,
+      "objective/kl": 14.953709602355957,
+      "objective/non_score_reward": -0.7476855516433716,
+      "objective/rlhf_reward": -4.990742206573486,
+      "objective/scores": -0.5,
+      "policy/approxkl_avg": 55.110633850097656,
+      "policy/clipfrac_avg": 1.0,
+      "policy/entropy_avg": 0.640625,
+      "step": 88,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 4,
+      "val/ratio": 1.9969828128814697
+    },
+    {
+      "episode": 1440,
+      "epoch": 0.008627817521659416,
+      "loss/policy_avg": 0.06303240358829498,
+      "lr": 9.94312372188139e-06,
+      "objective/entropy": 50.4556770324707,
+      "objective/kl": 19.505146026611328,
+      "objective/non_score_reward": -0.9752573370933533,
+      "objective/rlhf_reward": -3.9010292887687683,
+      "objective/scores": 0.0,
+      "policy/approxkl_avg": 34.922752380371094,
+      "policy/clipfrac_avg": 0.75,
+      "policy/entropy_avg": 0.607421875,
+      "step": 89,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 4,
+      "val/ratio": 1.9964232444763184
+    },
+    {
+      "episode": 1456,
+      "epoch": 0.008723682160788965,
+      "loss/policy_avg": 0.2796894907951355,
+      "lr": 9.942484662576688e-06,
+      "objective/entropy": 135.44993591308594,
+      "objective/kl": 22.230022430419922,
+      "objective/non_score_reward": -1.1115009784698486,
+      "objective/rlhf_reward": -4.446004092693329,
+      "objective/scores": 0.0,
+      "policy/approxkl_avg": 48.1524658203125,
+      "policy/clipfrac_avg": 1.0,
+      "policy/entropy_avg": 0.56640625,
+      "step": 90,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 3,
+      "val/ratio": 1.998495101928711
+    },
+    {
+      "episode": 1472,
+      "epoch": 0.008819546799918514,
+      "loss/policy_avg": 0.12483496963977814,
+      "lr": 9.941845603271985e-06,
+      "objective/entropy": 148.17709350585938,
+      "objective/kl": 17.60011100769043,
+      "objective/non_score_reward": -0.8800055384635925,
+      "objective/rlhf_reward": -5.520022392272949,
+      "objective/scores": -0.5,
+      "policy/approxkl_avg": 142.25204467773438,
+      "policy/clipfrac_avg": 1.5,
+      "policy/entropy_avg": 0.6171875,
+      "step": 91,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 1,
+      "val/ratio": 1.998748779296875
+    },
+    {
+      "episode": 1488,
+      "epoch": 0.008915411439048063,
+      "loss/policy_avg": 0.0684453696012497,
+      "lr": 9.941206543967281e-06,
+      "objective/entropy": 25.60771942138672,
+      "objective/kl": 17.530319213867188,
+      "objective/non_score_reward": -0.8765159845352173,
+      "objective/rlhf_reward": -5.506063938140869,
+      "objective/scores": -0.5,
+      "policy/approxkl_avg": 70.80863189697266,
+      "policy/clipfrac_avg": 1.5,
+      "policy/entropy_avg": 0.625,
+      "step": 92,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 2,
+      "val/ratio": 1.9999198913574219
+    },
+    {
+      "episode": 1504,
+      "epoch": 0.009011276078177614,
+      "loss/policy_avg": 0.13488999009132385,
+      "lr": 9.940567484662578e-06,
+      "objective/entropy": -75.2538070678711,
+      "objective/kl": 19.696504592895508,
+      "objective/non_score_reward": -0.9848252534866333,
+      "objective/rlhf_reward": -3.939300984144211,
+      "objective/scores": 0.0,
+      "policy/approxkl_avg": 214.8182373046875,
+      "policy/clipfrac_avg": 1.25,
+      "policy/entropy_avg": 0.515625,
+      "step": 93,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 3,
+      "val/ratio": 1.9989702701568604
+    },
+    {
+      "episode": 1520,
+      "epoch": 0.009107140717307163,
+      "loss/policy_avg": 0.02409663423895836,
+      "lr": 9.939928425357874e-06,
+      "objective/entropy": 8.831840515136719,
+      "objective/kl": 25.456069946289062,
+      "objective/non_score_reward": -1.272803544998169,
+      "objective/rlhf_reward": -3.5754421589695777,
+      "objective/scores": 0.37894294565112985,
+      "policy/approxkl_avg": 176.86953735351562,
+      "policy/clipfrac_avg": 1.25,
+      "policy/entropy_avg": 0.4873046875,
+      "step": 94,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 3,
+      "val/ratio": 1.9964553117752075
+    },
+    {
+      "episode": 1536,
+      "epoch": 0.009203005356436712,
+      "loss/policy_avg": 0.0426328219473362,
+      "lr": 9.93928936605317e-06,
+      "objective/entropy": 185.92372131347656,
+      "objective/kl": 19.176239013671875,
+      "objective/non_score_reward": -0.95881187915802,
+      "objective/rlhf_reward": -3.8352474570274353,
+      "objective/scores": 0.0,
+      "policy/approxkl_avg": 269.95849609375,
+      "policy/clipfrac_avg": 1.25,
+      "policy/entropy_avg": 0.70703125,
+      "step": 95,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 1,
+      "val/ratio": 1.9930847883224487
+    },
+    {
+      "episode": 1552,
+      "epoch": 0.009298869995566261,
+      "loss/policy_avg": 0.3135666251182556,
+      "lr": 9.938650306748467e-06,
+      "objective/entropy": -119.88722229003906,
+      "objective/kl": 18.911632537841797,
+      "objective/non_score_reward": -0.9455816745758057,
+      "objective/rlhf_reward": -3.782326579093933,
+      "objective/scores": 0.0,
+      "policy/approxkl_avg": 136.56689453125,
+      "policy/clipfrac_avg": 0.75,
+      "policy/entropy_avg": 0.630859375,
+      "step": 96,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 4,
+      "val/ratio": 2.0003180503845215
+    },
+    {
+      "episode": 1568,
+      "epoch": 0.00939473463469581,
+      "loss/policy_avg": 0.1893162876367569,
+      "lr": 9.938011247443764e-06,
+      "objective/entropy": 179.721435546875,
+      "objective/kl": 21.26153564453125,
+      "objective/non_score_reward": -1.0630767345428467,
+      "objective/rlhf_reward": -1.3285883411180701,
+      "objective/scores": 0.7309297535714575,
+      "policy/approxkl_avg": 203.61773681640625,
+      "policy/clipfrac_avg": 1.0,
+      "policy/entropy_avg": 0.70703125,
+      "step": 97,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 2,
+      "val/ratio": 1.994171142578125
+    },
+    {
+      "episode": 1584,
+      "epoch": 0.009490599273825359,
+      "loss/policy_avg": 0.5632504224777222,
+      "lr": 9.937372188139061e-06,
+      "objective/entropy": 3.3514366149902344,
+      "objective/kl": 19.21142578125,
+      "objective/non_score_reward": -0.9605712890625,
+      "objective/rlhf_reward": -2.483035289977474,
+      "objective/scores": 0.33981246656813147,
+      "policy/approxkl_avg": 74.39619445800781,
+      "policy/clipfrac_avg": 0.75,
+      "policy/entropy_avg": 0.6953125,
+      "step": 98,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 3,
+      "val/ratio": 1.99905526638031
+    },
+    {
+      "episode": 1600,
+      "epoch": 0.009586463912954908,
+      "loss/policy_avg": 0.20837292075157166,
+      "lr": 9.936733128834358e-06,
+      "objective/entropy": 121.03665161132812,
+      "objective/kl": 13.999438285827637,
+      "objective/non_score_reward": -0.6999719142913818,
+      "objective/rlhf_reward": -4.799887657165527,
+      "objective/scores": -0.5,
+      "policy/approxkl_avg": 32.508689880371094,
+      "policy/clipfrac_avg": 1.0,
+      "policy/entropy_avg": 0.375,
+      "step": 99,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 0,
+      "val/ratio": 1.9994678497314453
+    },
+    {
+      "episode": 1616,
+      "epoch": 0.009682328552084457,
+      "loss/policy_avg": 0.2726283669471741,
+      "lr": 9.936094069529653e-06,
+      "objective/entropy": 110.09475708007812,
+      "objective/kl": 15.960447311401367,
+      "objective/non_score_reward": -0.798022449016571,
+      "objective/rlhf_reward": -3.192089796066284,
+      "objective/scores": 0.0,
+      "policy/approxkl_avg": 61.767425537109375,
+      "policy/clipfrac_avg": 1.25,
+      "policy/entropy_avg": 0.55859375,
+      "step": 100,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 3,
+      "val/ratio": 2.0027127265930176
+    },
+    {
+      "episode": 1632,
+      "epoch": 0.009778193191214006,
+      "loss/policy_avg": 0.2845292091369629,
+      "lr": 9.93545501022495e-06,
+      "objective/entropy": -153.4110107421875,
+      "objective/kl": 16.05643081665039,
+      "objective/non_score_reward": -0.8028215765953064,
+      "objective/rlhf_reward": 1.1887137234210972,
+      "objective/scores": 1.1,
+      "policy/approxkl_avg": 95.32630157470703,
+      "policy/clipfrac_avg": 1.0,
+      "policy/entropy_avg": 0.669921875,
+      "step": 101,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 4,
+      "val/ratio": 1.997380256652832
+    },
+    {
+      "episode": 1648,
+      "epoch": 0.009874057830343555,
+      "loss/policy_avg": 0.18602727353572845,
+      "lr": 9.934815950920245e-06,
+      "objective/entropy": -13.683324813842773,
+      "objective/kl": 23.494054794311523,
+      "objective/non_score_reward": -1.174702763557434,
+      "objective/rlhf_reward": -6.698811054229736,
+      "objective/scores": -0.5,
+      "policy/approxkl_avg": 145.7095947265625,
+      "policy/clipfrac_avg": 0.75,
+      "policy/entropy_avg": 0.4912109375,
+      "step": 102,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 2,
+      "val/ratio": 1.9994981288909912
+    },
+    {
+      "episode": 1664,
+      "epoch": 0.009969922469473104,
+      "loss/policy_avg": 0.2709546983242035,
+      "lr": 9.934176891615542e-06,
+      "objective/entropy": -101.46907043457031,
+      "objective/kl": 22.274028778076172,
+      "objective/non_score_reward": -1.113701581954956,
+      "objective/rlhf_reward": -2.507394979672368,
+      "objective/scores": 0.4868528072345416,
+      "policy/approxkl_avg": 308.6561584472656,
+      "policy/clipfrac_avg": 1.0,
+      "policy/entropy_avg": 0.4912109375,
+      "step": 103,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 4,
+      "val/ratio": 1.9981049299240112
+    },
+    {
+      "episode": 1680,
+      "epoch": 0.010065787108602653,
+      "loss/policy_avg": 0.0334465391933918,
+      "lr": 9.933537832310839e-06,
+      "objective/entropy": 130.1453857421875,
+      "objective/kl": 29.715213775634766,
+      "objective/non_score_reward": -1.4857605695724487,
+      "objective/rlhf_reward": -4.601406863241821,
+      "objective/scores": 0.33540891336663825,
+      "policy/approxkl_avg": 351.42138671875,
+      "policy/clipfrac_avg": 1.25,
+      "policy/entropy_avg": 0.5234375,
+      "step": 104,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 1,
+      "val/ratio": 1.998427152633667
+    },
+    {
+      "episode": 1696,
+      "epoch": 0.010161651747732202,
+      "loss/policy_avg": 0.24828088283538818,
+      "lr": 9.932898773006136e-06,
+      "objective/entropy": 125.98516082763672,
+      "objective/kl": 15.857706069946289,
+      "objective/non_score_reward": -0.7928853034973145,
+      "objective/rlhf_reward": -1.720943163247451,
+      "objective/scores": 0.36264953503719355,
+      "policy/approxkl_avg": 74.20083618164062,
+      "policy/clipfrac_avg": 1.0,
+      "policy/entropy_avg": 0.578125,
+      "step": 105,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 2,
+      "val/ratio": 1.9993948936462402
+    },
+    {
+      "episode": 1712,
+      "epoch": 0.01025751638686175,
+      "loss/policy_avg": 0.2954871356487274,
+      "lr": 9.932259713701433e-06,
+      "objective/entropy": 97.68868255615234,
+      "objective/kl": 12.135580062866211,
+      "objective/non_score_reward": -0.6067790389060974,
+      "objective/rlhf_reward": -2.4271161258220673,
+      "objective/scores": 0.0,
+      "policy/approxkl_avg": 67.66595458984375,
+      "policy/clipfrac_avg": 1.25,
+      "policy/entropy_avg": 0.59375,
+      "step": 106,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 3,
+      "val/ratio": 1.9992032051086426
+    },
+    {
+      "episode": 1728,
+      "epoch": 0.0103533810259913,
+      "loss/policy_avg": 0.10418711602687836,
+      "lr": 9.931620654396728e-06,
+      "objective/entropy": -43.816890716552734,
+      "objective/kl": 19.110689163208008,
+      "objective/non_score_reward": -0.9555345773696899,
+      "objective/rlhf_reward": -5.82213830947876,
+      "objective/scores": -0.5,
+      "policy/approxkl_avg": 160.15283203125,
+      "policy/clipfrac_avg": 1.0,
+      "policy/entropy_avg": 0.6796875,
+      "step": 107,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 5,
+      "val/ratio": 1.9975645542144775
+    },
+    {
+      "episode": 1744,
+      "epoch": 0.01044924566512085,
+      "loss/policy_avg": 0.23229390382766724,
+      "lr": 9.930981595092025e-06,
+      "objective/entropy": 91.57461547851562,
+      "objective/kl": 18.9378662109375,
+      "objective/non_score_reward": -0.9468932747840881,
+      "objective/rlhf_reward": -3.787573218345642,
+      "objective/scores": 0.0,
+      "policy/approxkl_avg": 155.35989379882812,
+      "policy/clipfrac_avg": 1.0,
+      "policy/entropy_avg": 0.66796875,
+      "step": 108,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 1,
+      "val/ratio": 1.9963436126708984
+    },
+    {
+      "episode": 1760,
+      "epoch": 0.010545110304250399,
+      "loss/policy_avg": 0.3382238447666168,
+      "lr": 9.930342535787322e-06,
+      "objective/entropy": -49.52970886230469,
+      "objective/kl": 17.919204711914062,
+      "objective/non_score_reward": -0.89596027135849,
+      "objective/rlhf_reward": -5.583841323852539,
+      "objective/scores": -0.5,
+      "policy/approxkl_avg": 151.97140502929688,
+      "policy/clipfrac_avg": 1.0,
+      "policy/entropy_avg": 0.73828125,
+      "step": 109,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 4,
+      "val/ratio": 1.9960722923278809
+    },
+    {
+      "episode": 1776,
+      "epoch": 0.010640974943379948,
+      "loss/policy_avg": 0.16102033853530884,
+      "lr": 9.929703476482619e-06,
+      "objective/entropy": -40.16828155517578,
+      "objective/kl": 15.826179504394531,
+      "objective/non_score_reward": -0.7913089990615845,
+      "objective/rlhf_reward": -3.1652360260486603,
+      "objective/scores": 0.0,
+      "policy/approxkl_avg": 32.21597671508789,
+      "policy/clipfrac_avg": 1.0,
+      "policy/entropy_avg": 0.767578125,
+      "step": 110,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 4,
+      "val/ratio": 1.9961647987365723
+    },
+    {
+      "episode": 1792,
+      "epoch": 0.010736839582509497,
+      "loss/policy_avg": 0.08855805546045303,
+      "lr": 9.929064417177915e-06,
+      "objective/entropy": 187.74282836914062,
+      "objective/kl": 22.12034797668457,
+      "objective/non_score_reward": -1.1060173511505127,
+      "objective/rlhf_reward": -6.424069404602051,
+      "objective/scores": -0.5,
+      "policy/approxkl_avg": 89.00162506103516,
+      "policy/clipfrac_avg": 1.0,
+      "policy/entropy_avg": 0.80859375,
+      "step": 111,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 2,
+      "val/ratio": 1.9971915483474731
+    },
+    {
+      "episode": 1808,
+      "epoch": 0.010832704221639046,
+      "loss/policy_avg": 0.3315132260322571,
+      "lr": 9.928425357873212e-06,
+      "objective/entropy": -130.41551208496094,
+      "objective/kl": 20.600021362304688,
+      "objective/non_score_reward": -1.030001163482666,
+      "objective/rlhf_reward": -2.7414021278298915,
+      "objective/scores": 0.34465054211822604,
+      "policy/approxkl_avg": 231.54774475097656,
+      "policy/clipfrac_avg": 1.0,
+      "policy/entropy_avg": 0.671875,
+      "step": 112,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 4,
+      "val/ratio": 2.0009350776672363
+    },
+    {
+      "episode": 1824,
+      "epoch": 0.010928568860768595,
+      "loss/policy_avg": 0.5379164814949036,
+      "lr": 9.927786298568507e-06,
+      "objective/entropy": 122.76021575927734,
+      "objective/kl": 24.667219161987305,
+      "objective/non_score_reward": -1.2333608865737915,
+      "objective/rlhf_reward": -6.933443546295166,
+      "objective/scores": -0.5,
+      "policy/approxkl_avg": 214.82647705078125,
+      "policy/clipfrac_avg": 1.0,
+      "policy/entropy_avg": 0.4638671875,
+      "step": 113,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 1,
+      "val/ratio": 1.9977924823760986
+    },
+    {
+      "episode": 1840,
+      "epoch": 0.011024433499898144,
+      "loss/policy_avg": 0.19039300084114075,
+      "lr": 9.927147239263804e-06,
+      "objective/entropy": -26.283668518066406,
+      "objective/kl": 21.043611526489258,
+      "objective/non_score_reward": -1.0521806478500366,
+      "objective/rlhf_reward": -2.6929507491909828,
+      "objective/scores": 0.37894294565112985,
+      "policy/approxkl_avg": 337.57025146484375,
+      "policy/clipfrac_avg": 1.0,
+      "policy/entropy_avg": 0.62890625,
+      "step": 114,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 4,
+      "val/ratio": 1.9954912662506104
+    },
+    {
+      "episode": 1856,
+      "epoch": 0.011120298139027693,
+      "loss/policy_avg": 0.030586296692490578,
+      "lr": 9.926508179959101e-06,
+      "objective/entropy": 189.2314910888672,
+      "objective/kl": 18.47957992553711,
+      "objective/non_score_reward": -0.9239791035652161,
+      "objective/rlhf_reward": -3.6959164142608643,
+      "objective/scores": 0.0,
+      "policy/approxkl_avg": 158.6993865966797,
+      "policy/clipfrac_avg": 1.0,
+      "policy/entropy_avg": 0.8671875,
+      "step": 115,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 2,
+      "val/ratio": 1.9987783432006836
+    },
+    {
+      "episode": 1872,
+      "epoch": 0.011216162778157242,
+      "loss/policy_avg": 0.23665881156921387,
+      "lr": 9.925869120654398e-06,
+      "objective/entropy": 73.73204803466797,
+      "objective/kl": 19.052127838134766,
+      "objective/non_score_reward": -0.9526063203811646,
+      "objective/rlhf_reward": -5.810425281524658,
+      "objective/scores": -0.5,
+      "policy/approxkl_avg": 45.58797073364258,
+      "policy/clipfrac_avg": 1.0,
+      "policy/entropy_avg": 0.55859375,
+      "step": 116,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 3,
+      "val/ratio": 1.9978525638580322
+    },
+    {
+      "episode": 1888,
+      "epoch": 0.01131202741728679,
+      "loss/policy_avg": 0.11659398674964905,
+      "lr": 9.925230061349695e-06,
+      "objective/entropy": 128.39474487304688,
+      "objective/kl": 25.3045597076416,
+      "objective/non_score_reward": -1.265228033065796,
+      "objective/rlhf_reward": -2.6609121322631832,
+      "objective/scores": 0.6,
+      "policy/approxkl_avg": 76.14613342285156,
+      "policy/clipfrac_avg": 0.75,
+      "policy/entropy_avg": 0.64453125,
+      "step": 117,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 2,
+      "val/ratio": 1.9966604709625244
+    },
+    {
+      "episode": 1904,
+      "epoch": 0.01140789205641634,
+      "loss/policy_avg": 0.19203245639801025,
+      "lr": 9.92459100204499e-06,
+      "objective/entropy": 57.626686096191406,
+      "objective/kl": 30.407909393310547,
+      "objective/non_score_reward": -1.5203955173492432,
+      "objective/rlhf_reward": -1.6815817117691036,
+      "objective/scores": 1.1,
+      "policy/approxkl_avg": 324.47161865234375,
+      "policy/clipfrac_avg": 1.0,
+      "policy/entropy_avg": 0.544921875,
+      "step": 118,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 3,
+      "val/ratio": 2.0000627040863037
+    },
+    {
+      "episode": 1920,
+      "epoch": 0.011503756695545889,
+      "loss/policy_avg": -0.02956828847527504,
+      "lr": 9.923951942740287e-06,
+      "objective/entropy": 160.20449829101562,
+      "objective/kl": 13.33430290222168,
+      "objective/non_score_reward": -0.666715145111084,
+      "objective/rlhf_reward": -2.666860580444336,
+      "objective/scores": 0.0,
+      "policy/approxkl_avg": 62.17939758300781,
+      "policy/clipfrac_avg": 1.25,
+      "policy/entropy_avg": 0.587890625,
+      "step": 119,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 2,
+      "val/ratio": 2.003843307495117
+    },
+    {
+      "episode": 1936,
+      "epoch": 0.011599621334675438,
+      "loss/policy_avg": 0.3666956424713135,
+      "lr": 9.923312883435584e-06,
+      "objective/entropy": 173.73385620117188,
+      "objective/kl": 25.82461929321289,
+      "objective/non_score_reward": -1.2912311553955078,
+      "objective/rlhf_reward": -7.164924621582031,
+      "objective/scores": -0.5,
+      "policy/approxkl_avg": 248.4417724609375,
+      "policy/clipfrac_avg": 1.25,
+      "policy/entropy_avg": 0.71484375,
+      "step": 120,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 1,
+      "val/ratio": 2.0001635551452637
+    },
+    {
+      "episode": 1952,
+      "epoch": 0.011695485973804987,
+      "loss/policy_avg": 0.07095953077077866,
+      "lr": 9.92267382413088e-06,
+      "objective/entropy": 60.89289855957031,
+      "objective/kl": 21.512653350830078,
+      "objective/non_score_reward": -1.0756325721740723,
+      "objective/rlhf_reward": -2.851932506175384,
+      "objective/scores": 0.36264953503719355,
+      "policy/approxkl_avg": 187.26104736328125,
+      "policy/clipfrac_avg": 1.25,
+      "policy/entropy_avg": 0.654296875,
+      "step": 121,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 2,
+      "val/ratio": 1.9986932277679443
+    },
+    {
+      "episode": 1968,
+      "epoch": 0.011791350612934537,
+      "loss/policy_avg": 0.11872611939907074,
+      "lr": 9.922034764826178e-06,
+      "objective/entropy": -24.511760711669922,
+      "objective/kl": 22.253305435180664,
+      "objective/non_score_reward": -1.1126651763916016,
+      "objective/rlhf_reward": -4.450661063194275,
+      "objective/scores": 0.0,
+      "policy/approxkl_avg": 199.84397888183594,
+      "policy/clipfrac_avg": 1.0,
+      "policy/entropy_avg": 0.6484375,
+      "step": 122,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 5,
+      "val/ratio": 1.9950015544891357
+    },
+    {
+      "episode": 1984,
+      "epoch": 0.011887215252064086,
+      "loss/policy_avg": 0.5726426839828491,
+      "lr": 9.921395705521473e-06,
+      "objective/entropy": 102.35612487792969,
+      "objective/kl": 32.768287658691406,
+      "objective/non_score_reward": -1.6384142637252808,
+      "objective/rlhf_reward": -8.553656578063965,
+      "objective/scores": -0.5,
+      "policy/approxkl_avg": 327.3544921875,
+      "policy/clipfrac_avg": 1.0,
+      "policy/entropy_avg": 0.533203125,
+      "step": 123,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 2,
+      "val/ratio": 1.995248556137085
+    },
+    {
+      "episode": 2000,
+      "epoch": 0.011983079891193635,
+      "loss/policy_avg": 0.19069992005825043,
+      "lr": 9.92075664621677e-06,
+      "objective/entropy": 7.145952224731445,
+      "objective/kl": 17.727392196655273,
+      "objective/non_score_reward": -0.8863697052001953,
+      "objective/rlhf_reward": -3.545478705316782,
+      "objective/scores": 0.0,
+      "policy/approxkl_avg": 26.600868225097656,
+      "policy/clipfrac_avg": 1.0,
+      "policy/entropy_avg": 0.57421875,
+      "step": 124,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 3,
+      "val/ratio": 2.0011134147644043
+    },
+    {
+      "episode": 2016,
+      "epoch": 0.012078944530323184,
+      "loss/policy_avg": 0.2572447657585144,
+      "lr": 9.920117586912067e-06,
+      "objective/entropy": 109.04229736328125,
+      "objective/kl": 17.79098129272461,
+      "objective/non_score_reward": -0.8895490765571594,
+      "objective/rlhf_reward": -5.558196067810059,
+      "objective/scores": -0.5,
+      "policy/approxkl_avg": 111.66732788085938,
+      "policy/clipfrac_avg": 1.0,
+      "policy/entropy_avg": 0.763671875,
+      "step": 125,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 3,
+      "val/ratio": 1.9968197345733643
+    },
+    {
+      "episode": 2032,
+      "epoch": 0.012174809169452733,
+      "loss/policy_avg": 0.043444547802209854,
+      "lr": 9.919478527607362e-06,
+      "objective/entropy": 75.83810424804688,
+      "objective/kl": 27.20602798461914,
+      "objective/non_score_reward": -1.3603014945983887,
+      "objective/rlhf_reward": -2.51748690450308,
+      "objective/scores": 0.7309297535714575,
+      "policy/approxkl_avg": 274.793701171875,
+      "policy/clipfrac_avg": 1.25,
+      "policy/entropy_avg": 0.521484375,
+      "step": 126,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 2,
+      "val/ratio": 1.9991501569747925
+    },
+    {
+      "episode": 2048,
+      "epoch": 0.012270673808582282,
+      "loss/policy_avg": 0.2138219177722931,
+      "lr": 9.918839468302659e-06,
+      "objective/entropy": 21.247840881347656,
+      "objective/kl": 14.299978256225586,
+      "objective/non_score_reward": -0.7149989008903503,
+      "objective/rlhf_reward": -2.8599955439567566,
+      "objective/scores": 0.0,
+      "policy/approxkl_avg": 21.416780471801758,
+      "policy/clipfrac_avg": 1.0,
+      "policy/entropy_avg": 0.80078125,
+      "step": 127,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 5,
+      "val/ratio": 1.998870849609375
+    },
+    {
+      "episode": 2064,
+      "epoch": 0.012366538447711831,
+      "loss/policy_avg": 0.23010344803333282,
+      "lr": 9.918200408997956e-06,
+      "objective/entropy": -76.91316223144531,
+      "objective/kl": 13.382017135620117,
+      "objective/non_score_reward": -0.6691007614135742,
+      "objective/rlhf_reward": -2.676403224468231,
+      "objective/scores": 0.0,
+      "policy/approxkl_avg": 37.060523986816406,
+      "policy/clipfrac_avg": 1.0,
+      "policy/entropy_avg": 0.677734375,
+      "step": 128,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 4,
+      "val/ratio": 1.998915195465088
+    },
+    {
+      "episode": 2080,
+      "epoch": 0.01246240308684138,
+      "loss/policy_avg": 0.4017820954322815,
+      "lr": 9.917561349693252e-06,
+      "objective/entropy": 198.82456970214844,
+      "objective/kl": 22.337753295898438,
+      "objective/non_score_reward": -1.1168878078460693,
+      "objective/rlhf_reward": -2.9112917771011144,
+      "objective/scores": 0.38906482631788786,
+      "policy/approxkl_avg": 41.49570846557617,
+      "policy/clipfrac_avg": 1.0,
+      "policy/entropy_avg": 0.59765625,
+      "step": 129,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 0,
+      "val/ratio": 2.001622200012207
+    },
+    {
+      "episode": 2096,
+      "epoch": 0.012558267725970929,
+      "loss/policy_avg": 0.45664405822753906,
+      "lr": 9.91692229038855e-06,
+      "objective/entropy": 96.67610168457031,
+      "objective/kl": 13.830822944641113,
+      "objective/non_score_reward": -0.6915411353111267,
+      "objective/rlhf_reward": -2.7661644518375397,
+      "objective/scores": 0.0,
+      "policy/approxkl_avg": 49.5977783203125,
+      "policy/clipfrac_avg": 1.0,
+      "policy/entropy_avg": 0.87890625,
+      "step": 130,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 4,
+      "val/ratio": 2.0018720626831055
+    },
+    {
+      "episode": 2112,
+      "epoch": 0.012654132365100478,
+      "loss/policy_avg": 0.18199189007282257,
+      "lr": 9.916283231083844e-06,
+      "objective/entropy": 12.164558410644531,
+      "objective/kl": 17.693878173828125,
+      "objective/non_score_reward": -0.8846939206123352,
+      "objective/rlhf_reward": -3.538775682449341,
+      "objective/scores": 0.0,
+      "policy/approxkl_avg": 3.7435173988342285,
+      "policy/clipfrac_avg": 1.5,
+      "policy/entropy_avg": 0.62890625,
+      "step": 131,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 2,
+      "val/ratio": 2.0015268325805664
+    },
+    {
+      "episode": 2128,
+      "epoch": 0.012749997004230027,
+      "loss/policy_avg": 0.21469825506210327,
+      "lr": 9.915644171779141e-06,
+      "objective/entropy": -11.569038391113281,
+      "objective/kl": 14.204147338867188,
+      "objective/non_score_reward": -0.7102073431015015,
+      "objective/rlhf_reward": -1.481579744552059,
+      "objective/scores": 0.33981246656813147,
+      "policy/approxkl_avg": 91.71839141845703,
+      "policy/clipfrac_avg": 1.0,
+      "policy/entropy_avg": 0.658203125,
+      "step": 132,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 2,
+      "val/ratio": 2.0028629302978516
+    },
+    {
+      "episode": 2144,
+      "epoch": 0.012845861643359576,
+      "loss/policy_avg": 0.27063143253326416,
+      "lr": 9.915005112474438e-06,
+      "objective/entropy": 180.4578857421875,
+      "objective/kl": 24.935741424560547,
+      "objective/non_score_reward": -1.2467870712280273,
+      "objective/rlhf_reward": -6.987148284912109,
+      "objective/scores": -0.5,
+      "policy/approxkl_avg": 138.678955078125,
+      "policy/clipfrac_avg": 0.75,
+      "policy/entropy_avg": 0.529296875,
+      "step": 133,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 0,
+      "val/ratio": 1.9967637062072754
+    },
+    {
+      "episode": 2160,
+      "epoch": 0.012941726282489125,
+      "loss/policy_avg": 0.1394023448228836,
+      "lr": 9.914366053169735e-06,
+      "objective/entropy": -29.98552703857422,
+      "objective/kl": 13.385698318481445,
+      "objective/non_score_reward": -0.6692849397659302,
+      "objective/rlhf_reward": -2.6771397292613983,
+      "objective/scores": 0.0,
+      "policy/approxkl_avg": 73.47354125976562,
+      "policy/clipfrac_avg": 1.25,
+      "policy/entropy_avg": 0.6171875,
+      "step": 134,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 3,
+      "val/ratio": 2.000580310821533
+    },
+    {
+      "episode": 2176,
+      "epoch": 0.013037590921618674,
+      "loss/policy_avg": 0.0048561920411884785,
+      "lr": 9.913726993865032e-06,
+      "objective/entropy": 88.89292907714844,
+      "objective/kl": 28.03160858154297,
+      "objective/non_score_reward": -1.4015804529190063,
+      "objective/rlhf_reward": -4.227719643203121,
+      "objective/scores": 0.34465054211822604,
+      "policy/approxkl_avg": 370.16766357421875,
+      "policy/clipfrac_avg": 1.0,
+      "policy/entropy_avg": 0.666015625,
+      "step": 135,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 2,
+      "val/ratio": 1.995047926902771
+    },
+    {
+      "episode": 2192,
+      "epoch": 0.013133455560748224,
+      "loss/policy_avg": 0.1565648913383484,
+      "lr": 9.913087934560329e-06,
+      "objective/entropy": 48.28108596801758,
+      "objective/kl": 22.514755249023438,
+      "objective/non_score_reward": -1.1257379055023193,
+      "objective/rlhf_reward": -0.10295168161392176,
+      "objective/scores": 1.1,
+      "policy/approxkl_avg": 272.63470458984375,
+      "policy/clipfrac_avg": 0.75,
+      "policy/entropy_avg": 0.58203125,
+      "step": 136,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 5,
+      "val/ratio": 2.0003504753112793
+    },
+    {
+      "episode": 2208,
+      "epoch": 0.013229320199877773,
+      "loss/policy_avg": 0.1350177526473999,
+      "lr": 9.912448875255624e-06,
+      "objective/entropy": 184.51797485351562,
+      "objective/kl": 30.795909881591797,
+      "objective/non_score_reward": -1.5397955179214478,
+      "objective/rlhf_reward": -8.159181594848633,
+      "objective/scores": -0.5,
+      "policy/approxkl_avg": 407.7762145996094,
+      "policy/clipfrac_avg": 1.0,
+      "policy/entropy_avg": 0.578125,
+      "step": 137,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 0,
+      "val/ratio": 1.997571349143982
+    },
+    {
+      "episode": 2224,
+      "epoch": 0.013325184839007322,
+      "loss/policy_avg": 0.2587956190109253,
+      "lr": 9.911809815950921e-06,
+      "objective/entropy": 14.785064697265625,
+      "objective/kl": 23.858671188354492,
+      "objective/non_score_reward": -1.1929335594177246,
+      "objective/rlhf_reward": -4.77173438668251,
+      "objective/scores": 0.0,
+      "policy/approxkl_avg": 258.4976501464844,
+      "policy/clipfrac_avg": 1.0,
+      "policy/entropy_avg": 0.58984375,
+      "step": 138,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 2,
+      "val/ratio": 1.996340036392212
+    },
+    {
+      "episode": 2240,
+      "epoch": 0.013421049478136871,
+      "loss/policy_avg": 0.03932709991931915,
+      "lr": 9.911170756646218e-06,
+      "objective/entropy": -88.26953887939453,
+      "objective/kl": 11.428003311157227,
+      "objective/non_score_reward": -0.5714001655578613,
+      "objective/rlhf_reward": -4.285600662231445,
+      "objective/scores": -0.5,
+      "policy/approxkl_avg": 41.424224853515625,
+      "policy/clipfrac_avg": 1.0,
+      "policy/entropy_avg": 0.775390625,
+      "step": 139,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 4,
+      "val/ratio": 2.0010673999786377
+    },
+    {
+      "episode": 2256,
+      "epoch": 0.01351691411726642,
+      "loss/policy_avg": 0.20215287804603577,
+      "lr": 9.910531697341515e-06,
+      "objective/entropy": 66.65933227539062,
+      "objective/kl": 20.929710388183594,
+      "objective/non_score_reward": -1.046485424041748,
+      "objective/rlhf_reward": 0.2140582442283634,
+      "objective/scores": 1.1,
+      "policy/approxkl_avg": 124.57344055175781,
+      "policy/clipfrac_avg": 0.75,
+      "policy/entropy_avg": 0.7421875,
+      "step": 140,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 3,
+      "val/ratio": 1.9977262020111084
+    },
+    {
+      "episode": 2272,
+      "epoch": 0.01361277875639597,
+      "loss/policy_avg": 0.5699018836021423,
+      "lr": 9.909892638036812e-06,
+      "objective/entropy": 10.020034790039062,
+      "objective/kl": 16.74536895751953,
+      "objective/non_score_reward": -0.8372684717178345,
+      "objective/rlhf_reward": 1.0509260237216953,
+      "objective/scores": 1.1,
+      "policy/approxkl_avg": 193.53329467773438,
+      "policy/clipfrac_avg": 0.75,
+      "policy/entropy_avg": 0.640625,
+      "step": 141,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 3,
+      "val/ratio": 1.9986698627471924
+    },
+    {
+      "episode": 2288,
+      "epoch": 0.013708643395525518,
+      "loss/policy_avg": 0.15781471133232117,
+      "lr": 9.909253578732107e-06,
+      "objective/entropy": -44.31187438964844,
+      "objective/kl": 30.742799758911133,
+      "objective/non_score_reward": -1.537139892578125,
+      "objective/rlhf_reward": -8.1485595703125,
+      "objective/scores": -0.5,
+      "policy/approxkl_avg": 158.5760498046875,
+      "policy/clipfrac_avg": 1.25,
+      "policy/entropy_avg": 0.73828125,
+      "step": 142,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 4,
+      "val/ratio": 1.9987871646881104
+    },
+    {
+      "episode": 2304,
+      "epoch": 0.013804508034655067,
+      "loss/policy_avg": 0.09526471048593521,
+      "lr": 9.908614519427404e-06,
+      "objective/entropy": 52.471221923828125,
+      "objective/kl": 19.550655364990234,
+      "objective/non_score_reward": -0.9775327444076538,
+      "objective/rlhf_reward": -2.4595327778771967,
+      "objective/scores": 0.36264953503719355,
+      "policy/approxkl_avg": 176.07566833496094,
+      "policy/clipfrac_avg": 1.0,
+      "policy/entropy_avg": 0.91796875,
+      "step": 143,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 3,
+      "val/ratio": 1.9970924854278564
+    },
+    {
+      "episode": 2320,
+      "epoch": 0.013900372673784616,
+      "loss/policy_avg": 0.03243420645594597,
+      "lr": 9.9079754601227e-06,
+      "objective/entropy": 128.66928100585938,
+      "objective/kl": 21.24932861328125,
+      "objective/non_score_reward": -1.0624663829803467,
+      "objective/rlhf_reward": -1.849865472316742,
+      "objective/scores": 0.6,
+      "policy/approxkl_avg": 188.10623168945312,
+      "policy/clipfrac_avg": 1.25,
+      "policy/entropy_avg": 0.470703125,
+      "step": 144,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 1,
+      "val/ratio": 1.9996135234832764
+    },
+    {
+      "episode": 2336,
+      "epoch": 0.013996237312914165,
+      "loss/policy_avg": 0.25250673294067383,
+      "lr": 9.907336400817996e-06,
+      "objective/entropy": 198.611083984375,
+      "objective/kl": 21.650169372558594,
+      "objective/non_score_reward": -1.0825085639953613,
+      "objective/rlhf_reward": -6.330034255981445,
+      "objective/scores": -0.5,
+      "policy/approxkl_avg": 130.0052490234375,
+      "policy/clipfrac_avg": 1.25,
+      "policy/entropy_avg": 0.892578125,
+      "step": 145,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 1,
+      "val/ratio": 1.9994676113128662
+    },
+    {
+      "episode": 2352,
+      "epoch": 0.014092101952043714,
+      "loss/policy_avg": -0.034668684005737305,
+      "lr": 9.906697341513293e-06,
+      "objective/entropy": -27.681907653808594,
+      "objective/kl": 26.334529876708984,
+      "objective/non_score_reward": -1.316726565361023,
+      "objective/rlhf_reward": -3.907656395171566,
+      "objective/scores": 0.33981246656813147,
+      "policy/approxkl_avg": 210.81857299804688,
+      "policy/clipfrac_avg": 1.25,
+      "policy/entropy_avg": 0.552734375,
+      "step": 146,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 4,
+      "val/ratio": 1.9972717761993408
+    },
+    {
+      "episode": 2368,
+      "epoch": 0.014187966591173263,
+      "loss/policy_avg": 0.12787118554115295,
+      "lr": 9.90605828220859e-06,
+      "objective/entropy": -43.486568450927734,
+      "objective/kl": 20.714540481567383,
+      "objective/non_score_reward": -1.0357270240783691,
+      "objective/rlhf_reward": -4.142907917499542,
+      "objective/scores": 0.0,
+      "policy/approxkl_avg": 86.59358978271484,
+      "policy/clipfrac_avg": 1.25,
+      "policy/entropy_avg": 0.673828125,
+      "step": 147,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 4,
+      "val/ratio": 1.999436378479004
+    },
+    {
+      "episode": 2384,
+      "epoch": 0.014283831230302812,
+      "loss/policy_avg": 0.03302329033613205,
+      "lr": 9.905419222903886e-06,
+      "objective/entropy": 173.50836181640625,
+      "objective/kl": 23.32859230041504,
+      "objective/non_score_reward": -1.1664297580718994,
+      "objective/rlhf_reward": -3.3402058220206925,
+      "objective/scores": 0.3313782131597591,
+      "policy/approxkl_avg": 224.7312774658203,
+      "policy/clipfrac_avg": 1.0,
+      "policy/entropy_avg": 0.759765625,
+      "step": 148,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 2,
+      "val/ratio": 1.9986599683761597
+    },
+    {
+      "episode": 2400,
+      "epoch": 0.01437969586943236,
+      "loss/policy_avg": 0.04894339293241501,
+      "lr": 9.904780163599183e-06,
+      "objective/entropy": 48.957122802734375,
+      "objective/kl": 25.618064880371094,
+      "objective/non_score_reward": -1.2809032201766968,
+      "objective/rlhf_reward": -3.699780781467525,
+      "objective/scores": 0.35595802480981553,
+      "policy/approxkl_avg": 230.8973388671875,
+      "policy/clipfrac_avg": 1.25,
+      "policy/entropy_avg": 0.576171875,
+      "step": 149,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 5,
+      "val/ratio": 1.9963643550872803
+    },
+    {
+      "episode": 2416,
+      "epoch": 0.01447556050856191,
+      "loss/policy_avg": 0.36329030990600586,
+      "lr": 9.904141104294478e-06,
+      "objective/entropy": 157.50445556640625,
+      "objective/kl": 23.93838882446289,
+      "objective/non_score_reward": -1.1969194412231445,
+      "objective/rlhf_reward": -4.787678003311157,
+      "objective/scores": 0.0,
+      "policy/approxkl_avg": 200.30621337890625,
+      "policy/clipfrac_avg": 1.0,
+      "policy/entropy_avg": 0.6484375,
+      "step": 150,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 1,
+      "val/ratio": 1.9953033924102783
+    },
+    {
+      "episode": 2432,
+      "epoch": 0.01457142514769146,
+      "loss/policy_avg": 0.3649589419364929,
+      "lr": 9.903502044989775e-06,
+      "objective/entropy": 213.43943786621094,
+      "objective/kl": 19.777463912963867,
+      "objective/non_score_reward": -0.9888731837272644,
+      "objective/rlhf_reward": -2.6138571410471494,
+      "objective/scores": 0.33540891336663825,
+      "policy/approxkl_avg": 81.07418060302734,
+      "policy/clipfrac_avg": 1.25,
+      "policy/entropy_avg": 0.677734375,
+      "step": 151,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 0,
+      "val/ratio": 2.0006325244903564
+    },
+    {
+      "episode": 2448,
+      "epoch": 0.01466728978682101,
+      "loss/policy_avg": 0.35868164896965027,
+      "lr": 9.902862985685072e-06,
+      "objective/entropy": 2.8281936645507812,
+      "objective/kl": 23.836688995361328,
+      "objective/non_score_reward": -1.191834568977356,
+      "objective/rlhf_reward": -6.767337799072266,
+      "objective/scores": -0.5,
+      "policy/approxkl_avg": 91.45345306396484,
+      "policy/clipfrac_avg": 0.5,
+      "policy/entropy_avg": 0.5703125,
+      "step": 152,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 4,
+      "val/ratio": 1.9970684051513672
+    },
+    {
+      "episode": 2464,
+      "epoch": 0.014763154425950558,
+      "loss/policy_avg": 0.07516692578792572,
+      "lr": 9.902223926380369e-06,
+      "objective/entropy": 171.71034240722656,
+      "objective/kl": 21.95725440979004,
+      "objective/non_score_reward": -1.097862720489502,
+      "objective/rlhf_reward": -4.3914510905742645,
+      "objective/scores": 0.0,
+      "policy/approxkl_avg": 221.61212158203125,
+      "policy/clipfrac_avg": 1.0,
+      "policy/entropy_avg": 0.53515625,
+      "step": 153,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 0,
+      "val/ratio": 1.9973832368850708
+    },
+    {
+      "episode": 2480,
+      "epoch": 0.014859019065080107,
+      "loss/policy_avg": 0.01844581961631775,
+      "lr": 9.901584867075666e-06,
+      "objective/entropy": 64.6440200805664,
+      "objective/kl": 17.436233520507812,
+      "objective/non_score_reward": -0.8718117475509644,
+      "objective/rlhf_reward": -5.487246990203857,
+      "objective/scores": -0.5,
+      "policy/approxkl_avg": 70.52445983886719,
+      "policy/clipfrac_avg": 1.25,
+      "policy/entropy_avg": 0.755859375,
+      "step": 154,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 4,
+      "val/ratio": 2.000408172607422
+    },
+    {
+      "episode": 2496,
+      "epoch": 0.014954883704209656,
+      "loss/policy_avg": 0.05899505689740181,
+      "lr": 9.900945807770961e-06,
+      "objective/entropy": 123.7980728149414,
+      "objective/kl": 25.07213592529297,
+      "objective/non_score_reward": -1.253606915473938,
+      "objective/rlhf_reward": -7.014427661895752,
+      "objective/scores": -0.5,
+      "policy/approxkl_avg": 88.28120422363281,
+      "policy/clipfrac_avg": 1.25,
+      "policy/entropy_avg": 0.51171875,
+      "step": 155,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 1,
+      "val/ratio": 2.000736951828003
+    },
+    {
+      "episode": 2512,
+      "epoch": 0.015050748343339205,
+      "loss/policy_avg": 0.4545804560184479,
+      "lr": 9.900306748466258e-06,
+      "objective/entropy": 10.871131896972656,
+      "objective/kl": 26.028505325317383,
+      "objective/non_score_reward": -1.3014252185821533,
+      "objective/rlhf_reward": -5.205701023340225,
+      "objective/scores": 0.0,
+      "policy/approxkl_avg": 217.84939575195312,
+      "policy/clipfrac_avg": 1.0,
+      "policy/entropy_avg": 0.5859375,
+      "step": 156,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 2,
+      "val/ratio": 1.9983347654342651
+    },
+    {
+      "episode": 2528,
+      "epoch": 0.015146612982468754,
+      "loss/policy_avg": 0.4191577136516571,
+      "lr": 9.899667689161555e-06,
+      "objective/entropy": 109.52301025390625,
+      "objective/kl": 27.344154357910156,
+      "objective/non_score_reward": -1.3672077655792236,
+      "objective/rlhf_reward": -7.4688310623168945,
+      "objective/scores": -0.5,
+      "policy/approxkl_avg": 89.15927124023438,
+      "policy/clipfrac_avg": 1.0,
+      "policy/entropy_avg": 0.62109375,
+      "step": 157,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 1,
+      "val/ratio": 1.999230146408081
+    },
+    {
+      "episode": 2544,
+      "epoch": 0.015242477621598303,
+      "loss/policy_avg": 0.1439390629529953,
+      "lr": 9.899028629856852e-06,
+      "objective/entropy": 220.28952026367188,
+      "objective/kl": 19.178768157958984,
+      "objective/non_score_reward": -0.9589384198188782,
+      "objective/rlhf_reward": -3.8357537388801575,
+      "objective/scores": 0.0,
+      "policy/approxkl_avg": 71.46617126464844,
+      "policy/clipfrac_avg": 1.0,
+      "policy/entropy_avg": 0.8828125,
+      "step": 158,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 2,
+      "val/ratio": 1.997718334197998
+    },
+    {
+      "episode": 2560,
+      "epoch": 0.015338342260727852,
+      "loss/policy_avg": 0.30983591079711914,
+      "lr": 9.898389570552149e-06,
+      "objective/entropy": 104.64752197265625,
+      "objective/kl": 27.657455444335938,
+      "objective/non_score_reward": -1.3828728199005127,
+      "objective/rlhf_reward": -3.7981575886408487,
+      "objective/scores": 0.43333333333333335,
+      "policy/approxkl_avg": 138.6593017578125,
+      "policy/clipfrac_avg": 1.0,
+      "policy/entropy_avg": 0.560546875,
+      "step": 159,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 3,
+      "val/ratio": 1.9977858066558838
+    },
+    {
+      "episode": 2576,
+      "epoch": 0.015434206899857401,
+      "loss/policy_avg": 0.07094208896160126,
+      "lr": 9.897750511247446e-06,
+      "objective/entropy": 90.89056396484375,
+      "objective/kl": 21.30394744873047,
+      "objective/non_score_reward": -1.065197467803955,
+      "objective/rlhf_reward": -4.260790050029755,
+      "objective/scores": 0.0,
+      "policy/approxkl_avg": 297.8140563964844,
+      "policy/clipfrac_avg": 1.25,
+      "policy/entropy_avg": 0.80859375,
+      "step": 160,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 3,
+      "val/ratio": 2.0023467540740967
+    },
+    {
+      "episode": 2592,
+      "epoch": 0.01553007153898695,
+      "loss/policy_avg": 0.16327880322933197,
+      "lr": 9.89711145194274e-06,
+      "objective/entropy": 100.65301513671875,
+      "objective/kl": 24.78557586669922,
+      "objective/non_score_reward": -1.239278793334961,
+      "objective/rlhf_reward": -3.1322867229309788,
+      "objective/scores": 0.4562071871080222,
+      "policy/approxkl_avg": 208.9399871826172,
+      "policy/clipfrac_avg": 1.5,
+      "policy/entropy_avg": 0.625,
+      "step": 161,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 2,
+      "val/ratio": 2.0041985511779785
+    },
+    {
+      "episode": 2608,
+      "epoch": 0.0156259361781165,
+      "loss/policy_avg": 0.216099351644516,
+      "lr": 9.896472392638038e-06,
+      "objective/entropy": 57.33685302734375,
+      "objective/kl": 29.304649353027344,
+      "objective/non_score_reward": -1.4652326107025146,
+      "objective/rlhf_reward": -7.8609299659729,
+      "objective/scores": -0.5,
+      "policy/approxkl_avg": 207.6790313720703,
+      "policy/clipfrac_avg": 1.25,
+      "policy/entropy_avg": 0.833984375,
+      "step": 162,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 2,
+      "val/ratio": 1.9980382919311523
+    },
+    {
+      "episode": 2624,
+      "epoch": 0.01572180081724605,
+      "loss/policy_avg": 0.0953613817691803,
+      "lr": 9.895833333333334e-06,
+      "objective/entropy": -133.8697967529297,
+      "objective/kl": 16.73604965209961,
+      "objective/non_score_reward": -0.83680260181427,
+      "objective/rlhf_reward": -3.347210466861725,
+      "objective/scores": 0.0,
+      "policy/approxkl_avg": 145.06759643554688,
+      "policy/clipfrac_avg": 1.25,
+      "policy/entropy_avg": 0.802734375,
+      "step": 163,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 6,
+      "val/ratio": 1.9988582134246826
+    },
+    {
+      "episode": 2640,
+      "epoch": 0.0158176654563756,
+      "loss/policy_avg": 0.42890581488609314,
+      "lr": 9.895194274028631e-06,
+      "objective/entropy": 180.38433837890625,
+      "objective/kl": 25.974315643310547,
+      "objective/non_score_reward": -1.2987157106399536,
+      "objective/rlhf_reward": -7.1948628425598145,
+      "objective/scores": -0.5,
+      "policy/approxkl_avg": 126.24812316894531,
+      "policy/clipfrac_avg": 1.0,
+      "policy/entropy_avg": 0.73046875,
+      "step": 164,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 1,
+      "val/ratio": 1.9987969398498535
+    },
+    {
+      "episode": 2656,
+      "epoch": 0.015913530095505148,
+      "loss/policy_avg": 0.1958284080028534,
+      "lr": 9.894555214723928e-06,
+      "objective/entropy": 174.94032287597656,
+      "objective/kl": 25.64311981201172,
+      "objective/non_score_reward": -1.282155990600586,
+      "objective/rlhf_reward": -3.786988249331146,
+      "objective/scores": 0.33540891336663825,
+      "policy/approxkl_avg": 86.50934600830078,
+      "policy/clipfrac_avg": 0.75,
+      "policy/entropy_avg": 0.705078125,
+      "step": 165,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 1,
+      "val/ratio": 1.9944665431976318
+    },
+    {
+      "episode": 2672,
+      "epoch": 0.016009394734634697,
+      "loss/policy_avg": 0.3368389904499054,
+      "lr": 9.893916155419225e-06,
+      "objective/entropy": 10.10284423828125,
+      "objective/kl": 24.560279846191406,
+      "objective/non_score_reward": -1.2280139923095703,
+      "objective/rlhf_reward": -6.912055969238281,
+      "objective/scores": -0.5,
+      "policy/approxkl_avg": 80.86394500732422,
+      "policy/clipfrac_avg": 1.0,
+      "policy/entropy_avg": 0.720703125,
+      "step": 166,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 3,
+      "val/ratio": 1.9986299276351929
+    },
+    {
+      "episode": 2688,
+      "epoch": 0.016105259373764245,
+      "loss/policy_avg": 0.11198948323726654,
+      "lr": 9.89327709611452e-06,
+      "objective/entropy": 161.62661743164062,
+      "objective/kl": 39.22645568847656,
+      "objective/non_score_reward": -1.9613227844238281,
+      "objective/rlhf_reward": -7.845290899276733,
+      "objective/scores": 0.0,
+      "policy/approxkl_avg": 164.2472381591797,
+      "policy/clipfrac_avg": 1.25,
+      "policy/entropy_avg": 0.841796875,
+      "step": 167,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 2,
+      "val/ratio": 2.000530958175659
+    },
+    {
+      "episode": 2704,
+      "epoch": 0.016201124012893794,
+      "loss/policy_avg": 0.19894596934318542,
+      "lr": 9.892638036809815e-06,
+      "objective/entropy": 137.1651153564453,
+      "objective/kl": 24.861934661865234,
+      "objective/non_score_reward": -1.2430968284606934,
+      "objective/rlhf_reward": -3.147558684619974,
+      "objective/scores": 0.4562071871080222,
+      "policy/approxkl_avg": 133.68283081054688,
+      "policy/clipfrac_avg": 1.25,
+      "policy/entropy_avg": 0.66796875,
+      "step": 168,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 2,
+      "val/ratio": 1.997298240661621
+    },
+    {
+      "episode": 2720,
+      "epoch": 0.016296988652023343,
+      "loss/policy_avg": 0.12182526290416718,
+      "lr": 9.891998977505112e-06,
+      "objective/entropy": 18.006725311279297,
+      "objective/kl": 17.92361068725586,
+      "objective/non_score_reward": -0.8961805701255798,
+      "objective/rlhf_reward": -2.134124155254707,
+      "objective/scores": 0.36264953503719355,
+      "policy/approxkl_avg": 192.45278930664062,
+      "policy/clipfrac_avg": 0.5,
+      "policy/entropy_avg": 0.583984375,
+      "step": 169,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 3,
+      "val/ratio": 2.0006425380706787
+    },
+    {
+      "episode": 2736,
+      "epoch": 0.016392853291152892,
+      "loss/policy_avg": -0.012196972966194153,
+      "lr": 9.89135991820041e-06,
+      "objective/entropy": 115.16173553466797,
+      "objective/kl": 21.65146827697754,
+      "objective/non_score_reward": -1.082573413848877,
+      "objective/rlhf_reward": -2.5054651006785145,
+      "objective/scores": 0.4562071871080222,
+      "policy/approxkl_avg": 145.09487915039062,
+      "policy/clipfrac_avg": 1.25,
+      "policy/entropy_avg": 0.71484375,
+      "step": 170,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 3,
+      "val/ratio": 1.9983795881271362
+    },
+    {
+      "episode": 2752,
+      "epoch": 0.01648871793028244,
+      "loss/policy_avg": 0.5326859951019287,
+      "lr": 9.890720858895706e-06,
+      "objective/entropy": 95.13655853271484,
+      "objective/kl": 24.00056266784668,
+      "objective/non_score_reward": -1.2000280618667603,
+      "objective/rlhf_reward": -4.800112426280975,
+      "objective/scores": 0.0,
+      "policy/approxkl_avg": 119.50138854980469,
+      "policy/clipfrac_avg": 1.25,
+      "policy/entropy_avg": 0.634765625,
+      "step": 171,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 2,
+      "val/ratio": 2.0006933212280273
+    },
+    {
+      "episode": 2768,
+      "epoch": 0.01658458256941199,
+      "loss/policy_avg": 0.3668867349624634,
+      "lr": 9.890081799591003e-06,
+      "objective/entropy": 132.36126708984375,
+      "objective/kl": 21.386262893676758,
+      "objective/non_score_reward": -1.0693132877349854,
+      "objective/rlhf_reward": -1.8772529125213624,
+      "objective/scores": 0.6,
+      "policy/approxkl_avg": 48.470794677734375,
+      "policy/clipfrac_avg": 1.0,
+      "policy/entropy_avg": 0.666015625,
+      "step": 172,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 1,
+      "val/ratio": 1.9946337938308716
+    },
+    {
+      "episode": 2784,
+      "epoch": 0.01668044720854154,
+      "loss/policy_avg": 0.460104763507843,
+      "lr": 9.8894427402863e-06,
+      "objective/entropy": 129.8038330078125,
+      "objective/kl": 25.860858917236328,
+      "objective/non_score_reward": -1.2930430173873901,
+      "objective/rlhf_reward": -3.656400167735752,
+      "objective/scores": 0.37894294565112985,
+      "policy/approxkl_avg": 146.58050537109375,
+      "policy/clipfrac_avg": 0.75,
+      "policy/entropy_avg": 0.5625,
+      "step": 173,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 2,
+      "val/ratio": 1.9949872493743896
+    },
+    {
+      "episode": 2800,
+      "epoch": 0.016776311847671088,
+      "loss/policy_avg": 0.11980315297842026,
+      "lr": 9.888803680981595e-06,
+      "objective/entropy": 173.85202026367188,
+      "objective/kl": 23.159679412841797,
+      "objective/non_score_reward": -1.1579840183258057,
+      "objective/rlhf_reward": -6.631936073303223,
+      "objective/scores": -0.5,
+      "policy/approxkl_avg": 84.56037902832031,
+      "policy/clipfrac_avg": 1.0,
+      "policy/entropy_avg": 0.7734375,
+      "step": 174,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 3,
+      "val/ratio": 1.9972279071807861
+    },
+    {
+      "episode": 2816,
+      "epoch": 0.016872176486800637,
+      "loss/policy_avg": 0.060305699706077576,
+      "lr": 9.888164621676892e-06,
+      "objective/entropy": 90.63494110107422,
+      "objective/kl": 23.04631805419922,
+      "objective/non_score_reward": -1.1523159742355347,
+      "objective/rlhf_reward": -4.609263688325882,
+      "objective/scores": 0.0,
+      "policy/approxkl_avg": 204.8768310546875,
+      "policy/clipfrac_avg": 1.25,
+      "policy/entropy_avg": 0.564453125,
+      "step": 175,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 4,
+      "val/ratio": 1.9992055892944336
+    },
+    {
+      "episode": 2832,
+      "epoch": 0.016968041125930186,
+      "loss/policy_avg": 0.5130124092102051,
+      "lr": 9.887525562372189e-06,
+      "objective/entropy": 66.25984191894531,
+      "objective/kl": 32.70683288574219,
+      "objective/non_score_reward": -1.635341763496399,
+      "objective/rlhf_reward": -6.541367173194885,
+      "objective/scores": 0.0,
+      "policy/approxkl_avg": 265.0827941894531,
+      "policy/clipfrac_avg": 1.0,
+      "policy/entropy_avg": 0.609375,
+      "step": 176,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 1,
+      "val/ratio": 1.996222734451294
+    },
+    {
+      "episode": 2848,
+      "epoch": 0.017063905765059735,
+      "loss/policy_avg": 0.008577877655625343,
+      "lr": 9.886886503067486e-06,
+      "objective/entropy": -118.17359924316406,
+      "objective/kl": 21.02519989013672,
+      "objective/non_score_reward": -1.051259994506836,
+      "objective/rlhf_reward": -4.205039799213409,
+      "objective/scores": 0.0,
+      "policy/approxkl_avg": 260.4126892089844,
+      "policy/clipfrac_avg": 1.25,
+      "policy/entropy_avg": 0.66015625,
+      "step": 177,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 6,
+      "val/ratio": 2.000894069671631
+    },
+    {
+      "episode": 2864,
+      "epoch": 0.017159770404189284,
+      "loss/policy_avg": 0.18238189816474915,
+      "lr": 9.886247443762783e-06,
+      "objective/entropy": 130.0546875,
+      "objective/kl": 32.371009826660156,
+      "objective/non_score_reward": -1.6185506582260132,
+      "objective/rlhf_reward": -8.474203109741211,
+      "objective/scores": -0.5,
+      "policy/approxkl_avg": 228.2266387939453,
+      "policy/clipfrac_avg": 1.0,
+      "policy/entropy_avg": 0.7890625,
+      "step": 178,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 2,
+      "val/ratio": 1.9942381381988525
+    },
+    {
+      "episode": 2880,
+      "epoch": 0.017255635043318833,
+      "loss/policy_avg": 0.18286140263080597,
+      "lr": 9.88560838445808e-06,
+      "objective/entropy": -10.639881134033203,
+      "objective/kl": 29.253890991210938,
+      "objective/non_score_reward": -1.462694525718689,
+      "objective/rlhf_reward": -7.850778102874756,
+      "objective/scores": -0.5,
+      "policy/approxkl_avg": 148.62832641601562,
+      "policy/clipfrac_avg": 1.0,
+      "policy/entropy_avg": 0.798828125,
+      "step": 179,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 4,
+      "val/ratio": 1.9988160133361816
+    },
+    {
+      "episode": 2896,
+      "epoch": 0.017351499682448382,
+      "loss/policy_avg": 0.07891340553760529,
+      "lr": 9.884969325153375e-06,
+      "objective/entropy": -120.97007751464844,
+      "objective/kl": 21.97601890563965,
+      "objective/non_score_reward": -1.0988008975982666,
+      "objective/rlhf_reward": -2.2724974177041393,
+      "objective/scores": 0.5306765580733931,
+      "policy/approxkl_avg": 200.60455322265625,
+      "policy/clipfrac_avg": 1.25,
+      "policy/entropy_avg": 0.533203125,
+      "step": 180,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 5,
+      "val/ratio": 2.000365972518921
+    },
+    {
+      "episode": 2912,
+      "epoch": 0.01744736432157793,
+      "loss/policy_avg": 0.06744587421417236,
+      "lr": 9.884330265848671e-06,
+      "objective/entropy": 73.97216796875,
+      "objective/kl": 19.66523551940918,
+      "objective/non_score_reward": -0.9832619428634644,
+      "objective/rlhf_reward": -5.933047771453857,
+      "objective/scores": -0.5,
+      "policy/approxkl_avg": 168.08172607421875,
+      "policy/clipfrac_avg": 1.0,
+      "policy/entropy_avg": 0.603515625,
+      "step": 181,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 3,
+      "val/ratio": 1.997870683670044
+    },
+    {
+      "episode": 2928,
+      "epoch": 0.01754322896070748,
+      "loss/policy_avg": 0.12424597889184952,
+      "lr": 9.883691206543968e-06,
+      "objective/entropy": 77.82262420654297,
+      "objective/kl": 21.0150146484375,
+      "objective/non_score_reward": -1.050750732421875,
+      "objective/rlhf_reward": 0.1969969511032108,
+      "objective/scores": 1.1,
+      "policy/approxkl_avg": 109.60333251953125,
+      "policy/clipfrac_avg": 1.25,
+      "policy/entropy_avg": 0.693359375,
+      "step": 182,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 3,
+      "val/ratio": 1.996559500694275
+    },
+    {
+      "episode": 2944,
+      "epoch": 0.01763909359983703,
+      "loss/policy_avg": 0.2606327533721924,
+      "lr": 9.883052147239265e-06,
+      "objective/entropy": 172.60293579101562,
+      "objective/kl": 29.473426818847656,
+      "objective/non_score_reward": -1.4736714363098145,
+      "objective/rlhf_reward": -5.894685626029968,
+      "objective/scores": 0.0,
+      "policy/approxkl_avg": 194.51976013183594,
+      "policy/clipfrac_avg": 0.75,
+      "policy/entropy_avg": 0.818359375,
+      "step": 183,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 1,
+      "val/ratio": 1.9962568283081055
+    },
+    {
+      "episode": 2960,
+      "epoch": 0.017734958238966578,
+      "loss/policy_avg": 0.10910254716873169,
+      "lr": 9.882413087934562e-06,
+      "objective/entropy": 240.20162963867188,
+      "objective/kl": 15.176373481750488,
+      "objective/non_score_reward": -0.7588187456130981,
+      "objective/rlhf_reward": 1.3647250771522526,
+      "objective/scores": 1.1,
+      "policy/approxkl_avg": 43.88645935058594,
+      "policy/clipfrac_avg": 1.0,
+      "policy/entropy_avg": 0.8203125,
+      "step": 184,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 1,
+      "val/ratio": 2.0006511211395264
+    },
+    {
+      "episode": 2976,
+      "epoch": 0.017830822878096127,
+      "loss/policy_avg": 0.5547807812690735,
+      "lr": 9.881774028629857e-06,
+      "objective/entropy": 85.18072509765625,
+      "objective/kl": 21.537092208862305,
+      "objective/non_score_reward": -1.0768545866012573,
+      "objective/rlhf_reward": -6.307418346405029,
+      "objective/scores": -0.5,
+      "policy/approxkl_avg": 161.64654541015625,
+      "policy/clipfrac_avg": 0.75,
+      "policy/entropy_avg": 0.75390625,
+      "step": 185,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 2,
+      "val/ratio": 1.9996278285980225
+    },
+    {
+      "episode": 2992,
+      "epoch": 0.017926687517225676,
+      "loss/policy_avg": 0.3181283175945282,
+      "lr": 9.881134969325154e-06,
+      "objective/entropy": 129.28257751464844,
+      "objective/kl": 28.59075927734375,
+      "objective/non_score_reward": -1.4295378923416138,
+      "objective/rlhf_reward": -5.718151569366455,
+      "objective/scores": 0.0,
+      "policy/approxkl_avg": 53.536468505859375,
+      "policy/clipfrac_avg": 1.0,
+      "policy/entropy_avg": 0.705078125,
+      "step": 186,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 2,
+      "val/ratio": 1.998319149017334
+    },
+    {
+      "episode": 3008,
+      "epoch": 0.018022552156355228,
+      "loss/policy_avg": 0.37268152832984924,
+      "lr": 9.880495910020451e-06,
+      "objective/entropy": 183.25308227539062,
+      "objective/kl": 29.692989349365234,
+      "objective/non_score_reward": -1.484649419784546,
+      "objective/rlhf_reward": -7.938598155975342,
+      "objective/scores": -0.5,
+      "policy/approxkl_avg": 91.56600952148438,
+      "policy/clipfrac_avg": 1.0,
+      "policy/entropy_avg": 0.599609375,
+      "step": 187,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 2,
+      "val/ratio": 1.9983760118484497
+    },
+    {
+      "episode": 3024,
+      "epoch": 0.018118416795484777,
+      "loss/policy_avg": 0.7035294771194458,
+      "lr": 9.879856850715748e-06,
+      "objective/entropy": -141.20687866210938,
+      "objective/kl": 16.28227996826172,
+      "objective/non_score_reward": -0.8141138553619385,
+      "objective/rlhf_reward": -5.256455421447754,
+      "objective/scores": -0.5,
+      "policy/approxkl_avg": 39.17454528808594,
+      "policy/clipfrac_avg": 0.75,
+      "policy/entropy_avg": 0.912109375,
+      "step": 188,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 5,
+      "val/ratio": 2.000697612762451
+    },
+    {
+      "episode": 3040,
+      "epoch": 0.018214281434614326,
+      "loss/policy_avg": 0.23234151303768158,
+      "lr": 9.879217791411043e-06,
+      "objective/entropy": 41.34138107299805,
+      "objective/kl": 27.16008758544922,
+      "objective/non_score_reward": -1.3580043315887451,
+      "objective/rlhf_reward": -4.05341557511459,
+      "objective/scores": 0.34465054211822604,
+      "policy/approxkl_avg": 271.9233093261719,
+      "policy/clipfrac_avg": 0.75,
+      "policy/entropy_avg": 0.51171875,
+      "step": 189,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 1,
+      "val/ratio": 1.999565839767456
+    },
+    {
+      "episode": 3056,
+      "epoch": 0.018310146073743875,
+      "loss/policy_avg": 0.07211380451917648,
+      "lr": 9.87857873210634e-06,
+      "objective/entropy": 20.789365768432617,
+      "objective/kl": 19.183855056762695,
+      "objective/non_score_reward": -0.9591927528381348,
+      "objective/rlhf_reward": -5.836771011352539,
+      "objective/scores": -0.5,
+      "policy/approxkl_avg": 108.85220336914062,
+      "policy/clipfrac_avg": 0.5,
+      "policy/entropy_avg": 0.6328125,
+      "step": 190,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 3,
+      "val/ratio": 2.0007762908935547
+    },
+    {
+      "episode": 3072,
+      "epoch": 0.018406010712873424,
+      "loss/policy_avg": 0.7360332012176514,
+      "lr": 9.877939672801637e-06,
+      "objective/entropy": 219.01002502441406,
+      "objective/kl": 30.353984832763672,
+      "objective/non_score_reward": -1.5176992416381836,
+      "objective/rlhf_reward": -4.692195155707699,
+      "objective/scores": 0.34465054211822604,
+      "policy/approxkl_avg": 60.275230407714844,
+      "policy/clipfrac_avg": 1.0,
+      "policy/entropy_avg": 0.73046875,
+      "step": 191,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 1,
+      "val/ratio": 1.9968408346176147
+    },
+    {
+      "episode": 3088,
+      "epoch": 0.018501875352002973,
+      "loss/policy_avg": 0.7660672664642334,
+      "lr": 9.877300613496934e-06,
+      "objective/entropy": 192.5721435546875,
+      "objective/kl": 18.974138259887695,
+      "objective/non_score_reward": -0.948706865310669,
+      "objective/rlhf_reward": -5.794827461242676,
+      "objective/scores": -0.5,
+      "policy/approxkl_avg": 208.0977783203125,
+      "policy/clipfrac_avg": 0.75,
+      "policy/entropy_avg": 0.568359375,
+      "step": 192,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 0,
+      "val/ratio": 1.996788740158081
+    },
+    {
+      "episode": 3104,
+      "epoch": 0.018597739991132522,
+      "loss/policy_avg": 0.4530583620071411,
+      "lr": 9.876661554192229e-06,
+      "objective/entropy": 185.2235107421875,
+      "objective/kl": 24.102296829223633,
+      "objective/non_score_reward": -1.2051149606704712,
+      "objective/rlhf_reward": -0.4204598426818844,
+      "objective/scores": 1.1,
+      "policy/approxkl_avg": 34.94757080078125,
+      "policy/clipfrac_avg": 1.0,
+      "policy/entropy_avg": 0.728515625,
+      "step": 193,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 2,
+      "val/ratio": 1.9967865943908691
+    },
+    {
+      "episode": 3120,
+      "epoch": 0.01869360463026207,
+      "loss/policy_avg": 0.03487266227602959,
+      "lr": 9.876022494887526e-06,
+      "objective/entropy": 183.6469268798828,
+      "objective/kl": 20.133817672729492,
+      "objective/non_score_reward": -1.0066908597946167,
+      "objective/rlhf_reward": -4.026763558387756,
+      "objective/scores": 0.0,
+      "policy/approxkl_avg": 168.301025390625,
+      "policy/clipfrac_avg": 1.25,
+      "policy/entropy_avg": 0.7578125,
+      "step": 194,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 1,
+      "val/ratio": 2.0027122497558594
+    },
+    {
+      "episode": 3136,
+      "epoch": 0.01878946926939162,
+      "loss/policy_avg": -0.029073666781187057,
+      "lr": 9.875383435582823e-06,
+      "objective/entropy": 138.25656127929688,
+      "objective/kl": 18.322010040283203,
+      "objective/non_score_reward": -0.9161005020141602,
+      "objective/rlhf_reward": -5.664402008056641,
+      "objective/scores": -0.5,
+      "policy/approxkl_avg": 60.4761848449707,
+      "policy/clipfrac_avg": 1.25,
+      "policy/entropy_avg": 0.673828125,
+      "step": 195,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 2,
+      "val/ratio": 1.9991329908370972
+    },
+    {
+      "episode": 3152,
+      "epoch": 0.01888533390852117,
+      "loss/policy_avg": 0.14693962037563324,
+      "lr": 9.87474437627812e-06,
+      "objective/entropy": 71.7930908203125,
+      "objective/kl": 19.49433135986328,
+      "objective/non_score_reward": -0.9747166633605957,
+      "objective/rlhf_reward": -3.8988667130470276,
+      "objective/scores": 0.0,
+      "policy/approxkl_avg": 126.81082153320312,
+      "policy/clipfrac_avg": 1.25,
+      "policy/entropy_avg": 0.767578125,
+      "step": 196,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 3,
+      "val/ratio": 1.9977989196777344
+    },
+    {
+      "episode": 3168,
+      "epoch": 0.018981198547650718,
+      "loss/policy_avg": 0.6557031869888306,
+      "lr": 9.874105316973416e-06,
+      "objective/entropy": -9.503684997558594,
+      "objective/kl": 21.540775299072266,
+      "objective/non_score_reward": -1.0770388841629028,
+      "objective/rlhf_reward": -2.9295533085740626,
+      "objective/scores": 0.34465054211822604,
+      "policy/approxkl_avg": 100.91127014160156,
+      "policy/clipfrac_avg": 0.75,
+      "policy/entropy_avg": 0.619140625,
+      "step": 197,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 4,
+      "val/ratio": 1.9948209524154663
+    },
+    {
+      "episode": 3184,
+      "epoch": 0.019077063186780267,
+      "loss/policy_avg": 0.23461255431175232,
+      "lr": 9.873466257668712e-06,
+      "objective/entropy": -49.81024932861328,
+      "objective/kl": 20.112146377563477,
+      "objective/non_score_reward": -1.0056073665618896,
+      "objective/rlhf_reward": -6.0224289894104,
+      "objective/scores": -0.5,
+      "policy/approxkl_avg": 135.8631134033203,
+      "policy/clipfrac_avg": 1.0,
+      "policy/entropy_avg": 0.58203125,
+      "step": 198,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 3,
+      "val/ratio": 1.999420404434204
+    },
+    {
+      "episode": 3200,
+      "epoch": 0.019172927825909816,
+      "loss/policy_avg": -0.5007312297821045,
+      "lr": 9.872827198364009e-06,
+      "objective/entropy": -25.718414306640625,
+      "objective/kl": 15.317103385925293,
+      "objective/non_score_reward": -0.7658551931381226,
+      "objective/rlhf_reward": -5.06342077255249,
+      "objective/scores": -0.5,
+      "policy/approxkl_avg": 154.1348876953125,
+      "policy/clipfrac_avg": 1.75,
+      "policy/entropy_avg": 0.796875,
+      "step": 199,
+      "val/clipfrac_avg": 0.0,
+      "val/num_eos_tokens": 3,
+      "val/ratio": 2.01686954498291
+    }
+  ],
+  "logging_steps": 500,
+  "max_steps": 7824,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3.0,
+  "save_steps": 200,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0,
+  "train_batch_size": null,
+  "trial_name": null,
+  "trial_params": null
+}