| { | |
| "best_metric": null, | |
| "best_model_checkpoint": null, | |
| "epoch": 1.0, | |
| "eval_steps": 500, | |
| "global_step": 313, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "epoch": 0.003194888178913738, | |
| "grad_norm": 39.256392372009124, | |
| "learning_rate": 5.3191489361702125e-09, | |
| "logits/chosen": -3.34375, | |
| "logits/rejected": -3.609375, | |
| "logps/chosen": -40.5, | |
| "logps/rejected": -49.25, | |
| "loss": 0.6914, | |
| "rewards/accuracies": 0.0, | |
| "rewards/chosen": 0.0, | |
| "rewards/margins": 0.0, | |
| "rewards/rejected": 0.0, | |
| "step": 1 | |
| }, | |
| { | |
| "epoch": 0.03194888178913738, | |
| "grad_norm": 32.81412666334126, | |
| "learning_rate": 5.3191489361702123e-08, | |
| "logits/chosen": -3.09375, | |
| "logits/rejected": -2.859375, | |
| "logps/chosen": -41.25, | |
| "logps/rejected": -38.25, | |
| "loss": 0.6925, | |
| "rewards/accuracies": 0.0972222238779068, | |
| "rewards/chosen": -0.001739501953125, | |
| "rewards/margins": -0.00017261505126953125, | |
| "rewards/rejected": -0.00156402587890625, | |
| "step": 10 | |
| }, | |
| { | |
| "epoch": 0.06389776357827476, | |
| "grad_norm": 31.71771727943446, | |
| "learning_rate": 1.0638297872340425e-07, | |
| "logits/chosen": -3.03125, | |
| "logits/rejected": -2.8125, | |
| "logps/chosen": -42.0, | |
| "logps/rejected": -36.75, | |
| "loss": 0.6887, | |
| "rewards/accuracies": 0.15000000596046448, | |
| "rewards/chosen": -0.0001583099365234375, | |
| "rewards/margins": 0.0050048828125, | |
| "rewards/rejected": -0.005157470703125, | |
| "step": 20 | |
| }, | |
| { | |
| "epoch": 0.09584664536741214, | |
| "grad_norm": 33.89035773805732, | |
| "learning_rate": 1.5957446808510638e-07, | |
| "logits/chosen": -2.859375, | |
| "logits/rejected": -3.0625, | |
| "logps/chosen": -44.5, | |
| "logps/rejected": -42.25, | |
| "loss": 0.6852, | |
| "rewards/accuracies": 0.32499998807907104, | |
| "rewards/chosen": 0.0032958984375, | |
| "rewards/margins": 0.0284423828125, | |
| "rewards/rejected": -0.025146484375, | |
| "step": 30 | |
| }, | |
| { | |
| "epoch": 0.12779552715654952, | |
| "grad_norm": 33.79231622581764, | |
| "learning_rate": 2.127659574468085e-07, | |
| "logits/chosen": -3.109375, | |
| "logits/rejected": -2.921875, | |
| "logps/chosen": -47.5, | |
| "logps/rejected": -37.75, | |
| "loss": 0.6768, | |
| "rewards/accuracies": 0.26249998807907104, | |
| "rewards/chosen": 0.03759765625, | |
| "rewards/margins": 0.029296875, | |
| "rewards/rejected": 0.00830078125, | |
| "step": 40 | |
| }, | |
| { | |
| "epoch": 0.1597444089456869, | |
| "grad_norm": 33.962249009797624, | |
| "learning_rate": 2.659574468085106e-07, | |
| "logits/chosen": -3.046875, | |
| "logits/rejected": -2.828125, | |
| "logps/chosen": -39.0, | |
| "logps/rejected": -35.25, | |
| "loss": 0.6589, | |
| "rewards/accuracies": 0.32499998807907104, | |
| "rewards/chosen": 0.04638671875, | |
| "rewards/margins": 0.08740234375, | |
| "rewards/rejected": -0.040771484375, | |
| "step": 50 | |
| }, | |
| { | |
| "epoch": 0.19169329073482427, | |
| "grad_norm": 31.06082261346926, | |
| "learning_rate": 3.1914893617021275e-07, | |
| "logits/chosen": -2.9375, | |
| "logits/rejected": -2.890625, | |
| "logps/chosen": -36.0, | |
| "logps/rejected": -41.5, | |
| "loss": 0.6679, | |
| "rewards/accuracies": 0.36250001192092896, | |
| "rewards/chosen": 0.0947265625, | |
| "rewards/margins": 0.087890625, | |
| "rewards/rejected": 0.00640869140625, | |
| "step": 60 | |
| }, | |
| { | |
| "epoch": 0.22364217252396165, | |
| "grad_norm": 33.61384586606748, | |
| "learning_rate": 3.7234042553191484e-07, | |
| "logits/chosen": -3.25, | |
| "logits/rejected": -2.796875, | |
| "logps/chosen": -54.0, | |
| "logps/rejected": -39.5, | |
| "loss": 0.6207, | |
| "rewards/accuracies": 0.4375, | |
| "rewards/chosen": 0.054931640625, | |
| "rewards/margins": 0.296875, | |
| "rewards/rejected": -0.2421875, | |
| "step": 70 | |
| }, | |
| { | |
| "epoch": 0.25559105431309903, | |
| "grad_norm": 29.13019721736027, | |
| "learning_rate": 4.25531914893617e-07, | |
| "logits/chosen": -3.28125, | |
| "logits/rejected": -3.0, | |
| "logps/chosen": -49.0, | |
| "logps/rejected": -47.0, | |
| "loss": 0.6504, | |
| "rewards/accuracies": 0.42500001192092896, | |
| "rewards/chosen": -0.140625, | |
| "rewards/margins": 0.2734375, | |
| "rewards/rejected": -0.4140625, | |
| "step": 80 | |
| }, | |
| { | |
| "epoch": 0.28753993610223644, | |
| "grad_norm": 40.06738628161476, | |
| "learning_rate": 4.787234042553192e-07, | |
| "logits/chosen": -3.390625, | |
| "logits/rejected": -2.90625, | |
| "logps/chosen": -47.75, | |
| "logps/rejected": -38.25, | |
| "loss": 0.5986, | |
| "rewards/accuracies": 0.3125, | |
| "rewards/chosen": -0.00396728515625, | |
| "rewards/margins": 0.265625, | |
| "rewards/rejected": -0.26953125, | |
| "step": 90 | |
| }, | |
| { | |
| "epoch": 0.3194888178913738, | |
| "grad_norm": 26.980268633866878, | |
| "learning_rate": 4.964497041420119e-07, | |
| "logits/chosen": -3.1875, | |
| "logits/rejected": -3.0, | |
| "logps/chosen": -43.25, | |
| "logps/rejected": -43.25, | |
| "loss": 0.6019, | |
| "rewards/accuracies": 0.4124999940395355, | |
| "rewards/chosen": -0.146484375, | |
| "rewards/margins": 0.30859375, | |
| "rewards/rejected": -0.455078125, | |
| "step": 100 | |
| }, | |
| { | |
| "epoch": 0.3514376996805112, | |
| "grad_norm": 42.49654986796296, | |
| "learning_rate": 4.905325443786982e-07, | |
| "logits/chosen": -2.984375, | |
| "logits/rejected": -3.140625, | |
| "logps/chosen": -45.75, | |
| "logps/rejected": -53.5, | |
| "loss": 0.5866, | |
| "rewards/accuracies": 0.36250001192092896, | |
| "rewards/chosen": -0.1298828125, | |
| "rewards/margins": 0.546875, | |
| "rewards/rejected": -0.67578125, | |
| "step": 110 | |
| }, | |
| { | |
| "epoch": 0.38338658146964855, | |
| "grad_norm": 33.45575091341326, | |
| "learning_rate": 4.846153846153846e-07, | |
| "logits/chosen": -3.375, | |
| "logits/rejected": -3.109375, | |
| "logps/chosen": -55.75, | |
| "logps/rejected": -53.25, | |
| "loss": 0.5792, | |
| "rewards/accuracies": 0.4375, | |
| "rewards/chosen": -0.2890625, | |
| "rewards/margins": 0.59375, | |
| "rewards/rejected": -0.8828125, | |
| "step": 120 | |
| }, | |
| { | |
| "epoch": 0.41533546325878595, | |
| "grad_norm": 35.64973041956874, | |
| "learning_rate": 4.78698224852071e-07, | |
| "logits/chosen": -3.09375, | |
| "logits/rejected": -2.96875, | |
| "logps/chosen": -45.25, | |
| "logps/rejected": -47.0, | |
| "loss": 0.5991, | |
| "rewards/accuracies": 0.32499998807907104, | |
| "rewards/chosen": -0.255859375, | |
| "rewards/margins": 0.5, | |
| "rewards/rejected": -0.75390625, | |
| "step": 130 | |
| }, | |
| { | |
| "epoch": 0.4472843450479233, | |
| "grad_norm": 33.67312459569186, | |
| "learning_rate": 4.727810650887574e-07, | |
| "logits/chosen": -3.328125, | |
| "logits/rejected": -3.046875, | |
| "logps/chosen": -48.25, | |
| "logps/rejected": -52.75, | |
| "loss": 0.544, | |
| "rewards/accuracies": 0.44999998807907104, | |
| "rewards/chosen": -0.2412109375, | |
| "rewards/margins": 0.8359375, | |
| "rewards/rejected": -1.078125, | |
| "step": 140 | |
| }, | |
| { | |
| "epoch": 0.4792332268370607, | |
| "grad_norm": 16.223843544431134, | |
| "learning_rate": 4.668639053254438e-07, | |
| "logits/chosen": -3.078125, | |
| "logits/rejected": -3.03125, | |
| "logps/chosen": -48.25, | |
| "logps/rejected": -54.25, | |
| "loss": 0.5827, | |
| "rewards/accuracies": 0.44999998807907104, | |
| "rewards/chosen": -0.4609375, | |
| "rewards/margins": 0.8046875, | |
| "rewards/rejected": -1.265625, | |
| "step": 150 | |
| }, | |
| { | |
| "epoch": 0.5111821086261981, | |
| "grad_norm": 30.865326556253784, | |
| "learning_rate": 4.6094674556213014e-07, | |
| "logits/chosen": -2.984375, | |
| "logits/rejected": -2.75, | |
| "logps/chosen": -46.75, | |
| "logps/rejected": -51.0, | |
| "loss": 0.5442, | |
| "rewards/accuracies": 0.4749999940395355, | |
| "rewards/chosen": -0.4296875, | |
| "rewards/margins": 0.875, | |
| "rewards/rejected": -1.3046875, | |
| "step": 160 | |
| }, | |
| { | |
| "epoch": 0.5431309904153354, | |
| "grad_norm": 22.81312034142999, | |
| "learning_rate": 4.5502958579881655e-07, | |
| "logits/chosen": -3.09375, | |
| "logits/rejected": -2.8125, | |
| "logps/chosen": -44.0, | |
| "logps/rejected": -49.5, | |
| "loss": 0.5765, | |
| "rewards/accuracies": 0.42500001192092896, | |
| "rewards/chosen": -0.4609375, | |
| "rewards/margins": 0.8203125, | |
| "rewards/rejected": -1.28125, | |
| "step": 170 | |
| }, | |
| { | |
| "epoch": 0.5750798722044729, | |
| "grad_norm": 50.05578499574947, | |
| "learning_rate": 4.491124260355029e-07, | |
| "logits/chosen": -3.140625, | |
| "logits/rejected": -2.921875, | |
| "logps/chosen": -49.25, | |
| "logps/rejected": -56.25, | |
| "loss": 0.5266, | |
| "rewards/accuracies": 0.4375, | |
| "rewards/chosen": -0.42578125, | |
| "rewards/margins": 1.1796875, | |
| "rewards/rejected": -1.609375, | |
| "step": 180 | |
| }, | |
| { | |
| "epoch": 0.6070287539936102, | |
| "grad_norm": 20.57671100687128, | |
| "learning_rate": 4.4319526627218936e-07, | |
| "logits/chosen": -3.3125, | |
| "logits/rejected": -2.734375, | |
| "logps/chosen": -56.25, | |
| "logps/rejected": -45.5, | |
| "loss": 0.5544, | |
| "rewards/accuracies": 0.4000000059604645, | |
| "rewards/chosen": -0.5234375, | |
| "rewards/margins": 0.70703125, | |
| "rewards/rejected": -1.2265625, | |
| "step": 190 | |
| }, | |
| { | |
| "epoch": 0.6389776357827476, | |
| "grad_norm": 35.51097905099775, | |
| "learning_rate": 4.372781065088757e-07, | |
| "logits/chosen": -3.09375, | |
| "logits/rejected": -2.890625, | |
| "logps/chosen": -49.75, | |
| "logps/rejected": -53.0, | |
| "loss": 0.6029, | |
| "rewards/accuracies": 0.38749998807907104, | |
| "rewards/chosen": -0.5546875, | |
| "rewards/margins": 0.83984375, | |
| "rewards/rejected": -1.390625, | |
| "step": 200 | |
| }, | |
| { | |
| "epoch": 0.670926517571885, | |
| "grad_norm": 17.899965646039988, | |
| "learning_rate": 4.313609467455621e-07, | |
| "logits/chosen": -3.140625, | |
| "logits/rejected": -3.015625, | |
| "logps/chosen": -44.5, | |
| "logps/rejected": -50.0, | |
| "loss": 0.5345, | |
| "rewards/accuracies": 0.38749998807907104, | |
| "rewards/chosen": -0.54296875, | |
| "rewards/margins": 0.796875, | |
| "rewards/rejected": -1.34375, | |
| "step": 210 | |
| }, | |
| { | |
| "epoch": 0.7028753993610224, | |
| "grad_norm": 21.641681357355413, | |
| "learning_rate": 4.2544378698224847e-07, | |
| "logits/chosen": -2.828125, | |
| "logits/rejected": -2.890625, | |
| "logps/chosen": -48.5, | |
| "logps/rejected": -53.75, | |
| "loss": 0.5107, | |
| "rewards/accuracies": 0.4000000059604645, | |
| "rewards/chosen": -0.546875, | |
| "rewards/margins": 0.92578125, | |
| "rewards/rejected": -1.4765625, | |
| "step": 220 | |
| }, | |
| { | |
| "epoch": 0.7348242811501597, | |
| "grad_norm": 26.32082815714621, | |
| "learning_rate": 4.195266272189349e-07, | |
| "logits/chosen": -3.15625, | |
| "logits/rejected": -2.828125, | |
| "logps/chosen": -57.0, | |
| "logps/rejected": -54.25, | |
| "loss": 0.574, | |
| "rewards/accuracies": 0.42500001192092896, | |
| "rewards/chosen": -0.83984375, | |
| "rewards/margins": 0.97265625, | |
| "rewards/rejected": -1.8125, | |
| "step": 230 | |
| }, | |
| { | |
| "epoch": 0.7667731629392971, | |
| "grad_norm": 25.67855359237542, | |
| "learning_rate": 4.1360946745562133e-07, | |
| "logits/chosen": -3.0625, | |
| "logits/rejected": -2.609375, | |
| "logps/chosen": -47.75, | |
| "logps/rejected": -47.5, | |
| "loss": 0.5904, | |
| "rewards/accuracies": 0.32499998807907104, | |
| "rewards/chosen": -0.7109375, | |
| "rewards/margins": 0.625, | |
| "rewards/rejected": -1.3359375, | |
| "step": 240 | |
| }, | |
| { | |
| "epoch": 0.7987220447284346, | |
| "grad_norm": 9.719975462174387, | |
| "learning_rate": 4.076923076923077e-07, | |
| "logits/chosen": -3.25, | |
| "logits/rejected": -3.03125, | |
| "logps/chosen": -52.0, | |
| "logps/rejected": -62.5, | |
| "loss": 0.4791, | |
| "rewards/accuracies": 0.4749999940395355, | |
| "rewards/chosen": -0.5078125, | |
| "rewards/margins": 1.5234375, | |
| "rewards/rejected": -2.03125, | |
| "step": 250 | |
| }, | |
| { | |
| "epoch": 0.8306709265175719, | |
| "grad_norm": 29.21855201998469, | |
| "learning_rate": 4.017751479289941e-07, | |
| "logits/chosen": -3.125, | |
| "logits/rejected": -2.625, | |
| "logps/chosen": -48.5, | |
| "logps/rejected": -52.75, | |
| "loss": 0.4803, | |
| "rewards/accuracies": 0.4124999940395355, | |
| "rewards/chosen": -0.953125, | |
| "rewards/margins": 0.94140625, | |
| "rewards/rejected": -1.890625, | |
| "step": 260 | |
| }, | |
| { | |
| "epoch": 0.8626198083067093, | |
| "grad_norm": 25.205926846344735, | |
| "learning_rate": 3.9585798816568044e-07, | |
| "logits/chosen": -3.28125, | |
| "logits/rejected": -2.859375, | |
| "logps/chosen": -50.75, | |
| "logps/rejected": -60.25, | |
| "loss": 0.5525, | |
| "rewards/accuracies": 0.4749999940395355, | |
| "rewards/chosen": -0.87890625, | |
| "rewards/margins": 1.296875, | |
| "rewards/rejected": -2.171875, | |
| "step": 270 | |
| }, | |
| { | |
| "epoch": 0.8945686900958466, | |
| "grad_norm": 36.1167561440604, | |
| "learning_rate": 3.8994082840236685e-07, | |
| "logits/chosen": -3.125, | |
| "logits/rejected": -2.8125, | |
| "logps/chosen": -53.5, | |
| "logps/rejected": -54.5, | |
| "loss": 0.5372, | |
| "rewards/accuracies": 0.4625000059604645, | |
| "rewards/chosen": -0.546875, | |
| "rewards/margins": 1.265625, | |
| "rewards/rejected": -1.8125, | |
| "step": 280 | |
| }, | |
| { | |
| "epoch": 0.9265175718849841, | |
| "grad_norm": 30.60002887625237, | |
| "learning_rate": 3.840236686390532e-07, | |
| "logits/chosen": -3.234375, | |
| "logits/rejected": -2.875, | |
| "logps/chosen": -57.75, | |
| "logps/rejected": -64.5, | |
| "loss": 0.5178, | |
| "rewards/accuracies": 0.44999998807907104, | |
| "rewards/chosen": -0.8046875, | |
| "rewards/margins": 1.6328125, | |
| "rewards/rejected": -2.4375, | |
| "step": 290 | |
| }, | |
| { | |
| "epoch": 0.9584664536741214, | |
| "grad_norm": 8.950670028012786, | |
| "learning_rate": 3.7810650887573966e-07, | |
| "logits/chosen": -2.96875, | |
| "logits/rejected": -3.03125, | |
| "logps/chosen": -47.75, | |
| "logps/rejected": -69.5, | |
| "loss": 0.5006, | |
| "rewards/accuracies": 0.4625000059604645, | |
| "rewards/chosen": -0.96875, | |
| "rewards/margins": 1.5078125, | |
| "rewards/rejected": -2.484375, | |
| "step": 300 | |
| }, | |
| { | |
| "epoch": 0.9904153354632588, | |
| "grad_norm": 30.917476600395073, | |
| "learning_rate": 3.72189349112426e-07, | |
| "logits/chosen": -3.09375, | |
| "logits/rejected": -2.96875, | |
| "logps/chosen": -55.75, | |
| "logps/rejected": -67.0, | |
| "loss": 0.5163, | |
| "rewards/accuracies": 0.42500001192092896, | |
| "rewards/chosen": -0.74609375, | |
| "rewards/margins": 1.5546875, | |
| "rewards/rejected": -2.296875, | |
| "step": 310 | |
| }, | |
| { | |
| "epoch": 1.0, | |
| "eval_logits/chosen": -3.296875, | |
| "eval_logits/rejected": -2.671875, | |
| "eval_logps/chosen": -62.25, | |
| "eval_logps/rejected": -57.5, | |
| "eval_loss": 0.5627734661102295, | |
| "eval_rewards/accuracies": 0.4642857015132904, | |
| "eval_rewards/chosen": -0.9453125, | |
| "eval_rewards/margins": 1.0078125, | |
| "eval_rewards/rejected": -1.953125, | |
| "eval_runtime": 12.3094, | |
| "eval_samples_per_second": 16.248, | |
| "eval_steps_per_second": 0.569, | |
| "step": 313 | |
| } | |
| ], | |
| "logging_steps": 10, | |
| "max_steps": 939, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 3, | |
| "save_steps": 500, | |
| "stateful_callbacks": { | |
| "TrainerControl": { | |
| "args": { | |
| "should_epoch_stop": false, | |
| "should_evaluate": false, | |
| "should_log": false, | |
| "should_save": true, | |
| "should_training_stop": false | |
| }, | |
| "attributes": {} | |
| } | |
| }, | |
| "total_flos": 0.0, | |
| "train_batch_size": 8, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |