| { | |
| "best_metric": null, | |
| "best_model_checkpoint": null, | |
| "epoch": 3.0, | |
| "eval_steps": 500, | |
| "global_step": 504, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "epoch": 0.02976190476190476, | |
| "grad_norm": 1776.2471923828125, | |
| "learning_rate": 2.5000000000000004e-07, | |
| "log_odds_chosen": -0.3503778874874115, | |
| "log_odds_ratio": -1.0639952421188354, | |
| "logits/chosen": 125.49534606933594, | |
| "logits/rejected": 180.3563232421875, | |
| "logps/chosen": -15.494331359863281, | |
| "logps/rejected": -15.143954277038574, | |
| "loss": 14.9629, | |
| "nll_loss": 15.042287826538086, | |
| "rewards/accuracies": 0.25, | |
| "rewards/chosen": -0.7747165560722351, | |
| "rewards/margins": -0.01751876249909401, | |
| "rewards/rejected": -0.7571978569030762, | |
| "step": 5 | |
| }, | |
| { | |
| "epoch": 0.05952380952380952, | |
| "grad_norm": 1193.9168701171875, | |
| "learning_rate": 5.000000000000001e-07, | |
| "log_odds_chosen": 0.06253216415643692, | |
| "log_odds_ratio": -0.8492839932441711, | |
| "logits/chosen": 210.31631469726562, | |
| "logits/rejected": 245.1850128173828, | |
| "logps/chosen": -12.538459777832031, | |
| "logps/rejected": -12.600992202758789, | |
| "loss": 12.6111, | |
| "nll_loss": 12.388693809509277, | |
| "rewards/accuracies": 0.4000000059604645, | |
| "rewards/chosen": -0.6269229650497437, | |
| "rewards/margins": 0.0031266347505152225, | |
| "rewards/rejected": -0.6300495862960815, | |
| "step": 10 | |
| }, | |
| { | |
| "epoch": 0.08928571428571429, | |
| "grad_norm": 723.246826171875, | |
| "learning_rate": 7.5e-07, | |
| "log_odds_chosen": -0.26854413747787476, | |
| "log_odds_ratio": -0.9210551977157593, | |
| "logits/chosen": 300.58953857421875, | |
| "logits/rejected": 289.635009765625, | |
| "logps/chosen": -8.324029922485352, | |
| "logps/rejected": -8.055707931518555, | |
| "loss": 8.2823, | |
| "nll_loss": 8.378087997436523, | |
| "rewards/accuracies": 0.4000000059604645, | |
| "rewards/chosen": -0.41620150208473206, | |
| "rewards/margins": -0.013416108675301075, | |
| "rewards/rejected": -0.40278539061546326, | |
| "step": 15 | |
| }, | |
| { | |
| "epoch": 0.11904761904761904, | |
| "grad_norm": 216.30667114257812, | |
| "learning_rate": 1.0000000000000002e-06, | |
| "log_odds_chosen": -0.09513016790151596, | |
| "log_odds_ratio": -0.9796999096870422, | |
| "logits/chosen": 255.70156860351562, | |
| "logits/rejected": 235.23721313476562, | |
| "logps/chosen": -5.404101848602295, | |
| "logps/rejected": -5.310047149658203, | |
| "loss": 5.4458, | |
| "nll_loss": 5.430812835693359, | |
| "rewards/accuracies": 0.4000000059604645, | |
| "rewards/chosen": -0.2702050805091858, | |
| "rewards/margins": -0.004702714271843433, | |
| "rewards/rejected": -0.265502393245697, | |
| "step": 20 | |
| }, | |
| { | |
| "epoch": 0.1488095238095238, | |
| "grad_norm": 164.33297729492188, | |
| "learning_rate": 1.25e-06, | |
| "log_odds_chosen": 0.25551921129226685, | |
| "log_odds_ratio": -0.6588834524154663, | |
| "logits/chosen": 274.9960632324219, | |
| "logits/rejected": 302.8502502441406, | |
| "logps/chosen": -3.1612606048583984, | |
| "logps/rejected": -3.390745162963867, | |
| "loss": 3.5011, | |
| "nll_loss": 3.502755641937256, | |
| "rewards/accuracies": 0.6000000238418579, | |
| "rewards/chosen": -0.15806300938129425, | |
| "rewards/margins": 0.01147423591464758, | |
| "rewards/rejected": -0.1695372611284256, | |
| "step": 25 | |
| }, | |
| { | |
| "epoch": 0.17857142857142858, | |
| "grad_norm": 78.93386840820312, | |
| "learning_rate": 1.5e-06, | |
| "log_odds_chosen": 0.24210360646247864, | |
| "log_odds_ratio": -0.6352943181991577, | |
| "logits/chosen": 290.36334228515625, | |
| "logits/rejected": 385.3616638183594, | |
| "logps/chosen": -2.418250799179077, | |
| "logps/rejected": -2.630479097366333, | |
| "loss": 2.5553, | |
| "nll_loss": 2.509606122970581, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": -0.12091253697872162, | |
| "rewards/margins": 0.01061142049729824, | |
| "rewards/rejected": -0.1315239518880844, | |
| "step": 30 | |
| }, | |
| { | |
| "epoch": 0.20833333333333334, | |
| "grad_norm": 42.19651412963867, | |
| "learning_rate": 1.75e-06, | |
| "log_odds_chosen": 0.4038185477256775, | |
| "log_odds_ratio": -0.5650765299797058, | |
| "logits/chosen": 365.90753173828125, | |
| "logits/rejected": 389.90338134765625, | |
| "logps/chosen": -1.5057640075683594, | |
| "logps/rejected": -1.8460853099822998, | |
| "loss": 2.129, | |
| "nll_loss": 1.8491789102554321, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": -0.07528821378946304, | |
| "rewards/margins": 0.01701606810092926, | |
| "rewards/rejected": -0.0923042744398117, | |
| "step": 35 | |
| }, | |
| { | |
| "epoch": 0.23809523809523808, | |
| "grad_norm": 35.26750183105469, | |
| "learning_rate": 2.0000000000000003e-06, | |
| "log_odds_chosen": 0.15938052535057068, | |
| "log_odds_ratio": -0.6910892724990845, | |
| "logits/chosen": 331.43438720703125, | |
| "logits/rejected": 372.8791198730469, | |
| "logps/chosen": -1.6345170736312866, | |
| "logps/rejected": -1.7539236545562744, | |
| "loss": 1.935, | |
| "nll_loss": 1.968824028968811, | |
| "rewards/accuracies": 0.6000000238418579, | |
| "rewards/chosen": -0.08172585070133209, | |
| "rewards/margins": 0.005970318801701069, | |
| "rewards/rejected": -0.08769617974758148, | |
| "step": 40 | |
| }, | |
| { | |
| "epoch": 0.26785714285714285, | |
| "grad_norm": 87.45353698730469, | |
| "learning_rate": 2.25e-06, | |
| "log_odds_chosen": -0.15498922765254974, | |
| "log_odds_ratio": -0.8517929911613464, | |
| "logits/chosen": 386.3527526855469, | |
| "logits/rejected": 371.41851806640625, | |
| "logps/chosen": -1.4574557542800903, | |
| "logps/rejected": -1.3248956203460693, | |
| "loss": 1.8765, | |
| "nll_loss": 1.7655121088027954, | |
| "rewards/accuracies": 0.44999998807907104, | |
| "rewards/chosen": -0.07287278026342392, | |
| "rewards/margins": -0.006628001574426889, | |
| "rewards/rejected": -0.06624479591846466, | |
| "step": 45 | |
| }, | |
| { | |
| "epoch": 0.2976190476190476, | |
| "grad_norm": 41.48765563964844, | |
| "learning_rate": 2.5e-06, | |
| "log_odds_chosen": 0.043729472905397415, | |
| "log_odds_ratio": -0.7361106872558594, | |
| "logits/chosen": 437.00823974609375, | |
| "logits/rejected": 443.38232421875, | |
| "logps/chosen": -1.658603310585022, | |
| "logps/rejected": -1.6873445510864258, | |
| "loss": 1.8933, | |
| "nll_loss": 2.108198642730713, | |
| "rewards/accuracies": 0.550000011920929, | |
| "rewards/chosen": -0.08293016999959946, | |
| "rewards/margins": 0.0014370663557201624, | |
| "rewards/rejected": -0.08436723798513412, | |
| "step": 50 | |
| }, | |
| { | |
| "epoch": 0.3273809523809524, | |
| "grad_norm": 51.39496994018555, | |
| "learning_rate": 2.7500000000000004e-06, | |
| "log_odds_chosen": 0.10381323099136353, | |
| "log_odds_ratio": -0.7126073241233826, | |
| "logits/chosen": 365.14544677734375, | |
| "logits/rejected": 364.1676330566406, | |
| "logps/chosen": -1.4783101081848145, | |
| "logps/rejected": -1.550569772720337, | |
| "loss": 1.8734, | |
| "nll_loss": 1.8976672887802124, | |
| "rewards/accuracies": 0.6000000238418579, | |
| "rewards/chosen": -0.073915496468544, | |
| "rewards/margins": 0.003612985834479332, | |
| "rewards/rejected": -0.07752849161624908, | |
| "step": 55 | |
| }, | |
| { | |
| "epoch": 0.35714285714285715, | |
| "grad_norm": 47.07830810546875, | |
| "learning_rate": 3e-06, | |
| "log_odds_chosen": 0.20662184059619904, | |
| "log_odds_ratio": -0.6513880491256714, | |
| "logits/chosen": 386.7112731933594, | |
| "logits/rejected": 356.2147216796875, | |
| "logps/chosen": -1.1965901851654053, | |
| "logps/rejected": -1.3431804180145264, | |
| "loss": 1.6989, | |
| "nll_loss": 1.9430646896362305, | |
| "rewards/accuracies": 0.550000011920929, | |
| "rewards/chosen": -0.05982951074838638, | |
| "rewards/margins": 0.007329514715820551, | |
| "rewards/rejected": -0.0671590268611908, | |
| "step": 60 | |
| }, | |
| { | |
| "epoch": 0.3869047619047619, | |
| "grad_norm": 116.96641540527344, | |
| "learning_rate": 3.2500000000000002e-06, | |
| "log_odds_chosen": 0.029183167964220047, | |
| "log_odds_ratio": -0.7434911727905273, | |
| "logits/chosen": 385.2690734863281, | |
| "logits/rejected": 389.1334228515625, | |
| "logps/chosen": -1.3286149501800537, | |
| "logps/rejected": -1.3333098888397217, | |
| "loss": 1.636, | |
| "nll_loss": 1.6027374267578125, | |
| "rewards/accuracies": 0.6499999761581421, | |
| "rewards/chosen": -0.06643076241016388, | |
| "rewards/margins": 0.00023474842600990087, | |
| "rewards/rejected": -0.06666550040245056, | |
| "step": 65 | |
| }, | |
| { | |
| "epoch": 0.4166666666666667, | |
| "grad_norm": 40.99021530151367, | |
| "learning_rate": 3.5e-06, | |
| "log_odds_chosen": 0.045442335307598114, | |
| "log_odds_ratio": -0.7358086705207825, | |
| "logits/chosen": 395.22906494140625, | |
| "logits/rejected": 390.57305908203125, | |
| "logps/chosen": -1.7554155588150024, | |
| "logps/rejected": -1.765885591506958, | |
| "loss": 1.6688, | |
| "nll_loss": 1.9699609279632568, | |
| "rewards/accuracies": 0.550000011920929, | |
| "rewards/chosen": -0.08777078241109848, | |
| "rewards/margins": 0.0005234999698586762, | |
| "rewards/rejected": -0.08829428255558014, | |
| "step": 70 | |
| }, | |
| { | |
| "epoch": 0.44642857142857145, | |
| "grad_norm": 67.57577514648438, | |
| "learning_rate": 3.7500000000000005e-06, | |
| "log_odds_chosen": 0.24833233654499054, | |
| "log_odds_ratio": -0.6203548908233643, | |
| "logits/chosen": 386.412109375, | |
| "logits/rejected": 376.1787414550781, | |
| "logps/chosen": -1.1873780488967896, | |
| "logps/rejected": -1.3402230739593506, | |
| "loss": 1.5454, | |
| "nll_loss": 1.4503333568572998, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": -0.059368908405303955, | |
| "rewards/margins": 0.0076422421261668205, | |
| "rewards/rejected": -0.06701115518808365, | |
| "step": 75 | |
| }, | |
| { | |
| "epoch": 0.47619047619047616, | |
| "grad_norm": 29.741676330566406, | |
| "learning_rate": 4.000000000000001e-06, | |
| "log_odds_chosen": 0.29179519414901733, | |
| "log_odds_ratio": -0.6332293748855591, | |
| "logits/chosen": 389.09295654296875, | |
| "logits/rejected": 405.5395202636719, | |
| "logps/chosen": -1.2367178201675415, | |
| "logps/rejected": -1.3924906253814697, | |
| "loss": 1.5966, | |
| "nll_loss": 1.5792012214660645, | |
| "rewards/accuracies": 0.550000011920929, | |
| "rewards/chosen": -0.061835892498493195, | |
| "rewards/margins": 0.007788646034896374, | |
| "rewards/rejected": -0.06962453573942184, | |
| "step": 80 | |
| }, | |
| { | |
| "epoch": 0.5059523809523809, | |
| "grad_norm": 53.26546096801758, | |
| "learning_rate": 4.25e-06, | |
| "log_odds_chosen": -0.2370494306087494, | |
| "log_odds_ratio": -0.9606464505195618, | |
| "logits/chosen": 405.4431457519531, | |
| "logits/rejected": 386.801025390625, | |
| "logps/chosen": -1.5545951128005981, | |
| "logps/rejected": -1.2866960763931274, | |
| "loss": 1.6757, | |
| "nll_loss": 1.9027442932128906, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": -0.07772975414991379, | |
| "rewards/margins": -0.013394953683018684, | |
| "rewards/rejected": -0.06433480232954025, | |
| "step": 85 | |
| }, | |
| { | |
| "epoch": 0.5357142857142857, | |
| "grad_norm": 28.4445743560791, | |
| "learning_rate": 4.5e-06, | |
| "log_odds_chosen": 0.7336848378181458, | |
| "log_odds_ratio": -0.5354053378105164, | |
| "logits/chosen": 416.51580810546875, | |
| "logits/rejected": 425.767578125, | |
| "logps/chosen": -1.2461140155792236, | |
| "logps/rejected": -1.842694878578186, | |
| "loss": 1.4576, | |
| "nll_loss": 1.5203516483306885, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": -0.06230570003390312, | |
| "rewards/margins": 0.02982904389500618, | |
| "rewards/rejected": -0.0921347513794899, | |
| "step": 90 | |
| }, | |
| { | |
| "epoch": 0.5654761904761905, | |
| "grad_norm": 46.642581939697266, | |
| "learning_rate": 4.75e-06, | |
| "log_odds_chosen": 0.25576168298721313, | |
| "log_odds_ratio": -0.6427351832389832, | |
| "logits/chosen": 347.84832763671875, | |
| "logits/rejected": 380.0849609375, | |
| "logps/chosen": -1.119447946548462, | |
| "logps/rejected": -1.234407663345337, | |
| "loss": 1.523, | |
| "nll_loss": 1.489450454711914, | |
| "rewards/accuracies": 0.550000011920929, | |
| "rewards/chosen": -0.05597240477800369, | |
| "rewards/margins": 0.005747987423092127, | |
| "rewards/rejected": -0.06172039359807968, | |
| "step": 95 | |
| }, | |
| { | |
| "epoch": 0.5952380952380952, | |
| "grad_norm": 43.77986526489258, | |
| "learning_rate": 5e-06, | |
| "log_odds_chosen": 0.12196314334869385, | |
| "log_odds_ratio": -0.7031902074813843, | |
| "logits/chosen": 417.86639404296875, | |
| "logits/rejected": 436.10858154296875, | |
| "logps/chosen": -1.2873280048370361, | |
| "logps/rejected": -1.3734104633331299, | |
| "loss": 1.5418, | |
| "nll_loss": 1.5198986530303955, | |
| "rewards/accuracies": 0.6000000238418579, | |
| "rewards/chosen": -0.0643664002418518, | |
| "rewards/margins": 0.004304117523133755, | |
| "rewards/rejected": -0.06867052614688873, | |
| "step": 100 | |
| }, | |
| { | |
| "epoch": 0.625, | |
| "grad_norm": 32.244384765625, | |
| "learning_rate": 4.8795003647426654e-06, | |
| "log_odds_chosen": 0.27784407138824463, | |
| "log_odds_ratio": -0.6329095959663391, | |
| "logits/chosen": 412.528564453125, | |
| "logits/rejected": 394.52130126953125, | |
| "logps/chosen": -1.0501761436462402, | |
| "logps/rejected": -1.182625412940979, | |
| "loss": 1.5085, | |
| "nll_loss": 1.6539300680160522, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": -0.05250881239771843, | |
| "rewards/margins": 0.006622466258704662, | |
| "rewards/rejected": -0.059131283313035965, | |
| "step": 105 | |
| }, | |
| { | |
| "epoch": 0.6547619047619048, | |
| "grad_norm": 69.666259765625, | |
| "learning_rate": 4.767312946227961e-06, | |
| "log_odds_chosen": 0.30888259410858154, | |
| "log_odds_ratio": -0.5859929919242859, | |
| "logits/chosen": 375.71612548828125, | |
| "logits/rejected": 356.3163146972656, | |
| "logps/chosen": -1.0360513925552368, | |
| "logps/rejected": -1.2427626848220825, | |
| "loss": 1.5381, | |
| "nll_loss": 1.5933644771575928, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": -0.05180257558822632, | |
| "rewards/margins": 0.01033556554466486, | |
| "rewards/rejected": -0.06213812902569771, | |
| "step": 110 | |
| }, | |
| { | |
| "epoch": 0.6845238095238095, | |
| "grad_norm": 34.408199310302734, | |
| "learning_rate": 4.662524041201569e-06, | |
| "log_odds_chosen": 0.2710966467857361, | |
| "log_odds_ratio": -0.6335155367851257, | |
| "logits/chosen": 389.82244873046875, | |
| "logits/rejected": 437.4444274902344, | |
| "logps/chosen": -0.9292726516723633, | |
| "logps/rejected": -1.0246301889419556, | |
| "loss": 1.4878, | |
| "nll_loss": 1.3399275541305542, | |
| "rewards/accuracies": 0.6000000238418579, | |
| "rewards/chosen": -0.04646363481879234, | |
| "rewards/margins": 0.0047678761184215546, | |
| "rewards/rejected": -0.0512315109372139, | |
| "step": 115 | |
| }, | |
| { | |
| "epoch": 0.7142857142857143, | |
| "grad_norm": 41.90970230102539, | |
| "learning_rate": 4.564354645876385e-06, | |
| "log_odds_chosen": 0.2518194615840912, | |
| "log_odds_ratio": -0.6617739796638489, | |
| "logits/chosen": 393.5774841308594, | |
| "logits/rejected": 395.57147216796875, | |
| "logps/chosen": -1.0596468448638916, | |
| "logps/rejected": -1.1526801586151123, | |
| "loss": 1.5378, | |
| "nll_loss": 1.5159982442855835, | |
| "rewards/accuracies": 0.550000011920929, | |
| "rewards/chosen": -0.05298234894871712, | |
| "rewards/margins": 0.004651663359254599, | |
| "rewards/rejected": -0.05763401836156845, | |
| "step": 120 | |
| }, | |
| { | |
| "epoch": 0.7440476190476191, | |
| "grad_norm": 30.15009880065918, | |
| "learning_rate": 4.47213595499958e-06, | |
| "log_odds_chosen": -0.14496295154094696, | |
| "log_odds_ratio": -0.9012987017631531, | |
| "logits/chosen": 378.92242431640625, | |
| "logits/rejected": 388.03662109375, | |
| "logps/chosen": -1.3249809741973877, | |
| "logps/rejected": -1.1592341661453247, | |
| "loss": 1.4803, | |
| "nll_loss": 1.3684661388397217, | |
| "rewards/accuracies": 0.44999998807907104, | |
| "rewards/chosen": -0.0662490501999855, | |
| "rewards/margins": -0.00828734040260315, | |
| "rewards/rejected": -0.05796170234680176, | |
| "step": 125 | |
| }, | |
| { | |
| "epoch": 0.7738095238095238, | |
| "grad_norm": 24.6285457611084, | |
| "learning_rate": 4.385290096535147e-06, | |
| "log_odds_chosen": 0.3137187361717224, | |
| "log_odds_ratio": -0.6695318818092346, | |
| "logits/chosen": 401.2060546875, | |
| "logits/rejected": 372.0254821777344, | |
| "logps/chosen": -1.065645456314087, | |
| "logps/rejected": -1.1312780380249023, | |
| "loss": 1.5122, | |
| "nll_loss": 1.4193568229675293, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": -0.053282272070646286, | |
| "rewards/margins": 0.003281626384705305, | |
| "rewards/rejected": -0.05656389519572258, | |
| "step": 130 | |
| }, | |
| { | |
| "epoch": 0.8035714285714286, | |
| "grad_norm": 22.474136352539062, | |
| "learning_rate": 4.303314829119352e-06, | |
| "log_odds_chosen": 0.08262600004673004, | |
| "log_odds_ratio": -0.7314961552619934, | |
| "logits/chosen": 373.8929138183594, | |
| "logits/rejected": 405.0035705566406, | |
| "logps/chosen": -1.1277202367782593, | |
| "logps/rejected": -1.2345101833343506, | |
| "loss": 1.5189, | |
| "nll_loss": 1.4631812572479248, | |
| "rewards/accuracies": 0.550000011920929, | |
| "rewards/chosen": -0.056386012583971024, | |
| "rewards/margins": 0.005339500494301319, | |
| "rewards/rejected": -0.06172550842165947, | |
| "step": 135 | |
| }, | |
| { | |
| "epoch": 0.8333333333333334, | |
| "grad_norm": 23.9536190032959, | |
| "learning_rate": 4.2257712736425835e-06, | |
| "log_odds_chosen": -0.12259285151958466, | |
| "log_odds_ratio": -0.7952130436897278, | |
| "logits/chosen": 394.89117431640625, | |
| "logits/rejected": 361.0680847167969, | |
| "logps/chosen": -1.0518170595169067, | |
| "logps/rejected": -0.986822247505188, | |
| "loss": 1.5033, | |
| "nll_loss": 1.4649903774261475, | |
| "rewards/accuracies": 0.44999998807907104, | |
| "rewards/chosen": -0.052590854465961456, | |
| "rewards/margins": -0.003249742556363344, | |
| "rewards/rejected": -0.0493411123752594, | |
| "step": 140 | |
| }, | |
| { | |
| "epoch": 0.8630952380952381, | |
| "grad_norm": 22.85877227783203, | |
| "learning_rate": 4.1522739926869985e-06, | |
| "log_odds_chosen": -0.07754195481538773, | |
| "log_odds_ratio": -0.7574380040168762, | |
| "logits/chosen": 365.57147216796875, | |
| "logits/rejected": 380.41046142578125, | |
| "logps/chosen": -1.0989421606063843, | |
| "logps/rejected": -1.0417240858078003, | |
| "loss": 1.4892, | |
| "nll_loss": 1.3651247024536133, | |
| "rewards/accuracies": 0.550000011920929, | |
| "rewards/chosen": -0.05494711548089981, | |
| "rewards/margins": -0.0028609037399291992, | |
| "rewards/rejected": -0.052086204290390015, | |
| "step": 145 | |
| }, | |
| { | |
| "epoch": 0.8928571428571429, | |
| "grad_norm": 28.741422653198242, | |
| "learning_rate": 4.082482904638631e-06, | |
| "log_odds_chosen": 0.4838029742240906, | |
| "log_odds_ratio": -0.5155819654464722, | |
| "logits/chosen": 390.33367919921875, | |
| "logits/rejected": 395.64117431640625, | |
| "logps/chosen": -1.0308912992477417, | |
| "logps/rejected": -1.3601925373077393, | |
| "loss": 1.4181, | |
| "nll_loss": 1.4082247018814087, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": -0.051544565707445145, | |
| "rewards/margins": 0.016465062275528908, | |
| "rewards/rejected": -0.0680096298456192, | |
| "step": 150 | |
| }, | |
| { | |
| "epoch": 0.9226190476190477, | |
| "grad_norm": 29.028520584106445, | |
| "learning_rate": 4.016096644512495e-06, | |
| "log_odds_chosen": 0.07582991570234299, | |
| "log_odds_ratio": -0.7352056503295898, | |
| "logits/chosen": 384.6861572265625, | |
| "logits/rejected": 382.9585876464844, | |
| "logps/chosen": -1.1874030828475952, | |
| "logps/rejected": -1.2361079454421997, | |
| "loss": 1.4165, | |
| "nll_loss": 1.365952730178833, | |
| "rewards/accuracies": 0.550000011920929, | |
| "rewards/chosen": -0.05937015265226364, | |
| "rewards/margins": 0.0024352427572011948, | |
| "rewards/rejected": -0.06180540472269058, | |
| "step": 155 | |
| }, | |
| { | |
| "epoch": 0.9523809523809523, | |
| "grad_norm": 47.11198043823242, | |
| "learning_rate": 3.952847075210474e-06, | |
| "log_odds_chosen": 0.17520497739315033, | |
| "log_odds_ratio": -0.691135048866272, | |
| "logits/chosen": 389.0434265136719, | |
| "logits/rejected": 431.94989013671875, | |
| "logps/chosen": -0.9387935400009155, | |
| "logps/rejected": -1.0681307315826416, | |
| "loss": 1.3893, | |
| "nll_loss": 1.2998160123825073, | |
| "rewards/accuracies": 0.44999998807907104, | |
| "rewards/chosen": -0.046939678490161896, | |
| "rewards/margins": 0.006466855760663748, | |
| "rewards/rejected": -0.05340652912855148, | |
| "step": 160 | |
| }, | |
| { | |
| "epoch": 0.9821428571428571, | |
| "grad_norm": 16.736385345458984, | |
| "learning_rate": 3.892494720807615e-06, | |
| "log_odds_chosen": -0.01084871869534254, | |
| "log_odds_ratio": -0.759717583656311, | |
| "logits/chosen": 427.4103088378906, | |
| "logits/rejected": 431.9371032714844, | |
| "logps/chosen": -1.1341661214828491, | |
| "logps/rejected": -1.1380488872528076, | |
| "loss": 1.4218, | |
| "nll_loss": 1.4594279527664185, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": -0.05670831352472305, | |
| "rewards/margins": 0.00019413381232880056, | |
| "rewards/rejected": -0.0569024458527565, | |
| "step": 165 | |
| }, | |
| { | |
| "epoch": 1.0, | |
| "eval_log_odds_chosen": 0.23235350847244263, | |
| "eval_log_odds_ratio": -0.6711738109588623, | |
| "eval_logits/chosen": 326.8028564453125, | |
| "eval_logits/rejected": 273.7525634765625, | |
| "eval_logps/chosen": -1.007071614265442, | |
| "eval_logps/rejected": -1.159067988395691, | |
| "eval_loss": 1.4487849473953247, | |
| "eval_nll_loss": 1.455283284187317, | |
| "eval_rewards/accuracies": 0.5571428537368774, | |
| "eval_rewards/chosen": -0.050353582948446274, | |
| "eval_rewards/margins": 0.007599818520247936, | |
| "eval_rewards/rejected": -0.057953398674726486, | |
| "eval_runtime": 201.1698, | |
| "eval_samples_per_second": 2.749, | |
| "eval_steps_per_second": 0.348, | |
| "step": 168 | |
| }, | |
| { | |
| "epoch": 1.0119047619047619, | |
| "grad_norm": 19.479097366333008, | |
| "learning_rate": 3.834824944236852e-06, | |
| "log_odds_chosen": 0.4537879526615143, | |
| "log_odds_ratio": -0.5628765225410461, | |
| "logits/chosen": 367.5221252441406, | |
| "logits/rejected": 402.8450622558594, | |
| "logps/chosen": -0.841064453125, | |
| "logps/rejected": -1.091392159461975, | |
| "loss": 1.2782, | |
| "nll_loss": 1.1047321557998657, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": -0.0420532263815403, | |
| "rewards/margins": 0.012516376562416553, | |
| "rewards/rejected": -0.054569609463214874, | |
| "step": 170 | |
| }, | |
| { | |
| "epoch": 1.0416666666666667, | |
| "grad_norm": 18.193614959716797, | |
| "learning_rate": 3.7796447300922724e-06, | |
| "log_odds_chosen": 0.6258941292762756, | |
| "log_odds_ratio": -0.5433114171028137, | |
| "logits/chosen": 404.79443359375, | |
| "logits/rejected": 432.61151123046875, | |
| "logps/chosen": -0.7016412615776062, | |
| "logps/rejected": -1.0624569654464722, | |
| "loss": 1.05, | |
| "nll_loss": 1.2699940204620361, | |
| "rewards/accuracies": 0.8500000238418579, | |
| "rewards/chosen": -0.03508206456899643, | |
| "rewards/margins": 0.018040789291262627, | |
| "rewards/rejected": -0.053122855722904205, | |
| "step": 175 | |
| }, | |
| { | |
| "epoch": 1.0714285714285714, | |
| "grad_norm": 15.39194393157959, | |
| "learning_rate": 3.72677996249965e-06, | |
| "log_odds_chosen": 0.7378727197647095, | |
| "log_odds_ratio": -0.47085660696029663, | |
| "logits/chosen": 343.8995056152344, | |
| "logits/rejected": 338.2045593261719, | |
| "logps/chosen": -0.8047459721565247, | |
| "logps/rejected": -1.2078239917755127, | |
| "loss": 1.1048, | |
| "nll_loss": 1.2225919961929321, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": -0.04023730009794235, | |
| "rewards/margins": 0.020153898745775223, | |
| "rewards/rejected": -0.060391198843717575, | |
| "step": 180 | |
| }, | |
| { | |
| "epoch": 1.1011904761904763, | |
| "grad_norm": 20.72310447692871, | |
| "learning_rate": 3.6760731104690393e-06, | |
| "log_odds_chosen": 1.557734727859497, | |
| "log_odds_ratio": -0.26492685079574585, | |
| "logits/chosen": 435.18865966796875, | |
| "logits/rejected": 362.96868896484375, | |
| "logps/chosen": -0.6113921403884888, | |
| "logps/rejected": -1.4927384853363037, | |
| "loss": 0.9894, | |
| "nll_loss": 1.086829423904419, | |
| "rewards/accuracies": 0.949999988079071, | |
| "rewards/chosen": -0.03056960739195347, | |
| "rewards/margins": 0.044067323207855225, | |
| "rewards/rejected": -0.07463693618774414, | |
| "step": 185 | |
| }, | |
| { | |
| "epoch": 1.130952380952381, | |
| "grad_norm": 16.229610443115234, | |
| "learning_rate": 3.6273812505500587e-06, | |
| "log_odds_chosen": 0.5972079038619995, | |
| "log_odds_ratio": -0.5253661870956421, | |
| "logits/chosen": 386.9327087402344, | |
| "logits/rejected": 405.801513671875, | |
| "logps/chosen": -0.7007042169570923, | |
| "logps/rejected": -1.028989553451538, | |
| "loss": 1.1004, | |
| "nll_loss": 1.0136523246765137, | |
| "rewards/accuracies": 0.6499999761581421, | |
| "rewards/chosen": -0.03503521531820297, | |
| "rewards/margins": 0.01641426980495453, | |
| "rewards/rejected": -0.0514494851231575, | |
| "step": 190 | |
| }, | |
| { | |
| "epoch": 1.1607142857142858, | |
| "grad_norm": 21.483884811401367, | |
| "learning_rate": 3.5805743701971648e-06, | |
| "log_odds_chosen": 0.9149399995803833, | |
| "log_odds_ratio": -0.4483817219734192, | |
| "logits/chosen": 371.6880187988281, | |
| "logits/rejected": 403.46356201171875, | |
| "logps/chosen": -0.7223442196846008, | |
| "logps/rejected": -1.1754904985427856, | |
| "loss": 1.0787, | |
| "nll_loss": 1.0912402868270874, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": -0.03611721470952034, | |
| "rewards/margins": 0.022657308727502823, | |
| "rewards/rejected": -0.05877452343702316, | |
| "step": 195 | |
| }, | |
| { | |
| "epoch": 1.1904761904761905, | |
| "grad_norm": 19.23330307006836, | |
| "learning_rate": 3.5355339059327378e-06, | |
| "log_odds_chosen": 1.305513620376587, | |
| "log_odds_ratio": -0.32568031549453735, | |
| "logits/chosen": 431.67156982421875, | |
| "logits/rejected": 391.12017822265625, | |
| "logps/chosen": -0.4990530014038086, | |
| "logps/rejected": -1.1961959600448608, | |
| "loss": 1.0519, | |
| "nll_loss": 0.9312452077865601, | |
| "rewards/accuracies": 0.8999999761581421, | |
| "rewards/chosen": -0.02495265007019043, | |
| "rewards/margins": 0.03485715016722679, | |
| "rewards/rejected": -0.05980980396270752, | |
| "step": 200 | |
| }, | |
| { | |
| "epoch": 1.2202380952380953, | |
| "grad_norm": 18.404104232788086, | |
| "learning_rate": 3.4921514788478916e-06, | |
| "log_odds_chosen": 1.2872587442398071, | |
| "log_odds_ratio": -0.3010835349559784, | |
| "logits/chosen": 335.2537841796875, | |
| "logits/rejected": 387.68621826171875, | |
| "logps/chosen": -0.6490974426269531, | |
| "logps/rejected": -1.4184123277664185, | |
| "loss": 1.0519, | |
| "nll_loss": 1.0444796085357666, | |
| "rewards/accuracies": 0.949999988079071, | |
| "rewards/chosen": -0.032454878091812134, | |
| "rewards/margins": 0.03846573829650879, | |
| "rewards/rejected": -0.07092060893774033, | |
| "step": 205 | |
| }, | |
| { | |
| "epoch": 1.25, | |
| "grad_norm": 13.957087516784668, | |
| "learning_rate": 3.450327796711771e-06, | |
| "log_odds_chosen": 1.1957916021347046, | |
| "log_odds_ratio": -0.36826610565185547, | |
| "logits/chosen": 365.1176452636719, | |
| "logits/rejected": 363.3384094238281, | |
| "logps/chosen": -0.6256347298622131, | |
| "logps/rejected": -1.242398738861084, | |
| "loss": 0.9696, | |
| "nll_loss": 1.0164039134979248, | |
| "rewards/accuracies": 0.8500000238418579, | |
| "rewards/chosen": -0.031281743198633194, | |
| "rewards/margins": 0.03083820268511772, | |
| "rewards/rejected": -0.06211994215846062, | |
| "step": 210 | |
| }, | |
| { | |
| "epoch": 1.2797619047619047, | |
| "grad_norm": 25.830501556396484, | |
| "learning_rate": 3.409971697352368e-06, | |
| "log_odds_chosen": 1.1722948551177979, | |
| "log_odds_ratio": -0.3577140271663666, | |
| "logits/chosen": 410.3714904785156, | |
| "logits/rejected": 416.93621826171875, | |
| "logps/chosen": -0.7465067505836487, | |
| "logps/rejected": -1.4228246212005615, | |
| "loss": 1.0589, | |
| "nll_loss": 1.059061050415039, | |
| "rewards/accuracies": 0.949999988079071, | |
| "rewards/chosen": -0.037325333803892136, | |
| "rewards/margins": 0.0338158942759037, | |
| "rewards/rejected": -0.07114122807979584, | |
| "step": 215 | |
| }, | |
| { | |
| "epoch": 1.3095238095238095, | |
| "grad_norm": 15.7648344039917, | |
| "learning_rate": 3.3709993123162106e-06, | |
| "log_odds_chosen": 0.9515246152877808, | |
| "log_odds_ratio": -0.4343441128730774, | |
| "logits/chosen": 352.2547607421875, | |
| "logits/rejected": 370.08245849609375, | |
| "logps/chosen": -0.6594604253768921, | |
| "logps/rejected": -1.1094262599945068, | |
| "loss": 1.0138, | |
| "nll_loss": 0.9946004748344421, | |
| "rewards/accuracies": 0.8500000238418579, | |
| "rewards/chosen": -0.032973017543554306, | |
| "rewards/margins": 0.022498302161693573, | |
| "rewards/rejected": -0.05547132343053818, | |
| "step": 220 | |
| }, | |
| { | |
| "epoch": 1.3392857142857144, | |
| "grad_norm": 21.596298217773438, | |
| "learning_rate": 3.3333333333333333e-06, | |
| "log_odds_chosen": 0.4564495086669922, | |
| "log_odds_ratio": -0.5201828479766846, | |
| "logits/chosen": 387.99542236328125, | |
| "logits/rejected": 365.8666076660156, | |
| "logps/chosen": -1.059605360031128, | |
| "logps/rejected": -1.3452465534210205, | |
| "loss": 1.0353, | |
| "nll_loss": 1.2666301727294922, | |
| "rewards/accuracies": 0.8500000238418579, | |
| "rewards/chosen": -0.05298026651144028, | |
| "rewards/margins": 0.014282059855759144, | |
| "rewards/rejected": -0.06726232916116714, | |
| "step": 225 | |
| }, | |
| { | |
| "epoch": 1.369047619047619, | |
| "grad_norm": 16.963186264038086, | |
| "learning_rate": 3.296902366978936e-06, | |
| "log_odds_chosen": 1.108178973197937, | |
| "log_odds_ratio": -0.3512019217014313, | |
| "logits/chosen": 355.7499084472656, | |
| "logits/rejected": 366.8659362792969, | |
| "logps/chosen": -0.6812053322792053, | |
| "logps/rejected": -1.3409600257873535, | |
| "loss": 1.0135, | |
| "nll_loss": 0.873543381690979, | |
| "rewards/accuracies": 0.8500000238418579, | |
| "rewards/chosen": -0.034060269594192505, | |
| "rewards/margins": 0.03298773616552353, | |
| "rewards/rejected": -0.06704800575971603, | |
| "step": 230 | |
| }, | |
| { | |
| "epoch": 1.3988095238095237, | |
| "grad_norm": 22.207305908203125, | |
| "learning_rate": 3.2616403652672114e-06, | |
| "log_odds_chosen": 1.4102222919464111, | |
| "log_odds_ratio": -0.3304731547832489, | |
| "logits/chosen": 382.0793151855469, | |
| "logits/rejected": 389.06683349609375, | |
| "logps/chosen": -0.5401080846786499, | |
| "logps/rejected": -1.397993803024292, | |
| "loss": 1.0737, | |
| "nll_loss": 0.9223471879959106, | |
| "rewards/accuracies": 0.8999999761581421, | |
| "rewards/chosen": -0.027005404233932495, | |
| "rewards/margins": 0.04289429262280464, | |
| "rewards/rejected": -0.06989969313144684, | |
| "step": 235 | |
| }, | |
| { | |
| "epoch": 1.4285714285714286, | |
| "grad_norm": 16.992250442504883, | |
| "learning_rate": 3.2274861218395142e-06, | |
| "log_odds_chosen": 0.807520866394043, | |
| "log_odds_ratio": -0.4530153274536133, | |
| "logits/chosen": 414.99261474609375, | |
| "logits/rejected": 423.21392822265625, | |
| "logps/chosen": -0.7907418012619019, | |
| "logps/rejected": -1.2757242918014526, | |
| "loss": 1.0285, | |
| "nll_loss": 1.0256245136260986, | |
| "rewards/accuracies": 0.8500000238418579, | |
| "rewards/chosen": -0.03953709080815315, | |
| "rewards/margins": 0.0242491252720356, | |
| "rewards/rejected": -0.06378621608018875, | |
| "step": 240 | |
| }, | |
| { | |
| "epoch": 1.4583333333333333, | |
| "grad_norm": 17.553632736206055, | |
| "learning_rate": 3.1943828249997e-06, | |
| "log_odds_chosen": 0.7604994177818298, | |
| "log_odds_ratio": -0.42883071303367615, | |
| "logits/chosen": 386.42657470703125, | |
| "logits/rejected": 363.54931640625, | |
| "logps/chosen": -0.6057090163230896, | |
| "logps/rejected": -1.0264109373092651, | |
| "loss": 1.0502, | |
| "nll_loss": 1.1268291473388672, | |
| "rewards/accuracies": 0.8999999761581421, | |
| "rewards/chosen": -0.03028545156121254, | |
| "rewards/margins": 0.021035097539424896, | |
| "rewards/rejected": -0.05132054537534714, | |
| "step": 245 | |
| }, | |
| { | |
| "epoch": 1.4880952380952381, | |
| "grad_norm": 22.123960494995117, | |
| "learning_rate": 3.1622776601683796e-06, | |
| "log_odds_chosen": 1.0444424152374268, | |
| "log_odds_ratio": -0.38615161180496216, | |
| "logits/chosen": 369.87274169921875, | |
| "logits/rejected": 389.29583740234375, | |
| "logps/chosen": -0.7160728573799133, | |
| "logps/rejected": -1.2788223028182983, | |
| "loss": 0.9816, | |
| "nll_loss": 0.8135589361190796, | |
| "rewards/accuracies": 0.8999999761581421, | |
| "rewards/chosen": -0.035803645849227905, | |
| "rewards/margins": 0.02813747525215149, | |
| "rewards/rejected": -0.0639411136507988, | |
| "step": 250 | |
| }, | |
| { | |
| "epoch": 1.5178571428571428, | |
| "grad_norm": 26.341325759887695, | |
| "learning_rate": 3.131121455425748e-06, | |
| "log_odds_chosen": 1.0527693033218384, | |
| "log_odds_ratio": -0.35840553045272827, | |
| "logits/chosen": 404.608154296875, | |
| "logits/rejected": 410.81854248046875, | |
| "logps/chosen": -0.6503961682319641, | |
| "logps/rejected": -1.250218152999878, | |
| "loss": 1.0367, | |
| "nll_loss": 0.9888055920600891, | |
| "rewards/accuracies": 0.8999999761581421, | |
| "rewards/chosen": -0.032519809901714325, | |
| "rewards/margins": 0.029991086572408676, | |
| "rewards/rejected": -0.0625109076499939, | |
| "step": 255 | |
| }, | |
| { | |
| "epoch": 1.5476190476190477, | |
| "grad_norm": 25.059885025024414, | |
| "learning_rate": 3.1008683647302113e-06, | |
| "log_odds_chosen": 1.0121057033538818, | |
| "log_odds_ratio": -0.4098740220069885, | |
| "logits/chosen": 375.2867126464844, | |
| "logits/rejected": 426.01507568359375, | |
| "logps/chosen": -0.8014397621154785, | |
| "logps/rejected": -1.4673702716827393, | |
| "loss": 1.0023, | |
| "nll_loss": 1.092798113822937, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": -0.0400719977915287, | |
| "rewards/margins": 0.03329651802778244, | |
| "rewards/rejected": -0.07336851954460144, | |
| "step": 260 | |
| }, | |
| { | |
| "epoch": 1.5773809523809523, | |
| "grad_norm": 17.973594665527344, | |
| "learning_rate": 3.0714755841697565e-06, | |
| "log_odds_chosen": 0.830208957195282, | |
| "log_odds_ratio": -0.5377876162528992, | |
| "logits/chosen": 421.629150390625, | |
| "logits/rejected": 414.95703125, | |
| "logps/chosen": -0.8159686923027039, | |
| "logps/rejected": -1.314146876335144, | |
| "loss": 1.0967, | |
| "nll_loss": 1.1257387399673462, | |
| "rewards/accuracies": 0.6499999761581421, | |
| "rewards/chosen": -0.040798433125019073, | |
| "rewards/margins": 0.024908915162086487, | |
| "rewards/rejected": -0.06570734083652496, | |
| "step": 265 | |
| }, | |
| { | |
| "epoch": 1.6071428571428572, | |
| "grad_norm": 17.566476821899414, | |
| "learning_rate": 3.0429030972509227e-06, | |
| "log_odds_chosen": 0.7149374485015869, | |
| "log_odds_ratio": -0.45594334602355957, | |
| "logits/chosen": 354.78692626953125, | |
| "logits/rejected": 364.1693115234375, | |
| "logps/chosen": -0.836654543876648, | |
| "logps/rejected": -1.2243211269378662, | |
| "loss": 1.0429, | |
| "nll_loss": 1.1530247926712036, | |
| "rewards/accuracies": 0.949999988079071, | |
| "rewards/chosen": -0.041832733899354935, | |
| "rewards/margins": 0.019383331760764122, | |
| "rewards/rejected": -0.06121605634689331, | |
| "step": 270 | |
| }, | |
| { | |
| "epoch": 1.6369047619047619, | |
| "grad_norm": 16.325292587280273, | |
| "learning_rate": 3.0151134457776365e-06, | |
| "log_odds_chosen": 0.7199637293815613, | |
| "log_odds_ratio": -0.434516578912735, | |
| "logits/chosen": 322.65130615234375, | |
| "logits/rejected": 338.8601989746094, | |
| "logps/chosen": -0.5899932384490967, | |
| "logps/rejected": -0.9299219250679016, | |
| "loss": 1.076, | |
| "nll_loss": 0.9019113779067993, | |
| "rewards/accuracies": 0.8500000238418579, | |
| "rewards/chosen": -0.029499661177396774, | |
| "rewards/margins": 0.016996433958411217, | |
| "rewards/rejected": -0.04649610072374344, | |
| "step": 275 | |
| }, | |
| { | |
| "epoch": 1.6666666666666665, | |
| "grad_norm": 14.822625160217285, | |
| "learning_rate": 2.988071523335984e-06, | |
| "log_odds_chosen": 0.9259305000305176, | |
| "log_odds_ratio": -0.5428971648216248, | |
| "logits/chosen": 395.42230224609375, | |
| "logits/rejected": 360.65234375, | |
| "logps/chosen": -0.5504172444343567, | |
| "logps/rejected": -1.1162601709365845, | |
| "loss": 1.0232, | |
| "nll_loss": 0.8841646909713745, | |
| "rewards/accuracies": 0.8999999761581421, | |
| "rewards/chosen": -0.027520865201950073, | |
| "rewards/margins": 0.02829214558005333, | |
| "rewards/rejected": -0.0558130145072937, | |
| "step": 280 | |
| }, | |
| { | |
| "epoch": 1.6964285714285714, | |
| "grad_norm": 13.360857009887695, | |
| "learning_rate": 2.961744388795462e-06, | |
| "log_odds_chosen": 0.7002909183502197, | |
| "log_odds_ratio": -0.46074408292770386, | |
| "logits/chosen": 344.73712158203125, | |
| "logits/rejected": 373.6070556640625, | |
| "logps/chosen": -0.6551269292831421, | |
| "logps/rejected": -1.0137001276016235, | |
| "loss": 0.9926, | |
| "nll_loss": 0.9025999307632446, | |
| "rewards/accuracies": 0.8500000238418579, | |
| "rewards/chosen": -0.032756343483924866, | |
| "rewards/margins": 0.017928656190633774, | |
| "rewards/rejected": -0.05068499967455864, | |
| "step": 285 | |
| }, | |
| { | |
| "epoch": 1.7261904761904763, | |
| "grad_norm": 12.45747184753418, | |
| "learning_rate": 2.9361010975735177e-06, | |
| "log_odds_chosen": 1.1189637184143066, | |
| "log_odds_ratio": -0.3354729413986206, | |
| "logits/chosen": 367.6058349609375, | |
| "logits/rejected": 436.7281188964844, | |
| "logps/chosen": -0.7302691340446472, | |
| "logps/rejected": -1.3499362468719482, | |
| "loss": 1.0165, | |
| "nll_loss": 0.9672770500183105, | |
| "rewards/accuracies": 0.8999999761581421, | |
| "rewards/chosen": -0.03651345893740654, | |
| "rewards/margins": 0.03098336234688759, | |
| "rewards/rejected": -0.06749682128429413, | |
| "step": 290 | |
| }, | |
| { | |
| "epoch": 1.755952380952381, | |
| "grad_norm": 20.8387508392334, | |
| "learning_rate": 2.9111125486979104e-06, | |
| "log_odds_chosen": 0.8264445066452026, | |
| "log_odds_ratio": -0.45267271995544434, | |
| "logits/chosen": 380.7887268066406, | |
| "logits/rejected": 392.38018798828125, | |
| "logps/chosen": -0.7333472967147827, | |
| "logps/rejected": -1.1829124689102173, | |
| "loss": 1.0473, | |
| "nll_loss": 1.2072994709014893, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": -0.036667369306087494, | |
| "rewards/margins": 0.0224782545119524, | |
| "rewards/rejected": -0.059145621955394745, | |
| "step": 295 | |
| }, | |
| { | |
| "epoch": 1.7857142857142856, | |
| "grad_norm": 14.306224822998047, | |
| "learning_rate": 2.8867513459481293e-06, | |
| "log_odds_chosen": 1.0914583206176758, | |
| "log_odds_ratio": -0.3539409041404724, | |
| "logits/chosen": 376.03033447265625, | |
| "logits/rejected": 378.3951416015625, | |
| "logps/chosen": -0.58739173412323, | |
| "logps/rejected": -1.1421259641647339, | |
| "loss": 0.975, | |
| "nll_loss": 0.8565952181816101, | |
| "rewards/accuracies": 0.8500000238418579, | |
| "rewards/chosen": -0.02936958707869053, | |
| "rewards/margins": 0.027736714109778404, | |
| "rewards/rejected": -0.05710630491375923, | |
| "step": 300 | |
| }, | |
| { | |
| "epoch": 1.8154761904761905, | |
| "grad_norm": 15.119916915893555, | |
| "learning_rate": 2.862991671569341e-06, | |
| "log_odds_chosen": 0.3856947720050812, | |
| "log_odds_ratio": -0.5676103830337524, | |
| "logits/chosen": 432.51239013671875, | |
| "logits/rejected": 413.9574279785156, | |
| "logps/chosen": -1.001999855041504, | |
| "logps/rejected": -1.1941574811935425, | |
| "loss": 1.026, | |
| "nll_loss": 1.176692247390747, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": -0.050099991261959076, | |
| "rewards/margins": 0.009607886895537376, | |
| "rewards/rejected": -0.0597078800201416, | |
| "step": 305 | |
| }, | |
| { | |
| "epoch": 1.8452380952380953, | |
| "grad_norm": 13.858372688293457, | |
| "learning_rate": 2.839809171235324e-06, | |
| "log_odds_chosen": 0.9851268529891968, | |
| "log_odds_ratio": -0.46969637274742126, | |
| "logits/chosen": 347.86224365234375, | |
| "logits/rejected": 354.42236328125, | |
| "logps/chosen": -0.7486366629600525, | |
| "logps/rejected": -1.323521375656128, | |
| "loss": 1.0778, | |
| "nll_loss": 1.036675214767456, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": -0.037431832402944565, | |
| "rewards/margins": 0.028744244948029518, | |
| "rewards/rejected": -0.06617607176303864, | |
| "step": 310 | |
| }, | |
| { | |
| "epoch": 1.875, | |
| "grad_norm": 17.435771942138672, | |
| "learning_rate": 2.817180849095055e-06, | |
| "log_odds_chosen": 0.7195825576782227, | |
| "log_odds_ratio": -0.5751198530197144, | |
| "logits/chosen": 372.2769470214844, | |
| "logits/rejected": 380.11322021484375, | |
| "logps/chosen": -0.9793826937675476, | |
| "logps/rejected": -1.4906439781188965, | |
| "loss": 1.0867, | |
| "nll_loss": 1.2049812078475952, | |
| "rewards/accuracies": 0.6499999761581421, | |
| "rewards/chosen": -0.04896913468837738, | |
| "rewards/margins": 0.025563066825270653, | |
| "rewards/rejected": -0.07453219592571259, | |
| "step": 315 | |
| }, | |
| { | |
| "epoch": 1.9047619047619047, | |
| "grad_norm": 13.72044563293457, | |
| "learning_rate": 2.7950849718747376e-06, | |
| "log_odds_chosen": 1.036712884902954, | |
| "log_odds_ratio": -0.36927738785743713, | |
| "logits/chosen": 385.5801086425781, | |
| "logits/rejected": 423.3285217285156, | |
| "logps/chosen": -0.7053465247154236, | |
| "logps/rejected": -1.304882526397705, | |
| "loss": 0.9866, | |
| "nll_loss": 0.9042797088623047, | |
| "rewards/accuracies": 0.949999988079071, | |
| "rewards/chosen": -0.03526733070611954, | |
| "rewards/margins": 0.029976800084114075, | |
| "rewards/rejected": -0.06524413079023361, | |
| "step": 320 | |
| }, | |
| { | |
| "epoch": 1.9345238095238095, | |
| "grad_norm": 17.074472427368164, | |
| "learning_rate": 2.773500981126146e-06, | |
| "log_odds_chosen": 1.224524736404419, | |
| "log_odds_ratio": -0.35412582755088806, | |
| "logits/chosen": 397.96649169921875, | |
| "logits/rejected": 423.78094482421875, | |
| "logps/chosen": -0.6596122980117798, | |
| "logps/rejected": -1.2944166660308838, | |
| "loss": 0.9893, | |
| "nll_loss": 0.9460498690605164, | |
| "rewards/accuracies": 0.8500000238418579, | |
| "rewards/chosen": -0.03298061341047287, | |
| "rewards/margins": 0.0317402258515358, | |
| "rewards/rejected": -0.06472083181142807, | |
| "step": 325 | |
| }, | |
| { | |
| "epoch": 1.9642857142857144, | |
| "grad_norm": 23.940942764282227, | |
| "learning_rate": 2.752409412815902e-06, | |
| "log_odds_chosen": 1.0460882186889648, | |
| "log_odds_ratio": -0.416635125875473, | |
| "logits/chosen": 391.8972473144531, | |
| "logits/rejected": 401.4590148925781, | |
| "logps/chosen": -0.7760294675827026, | |
| "logps/rejected": -1.4733283519744873, | |
| "loss": 1.0094, | |
| "nll_loss": 0.9179704785346985, | |
| "rewards/accuracies": 0.8500000238418579, | |
| "rewards/chosen": -0.038801468908786774, | |
| "rewards/margins": 0.034864943474531174, | |
| "rewards/rejected": -0.07366641610860825, | |
| "step": 330 | |
| }, | |
| { | |
| "epoch": 1.994047619047619, | |
| "grad_norm": 18.142581939697266, | |
| "learning_rate": 2.7317918235407652e-06, | |
| "log_odds_chosen": 0.5594000816345215, | |
| "log_odds_ratio": -0.544743537902832, | |
| "logits/chosen": 377.28729248046875, | |
| "logits/rejected": 394.92413330078125, | |
| "logps/chosen": -0.9491284489631653, | |
| "logps/rejected": -1.2867456674575806, | |
| "loss": 1.0804, | |
| "nll_loss": 1.2649915218353271, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": -0.047456420958042145, | |
| "rewards/margins": 0.016880858689546585, | |
| "rewards/rejected": -0.06433728337287903, | |
| "step": 335 | |
| }, | |
| { | |
| "epoch": 2.0, | |
| "eval_log_odds_chosen": 0.21522486209869385, | |
| "eval_log_odds_ratio": -0.6897445917129517, | |
| "eval_logits/chosen": 330.50665283203125, | |
| "eval_logits/rejected": 278.2473449707031, | |
| "eval_logps/chosen": -1.022003173828125, | |
| "eval_logps/rejected": -1.1829901933670044, | |
| "eval_loss": 1.4225263595581055, | |
| "eval_nll_loss": 1.4082757234573364, | |
| "eval_rewards/accuracies": 0.5142857432365417, | |
| "eval_rewards/chosen": -0.05110016465187073, | |
| "eval_rewards/margins": 0.008049344643950462, | |
| "eval_rewards/rejected": -0.05914951115846634, | |
| "eval_runtime": 201.3887, | |
| "eval_samples_per_second": 2.746, | |
| "eval_steps_per_second": 0.348, | |
| "step": 336 | |
| }, | |
| { | |
| "epoch": 2.0238095238095237, | |
| "grad_norm": 25.43450927734375, | |
| "learning_rate": 2.711630722733202e-06, | |
| "log_odds_chosen": 1.8818715810775757, | |
| "log_odds_ratio": -0.2647668421268463, | |
| "logits/chosen": 383.9913330078125, | |
| "logits/rejected": 380.02801513671875, | |
| "logps/chosen": -0.47935551404953003, | |
| "logps/rejected": -1.454525351524353, | |
| "loss": 0.6307, | |
| "nll_loss": 0.6181924343109131, | |
| "rewards/accuracies": 0.949999988079071, | |
| "rewards/chosen": -0.023967772722244263, | |
| "rewards/margins": 0.04875849559903145, | |
| "rewards/rejected": -0.07272626459598541, | |
| "step": 340 | |
| }, | |
| { | |
| "epoch": 2.0535714285714284, | |
| "grad_norm": 16.127397537231445, | |
| "learning_rate": 2.691909510290828e-06, | |
| "log_odds_chosen": 2.698786497116089, | |
| "log_odds_ratio": -0.14153851568698883, | |
| "logits/chosen": 388.04534912109375, | |
| "logits/rejected": 355.0823669433594, | |
| "logps/chosen": -0.29280781745910645, | |
| "logps/rejected": -1.475531816482544, | |
| "loss": 0.5566, | |
| "nll_loss": 0.6942065954208374, | |
| "rewards/accuracies": 0.8999999761581421, | |
| "rewards/chosen": -0.014640390872955322, | |
| "rewards/margins": 0.059136200696229935, | |
| "rewards/rejected": -0.07377658784389496, | |
| "step": 345 | |
| }, | |
| { | |
| "epoch": 2.0833333333333335, | |
| "grad_norm": 13.852057456970215, | |
| "learning_rate": 2.6726124191242444e-06, | |
| "log_odds_chosen": 2.8243956565856934, | |
| "log_odds_ratio": -0.09990663826465607, | |
| "logits/chosen": 354.74615478515625, | |
| "logits/rejected": 387.60528564453125, | |
| "logps/chosen": -0.37845298647880554, | |
| "logps/rejected": -2.111022472381592, | |
| "loss": 0.5438, | |
| "nll_loss": 0.5637595057487488, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.018922649323940277, | |
| "rewards/margins": 0.08662847429513931, | |
| "rewards/rejected": -0.10555113852024078, | |
| "step": 350 | |
| }, | |
| { | |
| "epoch": 2.113095238095238, | |
| "grad_norm": 16.1456241607666, | |
| "learning_rate": 2.6537244621713765e-06, | |
| "log_odds_chosen": 2.1223108768463135, | |
| "log_odds_ratio": -0.14685922861099243, | |
| "logits/chosen": 359.0018005371094, | |
| "logits/rejected": 385.0552062988281, | |
| "logps/chosen": -0.425741970539093, | |
| "logps/rejected": -1.6288859844207764, | |
| "loss": 0.5366, | |
| "nll_loss": 0.5897840261459351, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.0212871003895998, | |
| "rewards/margins": 0.060157209634780884, | |
| "rewards/rejected": -0.08144429326057434, | |
| "step": 355 | |
| }, | |
| { | |
| "epoch": 2.142857142857143, | |
| "grad_norm": 11.441978454589844, | |
| "learning_rate": 2.6352313834736496e-06, | |
| "log_odds_chosen": 2.8245463371276855, | |
| "log_odds_ratio": -0.12314938008785248, | |
| "logits/chosen": 369.9977722167969, | |
| "logits/rejected": 407.8392333984375, | |
| "logps/chosen": -0.3003098964691162, | |
| "logps/rejected": -1.5362733602523804, | |
| "loss": 0.5329, | |
| "nll_loss": 0.5958473086357117, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.015015493147075176, | |
| "rewards/margins": 0.06179817393422127, | |
| "rewards/rejected": -0.07681366801261902, | |
| "step": 360 | |
| }, | |
| { | |
| "epoch": 2.1726190476190474, | |
| "grad_norm": 12.86784839630127, | |
| "learning_rate": 2.6171196129510684e-06, | |
| "log_odds_chosen": 2.503121852874756, | |
| "log_odds_ratio": -0.1363641321659088, | |
| "logits/chosen": 334.23468017578125, | |
| "logits/rejected": 311.822509765625, | |
| "logps/chosen": -0.3001948595046997, | |
| "logps/rejected": -1.4599236249923706, | |
| "loss": 0.5156, | |
| "nll_loss": 0.5041288137435913, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.0150097431614995, | |
| "rewards/margins": 0.05798644572496414, | |
| "rewards/rejected": -0.07299618422985077, | |
| "step": 365 | |
| }, | |
| { | |
| "epoch": 2.2023809523809526, | |
| "grad_norm": 13.525810241699219, | |
| "learning_rate": 2.599376224550182e-06, | |
| "log_odds_chosen": 1.8203538656234741, | |
| "log_odds_ratio": -0.19545204937458038, | |
| "logits/chosen": 313.6293029785156, | |
| "logits/rejected": 326.6126403808594, | |
| "logps/chosen": -0.4296380877494812, | |
| "logps/rejected": -1.4019193649291992, | |
| "loss": 0.5492, | |
| "nll_loss": 0.6865583062171936, | |
| "rewards/accuracies": 0.949999988079071, | |
| "rewards/chosen": -0.02148190513253212, | |
| "rewards/margins": 0.04861406981945038, | |
| "rewards/rejected": -0.0700959712266922, | |
| "step": 370 | |
| }, | |
| { | |
| "epoch": 2.232142857142857, | |
| "grad_norm": 14.674184799194336, | |
| "learning_rate": 2.5819888974716113e-06, | |
| "log_odds_chosen": 1.8546769618988037, | |
| "log_odds_ratio": -0.22947852313518524, | |
| "logits/chosen": 377.0060119628906, | |
| "logits/rejected": 399.48590087890625, | |
| "logps/chosen": -0.3884517252445221, | |
| "logps/rejected": -1.292641520500183, | |
| "loss": 0.5785, | |
| "nll_loss": 0.5873233079910278, | |
| "rewards/accuracies": 0.8999999761581421, | |
| "rewards/chosen": -0.019422587007284164, | |
| "rewards/margins": 0.045209504663944244, | |
| "rewards/rejected": -0.06463208049535751, | |
| "step": 375 | |
| }, | |
| { | |
| "epoch": 2.261904761904762, | |
| "grad_norm": 11.674169540405273, | |
| "learning_rate": 2.564945880212886e-06, | |
| "log_odds_chosen": 2.2116830348968506, | |
| "log_odds_ratio": -0.12257333844900131, | |
| "logits/chosen": 367.1352233886719, | |
| "logits/rejected": 336.3498840332031, | |
| "logps/chosen": -0.3219259977340698, | |
| "logps/rejected": -1.350303292274475, | |
| "loss": 0.5297, | |
| "nll_loss": 0.5648508071899414, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.01609629951417446, | |
| "rewards/margins": 0.051418863236904144, | |
| "rewards/rejected": -0.06751517206430435, | |
| "step": 380 | |
| }, | |
| { | |
| "epoch": 2.2916666666666665, | |
| "grad_norm": 13.154436111450195, | |
| "learning_rate": 2.5482359571881276e-06, | |
| "log_odds_chosen": 2.5731985569000244, | |
| "log_odds_ratio": -0.1179199069738388, | |
| "logits/chosen": 340.62567138671875, | |
| "logits/rejected": 364.66888427734375, | |
| "logps/chosen": -0.27915579080581665, | |
| "logps/rejected": -1.4784090518951416, | |
| "loss": 0.5102, | |
| "nll_loss": 0.49411922693252563, | |
| "rewards/accuracies": 0.949999988079071, | |
| "rewards/chosen": -0.013957786373794079, | |
| "rewards/margins": 0.059962667524814606, | |
| "rewards/rejected": -0.07392045855522156, | |
| "step": 385 | |
| }, | |
| { | |
| "epoch": 2.3214285714285716, | |
| "grad_norm": 11.973630905151367, | |
| "learning_rate": 2.5318484177091667e-06, | |
| "log_odds_chosen": 2.2964727878570557, | |
| "log_odds_ratio": -0.13063772022724152, | |
| "logits/chosen": 359.48345947265625, | |
| "logits/rejected": 400.7521667480469, | |
| "logps/chosen": -0.3307945430278778, | |
| "logps/rejected": -1.4601058959960938, | |
| "loss": 0.5685, | |
| "nll_loss": 0.5173304677009583, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.01653972640633583, | |
| "rewards/margins": 0.056465573608875275, | |
| "rewards/rejected": -0.0730053037405014, | |
| "step": 390 | |
| }, | |
| { | |
| "epoch": 2.3511904761904763, | |
| "grad_norm": 10.832448959350586, | |
| "learning_rate": 2.515773027133138e-06, | |
| "log_odds_chosen": 2.685337543487549, | |
| "log_odds_ratio": -0.10510516166687012, | |
| "logits/chosen": 360.8955383300781, | |
| "logits/rejected": 359.0787048339844, | |
| "logps/chosen": -0.2764904201030731, | |
| "logps/rejected": -1.409620761871338, | |
| "loss": 0.511, | |
| "nll_loss": 0.500057578086853, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.013824522495269775, | |
| "rewards/margins": 0.05665650963783264, | |
| "rewards/rejected": -0.07048103958368301, | |
| "step": 395 | |
| }, | |
| { | |
| "epoch": 2.380952380952381, | |
| "grad_norm": 12.639362335205078, | |
| "learning_rate": 2.5e-06, | |
| "log_odds_chosen": 2.6170766353607178, | |
| "log_odds_ratio": -0.1315409243106842, | |
| "logits/chosen": 342.3817138671875, | |
| "logits/rejected": 396.52410888671875, | |
| "logps/chosen": -0.328637957572937, | |
| "logps/rejected": -1.6732807159423828, | |
| "loss": 0.527, | |
| "nll_loss": 0.5181549787521362, | |
| "rewards/accuracies": 0.949999988079071, | |
| "rewards/chosen": -0.01643189787864685, | |
| "rewards/margins": 0.067232146859169, | |
| "rewards/rejected": -0.08366404473781586, | |
| "step": 400 | |
| }, | |
| { | |
| "epoch": 2.4107142857142856, | |
| "grad_norm": 14.651439666748047, | |
| "learning_rate": 2.484519974999767e-06, | |
| "log_odds_chosen": 2.2201273441314697, | |
| "log_odds_ratio": -0.16681890189647675, | |
| "logits/chosen": 424.72918701171875, | |
| "logits/rejected": 371.58966064453125, | |
| "logps/chosen": -0.30684176087379456, | |
| "logps/rejected": -1.3665101528167725, | |
| "loss": 0.5749, | |
| "nll_loss": 0.5082268714904785, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.015342088416218758, | |
| "rewards/margins": 0.05298342555761337, | |
| "rewards/rejected": -0.06832550466060638, | |
| "step": 405 | |
| }, | |
| { | |
| "epoch": 2.4404761904761907, | |
| "grad_norm": 11.325146675109863, | |
| "learning_rate": 2.4693239916239746e-06, | |
| "log_odds_chosen": 2.499159336090088, | |
| "log_odds_ratio": -0.21561208367347717, | |
| "logits/chosen": 356.08258056640625, | |
| "logits/rejected": 390.337890625, | |
| "logps/chosen": -0.372717022895813, | |
| "logps/rejected": -1.7423985004425049, | |
| "loss": 0.5384, | |
| "nll_loss": 0.5256937742233276, | |
| "rewards/accuracies": 0.949999988079071, | |
| "rewards/chosen": -0.01863585226237774, | |
| "rewards/margins": 0.06848406046628952, | |
| "rewards/rejected": -0.087119922041893, | |
| "step": 410 | |
| }, | |
| { | |
| "epoch": 2.4702380952380953, | |
| "grad_norm": 12.71360969543457, | |
| "learning_rate": 2.4544034683690802e-06, | |
| "log_odds_chosen": 2.8460097312927246, | |
| "log_odds_ratio": -0.09599516540765762, | |
| "logits/chosen": 386.35302734375, | |
| "logits/rejected": 372.95367431640625, | |
| "logps/chosen": -0.26167118549346924, | |
| "logps/rejected": -1.7513694763183594, | |
| "loss": 0.551, | |
| "nll_loss": 0.4728317856788635, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.013083559460937977, | |
| "rewards/margins": 0.0744849145412445, | |
| "rewards/rejected": -0.08756847679615021, | |
| "step": 415 | |
| }, | |
| { | |
| "epoch": 2.5, | |
| "grad_norm": 19.943899154663086, | |
| "learning_rate": 2.4397501823713327e-06, | |
| "log_odds_chosen": 2.2042346000671387, | |
| "log_odds_ratio": -0.17198964953422546, | |
| "logits/chosen": 370.604736328125, | |
| "logits/rejected": 320.15875244140625, | |
| "logps/chosen": -0.30728116631507874, | |
| "logps/rejected": -1.3875327110290527, | |
| "loss": 0.5247, | |
| "nll_loss": 0.6666563153266907, | |
| "rewards/accuracies": 0.949999988079071, | |
| "rewards/chosen": -0.015364060178399086, | |
| "rewards/margins": 0.05401257798075676, | |
| "rewards/rejected": -0.0693766325712204, | |
| "step": 420 | |
| }, | |
| { | |
| "epoch": 2.5297619047619047, | |
| "grad_norm": 12.133928298950195, | |
| "learning_rate": 2.4253562503633297e-06, | |
| "log_odds_chosen": 2.90407133102417, | |
| "log_odds_ratio": -0.06831072270870209, | |
| "logits/chosen": 342.95733642578125, | |
| "logits/rejected": 367.10894775390625, | |
| "logps/chosen": -0.3182448446750641, | |
| "logps/rejected": -1.8485863208770752, | |
| "loss": 0.507, | |
| "nll_loss": 0.6053184270858765, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.015912240371108055, | |
| "rewards/margins": 0.07651706039905548, | |
| "rewards/rejected": -0.09242931008338928, | |
| "step": 425 | |
| }, | |
| { | |
| "epoch": 2.5595238095238093, | |
| "grad_norm": 12.26125717163086, | |
| "learning_rate": 2.411214110852061e-06, | |
| "log_odds_chosen": 2.55293607711792, | |
| "log_odds_ratio": -0.10798110067844391, | |
| "logits/chosen": 391.63848876953125, | |
| "logits/rejected": 404.0960388183594, | |
| "logps/chosen": -0.27310800552368164, | |
| "logps/rejected": -1.4932405948638916, | |
| "loss": 0.5084, | |
| "nll_loss": 0.4452734887599945, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.013655401766300201, | |
| "rewards/margins": 0.06100662797689438, | |
| "rewards/rejected": -0.07466202974319458, | |
| "step": 430 | |
| }, | |
| { | |
| "epoch": 2.5892857142857144, | |
| "grad_norm": 13.624907493591309, | |
| "learning_rate": 2.3973165074269213e-06, | |
| "log_odds_chosen": 2.3622078895568848, | |
| "log_odds_ratio": -0.2018093764781952, | |
| "logits/chosen": 356.6548767089844, | |
| "logits/rejected": 360.92474365234375, | |
| "logps/chosen": -0.36742302775382996, | |
| "logps/rejected": -1.6279608011245728, | |
| "loss": 0.5587, | |
| "nll_loss": 0.5465327501296997, | |
| "rewards/accuracies": 0.949999988079071, | |
| "rewards/chosen": -0.018371151760220528, | |
| "rewards/margins": 0.06302689015865326, | |
| "rewards/rejected": -0.08139804750680923, | |
| "step": 435 | |
| }, | |
| { | |
| "epoch": 2.619047619047619, | |
| "grad_norm": 13.956061363220215, | |
| "learning_rate": 2.3836564731139807e-06, | |
| "log_odds_chosen": 2.7665021419525146, | |
| "log_odds_ratio": -0.10336035490036011, | |
| "logits/chosen": 352.8879089355469, | |
| "logits/rejected": 368.2629699707031, | |
| "logps/chosen": -0.2728215754032135, | |
| "logps/rejected": -1.5103857517242432, | |
| "loss": 0.5462, | |
| "nll_loss": 0.46932536363601685, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.01364107709378004, | |
| "rewards/margins": 0.06187821552157402, | |
| "rewards/rejected": -0.07551928609609604, | |
| "step": 440 | |
| }, | |
| { | |
| "epoch": 2.6488095238095237, | |
| "grad_norm": 11.78285026550293, | |
| "learning_rate": 2.3702273156998867e-06, | |
| "log_odds_chosen": 2.3770580291748047, | |
| "log_odds_ratio": -0.116610586643219, | |
| "logits/chosen": 337.458740234375, | |
| "logits/rejected": 374.8058776855469, | |
| "logps/chosen": -0.3450239896774292, | |
| "logps/rejected": -1.6122970581054688, | |
| "loss": 0.5447, | |
| "nll_loss": 0.5509533882141113, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.01725119911134243, | |
| "rewards/margins": 0.06336364895105362, | |
| "rewards/rejected": -0.0806148499250412, | |
| "step": 445 | |
| }, | |
| { | |
| "epoch": 2.678571428571429, | |
| "grad_norm": 14.717430114746094, | |
| "learning_rate": 2.357022603955159e-06, | |
| "log_odds_chosen": 2.9325308799743652, | |
| "log_odds_ratio": -0.08914806693792343, | |
| "logits/chosen": 375.6985778808594, | |
| "logits/rejected": 367.75787353515625, | |
| "logps/chosen": -0.35901179909706116, | |
| "logps/rejected": -2.006959915161133, | |
| "loss": 0.5752, | |
| "nll_loss": 0.49780726432800293, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.01795058883726597, | |
| "rewards/margins": 0.08239741623401642, | |
| "rewards/rejected": -0.10034799575805664, | |
| "step": 450 | |
| }, | |
| { | |
| "epoch": 2.7083333333333335, | |
| "grad_norm": 18.606168746948242, | |
| "learning_rate": 2.3440361546924774e-06, | |
| "log_odds_chosen": 2.5973100662231445, | |
| "log_odds_ratio": -0.09789351373910904, | |
| "logits/chosen": 371.6011962890625, | |
| "logits/rejected": 356.3915710449219, | |
| "logps/chosen": -0.27857938408851624, | |
| "logps/rejected": -1.5046110153198242, | |
| "loss": 0.5909, | |
| "nll_loss": 0.5146271586418152, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.013928967528045177, | |
| "rewards/margins": 0.061301589012145996, | |
| "rewards/rejected": -0.07523055374622345, | |
| "step": 455 | |
| }, | |
| { | |
| "epoch": 2.738095238095238, | |
| "grad_norm": 14.296780586242676, | |
| "learning_rate": 2.3312620206007847e-06, | |
| "log_odds_chosen": 2.4838976860046387, | |
| "log_odds_ratio": -0.13478006422519684, | |
| "logits/chosen": 412.72332763671875, | |
| "logits/rejected": 433.41070556640625, | |
| "logps/chosen": -0.3284691870212555, | |
| "logps/rejected": -1.5628798007965088, | |
| "loss": 0.5718, | |
| "nll_loss": 0.5497859716415405, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.016423460096120834, | |
| "rewards/margins": 0.061720531433820724, | |
| "rewards/rejected": -0.07814399152994156, | |
| "step": 460 | |
| }, | |
| { | |
| "epoch": 2.767857142857143, | |
| "grad_norm": 12.61571216583252, | |
| "learning_rate": 2.3186944788008413e-06, | |
| "log_odds_chosen": 2.7270541191101074, | |
| "log_odds_ratio": -0.11491680145263672, | |
| "logits/chosen": 401.2771301269531, | |
| "logits/rejected": 386.7737121582031, | |
| "logps/chosen": -0.25855037569999695, | |
| "logps/rejected": -1.5313479900360107, | |
| "loss": 0.5771, | |
| "nll_loss": 0.6066937446594238, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.012927519157528877, | |
| "rewards/margins": 0.06363988667726517, | |
| "rewards/rejected": -0.0765674039721489, | |
| "step": 465 | |
| }, | |
| { | |
| "epoch": 2.7976190476190474, | |
| "grad_norm": 11.338821411132812, | |
| "learning_rate": 2.3063280200722128e-06, | |
| "log_odds_chosen": 2.1205978393554688, | |
| "log_odds_ratio": -0.20822449028491974, | |
| "logits/chosen": 357.4671630859375, | |
| "logits/rejected": 335.1904296875, | |
| "logps/chosen": -0.36784037947654724, | |
| "logps/rejected": -1.465673804283142, | |
| "loss": 0.5175, | |
| "nll_loss": 0.5160819888114929, | |
| "rewards/accuracies": 0.8999999761581421, | |
| "rewards/chosen": -0.018392018973827362, | |
| "rewards/margins": 0.0548916831612587, | |
| "rewards/rejected": -0.07328370213508606, | |
| "step": 470 | |
| }, | |
| { | |
| "epoch": 2.8273809523809526, | |
| "grad_norm": 11.929247856140137, | |
| "learning_rate": 2.2941573387056174e-06, | |
| "log_odds_chosen": 2.6770148277282715, | |
| "log_odds_ratio": -0.0899493470788002, | |
| "logits/chosen": 325.32476806640625, | |
| "logits/rejected": 420.31280517578125, | |
| "logps/chosen": -0.3526560664176941, | |
| "logps/rejected": -1.848586082458496, | |
| "loss": 0.5393, | |
| "nll_loss": 0.48061132431030273, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.017632806673645973, | |
| "rewards/margins": 0.07479649782180786, | |
| "rewards/rejected": -0.09242929518222809, | |
| "step": 475 | |
| }, | |
| { | |
| "epoch": 2.857142857142857, | |
| "grad_norm": 13.174921035766602, | |
| "learning_rate": 2.2821773229381924e-06, | |
| "log_odds_chosen": 2.5945560932159424, | |
| "log_odds_ratio": -0.09449335187673569, | |
| "logits/chosen": 369.5697326660156, | |
| "logits/rejected": 390.010009765625, | |
| "logps/chosen": -0.26720184087753296, | |
| "logps/rejected": -1.5164011716842651, | |
| "loss": 0.4922, | |
| "nll_loss": 0.5011499524116516, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.013360092416405678, | |
| "rewards/margins": 0.06245996803045273, | |
| "rewards/rejected": -0.07582006603479385, | |
| "step": 480 | |
| }, | |
| { | |
| "epoch": 2.886904761904762, | |
| "grad_norm": 17.229890823364258, | |
| "learning_rate": 2.270383045932499e-06, | |
| "log_odds_chosen": 2.310241222381592, | |
| "log_odds_ratio": -0.1323787122964859, | |
| "logits/chosen": 352.09368896484375, | |
| "logits/rejected": 390.89129638671875, | |
| "logps/chosen": -0.41382861137390137, | |
| "logps/rejected": -1.641650915145874, | |
| "loss": 0.5059, | |
| "nll_loss": 0.5348228216171265, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.020691432058811188, | |
| "rewards/margins": 0.06139112263917923, | |
| "rewards/rejected": -0.08208255469799042, | |
| "step": 485 | |
| }, | |
| { | |
| "epoch": 2.9166666666666665, | |
| "grad_norm": 11.300827026367188, | |
| "learning_rate": 2.2587697572631284e-06, | |
| "log_odds_chosen": 1.7844845056533813, | |
| "log_odds_ratio": -0.32421550154685974, | |
| "logits/chosen": 398.19256591796875, | |
| "logits/rejected": 312.98248291015625, | |
| "logps/chosen": -0.5920530557632446, | |
| "logps/rejected": -1.449924349784851, | |
| "loss": 0.6026, | |
| "nll_loss": 0.5389891862869263, | |
| "rewards/accuracies": 0.8999999761581421, | |
| "rewards/chosen": -0.0296026524156332, | |
| "rewards/margins": 0.04289356991648674, | |
| "rewards/rejected": -0.07249622046947479, | |
| "step": 490 | |
| }, | |
| { | |
| "epoch": 2.946428571428571, | |
| "grad_norm": 13.277922630310059, | |
| "learning_rate": 2.2473328748774737e-06, | |
| "log_odds_chosen": 2.5535902976989746, | |
| "log_odds_ratio": -0.12692758440971375, | |
| "logits/chosen": 344.2357177734375, | |
| "logits/rejected": 398.01348876953125, | |
| "logps/chosen": -0.27592435479164124, | |
| "logps/rejected": -1.5041601657867432, | |
| "loss": 0.5166, | |
| "nll_loss": 0.4667043685913086, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.013796217739582062, | |
| "rewards/margins": 0.06141179800033569, | |
| "rewards/rejected": -0.07520802319049835, | |
| "step": 495 | |
| }, | |
| { | |
| "epoch": 2.9761904761904763, | |
| "grad_norm": 10.682828903198242, | |
| "learning_rate": 2.23606797749979e-06, | |
| "log_odds_chosen": 2.8015060424804688, | |
| "log_odds_ratio": -0.12307514250278473, | |
| "logits/chosen": 391.4792785644531, | |
| "logits/rejected": 362.94403076171875, | |
| "logps/chosen": -0.2695181965827942, | |
| "logps/rejected": -1.6667454242706299, | |
| "loss": 0.5651, | |
| "nll_loss": 0.40066272020339966, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.01347590796649456, | |
| "rewards/margins": 0.06986136734485626, | |
| "rewards/rejected": -0.08333728462457657, | |
| "step": 500 | |
| }, | |
| { | |
| "epoch": 3.0, | |
| "eval_log_odds_chosen": 0.3315908908843994, | |
| "eval_log_odds_ratio": -0.6744564771652222, | |
| "eval_logits/chosen": 304.2648620605469, | |
| "eval_logits/rejected": 249.09341430664062, | |
| "eval_logps/chosen": -1.1561349630355835, | |
| "eval_logps/rejected": -1.3795461654663086, | |
| "eval_loss": 1.6016675233840942, | |
| "eval_nll_loss": 1.5642986297607422, | |
| "eval_rewards/accuracies": 0.5714285969734192, | |
| "eval_rewards/chosen": -0.057806748896837234, | |
| "eval_rewards/margins": 0.011170565150678158, | |
| "eval_rewards/rejected": -0.06897731870412827, | |
| "eval_runtime": 201.3868, | |
| "eval_samples_per_second": 2.746, | |
| "eval_steps_per_second": 0.348, | |
| "step": 504 | |
| }, | |
| { | |
| "epoch": 3.0, | |
| "step": 504, | |
| "total_flos": 0.0, | |
| "train_loss": 1.4336541661667446, | |
| "train_runtime": 14833.1355, | |
| "train_samples_per_second": 1.085, | |
| "train_steps_per_second": 0.034 | |
| } | |
| ], | |
| "logging_steps": 5, | |
| "max_steps": 504, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 3, | |
| "save_steps": 500, | |
| "stateful_callbacks": { | |
| "TrainerControl": { | |
| "args": { | |
| "should_epoch_stop": false, | |
| "should_evaluate": false, | |
| "should_log": false, | |
| "should_save": false, | |
| "should_training_stop": false | |
| }, | |
| "attributes": {} | |
| } | |
| }, | |
| "total_flos": 0.0, | |
| "train_batch_size": 2, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |