zephyr-7b-alpha / trainer_state.json
lewtun's picture
lewtun HF staff
Add HuggingFaceH4/mistral-7b-dpo-v0.4 checkpoint
ed8ea81
raw
history blame
104 kB
{
"best_metric": null,
"best_model_checkpoint": null,
"epoch": 1.0,
"eval_steps": 100,
"global_step": 1937,
"is_hyper_param_search": false,
"is_local_process_zero": true,
"is_world_process_zero": true,
"log_history": [
{
"epoch": 0.0,
"learning_rate": 2.5773195876288657e-09,
"logits/chosen": -2.3950748443603516,
"logits/rejected": -2.440203905105591,
"logps/chosen": -134.41867065429688,
"logps/rejected": -130.99615478515625,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 1
},
{
"epoch": 0.01,
"learning_rate": 2.5773195876288656e-08,
"logits/chosen": -2.905548095703125,
"logits/rejected": -2.900480031967163,
"logps/chosen": -333.7513427734375,
"logps/rejected": -256.0840148925781,
"loss": 0.693,
"rewards/accuracies": 0.4444444477558136,
"rewards/chosen": 0.0011086573358625174,
"rewards/margins": 0.0019541841465979815,
"rewards/rejected": -0.0008455271599814296,
"step": 10
},
{
"epoch": 0.01,
"learning_rate": 5.154639175257731e-08,
"logits/chosen": -2.7942252159118652,
"logits/rejected": -2.5809876918792725,
"logps/chosen": -318.5552673339844,
"logps/rejected": -244.302734375,
"loss": 0.6874,
"rewards/accuracies": 0.6000000238418579,
"rewards/chosen": 0.007939901202917099,
"rewards/margins": 0.012212857604026794,
"rewards/rejected": -0.004272956866770983,
"step": 20
},
{
"epoch": 0.02,
"learning_rate": 7.731958762886598e-08,
"logits/chosen": -2.95878267288208,
"logits/rejected": -2.914288282394409,
"logps/chosen": -328.8043518066406,
"logps/rejected": -237.27151489257812,
"loss": 0.6807,
"rewards/accuracies": 0.5,
"rewards/chosen": 0.042353544384241104,
"rewards/margins": 0.013048592023551464,
"rewards/rejected": 0.029304955154657364,
"step": 30
},
{
"epoch": 0.02,
"learning_rate": 1.0309278350515462e-07,
"logits/chosen": -2.766305685043335,
"logits/rejected": -2.8168842792510986,
"logps/chosen": -298.6869812011719,
"logps/rejected": -283.5751037597656,
"loss": 0.6611,
"rewards/accuracies": 0.800000011920929,
"rewards/chosen": 0.053902607411146164,
"rewards/margins": 0.043827421963214874,
"rewards/rejected": 0.010075189173221588,
"step": 40
},
{
"epoch": 0.03,
"learning_rate": 1.2886597938144328e-07,
"logits/chosen": -2.940128803253174,
"logits/rejected": -2.9153294563293457,
"logps/chosen": -353.18408203125,
"logps/rejected": -278.1434326171875,
"loss": 0.6488,
"rewards/accuracies": 0.800000011920929,
"rewards/chosen": 0.10765328258275986,
"rewards/margins": 0.15659113228321075,
"rewards/rejected": -0.0489378497004509,
"step": 50
},
{
"epoch": 0.03,
"learning_rate": 1.5463917525773197e-07,
"logits/chosen": -2.9204277992248535,
"logits/rejected": -2.847609281539917,
"logps/chosen": -337.5159912109375,
"logps/rejected": -260.6918029785156,
"loss": 0.6253,
"rewards/accuracies": 0.8999999761581421,
"rewards/chosen": 0.17934152483940125,
"rewards/margins": 0.402275413274765,
"rewards/rejected": -0.22293388843536377,
"step": 60
},
{
"epoch": 0.04,
"learning_rate": 1.804123711340206e-07,
"logits/chosen": -2.9181199073791504,
"logits/rejected": -2.883514881134033,
"logps/chosen": -402.8895568847656,
"logps/rejected": -262.24609375,
"loss": 0.6168,
"rewards/accuracies": 0.8500000238418579,
"rewards/chosen": 0.14267697930335999,
"rewards/margins": 0.45374640822410583,
"rewards/rejected": -0.3110693693161011,
"step": 70
},
{
"epoch": 0.04,
"learning_rate": 2.0618556701030925e-07,
"logits/chosen": -3.0056729316711426,
"logits/rejected": -2.903585433959961,
"logps/chosen": -391.81524658203125,
"logps/rejected": -263.700439453125,
"loss": 0.5578,
"rewards/accuracies": 0.699999988079071,
"rewards/chosen": -0.3499735891819,
"rewards/margins": 0.6209262609481812,
"rewards/rejected": -0.970899760723114,
"step": 80
},
{
"epoch": 0.05,
"learning_rate": 2.3195876288659794e-07,
"logits/chosen": -2.8997607231140137,
"logits/rejected": -2.787290573120117,
"logps/chosen": -333.6437072753906,
"logps/rejected": -248.12149047851562,
"loss": 0.5714,
"rewards/accuracies": 0.8500000238418579,
"rewards/chosen": -0.30834370851516724,
"rewards/margins": 0.7723485231399536,
"rewards/rejected": -1.0806924104690552,
"step": 90
},
{
"epoch": 0.05,
"learning_rate": 2.5773195876288655e-07,
"logits/chosen": -2.729923725128174,
"logits/rejected": -2.668921947479248,
"logps/chosen": -299.4058837890625,
"logps/rejected": -299.69464111328125,
"loss": 0.5602,
"rewards/accuracies": 0.699999988079071,
"rewards/chosen": -0.3236086368560791,
"rewards/margins": 0.48320120573043823,
"rewards/rejected": -0.8068099021911621,
"step": 100
},
{
"epoch": 0.05,
"eval_logits/chosen": -2.679702043533325,
"eval_logits/rejected": -2.655423879623413,
"eval_logps/chosen": -293.716064453125,
"eval_logps/rejected": -306.2606506347656,
"eval_loss": 0.5588741302490234,
"eval_rewards/accuracies": 0.71875,
"eval_rewards/chosen": -0.3358975350856781,
"eval_rewards/margins": 0.48091739416122437,
"eval_rewards/rejected": -0.8168148994445801,
"eval_runtime": 59.9141,
"eval_samples_per_second": 16.691,
"eval_steps_per_second": 0.267,
"step": 100
},
{
"epoch": 0.06,
"learning_rate": 2.835051546391752e-07,
"logits/chosen": -2.91627836227417,
"logits/rejected": -2.970510482788086,
"logps/chosen": -383.3172302246094,
"logps/rejected": -347.4027404785156,
"loss": 0.5764,
"rewards/accuracies": 0.800000011920929,
"rewards/chosen": -0.4358394742012024,
"rewards/margins": 0.6925853490829468,
"rewards/rejected": -1.128424882888794,
"step": 110
},
{
"epoch": 0.06,
"learning_rate": 3.0927835051546394e-07,
"logits/chosen": -2.760704517364502,
"logits/rejected": -2.690364360809326,
"logps/chosen": -230.55453491210938,
"logps/rejected": -258.10711669921875,
"loss": 0.5352,
"rewards/accuracies": 0.550000011920929,
"rewards/chosen": -0.5467896461486816,
"rewards/margins": 0.3793262541294098,
"rewards/rejected": -0.926115870475769,
"step": 120
},
{
"epoch": 0.07,
"learning_rate": 3.3505154639175255e-07,
"logits/chosen": -2.9248669147491455,
"logits/rejected": -2.743758201599121,
"logps/chosen": -375.25665283203125,
"logps/rejected": -237.9368438720703,
"loss": 0.5478,
"rewards/accuracies": 0.6499999761581421,
"rewards/chosen": -0.2538636326789856,
"rewards/margins": 0.8244791030883789,
"rewards/rejected": -1.0783426761627197,
"step": 130
},
{
"epoch": 0.07,
"learning_rate": 3.608247422680412e-07,
"logits/chosen": -2.789468765258789,
"logits/rejected": -2.6087424755096436,
"logps/chosen": -341.793212890625,
"logps/rejected": -303.3952941894531,
"loss": 0.531,
"rewards/accuracies": 0.75,
"rewards/chosen": -0.5787044763565063,
"rewards/margins": 0.7447447776794434,
"rewards/rejected": -1.3234491348266602,
"step": 140
},
{
"epoch": 0.08,
"learning_rate": 3.865979381443299e-07,
"logits/chosen": -3.055696964263916,
"logits/rejected": -2.8961217403411865,
"logps/chosen": -342.10162353515625,
"logps/rejected": -362.9927978515625,
"loss": 0.5117,
"rewards/accuracies": 0.800000011920929,
"rewards/chosen": -0.22076380252838135,
"rewards/margins": 1.0446679592132568,
"rewards/rejected": -1.2654317617416382,
"step": 150
},
{
"epoch": 0.08,
"learning_rate": 4.123711340206185e-07,
"logits/chosen": -2.769010066986084,
"logits/rejected": -2.677729368209839,
"logps/chosen": -316.1387939453125,
"logps/rejected": -257.9635009765625,
"loss": 0.5264,
"rewards/accuracies": 0.6499999761581421,
"rewards/chosen": -0.5055423378944397,
"rewards/margins": 0.9351354837417603,
"rewards/rejected": -1.4406778812408447,
"step": 160
},
{
"epoch": 0.09,
"learning_rate": 4.381443298969072e-07,
"logits/chosen": -2.7085089683532715,
"logits/rejected": -2.600250005722046,
"logps/chosen": -276.8943176269531,
"logps/rejected": -258.60809326171875,
"loss": 0.478,
"rewards/accuracies": 0.8999999761581421,
"rewards/chosen": -0.2844000458717346,
"rewards/margins": 1.338111162185669,
"rewards/rejected": -1.6225111484527588,
"step": 170
},
{
"epoch": 0.09,
"learning_rate": 4.639175257731959e-07,
"logits/chosen": -2.8041911125183105,
"logits/rejected": -2.6761889457702637,
"logps/chosen": -272.099365234375,
"logps/rejected": -236.39675903320312,
"loss": 0.4904,
"rewards/accuracies": 0.800000011920929,
"rewards/chosen": -0.0498843789100647,
"rewards/margins": 1.0812078714370728,
"rewards/rejected": -1.1310923099517822,
"step": 180
},
{
"epoch": 0.1,
"learning_rate": 4.896907216494845e-07,
"logits/chosen": -2.7893078327178955,
"logits/rejected": -2.787668228149414,
"logps/chosen": -315.68365478515625,
"logps/rejected": -278.1927490234375,
"loss": 0.5809,
"rewards/accuracies": 0.75,
"rewards/chosen": -0.05461850017309189,
"rewards/margins": 0.6613677740097046,
"rewards/rejected": -0.7159863114356995,
"step": 190
},
{
"epoch": 0.1,
"learning_rate": 4.982788296041308e-07,
"logits/chosen": -2.7293038368225098,
"logits/rejected": -2.7431674003601074,
"logps/chosen": -319.84307861328125,
"logps/rejected": -285.5979309082031,
"loss": 0.4852,
"rewards/accuracies": 0.8500000238418579,
"rewards/chosen": -0.25821852684020996,
"rewards/margins": 0.9920754432678223,
"rewards/rejected": -1.2502939701080322,
"step": 200
},
{
"epoch": 0.1,
"eval_logits/chosen": -2.5956599712371826,
"eval_logits/rejected": -2.576249122619629,
"eval_logps/chosen": -297.6180725097656,
"eval_logps/rejected": -319.912353515625,
"eval_loss": 0.513595700263977,
"eval_rewards/accuracies": 0.8125,
"eval_rewards/chosen": -0.5309990048408508,
"eval_rewards/margins": 0.9684009552001953,
"eval_rewards/rejected": -1.4993999004364014,
"eval_runtime": 58.9562,
"eval_samples_per_second": 16.962,
"eval_steps_per_second": 0.271,
"step": 200
},
{
"epoch": 0.11,
"learning_rate": 4.954102122776821e-07,
"logits/chosen": -2.4963221549987793,
"logits/rejected": -2.2522215843200684,
"logps/chosen": -330.4200134277344,
"logps/rejected": -269.993896484375,
"loss": 0.5766,
"rewards/accuracies": 0.8500000238418579,
"rewards/chosen": -0.39517518877983093,
"rewards/margins": 1.0865150690078735,
"rewards/rejected": -1.4816901683807373,
"step": 210
},
{
"epoch": 0.11,
"learning_rate": 4.925415949512335e-07,
"logits/chosen": -2.709627628326416,
"logits/rejected": -2.6324925422668457,
"logps/chosen": -355.61602783203125,
"logps/rejected": -297.36346435546875,
"loss": 0.5109,
"rewards/accuracies": 0.75,
"rewards/chosen": -0.4435829222202301,
"rewards/margins": 0.6400429606437683,
"rewards/rejected": -1.0836259126663208,
"step": 220
},
{
"epoch": 0.12,
"learning_rate": 4.896729776247848e-07,
"logits/chosen": -2.7627291679382324,
"logits/rejected": -2.728057384490967,
"logps/chosen": -326.62017822265625,
"logps/rejected": -297.73150634765625,
"loss": 0.4948,
"rewards/accuracies": 0.6499999761581421,
"rewards/chosen": -0.0938631147146225,
"rewards/margins": 0.8200471997261047,
"rewards/rejected": -0.9139102697372437,
"step": 230
},
{
"epoch": 0.12,
"learning_rate": 4.868043602983362e-07,
"logits/chosen": -2.758730173110962,
"logits/rejected": -2.7346835136413574,
"logps/chosen": -399.9264221191406,
"logps/rejected": -287.3780212402344,
"loss": 0.5489,
"rewards/accuracies": 0.550000011920929,
"rewards/chosen": -0.5322622656822205,
"rewards/margins": 0.36636096239089966,
"rewards/rejected": -0.8986232876777649,
"step": 240
},
{
"epoch": 0.13,
"learning_rate": 4.839357429718875e-07,
"logits/chosen": -2.922417163848877,
"logits/rejected": -2.7935752868652344,
"logps/chosen": -267.0278015136719,
"logps/rejected": -201.1547088623047,
"loss": 0.509,
"rewards/accuracies": 0.949999988079071,
"rewards/chosen": 0.024714648723602295,
"rewards/margins": 1.111724615097046,
"rewards/rejected": -1.087010145187378,
"step": 250
},
{
"epoch": 0.13,
"learning_rate": 4.810671256454389e-07,
"logits/chosen": -2.9726548194885254,
"logits/rejected": -2.8518033027648926,
"logps/chosen": -345.22869873046875,
"logps/rejected": -239.1555938720703,
"loss": 0.5219,
"rewards/accuracies": 0.8500000238418579,
"rewards/chosen": 0.062345243990421295,
"rewards/margins": 1.2734425067901611,
"rewards/rejected": -1.211097240447998,
"step": 260
},
{
"epoch": 0.14,
"learning_rate": 4.781985083189902e-07,
"logits/chosen": -2.840888500213623,
"logits/rejected": -2.7617764472961426,
"logps/chosen": -343.1612243652344,
"logps/rejected": -278.6701354980469,
"loss": 0.4955,
"rewards/accuracies": 0.6000000238418579,
"rewards/chosen": -0.15016205608844757,
"rewards/margins": 0.7697200179100037,
"rewards/rejected": -0.9198821187019348,
"step": 270
},
{
"epoch": 0.14,
"learning_rate": 4.7532989099254154e-07,
"logits/chosen": -2.9551005363464355,
"logits/rejected": -2.8918609619140625,
"logps/chosen": -391.0706787109375,
"logps/rejected": -303.4861755371094,
"loss": 0.5101,
"rewards/accuracies": 0.699999988079071,
"rewards/chosen": -0.2836604118347168,
"rewards/margins": 0.7845603227615356,
"rewards/rejected": -1.068220853805542,
"step": 280
},
{
"epoch": 0.15,
"learning_rate": 4.724612736660929e-07,
"logits/chosen": -2.664578437805176,
"logits/rejected": -2.801819086074829,
"logps/chosen": -272.6911315917969,
"logps/rejected": -268.9104919433594,
"loss": 0.4815,
"rewards/accuracies": 0.6000000238418579,
"rewards/chosen": 0.15687711536884308,
"rewards/margins": 0.9703904986381531,
"rewards/rejected": -0.813513457775116,
"step": 290
},
{
"epoch": 0.15,
"learning_rate": 4.695926563396443e-07,
"logits/chosen": -2.7159955501556396,
"logits/rejected": -2.7721734046936035,
"logps/chosen": -444.1826171875,
"logps/rejected": -277.5564270019531,
"loss": 0.5212,
"rewards/accuracies": 0.8500000238418579,
"rewards/chosen": 0.18774226307868958,
"rewards/margins": 1.6489717960357666,
"rewards/rejected": -1.4612294435501099,
"step": 300
},
{
"epoch": 0.15,
"eval_logits/chosen": -2.7125377655029297,
"eval_logits/rejected": -2.6865103244781494,
"eval_logps/chosen": -290.3699035644531,
"eval_logps/rejected": -313.44439697265625,
"eval_loss": 0.5167616009712219,
"eval_rewards/accuracies": 0.78125,
"eval_rewards/chosen": -0.1685914546251297,
"eval_rewards/margins": 1.0074106454849243,
"eval_rewards/rejected": -1.1760022640228271,
"eval_runtime": 57.2558,
"eval_samples_per_second": 17.465,
"eval_steps_per_second": 0.279,
"step": 300
},
{
"epoch": 0.16,
"learning_rate": 4.6672403901319564e-07,
"logits/chosen": -2.9332168102264404,
"logits/rejected": -2.83400297164917,
"logps/chosen": -351.634033203125,
"logps/rejected": -267.8584899902344,
"loss": 0.5018,
"rewards/accuracies": 0.8999999761581421,
"rewards/chosen": -0.10320504754781723,
"rewards/margins": 1.208617091178894,
"rewards/rejected": -1.3118221759796143,
"step": 310
},
{
"epoch": 0.17,
"learning_rate": 4.63855421686747e-07,
"logits/chosen": -2.86641001701355,
"logits/rejected": -2.9064583778381348,
"logps/chosen": -401.264404296875,
"logps/rejected": -330.1752624511719,
"loss": 0.4492,
"rewards/accuracies": 0.8500000238418579,
"rewards/chosen": -0.5078498125076294,
"rewards/margins": 1.2765679359436035,
"rewards/rejected": -1.784417748451233,
"step": 320
},
{
"epoch": 0.17,
"learning_rate": 4.609868043602983e-07,
"logits/chosen": -2.7677345275878906,
"logits/rejected": -2.808917760848999,
"logps/chosen": -373.0283203125,
"logps/rejected": -278.2051086425781,
"loss": 0.4576,
"rewards/accuracies": 0.8500000238418579,
"rewards/chosen": -0.5256485939025879,
"rewards/margins": 1.5822279453277588,
"rewards/rejected": -2.1078765392303467,
"step": 330
},
{
"epoch": 0.18,
"learning_rate": 4.581181870338497e-07,
"logits/chosen": -2.5862860679626465,
"logits/rejected": -2.6729650497436523,
"logps/chosen": -264.1101989746094,
"logps/rejected": -302.21832275390625,
"loss": 0.5585,
"rewards/accuracies": 0.75,
"rewards/chosen": -0.3558864891529083,
"rewards/margins": 0.7631327509880066,
"rewards/rejected": -1.1190193891525269,
"step": 340
},
{
"epoch": 0.18,
"learning_rate": 4.55249569707401e-07,
"logits/chosen": -2.9297871589660645,
"logits/rejected": -2.8774681091308594,
"logps/chosen": -199.7515106201172,
"logps/rejected": -261.4060363769531,
"loss": 0.4773,
"rewards/accuracies": 0.6499999761581421,
"rewards/chosen": -0.46969500184059143,
"rewards/margins": 0.5962409973144531,
"rewards/rejected": -1.0659358501434326,
"step": 350
},
{
"epoch": 0.19,
"learning_rate": 4.5238095238095237e-07,
"logits/chosen": -2.9737091064453125,
"logits/rejected": -3.0415940284729004,
"logps/chosen": -417.215087890625,
"logps/rejected": -402.1128845214844,
"loss": 0.4545,
"rewards/accuracies": 0.8500000238418579,
"rewards/chosen": 0.06597085297107697,
"rewards/margins": 1.6805864572525024,
"rewards/rejected": -1.6146154403686523,
"step": 360
},
{
"epoch": 0.19,
"learning_rate": 4.495123350545037e-07,
"logits/chosen": -2.9340670108795166,
"logits/rejected": -2.8952622413635254,
"logps/chosen": -200.85787963867188,
"logps/rejected": -190.8145751953125,
"loss": 0.462,
"rewards/accuracies": 0.8500000238418579,
"rewards/chosen": -0.39831048250198364,
"rewards/margins": 1.7207963466644287,
"rewards/rejected": -2.1191065311431885,
"step": 370
},
{
"epoch": 0.2,
"learning_rate": 4.46643717728055e-07,
"logits/chosen": -2.7802371978759766,
"logits/rejected": -2.692594051361084,
"logps/chosen": -342.89556884765625,
"logps/rejected": -360.488037109375,
"loss": 0.5404,
"rewards/accuracies": 0.6000000238418579,
"rewards/chosen": -0.2285180538892746,
"rewards/margins": 0.9076235890388489,
"rewards/rejected": -1.1361417770385742,
"step": 380
},
{
"epoch": 0.2,
"learning_rate": 4.437751004016064e-07,
"logits/chosen": -2.8958873748779297,
"logits/rejected": -2.8253509998321533,
"logps/chosen": -338.5411071777344,
"logps/rejected": -302.31536865234375,
"loss": 0.4479,
"rewards/accuracies": 0.75,
"rewards/chosen": -0.5085374712944031,
"rewards/margins": 1.4444196224212646,
"rewards/rejected": -1.9529569149017334,
"step": 390
},
{
"epoch": 0.21,
"learning_rate": 4.4090648307515776e-07,
"logits/chosen": -3.072361469268799,
"logits/rejected": -3.0403876304626465,
"logps/chosen": -298.0294189453125,
"logps/rejected": -269.53759765625,
"loss": 0.5496,
"rewards/accuracies": 0.75,
"rewards/chosen": -0.7352712154388428,
"rewards/margins": 0.4947175979614258,
"rewards/rejected": -1.229988694190979,
"step": 400
},
{
"epoch": 0.21,
"eval_logits/chosen": -2.8217618465423584,
"eval_logits/rejected": -2.7946977615356445,
"eval_logps/chosen": -290.2325744628906,
"eval_logps/rejected": -324.2634582519531,
"eval_loss": 0.4835220277309418,
"eval_rewards/accuracies": 0.828125,
"eval_rewards/chosen": -0.16172367334365845,
"eval_rewards/margins": 1.5552303791046143,
"eval_rewards/rejected": -1.716953992843628,
"eval_runtime": 58.8529,
"eval_samples_per_second": 16.992,
"eval_steps_per_second": 0.272,
"step": 400
},
{
"epoch": 0.21,
"learning_rate": 4.380378657487091e-07,
"logits/chosen": -2.878788948059082,
"logits/rejected": -2.7508370876312256,
"logps/chosen": -361.15850830078125,
"logps/rejected": -244.7436065673828,
"loss": 0.4209,
"rewards/accuracies": 0.6499999761581421,
"rewards/chosen": -0.3598092496395111,
"rewards/margins": 0.8073941469192505,
"rewards/rejected": -1.1672031879425049,
"step": 410
},
{
"epoch": 0.22,
"learning_rate": 4.3516924842226045e-07,
"logits/chosen": -2.940977096557617,
"logits/rejected": -2.811171531677246,
"logps/chosen": -342.15802001953125,
"logps/rejected": -299.4014587402344,
"loss": 0.4994,
"rewards/accuracies": 0.550000011920929,
"rewards/chosen": -0.5423839092254639,
"rewards/margins": 1.0953930616378784,
"rewards/rejected": -1.6377769708633423,
"step": 420
},
{
"epoch": 0.22,
"learning_rate": 4.323006310958118e-07,
"logits/chosen": -2.928069591522217,
"logits/rejected": -2.839228391647339,
"logps/chosen": -246.8507080078125,
"logps/rejected": -235.521728515625,
"loss": 0.4726,
"rewards/accuracies": 0.75,
"rewards/chosen": -0.7062541246414185,
"rewards/margins": 0.9924257397651672,
"rewards/rejected": -1.6986801624298096,
"step": 430
},
{
"epoch": 0.23,
"learning_rate": 4.2943201376936315e-07,
"logits/chosen": -2.8603832721710205,
"logits/rejected": -2.7824721336364746,
"logps/chosen": -376.1601257324219,
"logps/rejected": -356.33905029296875,
"loss": 0.5524,
"rewards/accuracies": 0.75,
"rewards/chosen": -0.40359026193618774,
"rewards/margins": 0.40071097016334534,
"rewards/rejected": -0.8043011426925659,
"step": 440
},
{
"epoch": 0.23,
"learning_rate": 4.265633964429145e-07,
"logits/chosen": -2.9410994052886963,
"logits/rejected": -2.9663491249084473,
"logps/chosen": -346.1313171386719,
"logps/rejected": -281.6514587402344,
"loss": 0.469,
"rewards/accuracies": 0.6000000238418579,
"rewards/chosen": -0.5330409407615662,
"rewards/margins": 0.553443968296051,
"rewards/rejected": -1.0864850282669067,
"step": 450
},
{
"epoch": 0.24,
"learning_rate": 4.2369477911646584e-07,
"logits/chosen": -2.9534189701080322,
"logits/rejected": -2.9417405128479004,
"logps/chosen": -362.89105224609375,
"logps/rejected": -356.94598388671875,
"loss": 0.546,
"rewards/accuracies": 0.8999999761581421,
"rewards/chosen": -0.10002334415912628,
"rewards/margins": 1.6824315786361694,
"rewards/rejected": -1.7824550867080688,
"step": 460
},
{
"epoch": 0.24,
"learning_rate": 4.208261617900172e-07,
"logits/chosen": -2.8067574501037598,
"logits/rejected": -2.815100908279419,
"logps/chosen": -359.10791015625,
"logps/rejected": -276.3143615722656,
"loss": 0.4473,
"rewards/accuracies": 0.6499999761581421,
"rewards/chosen": -0.5546383857727051,
"rewards/margins": 0.8130642175674438,
"rewards/rejected": -1.367702603340149,
"step": 470
},
{
"epoch": 0.25,
"learning_rate": 4.179575444635686e-07,
"logits/chosen": -3.0283010005950928,
"logits/rejected": -3.0092482566833496,
"logps/chosen": -320.7592468261719,
"logps/rejected": -327.0148010253906,
"loss": 0.4589,
"rewards/accuracies": 0.6000000238418579,
"rewards/chosen": -0.023906176909804344,
"rewards/margins": 0.516281008720398,
"rewards/rejected": -0.5401870608329773,
"step": 480
},
{
"epoch": 0.25,
"learning_rate": 4.150889271371199e-07,
"logits/chosen": -2.9509072303771973,
"logits/rejected": -2.9088292121887207,
"logps/chosen": -359.8896789550781,
"logps/rejected": -298.53167724609375,
"loss": 0.4272,
"rewards/accuracies": 0.6499999761581421,
"rewards/chosen": -0.9165481328964233,
"rewards/margins": 0.8448396921157837,
"rewards/rejected": -1.761387586593628,
"step": 490
},
{
"epoch": 0.26,
"learning_rate": 4.1222030981067123e-07,
"logits/chosen": -3.1632120609283447,
"logits/rejected": -2.959442138671875,
"logps/chosen": -348.33709716796875,
"logps/rejected": -335.841552734375,
"loss": 0.5209,
"rewards/accuracies": 0.800000011920929,
"rewards/chosen": 0.01871591992676258,
"rewards/margins": 1.6875536441802979,
"rewards/rejected": -1.668837547302246,
"step": 500
},
{
"epoch": 0.26,
"eval_logits/chosen": -2.8667073249816895,
"eval_logits/rejected": -2.8388283252716064,
"eval_logps/chosen": -296.5545654296875,
"eval_logps/rejected": -323.13250732421875,
"eval_loss": 0.5053695440292358,
"eval_rewards/accuracies": 0.734375,
"eval_rewards/chosen": -0.4778231978416443,
"eval_rewards/margins": 1.1825826168060303,
"eval_rewards/rejected": -1.6604057550430298,
"eval_runtime": 53.6768,
"eval_samples_per_second": 18.63,
"eval_steps_per_second": 0.298,
"step": 500
},
{
"epoch": 0.26,
"learning_rate": 4.093516924842226e-07,
"logits/chosen": -2.5999233722686768,
"logits/rejected": -2.6488568782806396,
"logps/chosen": -279.40753173828125,
"logps/rejected": -288.4382019042969,
"loss": 0.5262,
"rewards/accuracies": 0.6499999761581421,
"rewards/chosen": -0.03970176726579666,
"rewards/margins": 0.8601824641227722,
"rewards/rejected": -0.8998842239379883,
"step": 510
},
{
"epoch": 0.27,
"learning_rate": 4.064830751577739e-07,
"logits/chosen": -3.004120349884033,
"logits/rejected": -2.983384132385254,
"logps/chosen": -223.1768035888672,
"logps/rejected": -233.73312377929688,
"loss": 0.5426,
"rewards/accuracies": 0.699999988079071,
"rewards/chosen": -0.22979409992694855,
"rewards/margins": 1.1553027629852295,
"rewards/rejected": -1.3850969076156616,
"step": 520
},
{
"epoch": 0.27,
"learning_rate": 4.036144578313253e-07,
"logits/chosen": -3.057387351989746,
"logits/rejected": -2.9704792499542236,
"logps/chosen": -325.57025146484375,
"logps/rejected": -283.2383117675781,
"loss": 0.5058,
"rewards/accuracies": 0.949999988079071,
"rewards/chosen": 0.17162951827049255,
"rewards/margins": 1.4068710803985596,
"rewards/rejected": -1.2352415323257446,
"step": 530
},
{
"epoch": 0.28,
"learning_rate": 4.007458405048766e-07,
"logits/chosen": -2.940493583679199,
"logits/rejected": -2.9570419788360596,
"logps/chosen": -227.3856964111328,
"logps/rejected": -289.257080078125,
"loss": 0.5339,
"rewards/accuracies": 0.6499999761581421,
"rewards/chosen": -0.835346519947052,
"rewards/margins": 0.5295913219451904,
"rewards/rejected": -1.3649379014968872,
"step": 540
},
{
"epoch": 0.28,
"learning_rate": 3.9787722317842796e-07,
"logits/chosen": -3.0390350818634033,
"logits/rejected": -3.0303795337677,
"logps/chosen": -389.0104675292969,
"logps/rejected": -340.8678283691406,
"loss": 0.5671,
"rewards/accuracies": 0.8500000238418579,
"rewards/chosen": -0.15655937790870667,
"rewards/margins": 1.444120168685913,
"rewards/rejected": -1.6006797552108765,
"step": 550
},
{
"epoch": 0.29,
"learning_rate": 3.950086058519793e-07,
"logits/chosen": -3.060770034790039,
"logits/rejected": -3.0684587955474854,
"logps/chosen": -366.5193786621094,
"logps/rejected": -359.5283203125,
"loss": 0.501,
"rewards/accuracies": 0.800000011920929,
"rewards/chosen": -0.8856328129768372,
"rewards/margins": 1.0001723766326904,
"rewards/rejected": -1.8858048915863037,
"step": 560
},
{
"epoch": 0.29,
"learning_rate": 3.9213998852553066e-07,
"logits/chosen": -3.022430896759033,
"logits/rejected": -2.831993341445923,
"logps/chosen": -379.4962463378906,
"logps/rejected": -278.2923889160156,
"loss": 0.4947,
"rewards/accuracies": 0.75,
"rewards/chosen": -0.7191627621650696,
"rewards/margins": 1.4330142736434937,
"rewards/rejected": -2.152177095413208,
"step": 570
},
{
"epoch": 0.3,
"learning_rate": 3.8927137119908206e-07,
"logits/chosen": -2.922940731048584,
"logits/rejected": -2.9428744316101074,
"logps/chosen": -361.6253356933594,
"logps/rejected": -278.06280517578125,
"loss": 0.5375,
"rewards/accuracies": 0.6499999761581421,
"rewards/chosen": -0.573477029800415,
"rewards/margins": 0.7330626249313354,
"rewards/rejected": -1.30653977394104,
"step": 580
},
{
"epoch": 0.3,
"learning_rate": 3.864027538726334e-07,
"logits/chosen": -2.9140264987945557,
"logits/rejected": -2.889644145965576,
"logps/chosen": -289.79058837890625,
"logps/rejected": -292.6978759765625,
"loss": 0.4726,
"rewards/accuracies": 0.8500000238418579,
"rewards/chosen": -0.1533357799053192,
"rewards/margins": 1.4288156032562256,
"rewards/rejected": -1.582151174545288,
"step": 590
},
{
"epoch": 0.31,
"learning_rate": 3.835341365461847e-07,
"logits/chosen": -2.869101047515869,
"logits/rejected": -2.88112735748291,
"logps/chosen": -243.238037109375,
"logps/rejected": -313.36260986328125,
"loss": 0.4617,
"rewards/accuracies": 0.8500000238418579,
"rewards/chosen": -0.5780342221260071,
"rewards/margins": 1.605724573135376,
"rewards/rejected": -2.1837587356567383,
"step": 600
},
{
"epoch": 0.31,
"eval_logits/chosen": -2.8521082401275635,
"eval_logits/rejected": -2.823408603668213,
"eval_logps/chosen": -294.47406005859375,
"eval_logps/rejected": -320.2847900390625,
"eval_loss": 0.4909786283969879,
"eval_rewards/accuracies": 0.765625,
"eval_rewards/chosen": -0.373797744512558,
"eval_rewards/margins": 1.1442232131958008,
"eval_rewards/rejected": -1.5180209875106812,
"eval_runtime": 59.0534,
"eval_samples_per_second": 16.934,
"eval_steps_per_second": 0.271,
"step": 600
},
{
"epoch": 0.31,
"learning_rate": 3.8066551921973605e-07,
"logits/chosen": -2.9201841354370117,
"logits/rejected": -2.845341205596924,
"logps/chosen": -338.87127685546875,
"logps/rejected": -337.95928955078125,
"loss": 0.5093,
"rewards/accuracies": 0.75,
"rewards/chosen": -0.7563498020172119,
"rewards/margins": 1.1776630878448486,
"rewards/rejected": -1.934012770652771,
"step": 610
},
{
"epoch": 0.32,
"learning_rate": 3.7779690189328745e-07,
"logits/chosen": -3.034162759780884,
"logits/rejected": -3.027251720428467,
"logps/chosen": -379.36590576171875,
"logps/rejected": -306.5363464355469,
"loss": 0.5269,
"rewards/accuracies": 0.8500000238418579,
"rewards/chosen": -0.6596435308456421,
"rewards/margins": 0.7433546185493469,
"rewards/rejected": -1.4029979705810547,
"step": 620
},
{
"epoch": 0.33,
"learning_rate": 3.749282845668388e-07,
"logits/chosen": -2.87540864944458,
"logits/rejected": -2.834453821182251,
"logps/chosen": -252.4665069580078,
"logps/rejected": -234.0683135986328,
"loss": 0.4806,
"rewards/accuracies": 0.75,
"rewards/chosen": -1.2540767192840576,
"rewards/margins": 1.1702220439910889,
"rewards/rejected": -2.4242987632751465,
"step": 630
},
{
"epoch": 0.33,
"learning_rate": 3.7205966724039014e-07,
"logits/chosen": -2.9034817218780518,
"logits/rejected": -2.8846733570098877,
"logps/chosen": -308.4912109375,
"logps/rejected": -294.0049743652344,
"loss": 0.5071,
"rewards/accuracies": 0.8999999761581421,
"rewards/chosen": -0.7412096261978149,
"rewards/margins": 1.7391561269760132,
"rewards/rejected": -2.480365753173828,
"step": 640
},
{
"epoch": 0.34,
"learning_rate": 3.6919104991394144e-07,
"logits/chosen": -2.931398868560791,
"logits/rejected": -2.982588291168213,
"logps/chosen": -284.8676452636719,
"logps/rejected": -328.3984069824219,
"loss": 0.5295,
"rewards/accuracies": 0.8500000238418579,
"rewards/chosen": -0.8696074485778809,
"rewards/margins": 1.0539840459823608,
"rewards/rejected": -1.9235913753509521,
"step": 650
},
{
"epoch": 0.34,
"learning_rate": 3.663224325874928e-07,
"logits/chosen": -3.033780097961426,
"logits/rejected": -2.9609603881835938,
"logps/chosen": -372.281005859375,
"logps/rejected": -324.37640380859375,
"loss": 0.4882,
"rewards/accuracies": 0.75,
"rewards/chosen": -0.7286888957023621,
"rewards/margins": 1.0642763376235962,
"rewards/rejected": -1.792965292930603,
"step": 660
},
{
"epoch": 0.35,
"learning_rate": 3.634538152610442e-07,
"logits/chosen": -2.93717622756958,
"logits/rejected": -2.9276621341705322,
"logps/chosen": -240.49267578125,
"logps/rejected": -235.82901000976562,
"loss": 0.5725,
"rewards/accuracies": 0.6000000238418579,
"rewards/chosen": -0.5567636489868164,
"rewards/margins": 0.7625681161880493,
"rewards/rejected": -1.3193317651748657,
"step": 670
},
{
"epoch": 0.35,
"learning_rate": 3.6058519793459553e-07,
"logits/chosen": -2.834033727645874,
"logits/rejected": -2.778200387954712,
"logps/chosen": -311.96209716796875,
"logps/rejected": -279.34210205078125,
"loss": 0.4695,
"rewards/accuracies": 0.75,
"rewards/chosen": -0.3849760890007019,
"rewards/margins": 1.1402137279510498,
"rewards/rejected": -1.5251898765563965,
"step": 680
},
{
"epoch": 0.36,
"learning_rate": 3.577165806081469e-07,
"logits/chosen": -3.0405967235565186,
"logits/rejected": -2.836169481277466,
"logps/chosen": -377.798828125,
"logps/rejected": -225.7850799560547,
"loss": 0.4235,
"rewards/accuracies": 0.8500000238418579,
"rewards/chosen": 0.16026358306407928,
"rewards/margins": 1.4788005352020264,
"rewards/rejected": -1.3185369968414307,
"step": 690
},
{
"epoch": 0.36,
"learning_rate": 3.5484796328169817e-07,
"logits/chosen": -2.914154052734375,
"logits/rejected": -2.837174415588379,
"logps/chosen": -486.6206970214844,
"logps/rejected": -388.7563171386719,
"loss": 0.4452,
"rewards/accuracies": 0.800000011920929,
"rewards/chosen": 0.3467257618904114,
"rewards/margins": 1.8637710809707642,
"rewards/rejected": -1.517045021057129,
"step": 700
},
{
"epoch": 0.36,
"eval_logits/chosen": -2.7652909755706787,
"eval_logits/rejected": -2.7400858402252197,
"eval_logps/chosen": -296.1795959472656,
"eval_logps/rejected": -323.0770263671875,
"eval_loss": 0.48382142186164856,
"eval_rewards/accuracies": 0.703125,
"eval_rewards/chosen": -0.45907530188560486,
"eval_rewards/margins": 1.19855797290802,
"eval_rewards/rejected": -1.6576331853866577,
"eval_runtime": 56.4234,
"eval_samples_per_second": 17.723,
"eval_steps_per_second": 0.284,
"step": 700
},
{
"epoch": 0.37,
"learning_rate": 3.519793459552495e-07,
"logits/chosen": -2.81813645362854,
"logits/rejected": -2.891169309616089,
"logps/chosen": -304.12921142578125,
"logps/rejected": -300.01275634765625,
"loss": 0.4796,
"rewards/accuracies": 0.800000011920929,
"rewards/chosen": -1.1196739673614502,
"rewards/margins": 0.9914043545722961,
"rewards/rejected": -2.1110780239105225,
"step": 710
},
{
"epoch": 0.37,
"learning_rate": 3.491107286288009e-07,
"logits/chosen": -2.8922622203826904,
"logits/rejected": -2.8977553844451904,
"logps/chosen": -317.526123046875,
"logps/rejected": -434.422119140625,
"loss": 0.4804,
"rewards/accuracies": 1.0,
"rewards/chosen": -0.36432236433029175,
"rewards/margins": 2.1755287647247314,
"rewards/rejected": -2.539850950241089,
"step": 720
},
{
"epoch": 0.38,
"learning_rate": 3.4624211130235227e-07,
"logits/chosen": -2.897742509841919,
"logits/rejected": -2.849132537841797,
"logps/chosen": -359.33770751953125,
"logps/rejected": -375.489990234375,
"loss": 0.4515,
"rewards/accuracies": 0.6499999761581421,
"rewards/chosen": -0.7771750688552856,
"rewards/margins": 0.7097036838531494,
"rewards/rejected": -1.4868788719177246,
"step": 730
},
{
"epoch": 0.38,
"learning_rate": 3.433734939759036e-07,
"logits/chosen": -2.8762059211730957,
"logits/rejected": -2.845869302749634,
"logps/chosen": -367.5733642578125,
"logps/rejected": -284.77886962890625,
"loss": 0.5371,
"rewards/accuracies": 0.8500000238418579,
"rewards/chosen": -0.7336939573287964,
"rewards/margins": 1.311545968055725,
"rewards/rejected": -2.0452399253845215,
"step": 740
},
{
"epoch": 0.39,
"learning_rate": 3.405048766494549e-07,
"logits/chosen": -2.8201470375061035,
"logits/rejected": -2.8354251384735107,
"logps/chosen": -340.94549560546875,
"logps/rejected": -293.8914794921875,
"loss": 0.4984,
"rewards/accuracies": 0.8500000238418579,
"rewards/chosen": -0.7219395637512207,
"rewards/margins": 1.4890177249908447,
"rewards/rejected": -2.2109572887420654,
"step": 750
},
{
"epoch": 0.39,
"learning_rate": 3.376362593230063e-07,
"logits/chosen": -2.7780914306640625,
"logits/rejected": -2.7091243267059326,
"logps/chosen": -291.79864501953125,
"logps/rejected": -293.62762451171875,
"loss": 0.5131,
"rewards/accuracies": 0.8500000238418579,
"rewards/chosen": -0.4624660611152649,
"rewards/margins": 1.375497579574585,
"rewards/rejected": -1.8379634618759155,
"step": 760
},
{
"epoch": 0.4,
"learning_rate": 3.3476764199655765e-07,
"logits/chosen": -2.9393422603607178,
"logits/rejected": -2.913353681564331,
"logps/chosen": -266.66033935546875,
"logps/rejected": -310.0787658691406,
"loss": 0.5009,
"rewards/accuracies": 0.699999988079071,
"rewards/chosen": -0.27197906374931335,
"rewards/margins": 1.5637094974517822,
"rewards/rejected": -1.835688591003418,
"step": 770
},
{
"epoch": 0.4,
"learning_rate": 3.31899024670109e-07,
"logits/chosen": -2.799229621887207,
"logits/rejected": -2.7572007179260254,
"logps/chosen": -322.958251953125,
"logps/rejected": -250.6745147705078,
"loss": 0.462,
"rewards/accuracies": 0.8999999761581421,
"rewards/chosen": -0.1813792735338211,
"rewards/margins": 1.8009974956512451,
"rewards/rejected": -1.9823768138885498,
"step": 780
},
{
"epoch": 0.41,
"learning_rate": 3.2903040734366035e-07,
"logits/chosen": -2.9396896362304688,
"logits/rejected": -2.8256068229675293,
"logps/chosen": -336.33111572265625,
"logps/rejected": -305.9268493652344,
"loss": 0.4724,
"rewards/accuracies": 0.75,
"rewards/chosen": -0.5670480728149414,
"rewards/margins": 1.4373046159744263,
"rewards/rejected": -2.004352569580078,
"step": 790
},
{
"epoch": 0.41,
"learning_rate": 3.261617900172117e-07,
"logits/chosen": -2.8622994422912598,
"logits/rejected": -2.8278136253356934,
"logps/chosen": -284.0804443359375,
"logps/rejected": -313.81939697265625,
"loss": 0.4674,
"rewards/accuracies": 0.8999999761581421,
"rewards/chosen": 0.13986745476722717,
"rewards/margins": 1.9817402362823486,
"rewards/rejected": -1.8418725728988647,
"step": 800
},
{
"epoch": 0.41,
"eval_logits/chosen": -2.6945066452026367,
"eval_logits/rejected": -2.6739954948425293,
"eval_logps/chosen": -298.38177490234375,
"eval_logps/rejected": -327.2416076660156,
"eval_loss": 0.5077354907989502,
"eval_rewards/accuracies": 0.765625,
"eval_rewards/chosen": -0.5691820383071899,
"eval_rewards/margins": 1.2966790199279785,
"eval_rewards/rejected": -1.865861177444458,
"eval_runtime": 55.4449,
"eval_samples_per_second": 18.036,
"eval_steps_per_second": 0.289,
"step": 800
},
{
"epoch": 0.42,
"learning_rate": 3.2329317269076304e-07,
"logits/chosen": -2.8082878589630127,
"logits/rejected": -2.783658981323242,
"logps/chosen": -226.2019500732422,
"logps/rejected": -294.66326904296875,
"loss": 0.5152,
"rewards/accuracies": 0.800000011920929,
"rewards/chosen": -0.869047999382019,
"rewards/margins": 1.269092321395874,
"rewards/rejected": -2.1381404399871826,
"step": 810
},
{
"epoch": 0.42,
"learning_rate": 3.204245553643144e-07,
"logits/chosen": -2.9127354621887207,
"logits/rejected": -2.894219160079956,
"logps/chosen": -230.79513549804688,
"logps/rejected": -305.60113525390625,
"loss": 0.5279,
"rewards/accuracies": 0.3499999940395355,
"rewards/chosen": -0.7637732028961182,
"rewards/margins": -0.17934365570545197,
"rewards/rejected": -0.5844296216964722,
"step": 820
},
{
"epoch": 0.43,
"learning_rate": 3.1755593803786574e-07,
"logits/chosen": -2.9059672355651855,
"logits/rejected": -2.9442248344421387,
"logps/chosen": -238.79248046875,
"logps/rejected": -285.3636474609375,
"loss": 0.5044,
"rewards/accuracies": 0.800000011920929,
"rewards/chosen": -0.5089541673660278,
"rewards/margins": 1.2097771167755127,
"rewards/rejected": -1.718731164932251,
"step": 830
},
{
"epoch": 0.43,
"learning_rate": 3.146873207114171e-07,
"logits/chosen": -2.953200340270996,
"logits/rejected": -2.8728442192077637,
"logps/chosen": -345.51593017578125,
"logps/rejected": -307.09002685546875,
"loss": 0.5092,
"rewards/accuracies": 0.550000011920929,
"rewards/chosen": -1.007508397102356,
"rewards/margins": 0.5991470813751221,
"rewards/rejected": -1.6066553592681885,
"step": 840
},
{
"epoch": 0.44,
"learning_rate": 3.1181870338496843e-07,
"logits/chosen": -2.8444714546203613,
"logits/rejected": -2.791147470474243,
"logps/chosen": -296.8919677734375,
"logps/rejected": -274.53082275390625,
"loss": 0.4805,
"rewards/accuracies": 0.8999999761581421,
"rewards/chosen": -0.953098475933075,
"rewards/margins": 1.4887616634368896,
"rewards/rejected": -2.4418601989746094,
"step": 850
},
{
"epoch": 0.44,
"learning_rate": 3.089500860585198e-07,
"logits/chosen": -2.6704187393188477,
"logits/rejected": -2.6690378189086914,
"logps/chosen": -249.43881225585938,
"logps/rejected": -302.4588928222656,
"loss": 0.5111,
"rewards/accuracies": 0.800000011920929,
"rewards/chosen": -1.3775177001953125,
"rewards/margins": 1.329897403717041,
"rewards/rejected": -2.7074151039123535,
"step": 860
},
{
"epoch": 0.45,
"learning_rate": 3.060814687320711e-07,
"logits/chosen": -2.839586019515991,
"logits/rejected": -2.834083318710327,
"logps/chosen": -304.1976318359375,
"logps/rejected": -256.19378662109375,
"loss": 0.4354,
"rewards/accuracies": 0.8999999761581421,
"rewards/chosen": -0.47585782408714294,
"rewards/margins": 2.021141529083252,
"rewards/rejected": -2.4969992637634277,
"step": 870
},
{
"epoch": 0.45,
"learning_rate": 3.0321285140562247e-07,
"logits/chosen": -2.7889621257781982,
"logits/rejected": -2.7733607292175293,
"logps/chosen": -280.20977783203125,
"logps/rejected": -347.7413635253906,
"loss": 0.4604,
"rewards/accuracies": 0.800000011920929,
"rewards/chosen": -0.9127300977706909,
"rewards/margins": 0.9622424244880676,
"rewards/rejected": -1.8749723434448242,
"step": 880
},
{
"epoch": 0.46,
"learning_rate": 3.003442340791738e-07,
"logits/chosen": -2.904717206954956,
"logits/rejected": -2.835303783416748,
"logps/chosen": -336.64776611328125,
"logps/rejected": -316.78045654296875,
"loss": 0.5003,
"rewards/accuracies": 0.699999988079071,
"rewards/chosen": -0.2770691215991974,
"rewards/margins": 1.1059465408325195,
"rewards/rejected": -1.383015751838684,
"step": 890
},
{
"epoch": 0.46,
"learning_rate": 2.974756167527252e-07,
"logits/chosen": -3.022934913635254,
"logits/rejected": -2.956709146499634,
"logps/chosen": -286.189208984375,
"logps/rejected": -264.89776611328125,
"loss": 0.4656,
"rewards/accuracies": 0.75,
"rewards/chosen": -0.7135367393493652,
"rewards/margins": 1.1268901824951172,
"rewards/rejected": -1.8404268026351929,
"step": 900
},
{
"epoch": 0.46,
"eval_logits/chosen": -2.8014752864837646,
"eval_logits/rejected": -2.781651258468628,
"eval_logps/chosen": -297.5553283691406,
"eval_logps/rejected": -323.15179443359375,
"eval_loss": 0.4927058219909668,
"eval_rewards/accuracies": 0.765625,
"eval_rewards/chosen": -0.5278611183166504,
"eval_rewards/margins": 1.1335095167160034,
"eval_rewards/rejected": -1.661370873451233,
"eval_runtime": 57.0998,
"eval_samples_per_second": 17.513,
"eval_steps_per_second": 0.28,
"step": 900
},
{
"epoch": 0.47,
"learning_rate": 2.946069994262765e-07,
"logits/chosen": -2.978717803955078,
"logits/rejected": -2.8918721675872803,
"logps/chosen": -336.64495849609375,
"logps/rejected": -247.2961883544922,
"loss": 0.4344,
"rewards/accuracies": 0.800000011920929,
"rewards/chosen": -0.6280319094657898,
"rewards/margins": 1.4678236246109009,
"rewards/rejected": -2.095855712890625,
"step": 910
},
{
"epoch": 0.47,
"learning_rate": 2.9173838209982786e-07,
"logits/chosen": -2.786372423171997,
"logits/rejected": -2.8410544395446777,
"logps/chosen": -298.88360595703125,
"logps/rejected": -302.8692626953125,
"loss": 0.4686,
"rewards/accuracies": 0.6499999761581421,
"rewards/chosen": -0.6036109924316406,
"rewards/margins": 0.7661906480789185,
"rewards/rejected": -1.369801640510559,
"step": 920
},
{
"epoch": 0.48,
"learning_rate": 2.888697647733792e-07,
"logits/chosen": -2.915067195892334,
"logits/rejected": -2.732910633087158,
"logps/chosen": -356.8029479980469,
"logps/rejected": -226.57974243164062,
"loss": 0.4128,
"rewards/accuracies": 0.8999999761581421,
"rewards/chosen": -0.25983041524887085,
"rewards/margins": 2.1089906692504883,
"rewards/rejected": -2.368821144104004,
"step": 930
},
{
"epoch": 0.49,
"learning_rate": 2.8600114744693055e-07,
"logits/chosen": -2.872128486633301,
"logits/rejected": -2.8349528312683105,
"logps/chosen": -276.8409729003906,
"logps/rejected": -234.0821990966797,
"loss": 0.4633,
"rewards/accuracies": 0.75,
"rewards/chosen": -0.33096224069595337,
"rewards/margins": 1.0401618480682373,
"rewards/rejected": -1.371124029159546,
"step": 940
},
{
"epoch": 0.49,
"learning_rate": 2.8313253012048195e-07,
"logits/chosen": -2.768357276916504,
"logits/rejected": -2.8650612831115723,
"logps/chosen": -286.5350341796875,
"logps/rejected": -295.7557678222656,
"loss": 0.4694,
"rewards/accuracies": 1.0,
"rewards/chosen": -0.42378902435302734,
"rewards/margins": 2.2293660640716553,
"rewards/rejected": -2.6531550884246826,
"step": 950
},
{
"epoch": 0.5,
"learning_rate": 2.802639127940333e-07,
"logits/chosen": -2.8344969749450684,
"logits/rejected": -2.8180437088012695,
"logps/chosen": -312.8253173828125,
"logps/rejected": -299.6221618652344,
"loss": 0.4998,
"rewards/accuracies": 0.75,
"rewards/chosen": -0.2208956480026245,
"rewards/margins": 1.7145334482192993,
"rewards/rejected": -1.9354288578033447,
"step": 960
},
{
"epoch": 0.5,
"learning_rate": 2.773952954675846e-07,
"logits/chosen": -2.809140682220459,
"logits/rejected": -2.699197292327881,
"logps/chosen": -369.69854736328125,
"logps/rejected": -293.45477294921875,
"loss": 0.4849,
"rewards/accuracies": 0.800000011920929,
"rewards/chosen": -0.28183019161224365,
"rewards/margins": 1.6505457162857056,
"rewards/rejected": -1.9323756694793701,
"step": 970
},
{
"epoch": 0.51,
"learning_rate": 2.7452667814113594e-07,
"logits/chosen": -2.8047561645507812,
"logits/rejected": -2.777113199234009,
"logps/chosen": -326.3648376464844,
"logps/rejected": -289.0523986816406,
"loss": 0.4543,
"rewards/accuracies": 0.949999988079071,
"rewards/chosen": -0.42743197083473206,
"rewards/margins": 1.8371082544326782,
"rewards/rejected": -2.264540195465088,
"step": 980
},
{
"epoch": 0.51,
"learning_rate": 2.716580608146873e-07,
"logits/chosen": -2.8819570541381836,
"logits/rejected": -2.8494954109191895,
"logps/chosen": -318.66204833984375,
"logps/rejected": -345.1630859375,
"loss": 0.4534,
"rewards/accuracies": 0.8500000238418579,
"rewards/chosen": -0.4583119750022888,
"rewards/margins": 1.3975725173950195,
"rewards/rejected": -1.8558847904205322,
"step": 990
},
{
"epoch": 0.52,
"learning_rate": 2.687894434882387e-07,
"logits/chosen": -2.8820109367370605,
"logits/rejected": -2.9153826236724854,
"logps/chosen": -251.1084747314453,
"logps/rejected": -264.5345153808594,
"loss": 0.4102,
"rewards/accuracies": 0.6499999761581421,
"rewards/chosen": -0.5280589461326599,
"rewards/margins": 1.0574977397918701,
"rewards/rejected": -1.5855568647384644,
"step": 1000
},
{
"epoch": 0.52,
"eval_logits/chosen": -2.745532512664795,
"eval_logits/rejected": -2.7159857749938965,
"eval_logps/chosen": -298.5311279296875,
"eval_logps/rejected": -331.2577819824219,
"eval_loss": 0.4771742522716522,
"eval_rewards/accuracies": 0.765625,
"eval_rewards/chosen": -0.5766510367393494,
"eval_rewards/margins": 1.4900195598602295,
"eval_rewards/rejected": -2.0666706562042236,
"eval_runtime": 60.3269,
"eval_samples_per_second": 16.576,
"eval_steps_per_second": 0.265,
"step": 1000
},
{
"epoch": 0.52,
"learning_rate": 2.6592082616179004e-07,
"logits/chosen": -2.753818988800049,
"logits/rejected": -2.6555442810058594,
"logps/chosen": -421.75408935546875,
"logps/rejected": -326.7854919433594,
"loss": 0.4969,
"rewards/accuracies": 0.8500000238418579,
"rewards/chosen": -0.7776181101799011,
"rewards/margins": 1.5239381790161133,
"rewards/rejected": -2.30155611038208,
"step": 1010
},
{
"epoch": 0.53,
"learning_rate": 2.6305220883534133e-07,
"logits/chosen": -2.795933961868286,
"logits/rejected": -2.875976085662842,
"logps/chosen": -359.01318359375,
"logps/rejected": -325.6358642578125,
"loss": 0.4814,
"rewards/accuracies": 0.699999988079071,
"rewards/chosen": -0.12685959041118622,
"rewards/margins": 1.6275579929351807,
"rewards/rejected": -1.7544174194335938,
"step": 1020
},
{
"epoch": 0.53,
"learning_rate": 2.601835915088927e-07,
"logits/chosen": -2.7979979515075684,
"logits/rejected": -2.7110390663146973,
"logps/chosen": -307.5633239746094,
"logps/rejected": -272.8973388671875,
"loss": 0.4354,
"rewards/accuracies": 0.8999999761581421,
"rewards/chosen": -0.4669399857521057,
"rewards/margins": 1.9002759456634521,
"rewards/rejected": -2.367216110229492,
"step": 1030
},
{
"epoch": 0.54,
"learning_rate": 2.573149741824441e-07,
"logits/chosen": -2.864424467086792,
"logits/rejected": -2.7874035835266113,
"logps/chosen": -301.7801513671875,
"logps/rejected": -366.8116149902344,
"loss": 0.4799,
"rewards/accuracies": 0.800000011920929,
"rewards/chosen": -0.38910773396492004,
"rewards/margins": 2.2144553661346436,
"rewards/rejected": -2.6035633087158203,
"step": 1040
},
{
"epoch": 0.54,
"learning_rate": 2.544463568559954e-07,
"logits/chosen": -2.8506453037261963,
"logits/rejected": -2.7827541828155518,
"logps/chosen": -332.9029846191406,
"logps/rejected": -288.8146667480469,
"loss": 0.4728,
"rewards/accuracies": 0.75,
"rewards/chosen": -1.1103570461273193,
"rewards/margins": 1.3038640022277832,
"rewards/rejected": -2.4142210483551025,
"step": 1050
},
{
"epoch": 0.55,
"learning_rate": 2.5157773952954677e-07,
"logits/chosen": -2.746716022491455,
"logits/rejected": -2.738701105117798,
"logps/chosen": -362.2853698730469,
"logps/rejected": -263.8562316894531,
"loss": 0.5158,
"rewards/accuracies": 0.8500000238418579,
"rewards/chosen": -0.4159720540046692,
"rewards/margins": 1.3381980657577515,
"rewards/rejected": -1.7541700601577759,
"step": 1060
},
{
"epoch": 0.55,
"learning_rate": 2.4870912220309807e-07,
"logits/chosen": -2.7854630947113037,
"logits/rejected": -2.815742015838623,
"logps/chosen": -434.0101623535156,
"logps/rejected": -322.96990966796875,
"loss": 0.5047,
"rewards/accuracies": 0.800000011920929,
"rewards/chosen": -0.38989701867103577,
"rewards/margins": 1.555858850479126,
"rewards/rejected": -1.9457557201385498,
"step": 1070
},
{
"epoch": 0.56,
"learning_rate": 2.4584050487664947e-07,
"logits/chosen": -2.9198765754699707,
"logits/rejected": -2.8616695404052734,
"logps/chosen": -290.16680908203125,
"logps/rejected": -285.9183349609375,
"loss": 0.4499,
"rewards/accuracies": 0.800000011920929,
"rewards/chosen": -0.7213519811630249,
"rewards/margins": 1.0302088260650635,
"rewards/rejected": -1.751560926437378,
"step": 1080
},
{
"epoch": 0.56,
"learning_rate": 2.429718875502008e-07,
"logits/chosen": -2.9124810695648193,
"logits/rejected": -2.816594362258911,
"logps/chosen": -285.74139404296875,
"logps/rejected": -237.6147003173828,
"loss": 0.4673,
"rewards/accuracies": 0.800000011920929,
"rewards/chosen": -0.599921464920044,
"rewards/margins": 1.2972980737686157,
"rewards/rejected": -1.8972194194793701,
"step": 1090
},
{
"epoch": 0.57,
"learning_rate": 2.4010327022375216e-07,
"logits/chosen": -2.8846845626831055,
"logits/rejected": -2.914504289627075,
"logps/chosen": -373.78887939453125,
"logps/rejected": -349.1485900878906,
"loss": 0.4663,
"rewards/accuracies": 0.8999999761581421,
"rewards/chosen": -0.4652225971221924,
"rewards/margins": 0.824032187461853,
"rewards/rejected": -1.289254903793335,
"step": 1100
},
{
"epoch": 0.57,
"eval_logits/chosen": -2.725741386413574,
"eval_logits/rejected": -2.6993935108184814,
"eval_logps/chosen": -303.0741271972656,
"eval_logps/rejected": -331.96044921875,
"eval_loss": 0.4740181565284729,
"eval_rewards/accuracies": 0.765625,
"eval_rewards/chosen": -0.803801417350769,
"eval_rewards/margins": 1.2980027198791504,
"eval_rewards/rejected": -2.10180401802063,
"eval_runtime": 57.8167,
"eval_samples_per_second": 17.296,
"eval_steps_per_second": 0.277,
"step": 1100
},
{
"epoch": 0.57,
"learning_rate": 2.3723465289730348e-07,
"logits/chosen": -2.9184155464172363,
"logits/rejected": -2.913644552230835,
"logps/chosen": -325.4616394042969,
"logps/rejected": -290.3975830078125,
"loss": 0.4519,
"rewards/accuracies": 0.75,
"rewards/chosen": -0.7317468523979187,
"rewards/margins": 0.9042754173278809,
"rewards/rejected": -1.6360222101211548,
"step": 1110
},
{
"epoch": 0.58,
"learning_rate": 2.3436603557085483e-07,
"logits/chosen": -2.845045328140259,
"logits/rejected": -2.6523425579071045,
"logps/chosen": -370.0478820800781,
"logps/rejected": -286.46112060546875,
"loss": 0.5856,
"rewards/accuracies": 0.699999988079071,
"rewards/chosen": -0.8687052726745605,
"rewards/margins": 1.1778913736343384,
"rewards/rejected": -2.0465967655181885,
"step": 1120
},
{
"epoch": 0.58,
"learning_rate": 2.314974182444062e-07,
"logits/chosen": -2.8266937732696533,
"logits/rejected": -2.6822288036346436,
"logps/chosen": -350.9194641113281,
"logps/rejected": -341.3634033203125,
"loss": 0.4118,
"rewards/accuracies": 0.8500000238418579,
"rewards/chosen": -1.1032984256744385,
"rewards/margins": 1.7717195749282837,
"rewards/rejected": -2.8750178813934326,
"step": 1130
},
{
"epoch": 0.59,
"learning_rate": 2.2862880091795752e-07,
"logits/chosen": -2.71614146232605,
"logits/rejected": -2.7234530448913574,
"logps/chosen": -332.0517272949219,
"logps/rejected": -334.45977783203125,
"loss": 0.5011,
"rewards/accuracies": 0.75,
"rewards/chosen": -1.2248785495758057,
"rewards/margins": 1.2211261987686157,
"rewards/rejected": -2.4460043907165527,
"step": 1140
},
{
"epoch": 0.59,
"learning_rate": 2.257601835915089e-07,
"logits/chosen": -2.746645450592041,
"logits/rejected": -2.7202837467193604,
"logps/chosen": -285.22576904296875,
"logps/rejected": -271.9557800292969,
"loss": 0.4743,
"rewards/accuracies": 0.699999988079071,
"rewards/chosen": -0.9407691955566406,
"rewards/margins": 1.4142510890960693,
"rewards/rejected": -2.3550198078155518,
"step": 1150
},
{
"epoch": 0.6,
"learning_rate": 2.2289156626506022e-07,
"logits/chosen": -2.6213831901550293,
"logits/rejected": -2.5203697681427,
"logps/chosen": -377.27337646484375,
"logps/rejected": -397.376953125,
"loss": 0.4551,
"rewards/accuracies": 0.800000011920929,
"rewards/chosen": -0.6566792726516724,
"rewards/margins": 1.3486577272415161,
"rewards/rejected": -2.0053369998931885,
"step": 1160
},
{
"epoch": 0.6,
"learning_rate": 2.200229489386116e-07,
"logits/chosen": -2.8519699573516846,
"logits/rejected": -2.9094433784484863,
"logps/chosen": -343.9521789550781,
"logps/rejected": -308.2449645996094,
"loss": 0.4565,
"rewards/accuracies": 0.75,
"rewards/chosen": -0.4127805829048157,
"rewards/margins": 0.8223036527633667,
"rewards/rejected": -1.2350841760635376,
"step": 1170
},
{
"epoch": 0.61,
"learning_rate": 2.1715433161216294e-07,
"logits/chosen": -2.8739943504333496,
"logits/rejected": -2.8903346061706543,
"logps/chosen": -301.22967529296875,
"logps/rejected": -251.42855834960938,
"loss": 0.4426,
"rewards/accuracies": 0.800000011920929,
"rewards/chosen": -0.35094451904296875,
"rewards/margins": 1.0077975988388062,
"rewards/rejected": -1.3587422370910645,
"step": 1180
},
{
"epoch": 0.61,
"learning_rate": 2.1428571428571426e-07,
"logits/chosen": -2.892425060272217,
"logits/rejected": -2.8500430583953857,
"logps/chosen": -328.62652587890625,
"logps/rejected": -307.386474609375,
"loss": 0.4722,
"rewards/accuracies": 0.699999988079071,
"rewards/chosen": -0.8684075474739075,
"rewards/margins": 1.0192383527755737,
"rewards/rejected": -1.887645959854126,
"step": 1190
},
{
"epoch": 0.62,
"learning_rate": 2.1141709695926563e-07,
"logits/chosen": -2.910890817642212,
"logits/rejected": -2.8874032497406006,
"logps/chosen": -265.87310791015625,
"logps/rejected": -210.3219451904297,
"loss": 0.4737,
"rewards/accuracies": 0.699999988079071,
"rewards/chosen": -0.5260100960731506,
"rewards/margins": 0.7490276098251343,
"rewards/rejected": -1.2750377655029297,
"step": 1200
},
{
"epoch": 0.62,
"eval_logits/chosen": -2.713484764099121,
"eval_logits/rejected": -2.6841983795166016,
"eval_logps/chosen": -294.5633850097656,
"eval_logps/rejected": -323.9544677734375,
"eval_loss": 0.47160398960113525,
"eval_rewards/accuracies": 0.796875,
"eval_rewards/chosen": -0.3782654404640198,
"eval_rewards/margins": 1.3232390880584717,
"eval_rewards/rejected": -1.7015043497085571,
"eval_runtime": 57.3455,
"eval_samples_per_second": 17.438,
"eval_steps_per_second": 0.279,
"step": 1200
},
{
"epoch": 0.62,
"learning_rate": 2.0854847963281698e-07,
"logits/chosen": -2.970987319946289,
"logits/rejected": -2.867969512939453,
"logps/chosen": -368.6965026855469,
"logps/rejected": -353.63604736328125,
"loss": 0.4535,
"rewards/accuracies": 0.75,
"rewards/chosen": -0.4251924455165863,
"rewards/margins": 1.3024253845214844,
"rewards/rejected": -1.727617859840393,
"step": 1210
},
{
"epoch": 0.63,
"learning_rate": 2.0567986230636832e-07,
"logits/chosen": -2.964089870452881,
"logits/rejected": -2.933443784713745,
"logps/chosen": -293.6631164550781,
"logps/rejected": -330.43878173828125,
"loss": 0.4637,
"rewards/accuracies": 0.75,
"rewards/chosen": -0.6487616300582886,
"rewards/margins": 1.531836986541748,
"rewards/rejected": -2.180598735809326,
"step": 1220
},
{
"epoch": 0.64,
"learning_rate": 2.0281124497991967e-07,
"logits/chosen": -2.9999537467956543,
"logits/rejected": -2.9158272743225098,
"logps/chosen": -342.1163330078125,
"logps/rejected": -396.15350341796875,
"loss": 0.4611,
"rewards/accuracies": 0.800000011920929,
"rewards/chosen": -0.16167068481445312,
"rewards/margins": 1.715951919555664,
"rewards/rejected": -1.8776228427886963,
"step": 1230
},
{
"epoch": 0.64,
"learning_rate": 1.9994262765347102e-07,
"logits/chosen": -2.9242331981658936,
"logits/rejected": -2.841064929962158,
"logps/chosen": -337.76983642578125,
"logps/rejected": -282.3847351074219,
"loss": 0.4917,
"rewards/accuracies": 0.6499999761581421,
"rewards/chosen": -1.049020528793335,
"rewards/margins": 0.9485428929328918,
"rewards/rejected": -1.9975636005401611,
"step": 1240
},
{
"epoch": 0.65,
"learning_rate": 1.9707401032702237e-07,
"logits/chosen": -2.7807137966156006,
"logits/rejected": -2.775256872177124,
"logps/chosen": -282.36151123046875,
"logps/rejected": -339.3646545410156,
"loss": 0.4765,
"rewards/accuracies": 0.800000011920929,
"rewards/chosen": -0.4363161623477936,
"rewards/margins": 1.3644092082977295,
"rewards/rejected": -1.8007253408432007,
"step": 1250
},
{
"epoch": 0.65,
"learning_rate": 1.942053930005737e-07,
"logits/chosen": -2.850567579269409,
"logits/rejected": -2.8491926193237305,
"logps/chosen": -355.41064453125,
"logps/rejected": -349.8357238769531,
"loss": 0.4739,
"rewards/accuracies": 0.75,
"rewards/chosen": -0.5196908712387085,
"rewards/margins": 1.9353296756744385,
"rewards/rejected": -2.4550204277038574,
"step": 1260
},
{
"epoch": 0.66,
"learning_rate": 1.9133677567412506e-07,
"logits/chosen": -2.6965737342834473,
"logits/rejected": -2.7178375720977783,
"logps/chosen": -253.52798461914062,
"logps/rejected": -266.63330078125,
"loss": 0.4285,
"rewards/accuracies": 0.75,
"rewards/chosen": -0.8542221784591675,
"rewards/margins": 1.580542802810669,
"rewards/rejected": -2.434765338897705,
"step": 1270
},
{
"epoch": 0.66,
"learning_rate": 1.884681583476764e-07,
"logits/chosen": -2.8671460151672363,
"logits/rejected": -2.8770196437835693,
"logps/chosen": -395.8647155761719,
"logps/rejected": -322.73565673828125,
"loss": 0.5017,
"rewards/accuracies": 0.699999988079071,
"rewards/chosen": -0.8471239805221558,
"rewards/margins": 0.8821918368339539,
"rewards/rejected": -1.729315996170044,
"step": 1280
},
{
"epoch": 0.67,
"learning_rate": 1.8559954102122778e-07,
"logits/chosen": -2.8296945095062256,
"logits/rejected": -2.8752951622009277,
"logps/chosen": -439.872314453125,
"logps/rejected": -365.8454284667969,
"loss": 0.4554,
"rewards/accuracies": 0.75,
"rewards/chosen": -0.5656521320343018,
"rewards/margins": 1.5323896408081055,
"rewards/rejected": -2.0980417728424072,
"step": 1290
},
{
"epoch": 0.67,
"learning_rate": 1.827309236947791e-07,
"logits/chosen": -2.946869134902954,
"logits/rejected": -2.8591227531433105,
"logps/chosen": -316.06512451171875,
"logps/rejected": -311.39892578125,
"loss": 0.4259,
"rewards/accuracies": 0.75,
"rewards/chosen": -0.757400631904602,
"rewards/margins": 1.6462675333023071,
"rewards/rejected": -2.40366792678833,
"step": 1300
},
{
"epoch": 0.67,
"eval_logits/chosen": -2.735586166381836,
"eval_logits/rejected": -2.7046470642089844,
"eval_logps/chosen": -299.4761047363281,
"eval_logps/rejected": -329.3312072753906,
"eval_loss": 0.486631840467453,
"eval_rewards/accuracies": 0.78125,
"eval_rewards/chosen": -0.6239006519317627,
"eval_rewards/margins": 1.346441626548767,
"eval_rewards/rejected": -1.9703422784805298,
"eval_runtime": 57.6013,
"eval_samples_per_second": 17.361,
"eval_steps_per_second": 0.278,
"step": 1300
},
{
"epoch": 0.68,
"learning_rate": 1.7986230636833047e-07,
"logits/chosen": -2.9283134937286377,
"logits/rejected": -2.797074556350708,
"logps/chosen": -277.39044189453125,
"logps/rejected": -298.46978759765625,
"loss": 0.4063,
"rewards/accuracies": 0.550000011920929,
"rewards/chosen": -0.8742395639419556,
"rewards/margins": 0.7818448543548584,
"rewards/rejected": -1.656084418296814,
"step": 1310
},
{
"epoch": 0.68,
"learning_rate": 1.769936890418818e-07,
"logits/chosen": -2.8751561641693115,
"logits/rejected": -2.8942153453826904,
"logps/chosen": -327.19158935546875,
"logps/rejected": -371.1598205566406,
"loss": 0.4662,
"rewards/accuracies": 0.800000011920929,
"rewards/chosen": -0.4851578176021576,
"rewards/margins": 1.8855422735214233,
"rewards/rejected": -2.3707003593444824,
"step": 1320
},
{
"epoch": 0.69,
"learning_rate": 1.7412507171543314e-07,
"logits/chosen": -2.869723081588745,
"logits/rejected": -2.791024684906006,
"logps/chosen": -299.9261474609375,
"logps/rejected": -240.87722778320312,
"loss": 0.5043,
"rewards/accuracies": 0.6499999761581421,
"rewards/chosen": -1.252393364906311,
"rewards/margins": 0.8984333872795105,
"rewards/rejected": -2.1508266925811768,
"step": 1330
},
{
"epoch": 0.69,
"learning_rate": 1.7125645438898452e-07,
"logits/chosen": -2.8324294090270996,
"logits/rejected": -2.8103926181793213,
"logps/chosen": -360.5182800292969,
"logps/rejected": -295.58953857421875,
"loss": 0.4544,
"rewards/accuracies": 0.75,
"rewards/chosen": -0.9667569994926453,
"rewards/margins": 0.8512316942214966,
"rewards/rejected": -1.8179887533187866,
"step": 1340
},
{
"epoch": 0.7,
"learning_rate": 1.6838783706253584e-07,
"logits/chosen": -2.848896026611328,
"logits/rejected": -2.7923531532287598,
"logps/chosen": -372.48175048828125,
"logps/rejected": -274.67169189453125,
"loss": 0.5062,
"rewards/accuracies": 0.699999988079071,
"rewards/chosen": -0.6907297372817993,
"rewards/margins": 1.647875189781189,
"rewards/rejected": -2.3386049270629883,
"step": 1350
},
{
"epoch": 0.7,
"learning_rate": 1.655192197360872e-07,
"logits/chosen": -3.0092597007751465,
"logits/rejected": -3.0115504264831543,
"logps/chosen": -414.8208923339844,
"logps/rejected": -411.04327392578125,
"loss": 0.5338,
"rewards/accuracies": 0.699999988079071,
"rewards/chosen": -0.2658500671386719,
"rewards/margins": 1.0859276056289673,
"rewards/rejected": -1.3517776727676392,
"step": 1360
},
{
"epoch": 0.71,
"learning_rate": 1.6265060240963853e-07,
"logits/chosen": -2.947781801223755,
"logits/rejected": -2.9552831649780273,
"logps/chosen": -383.8572082519531,
"logps/rejected": -299.138427734375,
"loss": 0.5202,
"rewards/accuracies": 0.699999988079071,
"rewards/chosen": -0.4735872149467468,
"rewards/margins": 1.4772822856903076,
"rewards/rejected": -1.9508693218231201,
"step": 1370
},
{
"epoch": 0.71,
"learning_rate": 1.597819850831899e-07,
"logits/chosen": -2.812499523162842,
"logits/rejected": -2.7452359199523926,
"logps/chosen": -339.11981201171875,
"logps/rejected": -291.80035400390625,
"loss": 0.513,
"rewards/accuracies": 0.75,
"rewards/chosen": -0.7339299917221069,
"rewards/margins": 1.1406757831573486,
"rewards/rejected": -1.8746061325073242,
"step": 1380
},
{
"epoch": 0.72,
"learning_rate": 1.5691336775674125e-07,
"logits/chosen": -2.7743217945098877,
"logits/rejected": -2.7072505950927734,
"logps/chosen": -318.3100891113281,
"logps/rejected": -287.02581787109375,
"loss": 0.5028,
"rewards/accuracies": 0.699999988079071,
"rewards/chosen": -0.30296844244003296,
"rewards/margins": 1.0583657026290894,
"rewards/rejected": -1.3613340854644775,
"step": 1390
},
{
"epoch": 0.72,
"learning_rate": 1.5404475043029257e-07,
"logits/chosen": -2.930776834487915,
"logits/rejected": -2.942199468612671,
"logps/chosen": -239.4253387451172,
"logps/rejected": -327.1017150878906,
"loss": 0.4935,
"rewards/accuracies": 0.699999988079071,
"rewards/chosen": -0.503673255443573,
"rewards/margins": 0.9641939401626587,
"rewards/rejected": -1.4678672552108765,
"step": 1400
},
{
"epoch": 0.72,
"eval_logits/chosen": -2.7443745136260986,
"eval_logits/rejected": -2.71527099609375,
"eval_logps/chosen": -298.24908447265625,
"eval_logps/rejected": -325.1242980957031,
"eval_loss": 0.4747045338153839,
"eval_rewards/accuracies": 0.78125,
"eval_rewards/chosen": -0.5625503063201904,
"eval_rewards/margins": 1.1974470615386963,
"eval_rewards/rejected": -1.7599973678588867,
"eval_runtime": 59.3107,
"eval_samples_per_second": 16.86,
"eval_steps_per_second": 0.27,
"step": 1400
},
{
"epoch": 0.73,
"learning_rate": 1.5117613310384395e-07,
"logits/chosen": -2.92527437210083,
"logits/rejected": -3.0139377117156982,
"logps/chosen": -237.60580444335938,
"logps/rejected": -268.05389404296875,
"loss": 0.4175,
"rewards/accuracies": 0.8500000238418579,
"rewards/chosen": -0.6132401823997498,
"rewards/margins": 1.2935967445373535,
"rewards/rejected": -1.9068371057510376,
"step": 1410
},
{
"epoch": 0.73,
"learning_rate": 1.483075157773953e-07,
"logits/chosen": -2.7750067710876465,
"logits/rejected": -2.552576780319214,
"logps/chosen": -268.2431945800781,
"logps/rejected": -310.37359619140625,
"loss": 0.531,
"rewards/accuracies": 0.699999988079071,
"rewards/chosen": -0.5963152647018433,
"rewards/margins": 0.873083770275116,
"rewards/rejected": -1.469399094581604,
"step": 1420
},
{
"epoch": 0.74,
"learning_rate": 1.4543889845094664e-07,
"logits/chosen": -2.900043249130249,
"logits/rejected": -2.9251770973205566,
"logps/chosen": -294.197998046875,
"logps/rejected": -420.86981201171875,
"loss": 0.5,
"rewards/accuracies": 0.75,
"rewards/chosen": -0.7220600247383118,
"rewards/margins": 1.1596812009811401,
"rewards/rejected": -1.8817412853240967,
"step": 1430
},
{
"epoch": 0.74,
"learning_rate": 1.42570281124498e-07,
"logits/chosen": -2.8284263610839844,
"logits/rejected": -2.861384630203247,
"logps/chosen": -330.3293762207031,
"logps/rejected": -352.83013916015625,
"loss": 0.5019,
"rewards/accuracies": 0.8500000238418579,
"rewards/chosen": -0.5237518548965454,
"rewards/margins": 1.484616756439209,
"rewards/rejected": -2.008368730545044,
"step": 1440
},
{
"epoch": 0.75,
"learning_rate": 1.3970166379804933e-07,
"logits/chosen": -2.9035301208496094,
"logits/rejected": -2.877016544342041,
"logps/chosen": -330.01885986328125,
"logps/rejected": -309.932373046875,
"loss": 0.4207,
"rewards/accuracies": 0.75,
"rewards/chosen": -0.6725677847862244,
"rewards/margins": 1.2416926622390747,
"rewards/rejected": -1.9142605066299438,
"step": 1450
},
{
"epoch": 0.75,
"learning_rate": 1.3683304647160068e-07,
"logits/chosen": -2.8522450923919678,
"logits/rejected": -2.804635763168335,
"logps/chosen": -315.32598876953125,
"logps/rejected": -319.85528564453125,
"loss": 0.484,
"rewards/accuracies": 0.8500000238418579,
"rewards/chosen": -0.6710207462310791,
"rewards/margins": 2.302441120147705,
"rewards/rejected": -2.973461627960205,
"step": 1460
},
{
"epoch": 0.76,
"learning_rate": 1.3396442914515203e-07,
"logits/chosen": -2.795161008834839,
"logits/rejected": -2.835954189300537,
"logps/chosen": -301.64306640625,
"logps/rejected": -345.019287109375,
"loss": 0.5058,
"rewards/accuracies": 0.8500000238418579,
"rewards/chosen": -0.31656789779663086,
"rewards/margins": 1.5656077861785889,
"rewards/rejected": -1.8821756839752197,
"step": 1470
},
{
"epoch": 0.76,
"learning_rate": 1.3109581181870338e-07,
"logits/chosen": -2.8382556438446045,
"logits/rejected": -2.7719860076904297,
"logps/chosen": -416.19622802734375,
"logps/rejected": -278.07989501953125,
"loss": 0.3959,
"rewards/accuracies": 0.75,
"rewards/chosen": -0.7636359930038452,
"rewards/margins": 2.003495216369629,
"rewards/rejected": -2.7671313285827637,
"step": 1480
},
{
"epoch": 0.77,
"learning_rate": 1.2822719449225472e-07,
"logits/chosen": -2.7587904930114746,
"logits/rejected": -2.747542381286621,
"logps/chosen": -255.3273468017578,
"logps/rejected": -239.77218627929688,
"loss": 0.4899,
"rewards/accuracies": 0.800000011920929,
"rewards/chosen": -0.0010371461976319551,
"rewards/margins": 2.0150508880615234,
"rewards/rejected": -2.0160880088806152,
"step": 1490
},
{
"epoch": 0.77,
"learning_rate": 1.253585771658061e-07,
"logits/chosen": -2.7033538818359375,
"logits/rejected": -2.7172279357910156,
"logps/chosen": -317.5029296875,
"logps/rejected": -332.539794921875,
"loss": 0.4211,
"rewards/accuracies": 0.800000011920929,
"rewards/chosen": -0.8115976452827454,
"rewards/margins": 1.1709970235824585,
"rewards/rejected": -1.9825948476791382,
"step": 1500
},
{
"epoch": 0.77,
"eval_logits/chosen": -2.7236168384552,
"eval_logits/rejected": -2.6944026947021484,
"eval_logps/chosen": -299.1958923339844,
"eval_logps/rejected": -329.9108581542969,
"eval_loss": 0.4644867181777954,
"eval_rewards/accuracies": 0.765625,
"eval_rewards/chosen": -0.6098894476890564,
"eval_rewards/margins": 1.389434814453125,
"eval_rewards/rejected": -1.9993242025375366,
"eval_runtime": 57.1288,
"eval_samples_per_second": 17.504,
"eval_steps_per_second": 0.28,
"step": 1500
},
{
"epoch": 0.78,
"learning_rate": 1.2248995983935742e-07,
"logits/chosen": -2.7994678020477295,
"logits/rejected": -2.593963384628296,
"logps/chosen": -300.65960693359375,
"logps/rejected": -338.9661560058594,
"loss": 0.4655,
"rewards/accuracies": 0.8999999761581421,
"rewards/chosen": -0.44014763832092285,
"rewards/margins": 2.050450325012207,
"rewards/rejected": -2.49059796333313,
"step": 1510
},
{
"epoch": 0.78,
"learning_rate": 1.1962134251290876e-07,
"logits/chosen": -2.8634955883026123,
"logits/rejected": -2.9231014251708984,
"logps/chosen": -350.2730407714844,
"logps/rejected": -394.0359191894531,
"loss": 0.4687,
"rewards/accuracies": 0.6000000238418579,
"rewards/chosen": -0.7544436454772949,
"rewards/margins": 0.6226231455802917,
"rewards/rejected": -1.377066731452942,
"step": 1520
},
{
"epoch": 0.79,
"learning_rate": 1.1675272518646012e-07,
"logits/chosen": -2.714735984802246,
"logits/rejected": -2.6250107288360596,
"logps/chosen": -271.64691162109375,
"logps/rejected": -272.4801330566406,
"loss": 0.3846,
"rewards/accuracies": 0.8999999761581421,
"rewards/chosen": 0.06765889376401901,
"rewards/margins": 1.8679250478744507,
"rewards/rejected": -1.800265908241272,
"step": 1530
},
{
"epoch": 0.8,
"learning_rate": 1.1388410786001147e-07,
"logits/chosen": -2.952043294906616,
"logits/rejected": -2.9354848861694336,
"logps/chosen": -237.10208129882812,
"logps/rejected": -248.58023071289062,
"loss": 0.4791,
"rewards/accuracies": 0.5,
"rewards/chosen": -0.8253666162490845,
"rewards/margins": 0.6920875906944275,
"rewards/rejected": -1.5174543857574463,
"step": 1540
},
{
"epoch": 0.8,
"learning_rate": 1.1101549053356282e-07,
"logits/chosen": -2.839200735092163,
"logits/rejected": -2.827392101287842,
"logps/chosen": -347.25390625,
"logps/rejected": -284.7652893066406,
"loss": 0.4351,
"rewards/accuracies": 0.8999999761581421,
"rewards/chosen": -0.3293851315975189,
"rewards/margins": 2.0348496437072754,
"rewards/rejected": -2.3642349243164062,
"step": 1550
},
{
"epoch": 0.81,
"learning_rate": 1.0814687320711418e-07,
"logits/chosen": -2.853760242462158,
"logits/rejected": -2.728318214416504,
"logps/chosen": -224.41683959960938,
"logps/rejected": -228.2812042236328,
"loss": 0.4528,
"rewards/accuracies": 0.8999999761581421,
"rewards/chosen": -0.3215061128139496,
"rewards/margins": 1.6524702310562134,
"rewards/rejected": -1.9739763736724854,
"step": 1560
},
{
"epoch": 0.81,
"learning_rate": 1.0527825588066551e-07,
"logits/chosen": -2.870434284210205,
"logits/rejected": -2.793577194213867,
"logps/chosen": -282.13787841796875,
"logps/rejected": -271.94915771484375,
"loss": 0.4144,
"rewards/accuracies": 0.949999988079071,
"rewards/chosen": -0.5694266557693481,
"rewards/margins": 1.3067474365234375,
"rewards/rejected": -1.8761742115020752,
"step": 1570
},
{
"epoch": 0.82,
"learning_rate": 1.0240963855421686e-07,
"logits/chosen": -2.73341703414917,
"logits/rejected": -2.7930026054382324,
"logps/chosen": -214.6037139892578,
"logps/rejected": -285.880859375,
"loss": 0.4287,
"rewards/accuracies": 0.75,
"rewards/chosen": -0.6492952108383179,
"rewards/margins": 1.3862565755844116,
"rewards/rejected": -2.0355517864227295,
"step": 1580
},
{
"epoch": 0.82,
"learning_rate": 9.95410212277682e-08,
"logits/chosen": -2.885023593902588,
"logits/rejected": -2.929999351501465,
"logps/chosen": -294.29364013671875,
"logps/rejected": -273.53729248046875,
"loss": 0.4548,
"rewards/accuracies": 0.75,
"rewards/chosen": -0.3889610171318054,
"rewards/margins": 1.3147281408309937,
"rewards/rejected": -1.7036889791488647,
"step": 1590
},
{
"epoch": 0.83,
"learning_rate": 9.667240390131957e-08,
"logits/chosen": -2.8551316261291504,
"logits/rejected": -2.913240432739258,
"logps/chosen": -301.7610168457031,
"logps/rejected": -309.9018249511719,
"loss": 0.4931,
"rewards/accuracies": 0.800000011920929,
"rewards/chosen": -0.7070005536079407,
"rewards/margins": 1.0328757762908936,
"rewards/rejected": -1.7398761510849,
"step": 1600
},
{
"epoch": 0.83,
"eval_logits/chosen": -2.7304749488830566,
"eval_logits/rejected": -2.7005884647369385,
"eval_logps/chosen": -300.5933532714844,
"eval_logps/rejected": -332.0889892578125,
"eval_loss": 0.4683745205402374,
"eval_rewards/accuracies": 0.765625,
"eval_rewards/chosen": -0.6797637343406677,
"eval_rewards/margins": 1.4284672737121582,
"eval_rewards/rejected": -2.1082310676574707,
"eval_runtime": 58.1263,
"eval_samples_per_second": 17.204,
"eval_steps_per_second": 0.275,
"step": 1600
},
{
"epoch": 0.83,
"learning_rate": 9.380378657487091e-08,
"logits/chosen": -2.786533832550049,
"logits/rejected": -2.8000881671905518,
"logps/chosen": -252.6224822998047,
"logps/rejected": -307.4824523925781,
"loss": 0.4282,
"rewards/accuracies": 0.75,
"rewards/chosen": -0.869725227355957,
"rewards/margins": 1.490140438079834,
"rewards/rejected": -2.359865665435791,
"step": 1610
},
{
"epoch": 0.84,
"learning_rate": 9.093516924842226e-08,
"logits/chosen": -2.9314494132995605,
"logits/rejected": -2.888124942779541,
"logps/chosen": -322.18988037109375,
"logps/rejected": -340.00958251953125,
"loss": 0.4095,
"rewards/accuracies": 0.8500000238418579,
"rewards/chosen": -0.7641150951385498,
"rewards/margins": 1.8909038305282593,
"rewards/rejected": -2.6550190448760986,
"step": 1620
},
{
"epoch": 0.84,
"learning_rate": 8.806655192197361e-08,
"logits/chosen": -2.91792368888855,
"logits/rejected": -2.8502402305603027,
"logps/chosen": -327.46807861328125,
"logps/rejected": -310.23199462890625,
"loss": 0.4625,
"rewards/accuracies": 0.800000011920929,
"rewards/chosen": -0.5610017776489258,
"rewards/margins": 1.5278574228286743,
"rewards/rejected": -2.0888590812683105,
"step": 1630
},
{
"epoch": 0.85,
"learning_rate": 8.519793459552494e-08,
"logits/chosen": -2.8847482204437256,
"logits/rejected": -2.8378400802612305,
"logps/chosen": -315.1554870605469,
"logps/rejected": -282.12652587890625,
"loss": 0.418,
"rewards/accuracies": 0.8999999761581421,
"rewards/chosen": -0.6100664138793945,
"rewards/margins": 1.6232362985610962,
"rewards/rejected": -2.233302593231201,
"step": 1640
},
{
"epoch": 0.85,
"learning_rate": 8.23293172690763e-08,
"logits/chosen": -2.943779230117798,
"logits/rejected": -2.894235372543335,
"logps/chosen": -380.72894287109375,
"logps/rejected": -279.9917907714844,
"loss": 0.4489,
"rewards/accuracies": 0.8999999761581421,
"rewards/chosen": 0.3063332736492157,
"rewards/margins": 2.2862744331359863,
"rewards/rejected": -1.9799413681030273,
"step": 1650
},
{
"epoch": 0.86,
"learning_rate": 7.946069994262765e-08,
"logits/chosen": -2.738778829574585,
"logits/rejected": -2.805850028991699,
"logps/chosen": -221.29507446289062,
"logps/rejected": -314.97076416015625,
"loss": 0.4268,
"rewards/accuracies": 0.800000011920929,
"rewards/chosen": -0.5129801034927368,
"rewards/margins": 1.8285270929336548,
"rewards/rejected": -2.3415069580078125,
"step": 1660
},
{
"epoch": 0.86,
"learning_rate": 7.6592082616179e-08,
"logits/chosen": -2.73237681388855,
"logits/rejected": -2.7697629928588867,
"logps/chosen": -295.9866638183594,
"logps/rejected": -282.50189208984375,
"loss": 0.5111,
"rewards/accuracies": 0.8500000238418579,
"rewards/chosen": -0.5220045447349548,
"rewards/margins": 1.3860810995101929,
"rewards/rejected": -1.908085823059082,
"step": 1670
},
{
"epoch": 0.87,
"learning_rate": 7.372346528973034e-08,
"logits/chosen": -2.9239017963409424,
"logits/rejected": -2.8694822788238525,
"logps/chosen": -266.6614074707031,
"logps/rejected": -303.3201599121094,
"loss": 0.5028,
"rewards/accuracies": 0.800000011920929,
"rewards/chosen": -0.5870487689971924,
"rewards/margins": 0.9326974749565125,
"rewards/rejected": -1.5197464227676392,
"step": 1680
},
{
"epoch": 0.87,
"learning_rate": 7.08548479632817e-08,
"logits/chosen": -2.8852458000183105,
"logits/rejected": -2.9049694538116455,
"logps/chosen": -256.247802734375,
"logps/rejected": -329.19305419921875,
"loss": 0.4537,
"rewards/accuracies": 0.800000011920929,
"rewards/chosen": -0.6385833024978638,
"rewards/margins": 1.4285590648651123,
"rewards/rejected": -2.0671424865722656,
"step": 1690
},
{
"epoch": 0.88,
"learning_rate": 6.798623063683305e-08,
"logits/chosen": -2.8753390312194824,
"logits/rejected": -2.7723491191864014,
"logps/chosen": -285.18365478515625,
"logps/rejected": -267.1637878417969,
"loss": 0.5029,
"rewards/accuracies": 0.800000011920929,
"rewards/chosen": -0.7313726544380188,
"rewards/margins": 1.1651675701141357,
"rewards/rejected": -1.8965400457382202,
"step": 1700
},
{
"epoch": 0.88,
"eval_logits/chosen": -2.740252733230591,
"eval_logits/rejected": -2.710822343826294,
"eval_logps/chosen": -297.1233215332031,
"eval_logps/rejected": -327.8266906738281,
"eval_loss": 0.4595467746257782,
"eval_rewards/accuracies": 0.78125,
"eval_rewards/chosen": -0.5062620639801025,
"eval_rewards/margins": 1.3888535499572754,
"eval_rewards/rejected": -1.895115852355957,
"eval_runtime": 55.2023,
"eval_samples_per_second": 18.115,
"eval_steps_per_second": 0.29,
"step": 1700
},
{
"epoch": 0.88,
"learning_rate": 6.511761331038438e-08,
"logits/chosen": -2.895554542541504,
"logits/rejected": -2.7735402584075928,
"logps/chosen": -375.2083435058594,
"logps/rejected": -386.50689697265625,
"loss": 0.4635,
"rewards/accuracies": 0.8500000238418579,
"rewards/chosen": -0.7999193072319031,
"rewards/margins": 1.7455612421035767,
"rewards/rejected": -2.545480728149414,
"step": 1710
},
{
"epoch": 0.89,
"learning_rate": 6.224899598393573e-08,
"logits/chosen": -2.897815704345703,
"logits/rejected": -2.833980083465576,
"logps/chosen": -349.2824401855469,
"logps/rejected": -320.26080322265625,
"loss": 0.5059,
"rewards/accuracies": 0.800000011920929,
"rewards/chosen": -0.8347634077072144,
"rewards/margins": 1.1928595304489136,
"rewards/rejected": -2.027622938156128,
"step": 1720
},
{
"epoch": 0.89,
"learning_rate": 5.9380378657487085e-08,
"logits/chosen": -2.747729539871216,
"logits/rejected": -2.749624490737915,
"logps/chosen": -389.26104736328125,
"logps/rejected": -379.7428283691406,
"loss": 0.4561,
"rewards/accuracies": 0.550000011920929,
"rewards/chosen": -0.46041926741600037,
"rewards/margins": 1.0110886096954346,
"rewards/rejected": -1.4715079069137573,
"step": 1730
},
{
"epoch": 0.9,
"learning_rate": 5.651176133103844e-08,
"logits/chosen": -2.9265472888946533,
"logits/rejected": -2.7790768146514893,
"logps/chosen": -344.9275817871094,
"logps/rejected": -267.4143371582031,
"loss": 0.4472,
"rewards/accuracies": 0.949999988079071,
"rewards/chosen": -0.3255007266998291,
"rewards/margins": 2.0611414909362793,
"rewards/rejected": -2.3866424560546875,
"step": 1740
},
{
"epoch": 0.9,
"learning_rate": 5.3643144004589786e-08,
"logits/chosen": -2.8731534481048584,
"logits/rejected": -2.7964863777160645,
"logps/chosen": -293.29681396484375,
"logps/rejected": -305.7369079589844,
"loss": 0.4481,
"rewards/accuracies": 0.8500000238418579,
"rewards/chosen": -0.3717958629131317,
"rewards/margins": 1.912921667098999,
"rewards/rejected": -2.2847177982330322,
"step": 1750
},
{
"epoch": 0.91,
"learning_rate": 5.077452667814113e-08,
"logits/chosen": -2.771488666534424,
"logits/rejected": -2.865757465362549,
"logps/chosen": -309.177978515625,
"logps/rejected": -285.9333801269531,
"loss": 0.4892,
"rewards/accuracies": 0.800000011920929,
"rewards/chosen": -0.7683748602867126,
"rewards/margins": 1.0694787502288818,
"rewards/rejected": -1.8378536701202393,
"step": 1760
},
{
"epoch": 0.91,
"learning_rate": 4.790590935169248e-08,
"logits/chosen": -2.8640894889831543,
"logits/rejected": -2.8574938774108887,
"logps/chosen": -270.6001281738281,
"logps/rejected": -282.6536865234375,
"loss": 0.4509,
"rewards/accuracies": 0.75,
"rewards/chosen": -0.5418082475662231,
"rewards/margins": 1.5019443035125732,
"rewards/rejected": -2.043752670288086,
"step": 1770
},
{
"epoch": 0.92,
"learning_rate": 4.5037292025243834e-08,
"logits/chosen": -2.916822910308838,
"logits/rejected": -2.945218086242676,
"logps/chosen": -330.62493896484375,
"logps/rejected": -340.20599365234375,
"loss": 0.4988,
"rewards/accuracies": 0.8500000238418579,
"rewards/chosen": -0.980034351348877,
"rewards/margins": 1.0700609683990479,
"rewards/rejected": -2.0500950813293457,
"step": 1780
},
{
"epoch": 0.92,
"learning_rate": 4.2168674698795174e-08,
"logits/chosen": -2.9247031211853027,
"logits/rejected": -2.9124653339385986,
"logps/chosen": -289.2491149902344,
"logps/rejected": -264.84283447265625,
"loss": 0.4577,
"rewards/accuracies": 0.800000011920929,
"rewards/chosen": -0.3735916018486023,
"rewards/margins": 0.9222102165222168,
"rewards/rejected": -1.2958018779754639,
"step": 1790
},
{
"epoch": 0.93,
"learning_rate": 3.930005737234653e-08,
"logits/chosen": -2.9057695865631104,
"logits/rejected": -2.819000005722046,
"logps/chosen": -337.3312683105469,
"logps/rejected": -276.67010498046875,
"loss": 0.4965,
"rewards/accuracies": 0.8500000238418579,
"rewards/chosen": -0.23481640219688416,
"rewards/margins": 1.076974868774414,
"rewards/rejected": -1.3117913007736206,
"step": 1800
},
{
"epoch": 0.93,
"eval_logits/chosen": -2.752323627471924,
"eval_logits/rejected": -2.7225875854492188,
"eval_logps/chosen": -298.1203308105469,
"eval_logps/rejected": -328.0830993652344,
"eval_loss": 0.46130359172821045,
"eval_rewards/accuracies": 0.78125,
"eval_rewards/chosen": -0.5561107397079468,
"eval_rewards/margins": 1.3518264293670654,
"eval_rewards/rejected": -1.9079371690750122,
"eval_runtime": 57.0782,
"eval_samples_per_second": 17.52,
"eval_steps_per_second": 0.28,
"step": 1800
},
{
"epoch": 0.93,
"learning_rate": 3.6431440045897875e-08,
"logits/chosen": -2.8899993896484375,
"logits/rejected": -2.886475086212158,
"logps/chosen": -212.6807861328125,
"logps/rejected": -243.3516082763672,
"loss": 0.4966,
"rewards/accuracies": 0.75,
"rewards/chosen": -0.7232971787452698,
"rewards/margins": 1.2528116703033447,
"rewards/rejected": -1.9761087894439697,
"step": 1810
},
{
"epoch": 0.94,
"learning_rate": 3.356282271944923e-08,
"logits/chosen": -2.936781406402588,
"logits/rejected": -2.913865566253662,
"logps/chosen": -259.8666076660156,
"logps/rejected": -268.0596008300781,
"loss": 0.4455,
"rewards/accuracies": 0.800000011920929,
"rewards/chosen": -0.7119141221046448,
"rewards/margins": 1.462996244430542,
"rewards/rejected": -2.174910306930542,
"step": 1820
},
{
"epoch": 0.94,
"learning_rate": 3.0694205393000576e-08,
"logits/chosen": -2.8601667881011963,
"logits/rejected": -2.8055875301361084,
"logps/chosen": -324.9699401855469,
"logps/rejected": -261.33355712890625,
"loss": 0.4484,
"rewards/accuracies": 0.800000011920929,
"rewards/chosen": -0.6698704361915588,
"rewards/margins": 1.4717018604278564,
"rewards/rejected": -2.1415722370147705,
"step": 1830
},
{
"epoch": 0.95,
"learning_rate": 2.782558806655192e-08,
"logits/chosen": -2.9527735710144043,
"logits/rejected": -2.9381892681121826,
"logps/chosen": -338.42889404296875,
"logps/rejected": -293.7242126464844,
"loss": 0.4437,
"rewards/accuracies": 0.6000000238418579,
"rewards/chosen": -0.5508059859275818,
"rewards/margins": 0.5796544551849365,
"rewards/rejected": -1.130460500717163,
"step": 1840
},
{
"epoch": 0.96,
"learning_rate": 2.4956970740103267e-08,
"logits/chosen": -2.8355095386505127,
"logits/rejected": -2.847282648086548,
"logps/chosen": -303.50482177734375,
"logps/rejected": -281.2980041503906,
"loss": 0.4709,
"rewards/accuracies": 0.949999988079071,
"rewards/chosen": -0.2886872887611389,
"rewards/margins": 2.1929566860198975,
"rewards/rejected": -2.4816439151763916,
"step": 1850
},
{
"epoch": 0.96,
"learning_rate": 2.2088353413654617e-08,
"logits/chosen": -2.8765408992767334,
"logits/rejected": -2.8786520957946777,
"logps/chosen": -321.27203369140625,
"logps/rejected": -308.62628173828125,
"loss": 0.4371,
"rewards/accuracies": 0.800000011920929,
"rewards/chosen": -0.6084128618240356,
"rewards/margins": 1.1289150714874268,
"rewards/rejected": -1.7373279333114624,
"step": 1860
},
{
"epoch": 0.97,
"learning_rate": 1.9219736087205964e-08,
"logits/chosen": -2.8658809661865234,
"logits/rejected": -2.86934232711792,
"logps/chosen": -271.0024719238281,
"logps/rejected": -249.09765625,
"loss": 0.4678,
"rewards/accuracies": 0.6499999761581421,
"rewards/chosen": -0.7377957105636597,
"rewards/margins": 0.9156107902526855,
"rewards/rejected": -1.6534065008163452,
"step": 1870
},
{
"epoch": 0.97,
"learning_rate": 1.6351118760757314e-08,
"logits/chosen": -2.7913639545440674,
"logits/rejected": -2.7344272136688232,
"logps/chosen": -299.87823486328125,
"logps/rejected": -340.60467529296875,
"loss": 0.4679,
"rewards/accuracies": 0.75,
"rewards/chosen": -0.9796409606933594,
"rewards/margins": 1.4006431102752686,
"rewards/rejected": -2.380284070968628,
"step": 1880
},
{
"epoch": 0.98,
"learning_rate": 1.3482501434308661e-08,
"logits/chosen": -2.9909162521362305,
"logits/rejected": -2.8788750171661377,
"logps/chosen": -316.1296081542969,
"logps/rejected": -329.0151062011719,
"loss": 0.4584,
"rewards/accuracies": 0.8500000238418579,
"rewards/chosen": -0.6042786836624146,
"rewards/margins": 1.349906325340271,
"rewards/rejected": -1.954184889793396,
"step": 1890
},
{
"epoch": 0.98,
"learning_rate": 1.061388410786001e-08,
"logits/chosen": -2.840143918991089,
"logits/rejected": -2.8358423709869385,
"logps/chosen": -347.96185302734375,
"logps/rejected": -336.83831787109375,
"loss": 0.4337,
"rewards/accuracies": 0.75,
"rewards/chosen": -0.25721627473831177,
"rewards/margins": 1.7318767309188843,
"rewards/rejected": -1.9890931844711304,
"step": 1900
},
{
"epoch": 0.98,
"eval_logits/chosen": -2.746859073638916,
"eval_logits/rejected": -2.717475175857544,
"eval_logps/chosen": -297.12957763671875,
"eval_logps/rejected": -327.35992431640625,
"eval_loss": 0.4607921838760376,
"eval_rewards/accuracies": 0.765625,
"eval_rewards/chosen": -0.5065746307373047,
"eval_rewards/margins": 1.365204095840454,
"eval_rewards/rejected": -1.871778964996338,
"eval_runtime": 59.1698,
"eval_samples_per_second": 16.901,
"eval_steps_per_second": 0.27,
"step": 1900
},
{
"epoch": 0.99,
"learning_rate": 7.745266781411359e-09,
"logits/chosen": -2.944617986679077,
"logits/rejected": -2.859666347503662,
"logps/chosen": -312.51654052734375,
"logps/rejected": -263.56494140625,
"loss": 0.4129,
"rewards/accuracies": 0.550000011920929,
"rewards/chosen": -0.29929789900779724,
"rewards/margins": 0.7224828004837036,
"rewards/rejected": -1.0217806100845337,
"step": 1910
},
{
"epoch": 0.99,
"learning_rate": 4.8766494549627085e-09,
"logits/chosen": -2.8194212913513184,
"logits/rejected": -2.776564598083496,
"logps/chosen": -363.00396728515625,
"logps/rejected": -299.03338623046875,
"loss": 0.4647,
"rewards/accuracies": 0.550000011920929,
"rewards/chosen": -0.38584208488464355,
"rewards/margins": 0.8067569732666016,
"rewards/rejected": -1.1925990581512451,
"step": 1920
},
{
"epoch": 1.0,
"learning_rate": 2.008032128514056e-09,
"logits/chosen": -2.8939270973205566,
"logits/rejected": -2.778672933578491,
"logps/chosen": -323.1568298339844,
"logps/rejected": -284.89361572265625,
"loss": 0.4376,
"rewards/accuracies": 0.8500000238418579,
"rewards/chosen": -0.581506609916687,
"rewards/margins": 1.8644784688949585,
"rewards/rejected": -2.4459850788116455,
"step": 1930
},
{
"epoch": 1.0,
"step": 1937,
"total_flos": 0.0,
"train_loss": 0.48803097129668166,
"train_runtime": 7971.1784,
"train_samples_per_second": 7.774,
"train_steps_per_second": 0.243
}
],
"logging_steps": 10,
"max_steps": 1937,
"num_train_epochs": 1,
"save_steps": 500,
"total_flos": 0.0,
"trial_name": null,
"trial_params": null
}