| { | |
| "best_metric": null, | |
| "best_model_checkpoint": null, | |
| "epoch": 0.23277467411545624, | |
| "eval_steps": 500, | |
| "global_step": 125, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "epoch": 0.00186219739292365, | |
| "grad_norm": 40.418399810791016, | |
| "learning_rate": 9.259259259259259e-08, | |
| "logps/chosen": -143.09214782714844, | |
| "logps/rejected": -159.23831176757812, | |
| "loss": 0.6931, | |
| "losses/dpo": 0.6931471824645996, | |
| "losses/sft": 1.9754583835601807, | |
| "losses/total": 0.6931471824645996, | |
| "ref_logps/chosen": -143.09214782714844, | |
| "ref_logps/rejected": -159.23831176757812, | |
| "rewards/accuracies": 0.0, | |
| "rewards/chosen": 0.0, | |
| "rewards/margins": 0.0, | |
| "rewards/rejected": 0.0, | |
| "step": 1 | |
| }, | |
| { | |
| "epoch": 0.0037243947858473, | |
| "grad_norm": 40.48415756225586, | |
| "learning_rate": 1.8518518518518518e-07, | |
| "logps/chosen": -139.04046630859375, | |
| "logps/rejected": -167.1217041015625, | |
| "loss": 0.6931, | |
| "losses/dpo": 0.6931471824645996, | |
| "losses/sft": 2.354458808898926, | |
| "losses/total": 0.6931471824645996, | |
| "ref_logps/chosen": -139.04046630859375, | |
| "ref_logps/rejected": -167.1217041015625, | |
| "rewards/accuracies": 0.0, | |
| "rewards/chosen": 0.0, | |
| "rewards/margins": 0.0, | |
| "rewards/rejected": 0.0, | |
| "step": 2 | |
| }, | |
| { | |
| "epoch": 0.00558659217877095, | |
| "grad_norm": 43.48381042480469, | |
| "learning_rate": 2.7777777777777776e-07, | |
| "logps/chosen": -145.57992553710938, | |
| "logps/rejected": -169.97146606445312, | |
| "loss": 0.6981, | |
| "losses/dpo": 0.6891708374023438, | |
| "losses/sft": 2.1739914417266846, | |
| "losses/total": 0.6891708374023438, | |
| "ref_logps/chosen": -145.53663635253906, | |
| "ref_logps/rejected": -170.02041625976562, | |
| "rewards/accuracies": 0.484375, | |
| "rewards/chosen": -0.00432817917317152, | |
| "rewards/margins": -0.009223854169249535, | |
| "rewards/rejected": 0.004895674996078014, | |
| "step": 3 | |
| }, | |
| { | |
| "epoch": 0.0074487895716946, | |
| "grad_norm": 40.21121597290039, | |
| "learning_rate": 3.7037037037037036e-07, | |
| "logps/chosen": -145.57131958007812, | |
| "logps/rejected": -162.80123901367188, | |
| "loss": 0.6915, | |
| "losses/dpo": 0.6871472597122192, | |
| "losses/sft": 2.114041805267334, | |
| "losses/total": 0.6871472597122192, | |
| "ref_logps/chosen": -145.52674865722656, | |
| "ref_logps/rejected": -162.71473693847656, | |
| "rewards/accuracies": 0.578125, | |
| "rewards/chosen": -0.004457235801964998, | |
| "rewards/margins": 0.004192234016954899, | |
| "rewards/rejected": -0.00864946935325861, | |
| "step": 4 | |
| }, | |
| { | |
| "epoch": 0.00931098696461825, | |
| "grad_norm": 43.5928955078125, | |
| "learning_rate": 4.6296296296296297e-07, | |
| "logps/chosen": -149.61972045898438, | |
| "logps/rejected": -171.2332763671875, | |
| "loss": 0.6897, | |
| "losses/dpo": 0.6956955194473267, | |
| "losses/sft": 1.9878240823745728, | |
| "losses/total": 0.6956955194473267, | |
| "ref_logps/chosen": -149.57879638671875, | |
| "ref_logps/rejected": -171.1157684326172, | |
| "rewards/accuracies": 0.5625, | |
| "rewards/chosen": -0.004093348979949951, | |
| "rewards/margins": 0.007657159119844437, | |
| "rewards/rejected": -0.011750508099794388, | |
| "step": 5 | |
| }, | |
| { | |
| "epoch": 0.0111731843575419, | |
| "grad_norm": 43.37643051147461, | |
| "learning_rate": 5.555555555555555e-07, | |
| "logps/chosen": -144.1446075439453, | |
| "logps/rejected": -165.30984497070312, | |
| "loss": 0.6839, | |
| "losses/dpo": 0.7011494636535645, | |
| "losses/sft": 2.2325987815856934, | |
| "losses/total": 0.7011494636535645, | |
| "ref_logps/chosen": -144.15261840820312, | |
| "ref_logps/rejected": -165.12521362304688, | |
| "rewards/accuracies": 0.609375, | |
| "rewards/chosen": 0.0007997989887371659, | |
| "rewards/margins": 0.019263559952378273, | |
| "rewards/rejected": -0.01846376061439514, | |
| "step": 6 | |
| }, | |
| { | |
| "epoch": 0.01303538175046555, | |
| "grad_norm": 41.759315490722656, | |
| "learning_rate": 6.481481481481481e-07, | |
| "logps/chosen": -140.94808959960938, | |
| "logps/rejected": -161.61846923828125, | |
| "loss": 0.6677, | |
| "losses/dpo": 0.6434986591339111, | |
| "losses/sft": 2.4416794776916504, | |
| "losses/total": 0.6434986591339111, | |
| "ref_logps/chosen": -140.9979248046875, | |
| "ref_logps/rejected": -161.13681030273438, | |
| "rewards/accuracies": 0.734375, | |
| "rewards/chosen": 0.004983758553862572, | |
| "rewards/margins": 0.05314992740750313, | |
| "rewards/rejected": -0.048166170716285706, | |
| "step": 7 | |
| }, | |
| { | |
| "epoch": 0.0148975791433892, | |
| "grad_norm": 40.88422393798828, | |
| "learning_rate": 7.407407407407407e-07, | |
| "logps/chosen": -150.63833618164062, | |
| "logps/rejected": -174.64923095703125, | |
| "loss": 0.6588, | |
| "losses/dpo": 0.639857292175293, | |
| "losses/sft": 2.3609848022460938, | |
| "losses/total": 0.639857292175293, | |
| "ref_logps/chosen": -150.88568115234375, | |
| "ref_logps/rejected": -174.18106079101562, | |
| "rewards/accuracies": 0.796875, | |
| "rewards/chosen": 0.02473468706011772, | |
| "rewards/margins": 0.0715511366724968, | |
| "rewards/rejected": -0.046816445887088776, | |
| "step": 8 | |
| }, | |
| { | |
| "epoch": 0.01675977653631285, | |
| "grad_norm": 36.576690673828125, | |
| "learning_rate": 8.333333333333333e-07, | |
| "logps/chosen": -146.96463012695312, | |
| "logps/rejected": -174.3262939453125, | |
| "loss": 0.6259, | |
| "losses/dpo": 0.6055787801742554, | |
| "losses/sft": 2.1100568771362305, | |
| "losses/total": 0.6055787801742554, | |
| "ref_logps/chosen": -147.0458221435547, | |
| "ref_logps/rejected": -172.9729461669922, | |
| "rewards/accuracies": 0.9375, | |
| "rewards/chosen": 0.008118623867630959, | |
| "rewards/margins": 0.14345335960388184, | |
| "rewards/rejected": -0.13533474504947662, | |
| "step": 9 | |
| }, | |
| { | |
| "epoch": 0.0186219739292365, | |
| "grad_norm": 41.190025329589844, | |
| "learning_rate": 9.259259259259259e-07, | |
| "logps/chosen": -154.90782165527344, | |
| "logps/rejected": -160.24624633789062, | |
| "loss": 0.5904, | |
| "losses/dpo": 0.5785373449325562, | |
| "losses/sft": 2.5960636138916016, | |
| "losses/total": 0.5785373449325562, | |
| "ref_logps/chosen": -155.20986938476562, | |
| "ref_logps/rejected": -158.2913055419922, | |
| "rewards/accuracies": 0.921875, | |
| "rewards/chosen": 0.030205056071281433, | |
| "rewards/margins": 0.2256987988948822, | |
| "rewards/rejected": -0.19549374282360077, | |
| "step": 10 | |
| }, | |
| { | |
| "epoch": 0.020484171322160148, | |
| "grad_norm": 32.080810546875, | |
| "learning_rate": 1.0185185185185185e-06, | |
| "logps/chosen": -138.2403106689453, | |
| "logps/rejected": -162.02291870117188, | |
| "loss": 0.5437, | |
| "losses/dpo": 0.5474151968955994, | |
| "losses/sft": 1.9134595394134521, | |
| "losses/total": 0.5474151968955994, | |
| "ref_logps/chosen": -138.54551696777344, | |
| "ref_logps/rejected": -158.88693237304688, | |
| "rewards/accuracies": 0.9375, | |
| "rewards/chosen": 0.030520910397171974, | |
| "rewards/margins": 0.34412044286727905, | |
| "rewards/rejected": -0.31359952688217163, | |
| "step": 11 | |
| }, | |
| { | |
| "epoch": 0.0223463687150838, | |
| "grad_norm": 29.643903732299805, | |
| "learning_rate": 1.111111111111111e-06, | |
| "logps/chosen": -154.15118408203125, | |
| "logps/rejected": -180.0041961669922, | |
| "loss": 0.4393, | |
| "losses/dpo": 0.4857974946498871, | |
| "losses/sft": 2.032329797744751, | |
| "losses/total": 0.4857974946498871, | |
| "ref_logps/chosen": -154.87046813964844, | |
| "ref_logps/rejected": -174.29629516601562, | |
| "rewards/accuracies": 0.96875, | |
| "rewards/chosen": 0.0719279944896698, | |
| "rewards/margins": 0.6427179574966431, | |
| "rewards/rejected": -0.5707899332046509, | |
| "step": 12 | |
| }, | |
| { | |
| "epoch": 0.024208566108007448, | |
| "grad_norm": 27.43463134765625, | |
| "learning_rate": 1.2037037037037037e-06, | |
| "logps/chosen": -146.56866455078125, | |
| "logps/rejected": -173.71633911132812, | |
| "loss": 0.4487, | |
| "losses/dpo": 0.4681214988231659, | |
| "losses/sft": 2.5795841217041016, | |
| "losses/total": 0.4681214988231659, | |
| "ref_logps/chosen": -146.011962890625, | |
| "ref_logps/rejected": -166.83184814453125, | |
| "rewards/accuracies": 0.9375, | |
| "rewards/chosen": -0.055669985711574554, | |
| "rewards/margins": 0.6327812075614929, | |
| "rewards/rejected": -0.6884511709213257, | |
| "step": 13 | |
| }, | |
| { | |
| "epoch": 0.0260707635009311, | |
| "grad_norm": 24.864574432373047, | |
| "learning_rate": 1.2962962962962962e-06, | |
| "logps/chosen": -151.02015686035156, | |
| "logps/rejected": -181.14610290527344, | |
| "loss": 0.3711, | |
| "losses/dpo": 0.4647141695022583, | |
| "losses/sft": 2.417238473892212, | |
| "losses/total": 0.4647141695022583, | |
| "ref_logps/chosen": -151.75128173828125, | |
| "ref_logps/rejected": -172.87350463867188, | |
| "rewards/accuracies": 0.96875, | |
| "rewards/chosen": 0.07311299443244934, | |
| "rewards/margins": 0.9003715515136719, | |
| "rewards/rejected": -0.8272585272789001, | |
| "step": 14 | |
| }, | |
| { | |
| "epoch": 0.027932960893854747, | |
| "grad_norm": 22.773723602294922, | |
| "learning_rate": 1.3888888888888892e-06, | |
| "logps/chosen": -146.68460083007812, | |
| "logps/rejected": -180.87896728515625, | |
| "loss": 0.3584, | |
| "losses/dpo": 0.33828622102737427, | |
| "losses/sft": 2.191209077835083, | |
| "losses/total": 0.33828622102737427, | |
| "ref_logps/chosen": -145.39768981933594, | |
| "ref_logps/rejected": -170.00677490234375, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": -0.12869027256965637, | |
| "rewards/margins": 0.9585269689559937, | |
| "rewards/rejected": -1.0872172117233276, | |
| "step": 15 | |
| }, | |
| { | |
| "epoch": 0.0297951582867784, | |
| "grad_norm": 17.319433212280273, | |
| "learning_rate": 1.4814814814814815e-06, | |
| "logps/chosen": -157.9118194580078, | |
| "logps/rejected": -180.584228515625, | |
| "loss": 0.253, | |
| "losses/dpo": 0.30005261301994324, | |
| "losses/sft": 2.218407154083252, | |
| "losses/total": 0.30005261301994324, | |
| "ref_logps/chosen": -155.03277587890625, | |
| "ref_logps/rejected": -161.18026733398438, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": -0.2879039943218231, | |
| "rewards/margins": 1.652491807937622, | |
| "rewards/rejected": -1.940395712852478, | |
| "step": 16 | |
| }, | |
| { | |
| "epoch": 0.03165735567970205, | |
| "grad_norm": 14.82257080078125, | |
| "learning_rate": 1.5740740740740742e-06, | |
| "logps/chosen": -151.82461547851562, | |
| "logps/rejected": -182.52401733398438, | |
| "loss": 0.2197, | |
| "losses/dpo": 0.2500099837779999, | |
| "losses/sft": 2.3733954429626465, | |
| "losses/total": 0.2500099837779999, | |
| "ref_logps/chosen": -147.43585205078125, | |
| "ref_logps/rejected": -158.81658935546875, | |
| "rewards/accuracies": 0.953125, | |
| "rewards/chosen": -0.438876748085022, | |
| "rewards/margins": 1.9318653345108032, | |
| "rewards/rejected": -2.3707423210144043, | |
| "step": 17 | |
| }, | |
| { | |
| "epoch": 0.0335195530726257, | |
| "grad_norm": 15.098052978515625, | |
| "learning_rate": 1.6666666666666667e-06, | |
| "logps/chosen": -146.62161254882812, | |
| "logps/rejected": -192.061767578125, | |
| "loss": 0.2086, | |
| "losses/dpo": 0.087049201130867, | |
| "losses/sft": 2.235438585281372, | |
| "losses/total": 0.087049201130867, | |
| "ref_logps/chosen": -142.35076904296875, | |
| "ref_logps/rejected": -164.73597717285156, | |
| "rewards/accuracies": 0.921875, | |
| "rewards/chosen": -0.42708471417427063, | |
| "rewards/margins": 2.305494785308838, | |
| "rewards/rejected": -2.732579469680786, | |
| "step": 18 | |
| }, | |
| { | |
| "epoch": 0.035381750465549346, | |
| "grad_norm": 14.139907836914062, | |
| "learning_rate": 1.7592592592592594e-06, | |
| "logps/chosen": -151.61148071289062, | |
| "logps/rejected": -204.13427734375, | |
| "loss": 0.1831, | |
| "losses/dpo": 0.2179492712020874, | |
| "losses/sft": 2.547309637069702, | |
| "losses/total": 0.2179492712020874, | |
| "ref_logps/chosen": -145.25982666015625, | |
| "ref_logps/rejected": -172.7918701171875, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": -0.6351636648178101, | |
| "rewards/margins": 2.4990787506103516, | |
| "rewards/rejected": -3.134242534637451, | |
| "step": 19 | |
| }, | |
| { | |
| "epoch": 0.037243947858473, | |
| "grad_norm": 15.214645385742188, | |
| "learning_rate": 1.8518518518518519e-06, | |
| "logps/chosen": -149.88876342773438, | |
| "logps/rejected": -218.4535675048828, | |
| "loss": 0.181, | |
| "losses/dpo": 0.18753334879875183, | |
| "losses/sft": 2.394906759262085, | |
| "losses/total": 0.18753334879875183, | |
| "ref_logps/chosen": -142.5288543701172, | |
| "ref_logps/rejected": -185.18878173828125, | |
| "rewards/accuracies": 0.9375, | |
| "rewards/chosen": -0.7359906435012817, | |
| "rewards/margins": 2.590489387512207, | |
| "rewards/rejected": -3.3264803886413574, | |
| "step": 20 | |
| }, | |
| { | |
| "epoch": 0.03910614525139665, | |
| "grad_norm": 15.793328285217285, | |
| "learning_rate": 1.944444444444445e-06, | |
| "logps/chosen": -155.686279296875, | |
| "logps/rejected": -189.35899353027344, | |
| "loss": 0.1946, | |
| "losses/dpo": 0.2072586715221405, | |
| "losses/sft": 2.1763200759887695, | |
| "losses/total": 0.2072586715221405, | |
| "ref_logps/chosen": -148.760986328125, | |
| "ref_logps/rejected": -153.56321716308594, | |
| "rewards/accuracies": 0.96875, | |
| "rewards/chosen": -0.6925297379493713, | |
| "rewards/margins": 2.88704776763916, | |
| "rewards/rejected": -3.5795774459838867, | |
| "step": 21 | |
| }, | |
| { | |
| "epoch": 0.040968342644320296, | |
| "grad_norm": 15.932718276977539, | |
| "learning_rate": 2.037037037037037e-06, | |
| "logps/chosen": -147.47117614746094, | |
| "logps/rejected": -206.65597534179688, | |
| "loss": 0.1655, | |
| "losses/dpo": 0.10909190773963928, | |
| "losses/sft": 2.1695094108581543, | |
| "losses/total": 0.10909190773963928, | |
| "ref_logps/chosen": -137.73451232910156, | |
| "ref_logps/rejected": -160.2368927001953, | |
| "rewards/accuracies": 0.9375, | |
| "rewards/chosen": -0.9736669063568115, | |
| "rewards/margins": 3.6682400703430176, | |
| "rewards/rejected": -4.641907691955566, | |
| "step": 22 | |
| }, | |
| { | |
| "epoch": 0.04283054003724395, | |
| "grad_norm": 11.131953239440918, | |
| "learning_rate": 2.1296296296296298e-06, | |
| "logps/chosen": -172.5790252685547, | |
| "logps/rejected": -234.46731567382812, | |
| "loss": 0.0974, | |
| "losses/dpo": 0.25966331362724304, | |
| "losses/sft": 2.784914255142212, | |
| "losses/total": 0.25966331362724304, | |
| "ref_logps/chosen": -154.8829345703125, | |
| "ref_logps/rejected": -172.43850708007812, | |
| "rewards/accuracies": 0.96875, | |
| "rewards/chosen": -1.7696071863174438, | |
| "rewards/margins": 4.433272361755371, | |
| "rewards/rejected": -6.202879905700684, | |
| "step": 23 | |
| }, | |
| { | |
| "epoch": 0.0446927374301676, | |
| "grad_norm": 9.715975761413574, | |
| "learning_rate": 2.222222222222222e-06, | |
| "logps/chosen": -161.45799255371094, | |
| "logps/rejected": -231.06072998046875, | |
| "loss": 0.0859, | |
| "losses/dpo": 0.04074842855334282, | |
| "losses/sft": 2.738722324371338, | |
| "losses/total": 0.04074842855334282, | |
| "ref_logps/chosen": -142.22097778320312, | |
| "ref_logps/rejected": -161.80909729003906, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": -1.9237014055252075, | |
| "rewards/margins": 5.001461029052734, | |
| "rewards/rejected": -6.9251627922058105, | |
| "step": 24 | |
| }, | |
| { | |
| "epoch": 0.04655493482309125, | |
| "grad_norm": 14.314465522766113, | |
| "learning_rate": 2.314814814814815e-06, | |
| "logps/chosen": -170.95504760742188, | |
| "logps/rejected": -257.62115478515625, | |
| "loss": 0.124, | |
| "losses/dpo": 0.004803886171430349, | |
| "losses/sft": 2.606781244277954, | |
| "losses/total": 0.004803886171430349, | |
| "ref_logps/chosen": -146.23434448242188, | |
| "ref_logps/rejected": -173.50958251953125, | |
| "rewards/accuracies": 0.953125, | |
| "rewards/chosen": -2.472069263458252, | |
| "rewards/margins": 5.939087867736816, | |
| "rewards/rejected": -8.411157608032227, | |
| "step": 25 | |
| }, | |
| { | |
| "epoch": 0.048417132216014895, | |
| "grad_norm": 16.382986068725586, | |
| "learning_rate": 2.4074074074074075e-06, | |
| "logps/chosen": -171.77096557617188, | |
| "logps/rejected": -249.99264526367188, | |
| "loss": 0.1927, | |
| "losses/dpo": 0.023840777575969696, | |
| "losses/sft": 2.574446201324463, | |
| "losses/total": 0.023840777575969696, | |
| "ref_logps/chosen": -146.96835327148438, | |
| "ref_logps/rejected": -168.13232421875, | |
| "rewards/accuracies": 0.96875, | |
| "rewards/chosen": -2.4802613258361816, | |
| "rewards/margins": 5.705770969390869, | |
| "rewards/rejected": -8.18603229522705, | |
| "step": 26 | |
| }, | |
| { | |
| "epoch": 0.05027932960893855, | |
| "grad_norm": 9.626073837280273, | |
| "learning_rate": 2.5e-06, | |
| "logps/chosen": -169.91448974609375, | |
| "logps/rejected": -251.7610626220703, | |
| "loss": 0.058, | |
| "losses/dpo": 0.0014569438062608242, | |
| "losses/sft": 2.5121726989746094, | |
| "losses/total": 0.0014569438062608242, | |
| "ref_logps/chosen": -141.93600463867188, | |
| "ref_logps/rejected": -160.8459014892578, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": -2.7978479862213135, | |
| "rewards/margins": 6.293667793273926, | |
| "rewards/rejected": -9.091516494750977, | |
| "step": 27 | |
| }, | |
| { | |
| "epoch": 0.0521415270018622, | |
| "grad_norm": 9.746648788452148, | |
| "learning_rate": 2.5925925925925925e-06, | |
| "logps/chosen": -173.24583435058594, | |
| "logps/rejected": -267.49298095703125, | |
| "loss": 0.0456, | |
| "losses/dpo": 0.010660952888429165, | |
| "losses/sft": 2.7988812923431396, | |
| "losses/total": 0.010660952888429165, | |
| "ref_logps/chosen": -147.74620056152344, | |
| "ref_logps/rejected": -170.100830078125, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": -2.5499653816223145, | |
| "rewards/margins": 7.189251899719238, | |
| "rewards/rejected": -9.739216804504395, | |
| "step": 28 | |
| }, | |
| { | |
| "epoch": 0.054003724394785846, | |
| "grad_norm": 19.729585647583008, | |
| "learning_rate": 2.6851851851851856e-06, | |
| "logps/chosen": -173.5810546875, | |
| "logps/rejected": -260.90985107421875, | |
| "loss": 0.1598, | |
| "losses/dpo": 0.11473169922828674, | |
| "losses/sft": 2.465557098388672, | |
| "losses/total": 0.11473169922828674, | |
| "ref_logps/chosen": -144.18157958984375, | |
| "ref_logps/rejected": -164.2210693359375, | |
| "rewards/accuracies": 0.890625, | |
| "rewards/chosen": -2.939948081970215, | |
| "rewards/margins": 6.72892951965332, | |
| "rewards/rejected": -9.668878555297852, | |
| "step": 29 | |
| }, | |
| { | |
| "epoch": 0.055865921787709494, | |
| "grad_norm": 15.30510425567627, | |
| "learning_rate": 2.7777777777777783e-06, | |
| "logps/chosen": -170.92425537109375, | |
| "logps/rejected": -277.4226379394531, | |
| "loss": 0.1033, | |
| "losses/dpo": 0.13019727170467377, | |
| "losses/sft": 2.4969606399536133, | |
| "losses/total": 0.13019727170467377, | |
| "ref_logps/chosen": -142.30502319335938, | |
| "ref_logps/rejected": -169.90554809570312, | |
| "rewards/accuracies": 0.9375, | |
| "rewards/chosen": -2.8619208335876465, | |
| "rewards/margins": 7.889788627624512, | |
| "rewards/rejected": -10.751708984375, | |
| "step": 30 | |
| }, | |
| { | |
| "epoch": 0.05772811918063315, | |
| "grad_norm": 6.729362487792969, | |
| "learning_rate": 2.8703703703703706e-06, | |
| "logps/chosen": -177.1944122314453, | |
| "logps/rejected": -277.68206787109375, | |
| "loss": 0.0363, | |
| "losses/dpo": 0.03610475733876228, | |
| "losses/sft": 2.655313491821289, | |
| "losses/total": 0.03610475733876228, | |
| "ref_logps/chosen": -140.73678588867188, | |
| "ref_logps/rejected": -164.96824645996094, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -3.6457643508911133, | |
| "rewards/margins": 7.625619888305664, | |
| "rewards/rejected": -11.271383285522461, | |
| "step": 31 | |
| }, | |
| { | |
| "epoch": 0.0595903165735568, | |
| "grad_norm": 15.916095733642578, | |
| "learning_rate": 2.962962962962963e-06, | |
| "logps/chosen": -168.07888793945312, | |
| "logps/rejected": -266.97467041015625, | |
| "loss": 0.1384, | |
| "losses/dpo": 0.003958229906857014, | |
| "losses/sft": 3.21946120262146, | |
| "losses/total": 0.003958229906857014, | |
| "ref_logps/chosen": -135.16700744628906, | |
| "ref_logps/rejected": -166.81027221679688, | |
| "rewards/accuracies": 0.9375, | |
| "rewards/chosen": -3.2911877632141113, | |
| "rewards/margins": 6.725253582000732, | |
| "rewards/rejected": -10.016440391540527, | |
| "step": 32 | |
| }, | |
| { | |
| "epoch": 0.061452513966480445, | |
| "grad_norm": 13.613255500793457, | |
| "learning_rate": 3.055555555555556e-06, | |
| "logps/chosen": -163.11419677734375, | |
| "logps/rejected": -267.9139099121094, | |
| "loss": 0.1, | |
| "losses/dpo": 0.0983101949095726, | |
| "losses/sft": 2.5492136478424072, | |
| "losses/total": 0.0983101949095726, | |
| "ref_logps/chosen": -140.14205932617188, | |
| "ref_logps/rejected": -160.53872680664062, | |
| "rewards/accuracies": 0.9375, | |
| "rewards/chosen": -2.297213554382324, | |
| "rewards/margins": 8.44030475616455, | |
| "rewards/rejected": -10.737519264221191, | |
| "step": 33 | |
| }, | |
| { | |
| "epoch": 0.0633147113594041, | |
| "grad_norm": 15.18605899810791, | |
| "learning_rate": 3.1481481481481483e-06, | |
| "logps/chosen": -167.43917846679688, | |
| "logps/rejected": -285.1209411621094, | |
| "loss": 0.1385, | |
| "losses/dpo": 0.010567452758550644, | |
| "losses/sft": 2.848440170288086, | |
| "losses/total": 0.010567452758550644, | |
| "ref_logps/chosen": -137.24496459960938, | |
| "ref_logps/rejected": -172.29844665527344, | |
| "rewards/accuracies": 0.96875, | |
| "rewards/chosen": -3.019423723220825, | |
| "rewards/margins": 8.262825965881348, | |
| "rewards/rejected": -11.282249450683594, | |
| "step": 34 | |
| }, | |
| { | |
| "epoch": 0.06517690875232775, | |
| "grad_norm": 14.179464340209961, | |
| "learning_rate": 3.240740740740741e-06, | |
| "logps/chosen": -163.87579345703125, | |
| "logps/rejected": -273.77777099609375, | |
| "loss": 0.1047, | |
| "losses/dpo": 0.010419070720672607, | |
| "losses/sft": 2.3946168422698975, | |
| "losses/total": 0.010419070720672607, | |
| "ref_logps/chosen": -144.74697875976562, | |
| "ref_logps/rejected": -167.2154541015625, | |
| "rewards/accuracies": 0.9375, | |
| "rewards/chosen": -1.9128817319869995, | |
| "rewards/margins": 8.743349075317383, | |
| "rewards/rejected": -10.656229972839355, | |
| "step": 35 | |
| }, | |
| { | |
| "epoch": 0.0670391061452514, | |
| "grad_norm": 7.81901741027832, | |
| "learning_rate": 3.3333333333333333e-06, | |
| "logps/chosen": -170.99649047851562, | |
| "logps/rejected": -260.6686096191406, | |
| "loss": 0.0408, | |
| "losses/dpo": 0.03330932930111885, | |
| "losses/sft": 2.9487311840057373, | |
| "losses/total": 0.03330932930111885, | |
| "ref_logps/chosen": -150.3196563720703, | |
| "ref_logps/rejected": -156.96363830566406, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": -2.0676848888397217, | |
| "rewards/margins": 8.302813529968262, | |
| "rewards/rejected": -10.370498657226562, | |
| "step": 36 | |
| }, | |
| { | |
| "epoch": 0.06890130353817504, | |
| "grad_norm": 14.016197204589844, | |
| "learning_rate": 3.4259259259259265e-06, | |
| "logps/chosen": -161.38865661621094, | |
| "logps/rejected": -281.09271240234375, | |
| "loss": 0.1047, | |
| "losses/dpo": 0.5629028677940369, | |
| "losses/sft": 2.4373741149902344, | |
| "losses/total": 0.5629028677940369, | |
| "ref_logps/chosen": -143.14517211914062, | |
| "ref_logps/rejected": -172.78611755371094, | |
| "rewards/accuracies": 0.953125, | |
| "rewards/chosen": -1.8243476152420044, | |
| "rewards/margins": 9.006311416625977, | |
| "rewards/rejected": -10.830659866333008, | |
| "step": 37 | |
| }, | |
| { | |
| "epoch": 0.07076350093109869, | |
| "grad_norm": 15.18514347076416, | |
| "learning_rate": 3.5185185185185187e-06, | |
| "logps/chosen": -160.60586547851562, | |
| "logps/rejected": -263.134765625, | |
| "loss": 0.1468, | |
| "losses/dpo": 0.018482662737369537, | |
| "losses/sft": 2.2324092388153076, | |
| "losses/total": 0.018482662737369537, | |
| "ref_logps/chosen": -141.36175537109375, | |
| "ref_logps/rejected": -164.61480712890625, | |
| "rewards/accuracies": 0.953125, | |
| "rewards/chosen": -1.9244120121002197, | |
| "rewards/margins": 7.92758321762085, | |
| "rewards/rejected": -9.851995468139648, | |
| "step": 38 | |
| }, | |
| { | |
| "epoch": 0.07262569832402235, | |
| "grad_norm": 8.755450248718262, | |
| "learning_rate": 3.6111111111111115e-06, | |
| "logps/chosen": -170.1685791015625, | |
| "logps/rejected": -288.41119384765625, | |
| "loss": 0.0421, | |
| "losses/dpo": 9.010350913740695e-05, | |
| "losses/sft": 2.3953020572662354, | |
| "losses/total": 9.010350913740695e-05, | |
| "ref_logps/chosen": -152.65139770507812, | |
| "ref_logps/rejected": -172.203857421875, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": -1.751717448234558, | |
| "rewards/margins": 9.86901569366455, | |
| "rewards/rejected": -11.620733261108398, | |
| "step": 39 | |
| }, | |
| { | |
| "epoch": 0.074487895716946, | |
| "grad_norm": 2.3320696353912354, | |
| "learning_rate": 3.7037037037037037e-06, | |
| "logps/chosen": -164.16981506347656, | |
| "logps/rejected": -263.49163818359375, | |
| "loss": 0.0103, | |
| "losses/dpo": 0.013380440883338451, | |
| "losses/sft": 2.408010482788086, | |
| "losses/total": 0.013380440883338451, | |
| "ref_logps/chosen": -153.0699462890625, | |
| "ref_logps/rejected": -166.4036407470703, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -1.1099885702133179, | |
| "rewards/margins": 8.598810195922852, | |
| "rewards/rejected": -9.708799362182617, | |
| "step": 40 | |
| }, | |
| { | |
| "epoch": 0.07635009310986965, | |
| "grad_norm": 13.349935531616211, | |
| "learning_rate": 3.796296296296297e-06, | |
| "logps/chosen": -162.9290771484375, | |
| "logps/rejected": -275.8412780761719, | |
| "loss": 0.0746, | |
| "losses/dpo": 0.0015751949977129698, | |
| "losses/sft": 2.3437771797180176, | |
| "losses/total": 0.0015751949977129698, | |
| "ref_logps/chosen": -151.49713134765625, | |
| "ref_logps/rejected": -168.54776000976562, | |
| "rewards/accuracies": 0.953125, | |
| "rewards/chosen": -1.1431937217712402, | |
| "rewards/margins": 9.586158752441406, | |
| "rewards/rejected": -10.729352951049805, | |
| "step": 41 | |
| }, | |
| { | |
| "epoch": 0.0782122905027933, | |
| "grad_norm": 7.708398818969727, | |
| "learning_rate": 3.88888888888889e-06, | |
| "logps/chosen": -170.53414916992188, | |
| "logps/rejected": -267.343017578125, | |
| "loss": 0.0415, | |
| "losses/dpo": 0.16293640434741974, | |
| "losses/sft": 2.4445619583129883, | |
| "losses/total": 0.16293640434741974, | |
| "ref_logps/chosen": -158.01504516601562, | |
| "ref_logps/rejected": -162.6872100830078, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": -1.2519094944000244, | |
| "rewards/margins": 9.213672637939453, | |
| "rewards/rejected": -10.465582847595215, | |
| "step": 42 | |
| }, | |
| { | |
| "epoch": 0.08007448789571694, | |
| "grad_norm": 8.482429504394531, | |
| "learning_rate": 3.9814814814814814e-06, | |
| "logps/chosen": -152.68186950683594, | |
| "logps/rejected": -256.3250427246094, | |
| "loss": 0.0489, | |
| "losses/dpo": 0.005869926419109106, | |
| "losses/sft": 2.2688705921173096, | |
| "losses/total": 0.005869926419109106, | |
| "ref_logps/chosen": -143.47671508789062, | |
| "ref_logps/rejected": -158.04229736328125, | |
| "rewards/accuracies": 0.96875, | |
| "rewards/chosen": -0.9205170273780823, | |
| "rewards/margins": 8.907758712768555, | |
| "rewards/rejected": -9.828275680541992, | |
| "step": 43 | |
| }, | |
| { | |
| "epoch": 0.08193668528864059, | |
| "grad_norm": 4.836800575256348, | |
| "learning_rate": 4.074074074074074e-06, | |
| "logps/chosen": -161.54238891601562, | |
| "logps/rejected": -249.66888427734375, | |
| "loss": 0.0293, | |
| "losses/dpo": 0.007885000668466091, | |
| "losses/sft": 2.5034093856811523, | |
| "losses/total": 0.007885000668466091, | |
| "ref_logps/chosen": -150.9593963623047, | |
| "ref_logps/rejected": -160.7205810546875, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": -1.05829918384552, | |
| "rewards/margins": 7.8365325927734375, | |
| "rewards/rejected": -8.894831657409668, | |
| "step": 44 | |
| }, | |
| { | |
| "epoch": 0.08379888268156424, | |
| "grad_norm": 7.306879997253418, | |
| "learning_rate": 4.166666666666667e-06, | |
| "logps/chosen": -168.57354736328125, | |
| "logps/rejected": -273.81622314453125, | |
| "loss": 0.0409, | |
| "losses/dpo": 0.044236376881599426, | |
| "losses/sft": 2.285489559173584, | |
| "losses/total": 0.044236376881599426, | |
| "ref_logps/chosen": -153.68539428710938, | |
| "ref_logps/rejected": -173.37750244140625, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -1.4888172149658203, | |
| "rewards/margins": 8.555052757263184, | |
| "rewards/rejected": -10.043869972229004, | |
| "step": 45 | |
| }, | |
| { | |
| "epoch": 0.0856610800744879, | |
| "grad_norm": 8.716976165771484, | |
| "learning_rate": 4.2592592592592596e-06, | |
| "logps/chosen": -157.63235473632812, | |
| "logps/rejected": -256.28546142578125, | |
| "loss": 0.0498, | |
| "losses/dpo": 0.047989651560783386, | |
| "losses/sft": 2.594020366668701, | |
| "losses/total": 0.047989651560783386, | |
| "ref_logps/chosen": -149.37562561035156, | |
| "ref_logps/rejected": -163.25059509277344, | |
| "rewards/accuracies": 0.96875, | |
| "rewards/chosen": -0.8256728053092957, | |
| "rewards/margins": 8.477815628051758, | |
| "rewards/rejected": -9.303489685058594, | |
| "step": 46 | |
| }, | |
| { | |
| "epoch": 0.08752327746741155, | |
| "grad_norm": 14.776224136352539, | |
| "learning_rate": 4.351851851851852e-06, | |
| "logps/chosen": -159.35926818847656, | |
| "logps/rejected": -254.82000732421875, | |
| "loss": 0.0947, | |
| "losses/dpo": 0.0015063448809087276, | |
| "losses/sft": 2.186452627182007, | |
| "losses/total": 0.0015063448809087276, | |
| "ref_logps/chosen": -153.80572509765625, | |
| "ref_logps/rejected": -160.63690185546875, | |
| "rewards/accuracies": 0.953125, | |
| "rewards/chosen": -0.555354654788971, | |
| "rewards/margins": 8.862955093383789, | |
| "rewards/rejected": -9.41831111907959, | |
| "step": 47 | |
| }, | |
| { | |
| "epoch": 0.0893854748603352, | |
| "grad_norm": 6.18978214263916, | |
| "learning_rate": 4.444444444444444e-06, | |
| "logps/chosen": -146.09950256347656, | |
| "logps/rejected": -269.2020568847656, | |
| "loss": 0.0442, | |
| "losses/dpo": 0.010637683793902397, | |
| "losses/sft": 2.3249499797821045, | |
| "losses/total": 0.010637683793902397, | |
| "ref_logps/chosen": -140.32867431640625, | |
| "ref_logps/rejected": -161.42984008789062, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": -0.5770829319953918, | |
| "rewards/margins": 10.200138092041016, | |
| "rewards/rejected": -10.7772216796875, | |
| "step": 48 | |
| }, | |
| { | |
| "epoch": 0.09124767225325885, | |
| "grad_norm": 8.178156852722168, | |
| "learning_rate": 4.537037037037038e-06, | |
| "logps/chosen": -150.715087890625, | |
| "logps/rejected": -288.4056396484375, | |
| "loss": 0.0446, | |
| "losses/dpo": 0.2217281013727188, | |
| "losses/sft": 2.4317214488983154, | |
| "losses/total": 0.2217281013727188, | |
| "ref_logps/chosen": -143.90017700195312, | |
| "ref_logps/rejected": -177.4407958984375, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": -0.6814901828765869, | |
| "rewards/margins": 10.414993286132812, | |
| "rewards/rejected": -11.09648323059082, | |
| "step": 49 | |
| }, | |
| { | |
| "epoch": 0.0931098696461825, | |
| "grad_norm": 11.958553314208984, | |
| "learning_rate": 4.62962962962963e-06, | |
| "logps/chosen": -156.93243408203125, | |
| "logps/rejected": -276.21051025390625, | |
| "loss": 0.0786, | |
| "losses/dpo": 0.020680470392107964, | |
| "losses/sft": 2.4808478355407715, | |
| "losses/total": 0.020680470392107964, | |
| "ref_logps/chosen": -146.65036010742188, | |
| "ref_logps/rejected": -166.49574279785156, | |
| "rewards/accuracies": 0.96875, | |
| "rewards/chosen": -1.0282070636749268, | |
| "rewards/margins": 9.943269729614258, | |
| "rewards/rejected": -10.971476554870605, | |
| "step": 50 | |
| }, | |
| { | |
| "epoch": 0.09497206703910614, | |
| "grad_norm": 2.3756279945373535, | |
| "learning_rate": 4.722222222222222e-06, | |
| "logps/chosen": -146.2957763671875, | |
| "logps/rejected": -290.5218505859375, | |
| "loss": 0.0115, | |
| "losses/dpo": 0.021228570491075516, | |
| "losses/sft": 2.0093941688537598, | |
| "losses/total": 0.021228570491075516, | |
| "ref_logps/chosen": -144.70184326171875, | |
| "ref_logps/rejected": -169.62815856933594, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.15939302742481232, | |
| "rewards/margins": 11.929975509643555, | |
| "rewards/rejected": -12.08936882019043, | |
| "step": 51 | |
| }, | |
| { | |
| "epoch": 0.09683426443202979, | |
| "grad_norm": 4.650004863739014, | |
| "learning_rate": 4.814814814814815e-06, | |
| "logps/chosen": -151.42132568359375, | |
| "logps/rejected": -283.79669189453125, | |
| "loss": 0.0183, | |
| "losses/dpo": 0.005594337359070778, | |
| "losses/sft": 2.4590542316436768, | |
| "losses/total": 0.005594337359070778, | |
| "ref_logps/chosen": -148.097412109375, | |
| "ref_logps/rejected": -164.08462524414062, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.33239230513572693, | |
| "rewards/margins": 11.638812065124512, | |
| "rewards/rejected": -11.97120475769043, | |
| "step": 52 | |
| }, | |
| { | |
| "epoch": 0.09869646182495345, | |
| "grad_norm": 7.309722423553467, | |
| "learning_rate": 4.907407407407408e-06, | |
| "logps/chosen": -163.71875, | |
| "logps/rejected": -280.90655517578125, | |
| "loss": 0.0367, | |
| "losses/dpo": 0.05281204357743263, | |
| "losses/sft": 2.374516725540161, | |
| "losses/total": 0.05281204357743263, | |
| "ref_logps/chosen": -151.98492431640625, | |
| "ref_logps/rejected": -164.55319213867188, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -1.173383116722107, | |
| "rewards/margins": 10.461955070495605, | |
| "rewards/rejected": -11.635337829589844, | |
| "step": 53 | |
| }, | |
| { | |
| "epoch": 0.1005586592178771, | |
| "grad_norm": 6.878481388092041, | |
| "learning_rate": 5e-06, | |
| "logps/chosen": -158.5335693359375, | |
| "logps/rejected": -270.8926086425781, | |
| "loss": 0.0462, | |
| "losses/dpo": 0.008242600597441196, | |
| "losses/sft": 2.6928701400756836, | |
| "losses/total": 0.008242600597441196, | |
| "ref_logps/chosen": -154.32550048828125, | |
| "ref_logps/rejected": -162.00128173828125, | |
| "rewards/accuracies": 0.96875, | |
| "rewards/chosen": -0.4208071529865265, | |
| "rewards/margins": 10.468324661254883, | |
| "rewards/rejected": -10.889131546020508, | |
| "step": 54 | |
| }, | |
| { | |
| "epoch": 0.10242085661080075, | |
| "grad_norm": 5.541274070739746, | |
| "learning_rate": 4.989648033126294e-06, | |
| "logps/chosen": -150.91574096679688, | |
| "logps/rejected": -279.86248779296875, | |
| "loss": 0.0329, | |
| "losses/dpo": 0.0699252262711525, | |
| "losses/sft": 2.2539589405059814, | |
| "losses/total": 0.0699252262711525, | |
| "ref_logps/chosen": -147.29766845703125, | |
| "ref_logps/rejected": -170.07183837890625, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": -0.36180704832077026, | |
| "rewards/margins": 10.61725902557373, | |
| "rewards/rejected": -10.97906494140625, | |
| "step": 55 | |
| }, | |
| { | |
| "epoch": 0.1042830540037244, | |
| "grad_norm": 6.928717613220215, | |
| "learning_rate": 4.9792960662525884e-06, | |
| "logps/chosen": -157.90420532226562, | |
| "logps/rejected": -274.0709228515625, | |
| "loss": 0.0401, | |
| "losses/dpo": 0.0023821014910936356, | |
| "losses/sft": 2.440980911254883, | |
| "losses/total": 0.0023821014910936356, | |
| "ref_logps/chosen": -145.0316925048828, | |
| "ref_logps/rejected": -164.71649169921875, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": -1.2872496843338013, | |
| "rewards/margins": 9.648194313049316, | |
| "rewards/rejected": -10.935443878173828, | |
| "step": 56 | |
| }, | |
| { | |
| "epoch": 0.10614525139664804, | |
| "grad_norm": 12.181221961975098, | |
| "learning_rate": 4.968944099378882e-06, | |
| "logps/chosen": -164.98733520507812, | |
| "logps/rejected": -292.568115234375, | |
| "loss": 0.0608, | |
| "losses/dpo": 0.2203754484653473, | |
| "losses/sft": 2.4729645252227783, | |
| "losses/total": 0.2203754484653473, | |
| "ref_logps/chosen": -145.78073120117188, | |
| "ref_logps/rejected": -170.54788208007812, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": -1.9206593036651611, | |
| "rewards/margins": 10.28136157989502, | |
| "rewards/rejected": -12.202020645141602, | |
| "step": 57 | |
| }, | |
| { | |
| "epoch": 0.10800744878957169, | |
| "grad_norm": 3.31730580329895, | |
| "learning_rate": 4.9585921325051765e-06, | |
| "logps/chosen": -144.76385498046875, | |
| "logps/rejected": -279.8059997558594, | |
| "loss": 0.0189, | |
| "losses/dpo": 0.023249687626957893, | |
| "losses/sft": 2.241105079650879, | |
| "losses/total": 0.023249687626957893, | |
| "ref_logps/chosen": -135.7710723876953, | |
| "ref_logps/rejected": -163.89041137695312, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.8992785215377808, | |
| "rewards/margins": 10.692280769348145, | |
| "rewards/rejected": -11.591558456420898, | |
| "step": 58 | |
| }, | |
| { | |
| "epoch": 0.10986964618249534, | |
| "grad_norm": 5.754706382751465, | |
| "learning_rate": 4.94824016563147e-06, | |
| "logps/chosen": -159.64266967773438, | |
| "logps/rejected": -272.6561584472656, | |
| "loss": 0.0327, | |
| "losses/dpo": 0.0525113008916378, | |
| "losses/sft": 2.5197577476501465, | |
| "losses/total": 0.0525113008916378, | |
| "ref_logps/chosen": -142.01556396484375, | |
| "ref_logps/rejected": -161.62533569335938, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -1.7627112865447998, | |
| "rewards/margins": 9.340368270874023, | |
| "rewards/rejected": -11.103079795837402, | |
| "step": 59 | |
| }, | |
| { | |
| "epoch": 0.11173184357541899, | |
| "grad_norm": 2.9009838104248047, | |
| "learning_rate": 4.9378881987577645e-06, | |
| "logps/chosen": -153.46998596191406, | |
| "logps/rejected": -300.505615234375, | |
| "loss": 0.0109, | |
| "losses/dpo": 0.05081873759627342, | |
| "losses/sft": 2.3817033767700195, | |
| "losses/total": 0.05081873759627342, | |
| "ref_logps/chosen": -137.58828735351562, | |
| "ref_logps/rejected": -164.65621948242188, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -1.5881686210632324, | |
| "rewards/margins": 11.996772766113281, | |
| "rewards/rejected": -13.584940910339355, | |
| "step": 60 | |
| }, | |
| { | |
| "epoch": 0.11359404096834265, | |
| "grad_norm": 10.7985258102417, | |
| "learning_rate": 4.927536231884059e-06, | |
| "logps/chosen": -161.22463989257812, | |
| "logps/rejected": -312.06597900390625, | |
| "loss": 0.0594, | |
| "losses/dpo": 0.02082718536257744, | |
| "losses/sft": 2.446563720703125, | |
| "losses/total": 0.02082718536257744, | |
| "ref_logps/chosen": -150.21656799316406, | |
| "ref_logps/rejected": -171.88088989257812, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": -1.1008076667785645, | |
| "rewards/margins": 12.917702674865723, | |
| "rewards/rejected": -14.018510818481445, | |
| "step": 61 | |
| }, | |
| { | |
| "epoch": 0.1154562383612663, | |
| "grad_norm": 16.700149536132812, | |
| "learning_rate": 4.9171842650103525e-06, | |
| "logps/chosen": -157.88522338867188, | |
| "logps/rejected": -296.93817138671875, | |
| "loss": 0.0929, | |
| "losses/dpo": 0.0005755929159931839, | |
| "losses/sft": 2.514739751815796, | |
| "losses/total": 0.0005755929159931839, | |
| "ref_logps/chosen": -140.2955780029297, | |
| "ref_logps/rejected": -165.69281005859375, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": -1.7589635848999023, | |
| "rewards/margins": 11.365575790405273, | |
| "rewards/rejected": -13.124539375305176, | |
| "step": 62 | |
| }, | |
| { | |
| "epoch": 0.11731843575418995, | |
| "grad_norm": 10.798019409179688, | |
| "learning_rate": 4.906832298136646e-06, | |
| "logps/chosen": -160.2606201171875, | |
| "logps/rejected": -316.3315734863281, | |
| "loss": 0.0485, | |
| "losses/dpo": 0.0018289771396666765, | |
| "losses/sft": 3.0345890522003174, | |
| "losses/total": 0.0018289771396666765, | |
| "ref_logps/chosen": -144.05250549316406, | |
| "ref_logps/rejected": -170.66879272460938, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": -1.6208107471466064, | |
| "rewards/margins": 12.945466041564941, | |
| "rewards/rejected": -14.566276550292969, | |
| "step": 63 | |
| }, | |
| { | |
| "epoch": 0.1191806331471136, | |
| "grad_norm": 8.646073341369629, | |
| "learning_rate": 4.896480331262941e-06, | |
| "logps/chosen": -163.75540161132812, | |
| "logps/rejected": -315.20458984375, | |
| "loss": 0.0576, | |
| "losses/dpo": 0.014824606478214264, | |
| "losses/sft": 3.261763572692871, | |
| "losses/total": 0.014824606478214264, | |
| "ref_logps/chosen": -149.48907470703125, | |
| "ref_logps/rejected": -173.37435913085938, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": -1.4266325235366821, | |
| "rewards/margins": 12.756391525268555, | |
| "rewards/rejected": -14.183023452758789, | |
| "step": 64 | |
| }, | |
| { | |
| "epoch": 0.12104283054003724, | |
| "grad_norm": 5.649742603302002, | |
| "learning_rate": 4.886128364389234e-06, | |
| "logps/chosen": -174.45712280273438, | |
| "logps/rejected": -324.653076171875, | |
| "loss": 0.018, | |
| "losses/dpo": 0.03694448992609978, | |
| "losses/sft": 2.400181293487549, | |
| "losses/total": 0.03694448992609978, | |
| "ref_logps/chosen": -155.43272399902344, | |
| "ref_logps/rejected": -175.6329345703125, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -1.902438759803772, | |
| "rewards/margins": 12.999577522277832, | |
| "rewards/rejected": -14.902015686035156, | |
| "step": 65 | |
| }, | |
| { | |
| "epoch": 0.12290502793296089, | |
| "grad_norm": 4.2772345542907715, | |
| "learning_rate": 4.875776397515528e-06, | |
| "logps/chosen": -160.1391143798828, | |
| "logps/rejected": -298.70098876953125, | |
| "loss": 0.0198, | |
| "losses/dpo": 0.009300258941948414, | |
| "losses/sft": 2.3741018772125244, | |
| "losses/total": 0.009300258941948414, | |
| "ref_logps/chosen": -143.62374877929688, | |
| "ref_logps/rejected": -162.72705078125, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -1.6515378952026367, | |
| "rewards/margins": 11.945855140686035, | |
| "rewards/rejected": -13.597392082214355, | |
| "step": 66 | |
| }, | |
| { | |
| "epoch": 0.12476722532588454, | |
| "grad_norm": 12.112936019897461, | |
| "learning_rate": 4.865424430641822e-06, | |
| "logps/chosen": -171.04205322265625, | |
| "logps/rejected": -303.92144775390625, | |
| "loss": 0.0626, | |
| "losses/dpo": 0.36290252208709717, | |
| "losses/sft": 2.4964911937713623, | |
| "losses/total": 0.36290252208709717, | |
| "ref_logps/chosen": -152.29478454589844, | |
| "ref_logps/rejected": -161.23959350585938, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": -1.8747248649597168, | |
| "rewards/margins": 12.393461227416992, | |
| "rewards/rejected": -14.268186569213867, | |
| "step": 67 | |
| }, | |
| { | |
| "epoch": 0.1266294227188082, | |
| "grad_norm": 5.256114959716797, | |
| "learning_rate": 4.855072463768117e-06, | |
| "logps/chosen": -174.6351776123047, | |
| "logps/rejected": -348.933837890625, | |
| "loss": 0.0251, | |
| "losses/dpo": 0.09207179397344589, | |
| "losses/sft": 2.219914436340332, | |
| "losses/total": 0.09207179397344589, | |
| "ref_logps/chosen": -151.7803955078125, | |
| "ref_logps/rejected": -185.33236694335938, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -2.285478353500366, | |
| "rewards/margins": 14.074670791625977, | |
| "rewards/rejected": -16.36014747619629, | |
| "step": 68 | |
| }, | |
| { | |
| "epoch": 0.12849162011173185, | |
| "grad_norm": 5.340096950531006, | |
| "learning_rate": 4.84472049689441e-06, | |
| "logps/chosen": -161.56605529785156, | |
| "logps/rejected": -303.75201416015625, | |
| "loss": 0.027, | |
| "losses/dpo": 0.013381706550717354, | |
| "losses/sft": 2.1893093585968018, | |
| "losses/total": 0.013381706550717354, | |
| "ref_logps/chosen": -150.4385986328125, | |
| "ref_logps/rejected": -168.53863525390625, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -1.1127451658248901, | |
| "rewards/margins": 12.40859317779541, | |
| "rewards/rejected": -13.521337509155273, | |
| "step": 69 | |
| }, | |
| { | |
| "epoch": 0.1303538175046555, | |
| "grad_norm": 13.288958549499512, | |
| "learning_rate": 4.834368530020705e-06, | |
| "logps/chosen": -172.6832275390625, | |
| "logps/rejected": -334.8218078613281, | |
| "loss": 0.1264, | |
| "losses/dpo": 0.0077868797816336155, | |
| "losses/sft": 2.6989755630493164, | |
| "losses/total": 0.0077868797816336155, | |
| "ref_logps/chosen": -152.52114868164062, | |
| "ref_logps/rejected": -174.5692901611328, | |
| "rewards/accuracies": 0.96875, | |
| "rewards/chosen": -2.0162088871002197, | |
| "rewards/margins": 14.009044647216797, | |
| "rewards/rejected": -16.02525520324707, | |
| "step": 70 | |
| }, | |
| { | |
| "epoch": 0.13221601489757914, | |
| "grad_norm": 6.7856059074401855, | |
| "learning_rate": 4.824016563146998e-06, | |
| "logps/chosen": -166.0758056640625, | |
| "logps/rejected": -302.7711181640625, | |
| "loss": 0.0375, | |
| "losses/dpo": 0.008599307388067245, | |
| "losses/sft": 2.780581474304199, | |
| "losses/total": 0.008599307388067245, | |
| "ref_logps/chosen": -144.70916748046875, | |
| "ref_logps/rejected": -158.5432586669922, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": -2.136662244796753, | |
| "rewards/margins": 12.286121368408203, | |
| "rewards/rejected": -14.422782897949219, | |
| "step": 71 | |
| }, | |
| { | |
| "epoch": 0.1340782122905028, | |
| "grad_norm": 7.3059282302856445, | |
| "learning_rate": 4.813664596273293e-06, | |
| "logps/chosen": -175.05950927734375, | |
| "logps/rejected": -346.2865905761719, | |
| "loss": 0.0272, | |
| "losses/dpo": 0.06259185075759888, | |
| "losses/sft": 2.080183982849121, | |
| "losses/total": 0.06259185075759888, | |
| "ref_logps/chosen": -153.53565979003906, | |
| "ref_logps/rejected": -175.8729248046875, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": -2.152385950088501, | |
| "rewards/margins": 14.888980865478516, | |
| "rewards/rejected": -17.041364669799805, | |
| "step": 72 | |
| }, | |
| { | |
| "epoch": 0.13594040968342644, | |
| "grad_norm": 14.326214790344238, | |
| "learning_rate": 4.803312629399586e-06, | |
| "logps/chosen": -159.51559448242188, | |
| "logps/rejected": -303.34576416015625, | |
| "loss": 0.1862, | |
| "losses/dpo": 0.1816004067659378, | |
| "losses/sft": 2.4103786945343018, | |
| "losses/total": 0.1816004067659378, | |
| "ref_logps/chosen": -137.89926147460938, | |
| "ref_logps/rejected": -151.60308837890625, | |
| "rewards/accuracies": 0.9375, | |
| "rewards/chosen": -2.1616339683532715, | |
| "rewards/margins": 13.0126371383667, | |
| "rewards/rejected": -15.174270629882812, | |
| "step": 73 | |
| }, | |
| { | |
| "epoch": 0.1378026070763501, | |
| "grad_norm": 11.816011428833008, | |
| "learning_rate": 4.79296066252588e-06, | |
| "logps/chosen": -163.58856201171875, | |
| "logps/rejected": -307.9797058105469, | |
| "loss": 0.076, | |
| "losses/dpo": 0.04001971334218979, | |
| "losses/sft": 2.3666553497314453, | |
| "losses/total": 0.04001971334218979, | |
| "ref_logps/chosen": -140.401123046875, | |
| "ref_logps/rejected": -169.53005981445312, | |
| "rewards/accuracies": 0.96875, | |
| "rewards/chosen": -2.3187437057495117, | |
| "rewards/margins": 11.52622127532959, | |
| "rewards/rejected": -13.844964981079102, | |
| "step": 74 | |
| }, | |
| { | |
| "epoch": 0.13966480446927373, | |
| "grad_norm": 13.491276741027832, | |
| "learning_rate": 4.782608695652174e-06, | |
| "logps/chosen": -163.01483154296875, | |
| "logps/rejected": -305.1317443847656, | |
| "loss": 0.1175, | |
| "losses/dpo": 0.021199261769652367, | |
| "losses/sft": 2.5555906295776367, | |
| "losses/total": 0.021199261769652367, | |
| "ref_logps/chosen": -135.76666259765625, | |
| "ref_logps/rejected": -159.01651000976562, | |
| "rewards/accuracies": 0.96875, | |
| "rewards/chosen": -2.72481632232666, | |
| "rewards/margins": 11.88670825958252, | |
| "rewards/rejected": -14.61152458190918, | |
| "step": 75 | |
| }, | |
| { | |
| "epoch": 0.14152700186219738, | |
| "grad_norm": 13.310988426208496, | |
| "learning_rate": 4.772256728778468e-06, | |
| "logps/chosen": -166.88494873046875, | |
| "logps/rejected": -300.84918212890625, | |
| "loss": 0.0695, | |
| "losses/dpo": 0.23336642980575562, | |
| "losses/sft": 2.7684144973754883, | |
| "losses/total": 0.23336642980575562, | |
| "ref_logps/chosen": -145.76715087890625, | |
| "ref_logps/rejected": -162.08209228515625, | |
| "rewards/accuracies": 0.953125, | |
| "rewards/chosen": -2.1117796897888184, | |
| "rewards/margins": 11.764933586120605, | |
| "rewards/rejected": -13.876713752746582, | |
| "step": 76 | |
| }, | |
| { | |
| "epoch": 0.14338919925512103, | |
| "grad_norm": 5.915053844451904, | |
| "learning_rate": 4.761904761904762e-06, | |
| "logps/chosen": -166.23074340820312, | |
| "logps/rejected": -321.7993469238281, | |
| "loss": 0.0245, | |
| "losses/dpo": 0.00781274028122425, | |
| "losses/sft": 2.6997854709625244, | |
| "losses/total": 0.00781274028122425, | |
| "ref_logps/chosen": -146.89723205566406, | |
| "ref_logps/rejected": -168.7777099609375, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -1.9333515167236328, | |
| "rewards/margins": 13.368810653686523, | |
| "rewards/rejected": -15.302162170410156, | |
| "step": 77 | |
| }, | |
| { | |
| "epoch": 0.1452513966480447, | |
| "grad_norm": 8.788707733154297, | |
| "learning_rate": 4.751552795031056e-06, | |
| "logps/chosen": -175.02999877929688, | |
| "logps/rejected": -321.0180969238281, | |
| "loss": 0.0271, | |
| "losses/dpo": 0.14333294332027435, | |
| "losses/sft": 2.538876533508301, | |
| "losses/total": 0.14333294332027435, | |
| "ref_logps/chosen": -157.10501098632812, | |
| "ref_logps/rejected": -181.61178588867188, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": -1.7924991846084595, | |
| "rewards/margins": 12.148136138916016, | |
| "rewards/rejected": -13.940634727478027, | |
| "step": 78 | |
| }, | |
| { | |
| "epoch": 0.14711359404096835, | |
| "grad_norm": 3.836193323135376, | |
| "learning_rate": 4.74120082815735e-06, | |
| "logps/chosen": -161.6851806640625, | |
| "logps/rejected": -307.22308349609375, | |
| "loss": 0.018, | |
| "losses/dpo": 0.005441778339445591, | |
| "losses/sft": 2.4244680404663086, | |
| "losses/total": 0.005441778339445591, | |
| "ref_logps/chosen": -141.81394958496094, | |
| "ref_logps/rejected": -166.87547302246094, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -1.9871225357055664, | |
| "rewards/margins": 12.047636985778809, | |
| "rewards/rejected": -14.034758567810059, | |
| "step": 79 | |
| }, | |
| { | |
| "epoch": 0.148975791433892, | |
| "grad_norm": 1.0960124731063843, | |
| "learning_rate": 4.730848861283645e-06, | |
| "logps/chosen": -164.8990020751953, | |
| "logps/rejected": -306.0439453125, | |
| "loss": 0.0046, | |
| "losses/dpo": 0.00012883917952422053, | |
| "losses/sft": 2.337103843688965, | |
| "losses/total": 0.00012883917952422053, | |
| "ref_logps/chosen": -150.0802001953125, | |
| "ref_logps/rejected": -169.78543090820312, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -1.48188054561615, | |
| "rewards/margins": 12.143974304199219, | |
| "rewards/rejected": -13.6258544921875, | |
| "step": 80 | |
| }, | |
| { | |
| "epoch": 0.15083798882681565, | |
| "grad_norm": 10.833060264587402, | |
| "learning_rate": 4.7204968944099384e-06, | |
| "logps/chosen": -166.50360107421875, | |
| "logps/rejected": -291.5777893066406, | |
| "loss": 0.0699, | |
| "losses/dpo": 0.3678208291530609, | |
| "losses/sft": 2.360893726348877, | |
| "losses/total": 0.3678208291530609, | |
| "ref_logps/chosen": -139.50567626953125, | |
| "ref_logps/rejected": -159.87924194335938, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": -2.699791193008423, | |
| "rewards/margins": 10.470064163208008, | |
| "rewards/rejected": -13.169855117797852, | |
| "step": 81 | |
| }, | |
| { | |
| "epoch": 0.1527001862197393, | |
| "grad_norm": 10.486247062683105, | |
| "learning_rate": 4.710144927536232e-06, | |
| "logps/chosen": -158.02520751953125, | |
| "logps/rejected": -298.325927734375, | |
| "loss": 0.0641, | |
| "losses/dpo": 0.04021751880645752, | |
| "losses/sft": 2.221280574798584, | |
| "losses/total": 0.04021751880645752, | |
| "ref_logps/chosen": -147.9010009765625, | |
| "ref_logps/rejected": -168.83761596679688, | |
| "rewards/accuracies": 0.96875, | |
| "rewards/chosen": -1.0124225616455078, | |
| "rewards/margins": 11.936410903930664, | |
| "rewards/rejected": -12.948833465576172, | |
| "step": 82 | |
| }, | |
| { | |
| "epoch": 0.15456238361266295, | |
| "grad_norm": 4.4983320236206055, | |
| "learning_rate": 4.6997929606625265e-06, | |
| "logps/chosen": -157.27609252929688, | |
| "logps/rejected": -284.30670166015625, | |
| "loss": 0.0134, | |
| "losses/dpo": 0.07455439865589142, | |
| "losses/sft": 2.680065631866455, | |
| "losses/total": 0.07455439865589142, | |
| "ref_logps/chosen": -143.0989990234375, | |
| "ref_logps/rejected": -166.63612365722656, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -1.4177087545394897, | |
| "rewards/margins": 10.349348068237305, | |
| "rewards/rejected": -11.767056465148926, | |
| "step": 83 | |
| }, | |
| { | |
| "epoch": 0.1564245810055866, | |
| "grad_norm": 5.925443172454834, | |
| "learning_rate": 4.68944099378882e-06, | |
| "logps/chosen": -166.75436401367188, | |
| "logps/rejected": -307.4884338378906, | |
| "loss": 0.0294, | |
| "losses/dpo": 0.018268397077918053, | |
| "losses/sft": 2.3180205821990967, | |
| "losses/total": 0.018268397077918053, | |
| "ref_logps/chosen": -146.59835815429688, | |
| "ref_logps/rejected": -171.87164306640625, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": -2.015601634979248, | |
| "rewards/margins": 11.546079635620117, | |
| "rewards/rejected": -13.561680793762207, | |
| "step": 84 | |
| }, | |
| { | |
| "epoch": 0.15828677839851024, | |
| "grad_norm": 10.823968887329102, | |
| "learning_rate": 4.679089026915114e-06, | |
| "logps/chosen": -164.76190185546875, | |
| "logps/rejected": -291.744873046875, | |
| "loss": 0.0787, | |
| "losses/dpo": 0.0033458764664828777, | |
| "losses/sft": 2.6443734169006348, | |
| "losses/total": 0.0033458764664828777, | |
| "ref_logps/chosen": -147.3948516845703, | |
| "ref_logps/rejected": -165.80938720703125, | |
| "rewards/accuracies": 0.96875, | |
| "rewards/chosen": -1.73670494556427, | |
| "rewards/margins": 10.856843948364258, | |
| "rewards/rejected": -12.593548774719238, | |
| "step": 85 | |
| }, | |
| { | |
| "epoch": 0.1601489757914339, | |
| "grad_norm": 10.07721996307373, | |
| "learning_rate": 4.668737060041408e-06, | |
| "logps/chosen": -170.19273376464844, | |
| "logps/rejected": -289.0329284667969, | |
| "loss": 0.0578, | |
| "losses/dpo": 0.22292722761631012, | |
| "losses/sft": 2.8669309616088867, | |
| "losses/total": 0.22292722761631012, | |
| "ref_logps/chosen": -155.04888916015625, | |
| "ref_logps/rejected": -169.506103515625, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": -1.5143851041793823, | |
| "rewards/margins": 10.438299179077148, | |
| "rewards/rejected": -11.95268440246582, | |
| "step": 86 | |
| }, | |
| { | |
| "epoch": 0.16201117318435754, | |
| "grad_norm": 7.202996730804443, | |
| "learning_rate": 4.6583850931677025e-06, | |
| "logps/chosen": -152.85894775390625, | |
| "logps/rejected": -276.5400390625, | |
| "loss": 0.0464, | |
| "losses/dpo": 0.0008024298003874719, | |
| "losses/sft": 2.105069637298584, | |
| "losses/total": 0.0008024298003874719, | |
| "ref_logps/chosen": -135.02166748046875, | |
| "ref_logps/rejected": -162.55917358398438, | |
| "rewards/accuracies": 0.96875, | |
| "rewards/chosen": -1.783726453781128, | |
| "rewards/margins": 9.614358901977539, | |
| "rewards/rejected": -11.39808464050293, | |
| "step": 87 | |
| }, | |
| { | |
| "epoch": 0.16387337057728119, | |
| "grad_norm": 10.529645919799805, | |
| "learning_rate": 4.648033126293996e-06, | |
| "logps/chosen": -167.41766357421875, | |
| "logps/rejected": -312.9851989746094, | |
| "loss": 0.1059, | |
| "losses/dpo": 1.8586399164632894e-05, | |
| "losses/sft": 2.3417227268218994, | |
| "losses/total": 1.8586399164632894e-05, | |
| "ref_logps/chosen": -150.24447631835938, | |
| "ref_logps/rejected": -171.88534545898438, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": -1.7173188924789429, | |
| "rewards/margins": 12.392667770385742, | |
| "rewards/rejected": -14.1099853515625, | |
| "step": 88 | |
| }, | |
| { | |
| "epoch": 0.16573556797020483, | |
| "grad_norm": 5.379014492034912, | |
| "learning_rate": 4.637681159420291e-06, | |
| "logps/chosen": -149.58767700195312, | |
| "logps/rejected": -283.6622009277344, | |
| "loss": 0.0265, | |
| "losses/dpo": 0.0076700253412127495, | |
| "losses/sft": 2.0566470623016357, | |
| "losses/total": 0.0076700253412127495, | |
| "ref_logps/chosen": -142.82745361328125, | |
| "ref_logps/rejected": -166.57040405273438, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.6760219931602478, | |
| "rewards/margins": 11.03315544128418, | |
| "rewards/rejected": -11.709177017211914, | |
| "step": 89 | |
| }, | |
| { | |
| "epoch": 0.16759776536312848, | |
| "grad_norm": 7.1271820068359375, | |
| "learning_rate": 4.627329192546584e-06, | |
| "logps/chosen": -166.07994079589844, | |
| "logps/rejected": -288.1270446777344, | |
| "loss": 0.0383, | |
| "losses/dpo": 0.006637689657509327, | |
| "losses/sft": 2.695986270904541, | |
| "losses/total": 0.006637689657509327, | |
| "ref_logps/chosen": -152.90316772460938, | |
| "ref_logps/rejected": -167.97433471679688, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": -1.3176772594451904, | |
| "rewards/margins": 10.697591781616211, | |
| "rewards/rejected": -12.01526927947998, | |
| "step": 90 | |
| }, | |
| { | |
| "epoch": 0.16945996275605213, | |
| "grad_norm": 3.960089683532715, | |
| "learning_rate": 4.616977225672879e-06, | |
| "logps/chosen": -155.234619140625, | |
| "logps/rejected": -274.2052917480469, | |
| "loss": 0.0283, | |
| "losses/dpo": 0.029921425506472588, | |
| "losses/sft": 2.448662757873535, | |
| "losses/total": 0.029921425506472588, | |
| "ref_logps/chosen": -146.57998657226562, | |
| "ref_logps/rejected": -161.9693603515625, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.8654617071151733, | |
| "rewards/margins": 10.35813045501709, | |
| "rewards/rejected": -11.223592758178711, | |
| "step": 91 | |
| }, | |
| { | |
| "epoch": 0.1713221601489758, | |
| "grad_norm": 8.884647369384766, | |
| "learning_rate": 4.606625258799172e-06, | |
| "logps/chosen": -156.59239196777344, | |
| "logps/rejected": -269.66265869140625, | |
| "loss": 0.0549, | |
| "losses/dpo": 0.000325945409713313, | |
| "losses/sft": 2.27170991897583, | |
| "losses/total": 0.000325945409713313, | |
| "ref_logps/chosen": -143.44590759277344, | |
| "ref_logps/rejected": -160.02975463867188, | |
| "rewards/accuracies": 0.96875, | |
| "rewards/chosen": -1.3146480321884155, | |
| "rewards/margins": 9.648642539978027, | |
| "rewards/rejected": -10.963289260864258, | |
| "step": 92 | |
| }, | |
| { | |
| "epoch": 0.17318435754189945, | |
| "grad_norm": 8.483807563781738, | |
| "learning_rate": 4.596273291925466e-06, | |
| "logps/chosen": -157.52755737304688, | |
| "logps/rejected": -302.79736328125, | |
| "loss": 0.0525, | |
| "losses/dpo": 0.007826759479939938, | |
| "losses/sft": 2.404571533203125, | |
| "losses/total": 0.007826759479939938, | |
| "ref_logps/chosen": -151.92408752441406, | |
| "ref_logps/rejected": -174.8727569580078, | |
| "rewards/accuracies": 0.96875, | |
| "rewards/chosen": -0.5603480339050293, | |
| "rewards/margins": 12.232114791870117, | |
| "rewards/rejected": -12.792463302612305, | |
| "step": 93 | |
| }, | |
| { | |
| "epoch": 0.1750465549348231, | |
| "grad_norm": 2.803177833557129, | |
| "learning_rate": 4.58592132505176e-06, | |
| "logps/chosen": -154.108154296875, | |
| "logps/rejected": -316.1098937988281, | |
| "loss": 0.0115, | |
| "losses/dpo": 0.014974968507885933, | |
| "losses/sft": 2.5976779460906982, | |
| "losses/total": 0.014974968507885933, | |
| "ref_logps/chosen": -139.6196746826172, | |
| "ref_logps/rejected": -177.22061157226562, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -1.4488475322723389, | |
| "rewards/margins": 12.440080642700195, | |
| "rewards/rejected": -13.88892936706543, | |
| "step": 94 | |
| }, | |
| { | |
| "epoch": 0.17690875232774675, | |
| "grad_norm": 10.011286735534668, | |
| "learning_rate": 4.575569358178054e-06, | |
| "logps/chosen": -161.99615478515625, | |
| "logps/rejected": -290.70928955078125, | |
| "loss": 0.0521, | |
| "losses/dpo": 0.004248100332915783, | |
| "losses/sft": 2.6869282722473145, | |
| "losses/total": 0.004248100332915783, | |
| "ref_logps/chosen": -150.13644409179688, | |
| "ref_logps/rejected": -170.98199462890625, | |
| "rewards/accuracies": 0.96875, | |
| "rewards/chosen": -1.1859712600708008, | |
| "rewards/margins": 10.786758422851562, | |
| "rewards/rejected": -11.972728729248047, | |
| "step": 95 | |
| }, | |
| { | |
| "epoch": 0.1787709497206704, | |
| "grad_norm": 13.34592342376709, | |
| "learning_rate": 4.565217391304348e-06, | |
| "logps/chosen": -157.46038818359375, | |
| "logps/rejected": -293.8796691894531, | |
| "loss": 0.0613, | |
| "losses/dpo": 0.0029001892544329166, | |
| "losses/sft": 2.191887378692627, | |
| "losses/total": 0.0029001892544329166, | |
| "ref_logps/chosen": -147.4150390625, | |
| "ref_logps/rejected": -175.098876953125, | |
| "rewards/accuracies": 0.96875, | |
| "rewards/chosen": -1.004532814025879, | |
| "rewards/margins": 10.873546600341797, | |
| "rewards/rejected": -11.878079414367676, | |
| "step": 96 | |
| }, | |
| { | |
| "epoch": 0.18063314711359404, | |
| "grad_norm": 13.375811576843262, | |
| "learning_rate": 4.554865424430642e-06, | |
| "logps/chosen": -166.24191284179688, | |
| "logps/rejected": -305.58819580078125, | |
| "loss": 0.0981, | |
| "losses/dpo": 0.0007697511464357376, | |
| "losses/sft": 2.3007588386535645, | |
| "losses/total": 0.0007697511464357376, | |
| "ref_logps/chosen": -144.6219482421875, | |
| "ref_logps/rejected": -174.1811981201172, | |
| "rewards/accuracies": 0.953125, | |
| "rewards/chosen": -2.1619954109191895, | |
| "rewards/margins": 10.978706359863281, | |
| "rewards/rejected": -13.140701293945312, | |
| "step": 97 | |
| }, | |
| { | |
| "epoch": 0.1824953445065177, | |
| "grad_norm": 11.116385459899902, | |
| "learning_rate": 4.544513457556936e-06, | |
| "logps/chosen": -151.30795288085938, | |
| "logps/rejected": -303.7850341796875, | |
| "loss": 0.0838, | |
| "losses/dpo": 0.006321331951767206, | |
| "losses/sft": 2.45919132232666, | |
| "losses/total": 0.006321331951767206, | |
| "ref_logps/chosen": -141.2546844482422, | |
| "ref_logps/rejected": -166.2907257080078, | |
| "rewards/accuracies": 0.96875, | |
| "rewards/chosen": -1.0053260326385498, | |
| "rewards/margins": 12.744101524353027, | |
| "rewards/rejected": -13.749427795410156, | |
| "step": 98 | |
| }, | |
| { | |
| "epoch": 0.18435754189944134, | |
| "grad_norm": 8.1275053024292, | |
| "learning_rate": 4.534161490683231e-06, | |
| "logps/chosen": -169.12844848632812, | |
| "logps/rejected": -291.58355712890625, | |
| "loss": 0.0379, | |
| "losses/dpo": 0.04353442043066025, | |
| "losses/sft": 2.9029669761657715, | |
| "losses/total": 0.04353442043066025, | |
| "ref_logps/chosen": -148.775390625, | |
| "ref_logps/rejected": -162.24786376953125, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": -2.0353074073791504, | |
| "rewards/margins": 10.898263931274414, | |
| "rewards/rejected": -12.933570861816406, | |
| "step": 99 | |
| }, | |
| { | |
| "epoch": 0.186219739292365, | |
| "grad_norm": 3.8263306617736816, | |
| "learning_rate": 4.523809523809524e-06, | |
| "logps/chosen": -160.37161254882812, | |
| "logps/rejected": -291.48577880859375, | |
| "loss": 0.0159, | |
| "losses/dpo": 0.06080714985728264, | |
| "losses/sft": 2.056828498840332, | |
| "losses/total": 0.06080714985728264, | |
| "ref_logps/chosen": -143.4698486328125, | |
| "ref_logps/rejected": -165.9519500732422, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -1.6901748180389404, | |
| "rewards/margins": 10.863210678100586, | |
| "rewards/rejected": -12.553384780883789, | |
| "step": 100 | |
| }, | |
| { | |
| "epoch": 0.18808193668528864, | |
| "grad_norm": 9.816635131835938, | |
| "learning_rate": 4.513457556935818e-06, | |
| "logps/chosen": -177.78240966796875, | |
| "logps/rejected": -330.1160888671875, | |
| "loss": 0.0489, | |
| "losses/dpo": 0.2933081388473511, | |
| "losses/sft": 2.7717084884643555, | |
| "losses/total": 0.2933081388473511, | |
| "ref_logps/chosen": -155.60540771484375, | |
| "ref_logps/rejected": -177.44467163085938, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": -2.2176995277404785, | |
| "rewards/margins": 13.04944133758545, | |
| "rewards/rejected": -15.267141342163086, | |
| "step": 101 | |
| }, | |
| { | |
| "epoch": 0.18994413407821228, | |
| "grad_norm": 1.3710126876831055, | |
| "learning_rate": 4.503105590062112e-06, | |
| "logps/chosen": -166.66590881347656, | |
| "logps/rejected": -290.761962890625, | |
| "loss": 0.0054, | |
| "losses/dpo": 0.02758178301155567, | |
| "losses/sft": 3.162290096282959, | |
| "losses/total": 0.02758178301155567, | |
| "ref_logps/chosen": -147.08917236328125, | |
| "ref_logps/rejected": -151.07664489746094, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -1.9576739072799683, | |
| "rewards/margins": 12.010858535766602, | |
| "rewards/rejected": -13.96853256225586, | |
| "step": 102 | |
| }, | |
| { | |
| "epoch": 0.19180633147113593, | |
| "grad_norm": 7.93828010559082, | |
| "learning_rate": 4.492753623188406e-06, | |
| "logps/chosen": -146.8543701171875, | |
| "logps/rejected": -312.2845764160156, | |
| "loss": 0.0425, | |
| "losses/dpo": 0.2460930347442627, | |
| "losses/sft": 2.2462611198425293, | |
| "losses/total": 0.2460930347442627, | |
| "ref_logps/chosen": -135.14158630371094, | |
| "ref_logps/rejected": -168.27618408203125, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": -1.1712779998779297, | |
| "rewards/margins": 13.229562759399414, | |
| "rewards/rejected": -14.400840759277344, | |
| "step": 103 | |
| }, | |
| { | |
| "epoch": 0.19366852886405958, | |
| "grad_norm": 13.03959846496582, | |
| "learning_rate": 4.4824016563146996e-06, | |
| "logps/chosen": -163.9506378173828, | |
| "logps/rejected": -338.9526062011719, | |
| "loss": 0.105, | |
| "losses/dpo": 1.4241080862120725e-05, | |
| "losses/sft": 2.1639623641967773, | |
| "losses/total": 1.4241080862120725e-05, | |
| "ref_logps/chosen": -144.27719116210938, | |
| "ref_logps/rejected": -172.1199951171875, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": -1.9673457145690918, | |
| "rewards/margins": 14.71591567993164, | |
| "rewards/rejected": -16.683259963989258, | |
| "step": 104 | |
| }, | |
| { | |
| "epoch": 0.19553072625698323, | |
| "grad_norm": 12.940238952636719, | |
| "learning_rate": 4.472049689440994e-06, | |
| "logps/chosen": -167.60537719726562, | |
| "logps/rejected": -305.2105407714844, | |
| "loss": 0.1266, | |
| "losses/dpo": 0.016943011432886124, | |
| "losses/sft": 2.568720817565918, | |
| "losses/total": 0.016943011432886124, | |
| "ref_logps/chosen": -144.21282958984375, | |
| "ref_logps/rejected": -158.3541259765625, | |
| "rewards/accuracies": 0.953125, | |
| "rewards/chosen": -2.339254140853882, | |
| "rewards/margins": 12.34638786315918, | |
| "rewards/rejected": -14.685641288757324, | |
| "step": 105 | |
| }, | |
| { | |
| "epoch": 0.1973929236499069, | |
| "grad_norm": 9.413414001464844, | |
| "learning_rate": 4.4616977225672884e-06, | |
| "logps/chosen": -171.18016052246094, | |
| "logps/rejected": -302.7274475097656, | |
| "loss": 0.0475, | |
| "losses/dpo": 0.0004924152162857354, | |
| "losses/sft": 2.5999226570129395, | |
| "losses/total": 0.0004924152162857354, | |
| "ref_logps/chosen": -144.5438690185547, | |
| "ref_logps/rejected": -162.2921905517578, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": -2.6636300086975098, | |
| "rewards/margins": 11.37989616394043, | |
| "rewards/rejected": -14.043526649475098, | |
| "step": 106 | |
| }, | |
| { | |
| "epoch": 0.19925512104283055, | |
| "grad_norm": 1.439731240272522, | |
| "learning_rate": 4.451345755693582e-06, | |
| "logps/chosen": -169.63963317871094, | |
| "logps/rejected": -335.1871337890625, | |
| "loss": 0.0046, | |
| "losses/dpo": 0.01760055683553219, | |
| "losses/sft": 2.191661834716797, | |
| "losses/total": 0.01760055683553219, | |
| "ref_logps/chosen": -148.11293029785156, | |
| "ref_logps/rejected": -171.40560913085938, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -2.1526713371276855, | |
| "rewards/margins": 14.225481986999512, | |
| "rewards/rejected": -16.37815284729004, | |
| "step": 107 | |
| }, | |
| { | |
| "epoch": 0.2011173184357542, | |
| "grad_norm": 12.107815742492676, | |
| "learning_rate": 4.4409937888198765e-06, | |
| "logps/chosen": -174.36929321289062, | |
| "logps/rejected": -322.1456604003906, | |
| "loss": 0.1067, | |
| "losses/dpo": 0.0005443975096568465, | |
| "losses/sft": 2.864640712738037, | |
| "losses/total": 0.0005443975096568465, | |
| "ref_logps/chosen": -149.93984985351562, | |
| "ref_logps/rejected": -166.4761962890625, | |
| "rewards/accuracies": 0.953125, | |
| "rewards/chosen": -2.442944288253784, | |
| "rewards/margins": 13.124004364013672, | |
| "rewards/rejected": -15.566947937011719, | |
| "step": 108 | |
| }, | |
| { | |
| "epoch": 0.20297951582867785, | |
| "grad_norm": 7.122522354125977, | |
| "learning_rate": 4.43064182194617e-06, | |
| "logps/chosen": -155.75596618652344, | |
| "logps/rejected": -314.00762939453125, | |
| "loss": 0.0363, | |
| "losses/dpo": 0.1575506329536438, | |
| "losses/sft": 2.296907424926758, | |
| "losses/total": 0.1575506329536438, | |
| "ref_logps/chosen": -143.9917755126953, | |
| "ref_logps/rejected": -175.87164306640625, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": -1.1764178276062012, | |
| "rewards/margins": 12.637179374694824, | |
| "rewards/rejected": -13.813596725463867, | |
| "step": 109 | |
| }, | |
| { | |
| "epoch": 0.2048417132216015, | |
| "grad_norm": 1.012307047843933, | |
| "learning_rate": 4.4202898550724645e-06, | |
| "logps/chosen": -164.65115356445312, | |
| "logps/rejected": -318.34326171875, | |
| "loss": 0.0034, | |
| "losses/dpo": 0.011834606528282166, | |
| "losses/sft": 2.363070249557495, | |
| "losses/total": 0.011834606528282166, | |
| "ref_logps/chosen": -149.04367065429688, | |
| "ref_logps/rejected": -166.6707000732422, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -1.5607484579086304, | |
| "rewards/margins": 13.606508255004883, | |
| "rewards/rejected": -15.167256355285645, | |
| "step": 110 | |
| }, | |
| { | |
| "epoch": 0.20670391061452514, | |
| "grad_norm": 5.557791233062744, | |
| "learning_rate": 4.409937888198758e-06, | |
| "logps/chosen": -150.85084533691406, | |
| "logps/rejected": -309.1524658203125, | |
| "loss": 0.0232, | |
| "losses/dpo": 0.014252948574721813, | |
| "losses/sft": 2.4658865928649902, | |
| "losses/total": 0.014252948574721813, | |
| "ref_logps/chosen": -137.03976440429688, | |
| "ref_logps/rejected": -161.1869659423828, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": -1.381108045578003, | |
| "rewards/margins": 13.415441513061523, | |
| "rewards/rejected": -14.796548843383789, | |
| "step": 111 | |
| }, | |
| { | |
| "epoch": 0.2085661080074488, | |
| "grad_norm": 9.207598686218262, | |
| "learning_rate": 4.399585921325052e-06, | |
| "logps/chosen": -167.4930419921875, | |
| "logps/rejected": -303.5692138671875, | |
| "loss": 0.0453, | |
| "losses/dpo": 0.0007799149607308209, | |
| "losses/sft": 2.518667459487915, | |
| "losses/total": 0.0007799149607308209, | |
| "ref_logps/chosen": -149.21981811523438, | |
| "ref_logps/rejected": -170.2943115234375, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": -1.8273215293884277, | |
| "rewards/margins": 11.500171661376953, | |
| "rewards/rejected": -13.327491760253906, | |
| "step": 112 | |
| }, | |
| { | |
| "epoch": 0.21042830540037244, | |
| "grad_norm": 7.050368309020996, | |
| "learning_rate": 4.389233954451346e-06, | |
| "logps/chosen": -159.51071166992188, | |
| "logps/rejected": -307.4903869628906, | |
| "loss": 0.0301, | |
| "losses/dpo": 0.0039618429727852345, | |
| "losses/sft": 2.5831570625305176, | |
| "losses/total": 0.0039618429727852345, | |
| "ref_logps/chosen": -137.70912170410156, | |
| "ref_logps/rejected": -167.49789428710938, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": -2.180156946182251, | |
| "rewards/margins": 11.819091796875, | |
| "rewards/rejected": -13.999248504638672, | |
| "step": 113 | |
| }, | |
| { | |
| "epoch": 0.2122905027932961, | |
| "grad_norm": 11.241766929626465, | |
| "learning_rate": 4.37888198757764e-06, | |
| "logps/chosen": -164.54208374023438, | |
| "logps/rejected": -306.73822021484375, | |
| "loss": 0.0491, | |
| "losses/dpo": 0.17672446370124817, | |
| "losses/sft": 2.2986695766448975, | |
| "losses/total": 0.17672446370124817, | |
| "ref_logps/chosen": -145.33531188964844, | |
| "ref_logps/rejected": -164.6031951904297, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": -1.92067551612854, | |
| "rewards/margins": 12.292827606201172, | |
| "rewards/rejected": -14.213502883911133, | |
| "step": 114 | |
| }, | |
| { | |
| "epoch": 0.21415270018621974, | |
| "grad_norm": 4.248304843902588, | |
| "learning_rate": 4.368530020703934e-06, | |
| "logps/chosen": -160.98765563964844, | |
| "logps/rejected": -324.66845703125, | |
| "loss": 0.0157, | |
| "losses/dpo": 0.028906403109431267, | |
| "losses/sft": 2.51163387298584, | |
| "losses/total": 0.028906403109431267, | |
| "ref_logps/chosen": -142.23577880859375, | |
| "ref_logps/rejected": -178.46255493164062, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -1.875186800956726, | |
| "rewards/margins": 12.745403289794922, | |
| "rewards/rejected": -14.620590209960938, | |
| "step": 115 | |
| }, | |
| { | |
| "epoch": 0.21601489757914338, | |
| "grad_norm": 11.712361335754395, | |
| "learning_rate": 4.358178053830228e-06, | |
| "logps/chosen": -176.92311096191406, | |
| "logps/rejected": -309.1515808105469, | |
| "loss": 0.0532, | |
| "losses/dpo": 0.0009635839378461242, | |
| "losses/sft": 2.519484758377075, | |
| "losses/total": 0.0009635839378461242, | |
| "ref_logps/chosen": -155.59686279296875, | |
| "ref_logps/rejected": -167.6163787841797, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": -2.132624626159668, | |
| "rewards/margins": 12.020895004272461, | |
| "rewards/rejected": -14.153520584106445, | |
| "step": 116 | |
| }, | |
| { | |
| "epoch": 0.21787709497206703, | |
| "grad_norm": 12.869205474853516, | |
| "learning_rate": 4.347826086956522e-06, | |
| "logps/chosen": -164.499755859375, | |
| "logps/rejected": -322.2271728515625, | |
| "loss": 0.0481, | |
| "losses/dpo": 0.026883596554398537, | |
| "losses/sft": 2.4711999893188477, | |
| "losses/total": 0.026883596554398537, | |
| "ref_logps/chosen": -141.9335174560547, | |
| "ref_logps/rejected": -174.70437622070312, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": -2.256622791290283, | |
| "rewards/margins": 12.495659828186035, | |
| "rewards/rejected": -14.75228214263916, | |
| "step": 117 | |
| }, | |
| { | |
| "epoch": 0.21973929236499068, | |
| "grad_norm": 7.2970499992370605, | |
| "learning_rate": 4.337474120082817e-06, | |
| "logps/chosen": -154.8804931640625, | |
| "logps/rejected": -307.47796630859375, | |
| "loss": 0.0217, | |
| "losses/dpo": 0.005061456002295017, | |
| "losses/sft": 2.7621004581451416, | |
| "losses/total": 0.005061456002295017, | |
| "ref_logps/chosen": -144.57373046875, | |
| "ref_logps/rejected": -164.53269958496094, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": -1.0306763648986816, | |
| "rewards/margins": 13.263851165771484, | |
| "rewards/rejected": -14.294527053833008, | |
| "step": 118 | |
| }, | |
| { | |
| "epoch": 0.22160148975791433, | |
| "grad_norm": 12.324630737304688, | |
| "learning_rate": 4.32712215320911e-06, | |
| "logps/chosen": -169.20716857910156, | |
| "logps/rejected": -327.8011474609375, | |
| "loss": 0.0692, | |
| "losses/dpo": 0.05636470764875412, | |
| "losses/sft": 2.663618326187134, | |
| "losses/total": 0.05636470764875412, | |
| "ref_logps/chosen": -141.14559936523438, | |
| "ref_logps/rejected": -169.51641845703125, | |
| "rewards/accuracies": 0.96875, | |
| "rewards/chosen": -2.806156635284424, | |
| "rewards/margins": 13.022315979003906, | |
| "rewards/rejected": -15.828473091125488, | |
| "step": 119 | |
| }, | |
| { | |
| "epoch": 0.22346368715083798, | |
| "grad_norm": 7.082119464874268, | |
| "learning_rate": 4.316770186335404e-06, | |
| "logps/chosen": -162.06655883789062, | |
| "logps/rejected": -312.4548645019531, | |
| "loss": 0.0211, | |
| "losses/dpo": 0.14828312397003174, | |
| "losses/sft": 2.6526529788970947, | |
| "losses/total": 0.14828312397003174, | |
| "ref_logps/chosen": -147.12899780273438, | |
| "ref_logps/rejected": -163.53634643554688, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": -1.4937548637390137, | |
| "rewards/margins": 13.39809799194336, | |
| "rewards/rejected": -14.891853332519531, | |
| "step": 120 | |
| }, | |
| { | |
| "epoch": 0.22532588454376165, | |
| "grad_norm": 7.707842826843262, | |
| "learning_rate": 4.306418219461698e-06, | |
| "logps/chosen": -169.38192749023438, | |
| "logps/rejected": -323.21514892578125, | |
| "loss": 0.0198, | |
| "losses/dpo": 0.1473950445652008, | |
| "losses/sft": 2.6331777572631836, | |
| "losses/total": 0.1473950445652008, | |
| "ref_logps/chosen": -149.7891845703125, | |
| "ref_logps/rejected": -174.02752685546875, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": -1.9592747688293457, | |
| "rewards/margins": 12.959487915039062, | |
| "rewards/rejected": -14.918761253356934, | |
| "step": 121 | |
| }, | |
| { | |
| "epoch": 0.2271880819366853, | |
| "grad_norm": 6.156428337097168, | |
| "learning_rate": 4.296066252587992e-06, | |
| "logps/chosen": -177.64190673828125, | |
| "logps/rejected": -320.0240478515625, | |
| "loss": 0.0313, | |
| "losses/dpo": 0.0023224614560604095, | |
| "losses/sft": 2.282505512237549, | |
| "losses/total": 0.0023224614560604095, | |
| "ref_logps/chosen": -150.99745178222656, | |
| "ref_logps/rejected": -166.0125732421875, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": -2.664444923400879, | |
| "rewards/margins": 12.736701011657715, | |
| "rewards/rejected": -15.40114688873291, | |
| "step": 122 | |
| }, | |
| { | |
| "epoch": 0.22905027932960895, | |
| "grad_norm": 12.972006797790527, | |
| "learning_rate": 4.2857142857142855e-06, | |
| "logps/chosen": -171.61099243164062, | |
| "logps/rejected": -314.13824462890625, | |
| "loss": 0.204, | |
| "losses/dpo": 0.11684274673461914, | |
| "losses/sft": 2.5414156913757324, | |
| "losses/total": 0.11684274673461914, | |
| "ref_logps/chosen": -152.2470703125, | |
| "ref_logps/rejected": -169.70111083984375, | |
| "rewards/accuracies": 0.953125, | |
| "rewards/chosen": -1.936392903327942, | |
| "rewards/margins": 12.507322311401367, | |
| "rewards/rejected": -14.443716049194336, | |
| "step": 123 | |
| }, | |
| { | |
| "epoch": 0.2309124767225326, | |
| "grad_norm": 8.865942001342773, | |
| "learning_rate": 4.27536231884058e-06, | |
| "logps/chosen": -178.2666015625, | |
| "logps/rejected": -294.51971435546875, | |
| "loss": 0.0887, | |
| "losses/dpo": 0.04358779639005661, | |
| "losses/sft": 2.905651569366455, | |
| "losses/total": 0.04358779639005661, | |
| "ref_logps/chosen": -146.95541381835938, | |
| "ref_logps/rejected": -152.81776428222656, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": -3.1311190128326416, | |
| "rewards/margins": 11.039077758789062, | |
| "rewards/rejected": -14.170197486877441, | |
| "step": 124 | |
| }, | |
| { | |
| "epoch": 0.23277467411545624, | |
| "grad_norm": 14.640778541564941, | |
| "learning_rate": 4.265010351966874e-06, | |
| "logps/chosen": -176.9116668701172, | |
| "logps/rejected": -332.41241455078125, | |
| "loss": 0.1211, | |
| "losses/dpo": 0.053478606045246124, | |
| "losses/sft": 2.7518341541290283, | |
| "losses/total": 0.053478606045246124, | |
| "ref_logps/chosen": -150.49269104003906, | |
| "ref_logps/rejected": -171.48587036132812, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": -2.641897678375244, | |
| "rewards/margins": 13.450759887695312, | |
| "rewards/rejected": -16.0926570892334, | |
| "step": 125 | |
| } | |
| ], | |
| "logging_steps": 1.0, | |
| "max_steps": 537, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 1, | |
| "save_steps": 25, | |
| "stateful_callbacks": { | |
| "TrainerControl": { | |
| "args": { | |
| "should_epoch_stop": false, | |
| "should_evaluate": false, | |
| "should_log": false, | |
| "should_save": true, | |
| "should_training_stop": false | |
| }, | |
| "attributes": {} | |
| } | |
| }, | |
| "total_flos": 0.0, | |
| "train_batch_size": 2, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |