| { | |
| "best_metric": null, | |
| "best_model_checkpoint": null, | |
| "epoch": 2.9952, | |
| "eval_steps": 500, | |
| "global_step": 936, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "epoch": 0.016, | |
| "grad_norm": 38.78816510379247, | |
| "learning_rate": 2.6595744680851066e-07, | |
| "logits/chosen": -0.9742305874824524, | |
| "logits/rejected": -1.0986480712890625, | |
| "logps/chosen": -1.3971922397613525, | |
| "logps/rejected": -2.0653083324432373, | |
| "loss": 1.7192, | |
| "odds_ratio_loss": 0.5222790241241455, | |
| "rewards/accuracies": 0.7250000238418579, | |
| "rewards/chosen": -0.06985961645841599, | |
| "rewards/margins": 0.03340579569339752, | |
| "rewards/rejected": -0.1032654196023941, | |
| "sft_loss": 1.3971922397613525, | |
| "step": 5 | |
| }, | |
| { | |
| "epoch": 0.032, | |
| "grad_norm": 34.938822021789555, | |
| "learning_rate": 5.319148936170213e-07, | |
| "logits/chosen": -0.9504960775375366, | |
| "logits/rejected": -0.9644749760627747, | |
| "logps/chosen": -1.5183446407318115, | |
| "logps/rejected": -1.7423614263534546, | |
| "loss": 1.7, | |
| "odds_ratio_loss": 0.6908475756645203, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": -0.07591722905635834, | |
| "rewards/margins": 0.011200849898159504, | |
| "rewards/rejected": -0.08711807429790497, | |
| "sft_loss": 1.5183446407318115, | |
| "step": 10 | |
| }, | |
| { | |
| "epoch": 0.048, | |
| "grad_norm": 16.339902176867668, | |
| "learning_rate": 7.97872340425532e-07, | |
| "logits/chosen": -0.9059197306632996, | |
| "logits/rejected": -0.9509723782539368, | |
| "logps/chosen": -1.3207223415374756, | |
| "logps/rejected": -2.0547330379486084, | |
| "loss": 1.6089, | |
| "odds_ratio_loss": 0.6031222343444824, | |
| "rewards/accuracies": 0.6499999761581421, | |
| "rewards/chosen": -0.06603612005710602, | |
| "rewards/margins": 0.036700539290905, | |
| "rewards/rejected": -0.10273666679859161, | |
| "sft_loss": 1.3207223415374756, | |
| "step": 15 | |
| }, | |
| { | |
| "epoch": 0.064, | |
| "grad_norm": 20.917533328584984, | |
| "learning_rate": 1.0638297872340427e-06, | |
| "logits/chosen": -0.9647526741027832, | |
| "logits/rejected": -1.0629401206970215, | |
| "logps/chosen": -1.4611456394195557, | |
| "logps/rejected": -2.1091372966766357, | |
| "loss": 1.4991, | |
| "odds_ratio_loss": 0.5197792649269104, | |
| "rewards/accuracies": 0.675000011920929, | |
| "rewards/chosen": -0.07305728644132614, | |
| "rewards/margins": 0.03239959105849266, | |
| "rewards/rejected": -0.1054568737745285, | |
| "sft_loss": 1.4611456394195557, | |
| "step": 20 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "grad_norm": 18.664826990233166, | |
| "learning_rate": 1.3297872340425533e-06, | |
| "logits/chosen": -1.054176688194275, | |
| "logits/rejected": -1.3495676517486572, | |
| "logps/chosen": -1.3905874490737915, | |
| "logps/rejected": -1.7380883693695068, | |
| "loss": 1.3634, | |
| "odds_ratio_loss": 0.5891755819320679, | |
| "rewards/accuracies": 0.6000000238418579, | |
| "rewards/chosen": -0.06952936947345734, | |
| "rewards/margins": 0.017375053837895393, | |
| "rewards/rejected": -0.08690442144870758, | |
| "sft_loss": 1.3905874490737915, | |
| "step": 25 | |
| }, | |
| { | |
| "epoch": 0.096, | |
| "grad_norm": 10.604261736039259, | |
| "learning_rate": 1.595744680851064e-06, | |
| "logits/chosen": -0.9233123660087585, | |
| "logits/rejected": -0.9947792887687683, | |
| "logps/chosen": -1.3364614248275757, | |
| "logps/rejected": -1.4293016195297241, | |
| "loss": 1.3477, | |
| "odds_ratio_loss": 0.7917504906654358, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": -0.0668230801820755, | |
| "rewards/margins": 0.0046419971622526646, | |
| "rewards/rejected": -0.07146507501602173, | |
| "sft_loss": 1.3364614248275757, | |
| "step": 30 | |
| }, | |
| { | |
| "epoch": 0.112, | |
| "grad_norm": 9.23627597385138, | |
| "learning_rate": 1.8617021276595745e-06, | |
| "logits/chosen": -1.1354405879974365, | |
| "logits/rejected": -1.1504920721054077, | |
| "logps/chosen": -1.2876451015472412, | |
| "logps/rejected": -1.394587755203247, | |
| "loss": 1.3323, | |
| "odds_ratio_loss": 0.7012001276016235, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": -0.06438224762678146, | |
| "rewards/margins": 0.005347140599042177, | |
| "rewards/rejected": -0.06972938776016235, | |
| "sft_loss": 1.2876451015472412, | |
| "step": 35 | |
| }, | |
| { | |
| "epoch": 0.128, | |
| "grad_norm": 10.737738874884705, | |
| "learning_rate": 2.1276595744680853e-06, | |
| "logits/chosen": -1.0701355934143066, | |
| "logits/rejected": -1.2617167234420776, | |
| "logps/chosen": -1.345517873764038, | |
| "logps/rejected": -1.5351002216339111, | |
| "loss": 1.3194, | |
| "odds_ratio_loss": 0.6834388971328735, | |
| "rewards/accuracies": 0.550000011920929, | |
| "rewards/chosen": -0.06727589666843414, | |
| "rewards/margins": 0.009479111060500145, | |
| "rewards/rejected": -0.07675500959157944, | |
| "sft_loss": 1.345517873764038, | |
| "step": 40 | |
| }, | |
| { | |
| "epoch": 0.144, | |
| "grad_norm": 3.92140979781429, | |
| "learning_rate": 2.393617021276596e-06, | |
| "logits/chosen": -0.8892070651054382, | |
| "logits/rejected": -0.9303566217422485, | |
| "logps/chosen": -1.1515204906463623, | |
| "logps/rejected": -1.311240315437317, | |
| "loss": 1.3284, | |
| "odds_ratio_loss": 0.6429694890975952, | |
| "rewards/accuracies": 0.675000011920929, | |
| "rewards/chosen": -0.05757603049278259, | |
| "rewards/margins": 0.007985993288457394, | |
| "rewards/rejected": -0.06556202471256256, | |
| "sft_loss": 1.1515204906463623, | |
| "step": 45 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "grad_norm": 12.131879118457386, | |
| "learning_rate": 2.6595744680851065e-06, | |
| "logits/chosen": -0.9589262008666992, | |
| "logits/rejected": -1.0993459224700928, | |
| "logps/chosen": -1.3476245403289795, | |
| "logps/rejected": -1.4529037475585938, | |
| "loss": 1.3627, | |
| "odds_ratio_loss": 0.6757664680480957, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": -0.06738122552633286, | |
| "rewards/margins": 0.005263959523290396, | |
| "rewards/rejected": -0.07264517992734909, | |
| "sft_loss": 1.3476245403289795, | |
| "step": 50 | |
| }, | |
| { | |
| "epoch": 0.176, | |
| "grad_norm": 6.920195139152275, | |
| "learning_rate": 2.9255319148936174e-06, | |
| "logits/chosen": -0.9547260403633118, | |
| "logits/rejected": -1.137032151222229, | |
| "logps/chosen": -1.3528764247894287, | |
| "logps/rejected": -1.5240767002105713, | |
| "loss": 1.2606, | |
| "odds_ratio_loss": 0.6684664487838745, | |
| "rewards/accuracies": 0.6000000238418579, | |
| "rewards/chosen": -0.06764382123947144, | |
| "rewards/margins": 0.008560018613934517, | |
| "rewards/rejected": -0.0762038379907608, | |
| "sft_loss": 1.3528764247894287, | |
| "step": 55 | |
| }, | |
| { | |
| "epoch": 0.192, | |
| "grad_norm": 5.7098162603211104, | |
| "learning_rate": 3.191489361702128e-06, | |
| "logits/chosen": -0.982739269733429, | |
| "logits/rejected": -1.1074873208999634, | |
| "logps/chosen": -1.4583510160446167, | |
| "logps/rejected": -1.5452958345413208, | |
| "loss": 1.3274, | |
| "odds_ratio_loss": 0.7071143388748169, | |
| "rewards/accuracies": 0.4749999940395355, | |
| "rewards/chosen": -0.07291755080223083, | |
| "rewards/margins": 0.00434724148362875, | |
| "rewards/rejected": -0.07726480066776276, | |
| "sft_loss": 1.4583510160446167, | |
| "step": 60 | |
| }, | |
| { | |
| "epoch": 0.208, | |
| "grad_norm": 4.824404122964849, | |
| "learning_rate": 3.457446808510639e-06, | |
| "logits/chosen": -0.9817106127738953, | |
| "logits/rejected": -1.111875295639038, | |
| "logps/chosen": -1.3345075845718384, | |
| "logps/rejected": -1.5089738368988037, | |
| "loss": 1.2879, | |
| "odds_ratio_loss": 0.6608881950378418, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": -0.06672537326812744, | |
| "rewards/margins": 0.008723323233425617, | |
| "rewards/rejected": -0.07544869929552078, | |
| "sft_loss": 1.3345075845718384, | |
| "step": 65 | |
| }, | |
| { | |
| "epoch": 0.224, | |
| "grad_norm": 6.021701349784469, | |
| "learning_rate": 3.723404255319149e-06, | |
| "logits/chosen": -1.1069040298461914, | |
| "logits/rejected": -1.075409173965454, | |
| "logps/chosen": -1.2757076025009155, | |
| "logps/rejected": -1.4533952474594116, | |
| "loss": 1.3134, | |
| "odds_ratio_loss": 0.7258908152580261, | |
| "rewards/accuracies": 0.5249999761581421, | |
| "rewards/chosen": -0.0637853816151619, | |
| "rewards/margins": 0.008884383365511894, | |
| "rewards/rejected": -0.07266975939273834, | |
| "sft_loss": 1.2757076025009155, | |
| "step": 70 | |
| }, | |
| { | |
| "epoch": 0.24, | |
| "grad_norm": 5.02350299089495, | |
| "learning_rate": 3.98936170212766e-06, | |
| "logits/chosen": -0.9449170231819153, | |
| "logits/rejected": -1.1291000843048096, | |
| "logps/chosen": -1.2914139032363892, | |
| "logps/rejected": -1.484882116317749, | |
| "loss": 1.2797, | |
| "odds_ratio_loss": 0.6280355453491211, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": -0.06457068026065826, | |
| "rewards/margins": 0.009673424996435642, | |
| "rewards/rejected": -0.07424411177635193, | |
| "sft_loss": 1.2914139032363892, | |
| "step": 75 | |
| }, | |
| { | |
| "epoch": 0.256, | |
| "grad_norm": 5.308377873815178, | |
| "learning_rate": 4.255319148936171e-06, | |
| "logits/chosen": -0.9751418828964233, | |
| "logits/rejected": -1.094626545906067, | |
| "logps/chosen": -1.0752859115600586, | |
| "logps/rejected": -1.334636926651001, | |
| "loss": 1.2678, | |
| "odds_ratio_loss": 0.5908417701721191, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": -0.05376429483294487, | |
| "rewards/margins": 0.012967551127076149, | |
| "rewards/rejected": -0.06673184782266617, | |
| "sft_loss": 1.0752859115600586, | |
| "step": 80 | |
| }, | |
| { | |
| "epoch": 0.272, | |
| "grad_norm": 6.346400541372991, | |
| "learning_rate": 4.521276595744681e-06, | |
| "logits/chosen": -0.9909063577651978, | |
| "logits/rejected": -1.0453189611434937, | |
| "logps/chosen": -1.0384023189544678, | |
| "logps/rejected": -1.2738986015319824, | |
| "loss": 1.2615, | |
| "odds_ratio_loss": 0.6149815320968628, | |
| "rewards/accuracies": 0.6000000238418579, | |
| "rewards/chosen": -0.051920123398303986, | |
| "rewards/margins": 0.011774817481637001, | |
| "rewards/rejected": -0.06369493901729584, | |
| "sft_loss": 1.0384023189544678, | |
| "step": 85 | |
| }, | |
| { | |
| "epoch": 0.288, | |
| "grad_norm": 6.055157418934933, | |
| "learning_rate": 4.787234042553192e-06, | |
| "logits/chosen": -0.9102107882499695, | |
| "logits/rejected": -0.9935488700866699, | |
| "logps/chosen": -1.1961522102355957, | |
| "logps/rejected": -1.415398359298706, | |
| "loss": 1.212, | |
| "odds_ratio_loss": 0.6041514277458191, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": -0.059807609766721725, | |
| "rewards/margins": 0.010962305590510368, | |
| "rewards/rejected": -0.07076992094516754, | |
| "sft_loss": 1.1961522102355957, | |
| "step": 90 | |
| }, | |
| { | |
| "epoch": 0.304, | |
| "grad_norm": 5.552038237166853, | |
| "learning_rate": 4.999982598564682e-06, | |
| "logits/chosen": -0.9868474006652832, | |
| "logits/rejected": -1.0573745965957642, | |
| "logps/chosen": -1.2718483209609985, | |
| "logps/rejected": -1.5264861583709717, | |
| "loss": 1.2618, | |
| "odds_ratio_loss": 0.640211820602417, | |
| "rewards/accuracies": 0.574999988079071, | |
| "rewards/chosen": -0.06359241157770157, | |
| "rewards/margins": 0.012731892988085747, | |
| "rewards/rejected": -0.07632430642843246, | |
| "sft_loss": 1.2718483209609985, | |
| "step": 95 | |
| }, | |
| { | |
| "epoch": 0.32, | |
| "grad_norm": 7.727039615086689, | |
| "learning_rate": 4.999373573764188e-06, | |
| "logits/chosen": -1.0175714492797852, | |
| "logits/rejected": -0.8734873533248901, | |
| "logps/chosen": -1.199947476387024, | |
| "logps/rejected": -1.3050801753997803, | |
| "loss": 1.2807, | |
| "odds_ratio_loss": 0.7060422897338867, | |
| "rewards/accuracies": 0.5249999761581421, | |
| "rewards/chosen": -0.059997379779815674, | |
| "rewards/margins": 0.005256640259176493, | |
| "rewards/rejected": -0.06525401026010513, | |
| "sft_loss": 1.199947476387024, | |
| "step": 100 | |
| }, | |
| { | |
| "epoch": 0.336, | |
| "grad_norm": 6.365580704740442, | |
| "learning_rate": 4.997894719430564e-06, | |
| "logits/chosen": -0.8132748603820801, | |
| "logits/rejected": -0.9986258745193481, | |
| "logps/chosen": -1.1854135990142822, | |
| "logps/rejected": -1.5025197267532349, | |
| "loss": 1.2805, | |
| "odds_ratio_loss": 0.6134302020072937, | |
| "rewards/accuracies": 0.550000011920929, | |
| "rewards/chosen": -0.05927068740129471, | |
| "rewards/margins": 0.015855297446250916, | |
| "rewards/rejected": -0.07512599229812622, | |
| "sft_loss": 1.1854135990142822, | |
| "step": 105 | |
| }, | |
| { | |
| "epoch": 0.352, | |
| "grad_norm": 6.860280491199452, | |
| "learning_rate": 4.995546550233241e-06, | |
| "logits/chosen": -0.9532275199890137, | |
| "logits/rejected": -1.136918306350708, | |
| "logps/chosen": -1.408894658088684, | |
| "logps/rejected": -1.558379888534546, | |
| "loss": 1.2769, | |
| "odds_ratio_loss": 0.7106048464775085, | |
| "rewards/accuracies": 0.550000011920929, | |
| "rewards/chosen": -0.0704447329044342, | |
| "rewards/margins": 0.007474270649254322, | |
| "rewards/rejected": -0.07791899144649506, | |
| "sft_loss": 1.408894658088684, | |
| "step": 110 | |
| }, | |
| { | |
| "epoch": 0.368, | |
| "grad_norm": 5.854849184237901, | |
| "learning_rate": 4.992329883379755e-06, | |
| "logits/chosen": -0.9923961758613586, | |
| "logits/rejected": -1.2076470851898193, | |
| "logps/chosen": -1.4104652404785156, | |
| "logps/rejected": -1.4293034076690674, | |
| "loss": 1.2514, | |
| "odds_ratio_loss": 0.7473000288009644, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": -0.07052326202392578, | |
| "rewards/margins": 0.0009419171256013215, | |
| "rewards/rejected": -0.07146517932415009, | |
| "sft_loss": 1.4104652404785156, | |
| "step": 115 | |
| }, | |
| { | |
| "epoch": 0.384, | |
| "grad_norm": 4.334718029265899, | |
| "learning_rate": 4.988245838331339e-06, | |
| "logits/chosen": -0.9599634408950806, | |
| "logits/rejected": -0.9879687428474426, | |
| "logps/chosen": -1.2545793056488037, | |
| "logps/rejected": -1.4001166820526123, | |
| "loss": 1.2204, | |
| "odds_ratio_loss": 0.6919530630111694, | |
| "rewards/accuracies": 0.5249999761581421, | |
| "rewards/chosen": -0.06272897124290466, | |
| "rewards/margins": 0.007276860065758228, | |
| "rewards/rejected": -0.07000583410263062, | |
| "sft_loss": 1.2545793056488037, | |
| "step": 120 | |
| }, | |
| { | |
| "epoch": 0.4, | |
| "grad_norm": 6.640886415637415, | |
| "learning_rate": 4.983295836413337e-06, | |
| "logits/chosen": -0.8892742395401001, | |
| "logits/rejected": -0.8854384422302246, | |
| "logps/chosen": -1.1642634868621826, | |
| "logps/rejected": -1.2482268810272217, | |
| "loss": 1.225, | |
| "odds_ratio_loss": 0.7551355361938477, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": -0.05821318179368973, | |
| "rewards/margins": 0.004198164213448763, | |
| "rewards/rejected": -0.0624113492667675, | |
| "sft_loss": 1.1642634868621826, | |
| "step": 125 | |
| }, | |
| { | |
| "epoch": 0.416, | |
| "grad_norm": 3.3977384471741727, | |
| "learning_rate": 4.977481600320545e-06, | |
| "logits/chosen": -1.0043504238128662, | |
| "logits/rejected": -1.0277382135391235, | |
| "logps/chosen": -1.2277835607528687, | |
| "logps/rejected": -1.3244746923446655, | |
| "loss": 1.2696, | |
| "odds_ratio_loss": 0.726096510887146, | |
| "rewards/accuracies": 0.550000011920929, | |
| "rewards/chosen": -0.06138917803764343, | |
| "rewards/margins": 0.0048345597460865974, | |
| "rewards/rejected": -0.06622373312711716, | |
| "sft_loss": 1.2277835607528687, | |
| "step": 130 | |
| }, | |
| { | |
| "epoch": 0.432, | |
| "grad_norm": 29.84907862640638, | |
| "learning_rate": 4.970805153517692e-06, | |
| "logits/chosen": -0.906798243522644, | |
| "logits/rejected": -1.0836145877838135, | |
| "logps/chosen": -1.273900032043457, | |
| "logps/rejected": -1.6986215114593506, | |
| "loss": 1.2562, | |
| "odds_ratio_loss": 0.6722120642662048, | |
| "rewards/accuracies": 0.550000011920929, | |
| "rewards/chosen": -0.06369500607252121, | |
| "rewards/margins": 0.02123607136309147, | |
| "rewards/rejected": -0.08493108302354813, | |
| "sft_loss": 1.273900032043457, | |
| "step": 135 | |
| }, | |
| { | |
| "epoch": 0.448, | |
| "grad_norm": 9.009223423852758, | |
| "learning_rate": 4.963268819535228e-06, | |
| "logits/chosen": -0.9239026308059692, | |
| "logits/rejected": -0.9300885200500488, | |
| "logps/chosen": -1.0778651237487793, | |
| "logps/rejected": -1.2015340328216553, | |
| "loss": 1.2297, | |
| "odds_ratio_loss": 0.6974226832389832, | |
| "rewards/accuracies": 0.44999998807907104, | |
| "rewards/chosen": -0.053893256932497025, | |
| "rewards/margins": 0.006183440797030926, | |
| "rewards/rejected": -0.06007670238614082, | |
| "sft_loss": 1.0778651237487793, | |
| "step": 140 | |
| }, | |
| { | |
| "epoch": 0.464, | |
| "grad_norm": 6.881851804175557, | |
| "learning_rate": 4.954875221160695e-06, | |
| "logits/chosen": -0.9062795639038086, | |
| "logits/rejected": -0.9437534213066101, | |
| "logps/chosen": -1.151301622390747, | |
| "logps/rejected": -1.376366376876831, | |
| "loss": 1.1857, | |
| "odds_ratio_loss": 0.6390897035598755, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": -0.05756508558988571, | |
| "rewards/margins": 0.01125323586165905, | |
| "rewards/rejected": -0.06881832331418991, | |
| "sft_loss": 1.151301622390747, | |
| "step": 145 | |
| }, | |
| { | |
| "epoch": 0.48, | |
| "grad_norm": 6.024680716397588, | |
| "learning_rate": 4.945627279525943e-06, | |
| "logits/chosen": -0.8047590255737305, | |
| "logits/rejected": -1.0275001525878906, | |
| "logps/chosen": -1.1844927072525024, | |
| "logps/rejected": -1.1749188899993896, | |
| "loss": 1.2559, | |
| "odds_ratio_loss": 0.7716752886772156, | |
| "rewards/accuracies": 0.4749999940395355, | |
| "rewards/chosen": -0.05922463536262512, | |
| "rewards/margins": -0.0004786819336004555, | |
| "rewards/rejected": -0.05874595046043396, | |
| "sft_loss": 1.1844927072525024, | |
| "step": 150 | |
| }, | |
| { | |
| "epoch": 0.496, | |
| "grad_norm": 4.969594803216186, | |
| "learning_rate": 4.935528213090526e-06, | |
| "logits/chosen": -0.8657525181770325, | |
| "logits/rejected": -0.9782077074050903, | |
| "logps/chosen": -1.2747188806533813, | |
| "logps/rejected": -1.331703543663025, | |
| "loss": 1.2301, | |
| "odds_ratio_loss": 0.720513641834259, | |
| "rewards/accuracies": 0.5249999761581421, | |
| "rewards/chosen": -0.06373593956232071, | |
| "rewards/margins": 0.0028492447454482317, | |
| "rewards/rejected": -0.06658518314361572, | |
| "sft_loss": 1.2747188806533813, | |
| "step": 155 | |
| }, | |
| { | |
| "epoch": 0.512, | |
| "grad_norm": 5.691340876461069, | |
| "learning_rate": 4.9245815365216115e-06, | |
| "logits/chosen": -0.7511823773384094, | |
| "logits/rejected": -1.0508475303649902, | |
| "logps/chosen": -1.1776434183120728, | |
| "logps/rejected": -1.6007537841796875, | |
| "loss": 1.221, | |
| "odds_ratio_loss": 0.5590518116950989, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": -0.05888216570019722, | |
| "rewards/margins": 0.021155523136258125, | |
| "rewards/rejected": -0.08003769814968109, | |
| "sft_loss": 1.1776434183120728, | |
| "step": 160 | |
| }, | |
| { | |
| "epoch": 0.528, | |
| "grad_norm": 6.0040479808042235, | |
| "learning_rate": 4.912791059470815e-06, | |
| "logits/chosen": -0.9242817163467407, | |
| "logits/rejected": -0.8517144918441772, | |
| "logps/chosen": -1.0615520477294922, | |
| "logps/rejected": -1.288195252418518, | |
| "loss": 1.1745, | |
| "odds_ratio_loss": 0.6426531076431274, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": -0.05307760834693909, | |
| "rewards/margins": 0.011332156136631966, | |
| "rewards/rejected": -0.0644097551703453, | |
| "sft_loss": 1.0615520477294922, | |
| "step": 165 | |
| }, | |
| { | |
| "epoch": 0.544, | |
| "grad_norm": 5.529927712259793, | |
| "learning_rate": 4.900160885248363e-06, | |
| "logits/chosen": -0.9031238555908203, | |
| "logits/rejected": -0.8960297703742981, | |
| "logps/chosen": -1.2523874044418335, | |
| "logps/rejected": -1.4036993980407715, | |
| "loss": 1.194, | |
| "odds_ratio_loss": 0.6845428347587585, | |
| "rewards/accuracies": 0.6499999761581421, | |
| "rewards/chosen": -0.06261937320232391, | |
| "rewards/margins": 0.007565596606582403, | |
| "rewards/rejected": -0.07018496841192245, | |
| "sft_loss": 1.2523874044418335, | |
| "step": 170 | |
| }, | |
| { | |
| "epoch": 0.56, | |
| "grad_norm": 4.1316911858664644, | |
| "learning_rate": 4.886695409395068e-06, | |
| "logits/chosen": -0.853635311126709, | |
| "logits/rejected": -1.0007998943328857, | |
| "logps/chosen": -1.158414602279663, | |
| "logps/rejected": -1.1915799379348755, | |
| "loss": 1.2303, | |
| "odds_ratio_loss": 0.7887060046195984, | |
| "rewards/accuracies": 0.4749999940395355, | |
| "rewards/chosen": -0.05792072415351868, | |
| "rewards/margins": 0.0016582731623202562, | |
| "rewards/rejected": -0.059578996151685715, | |
| "sft_loss": 1.158414602279663, | |
| "step": 175 | |
| }, | |
| { | |
| "epoch": 0.576, | |
| "grad_norm": 4.185731700055316, | |
| "learning_rate": 4.872399318152594e-06, | |
| "logits/chosen": -0.9244221448898315, | |
| "logits/rejected": -0.8243404626846313, | |
| "logps/chosen": -1.124448537826538, | |
| "logps/rejected": -1.1855733394622803, | |
| "loss": 1.2359, | |
| "odds_ratio_loss": 0.6966110467910767, | |
| "rewards/accuracies": 0.574999988079071, | |
| "rewards/chosen": -0.05622243881225586, | |
| "rewards/margins": 0.0030562314204871655, | |
| "rewards/rejected": -0.059278666973114014, | |
| "sft_loss": 1.124448537826538, | |
| "step": 180 | |
| }, | |
| { | |
| "epoch": 0.592, | |
| "grad_norm": 5.534373355221194, | |
| "learning_rate": 4.857277586832556e-06, | |
| "logits/chosen": -0.8700960278511047, | |
| "logits/rejected": -1.0795601606369019, | |
| "logps/chosen": -1.1355112791061401, | |
| "logps/rejected": -1.3763678073883057, | |
| "loss": 1.2299, | |
| "odds_ratio_loss": 0.61272132396698, | |
| "rewards/accuracies": 0.7250000238418579, | |
| "rewards/chosen": -0.05677555873990059, | |
| "rewards/margins": 0.012042825110256672, | |
| "rewards/rejected": -0.06881839036941528, | |
| "sft_loss": 1.1355112791061401, | |
| "step": 185 | |
| }, | |
| { | |
| "epoch": 0.608, | |
| "grad_norm": 8.320294434791975, | |
| "learning_rate": 4.841335478085015e-06, | |
| "logits/chosen": -0.837981104850769, | |
| "logits/rejected": -1.041286587715149, | |
| "logps/chosen": -1.0703322887420654, | |
| "logps/rejected": -1.6396074295043945, | |
| "loss": 1.1951, | |
| "odds_ratio_loss": 0.5467146635055542, | |
| "rewards/accuracies": 0.675000011920929, | |
| "rewards/chosen": -0.05351661518216133, | |
| "rewards/margins": 0.028463756665587425, | |
| "rewards/rejected": -0.08198036998510361, | |
| "sft_loss": 1.0703322887420654, | |
| "step": 190 | |
| }, | |
| { | |
| "epoch": 0.624, | |
| "grad_norm": 4.60123509876027, | |
| "learning_rate": 4.824578540066981e-06, | |
| "logits/chosen": -0.93339604139328, | |
| "logits/rejected": -0.8643118739128113, | |
| "logps/chosen": -1.3019145727157593, | |
| "logps/rejected": -1.2324789762496948, | |
| "loss": 1.2285, | |
| "odds_ratio_loss": 0.8142224550247192, | |
| "rewards/accuracies": 0.4749999940395355, | |
| "rewards/chosen": -0.06509573012590408, | |
| "rewards/margins": -0.003471782896667719, | |
| "rewards/rejected": -0.061623942106962204, | |
| "sft_loss": 1.3019145727157593, | |
| "step": 195 | |
| }, | |
| { | |
| "epoch": 0.64, | |
| "grad_norm": 21.66363438026074, | |
| "learning_rate": 4.807012604511542e-06, | |
| "logits/chosen": -0.6915689706802368, | |
| "logits/rejected": -0.897596538066864, | |
| "logps/chosen": -1.1985743045806885, | |
| "logps/rejected": -1.3054250478744507, | |
| "loss": 1.2092, | |
| "odds_ratio_loss": 0.6641479730606079, | |
| "rewards/accuracies": 0.550000011920929, | |
| "rewards/chosen": -0.059928715229034424, | |
| "rewards/margins": 0.005342531017959118, | |
| "rewards/rejected": -0.06527124345302582, | |
| "sft_loss": 1.1985743045806885, | |
| "step": 200 | |
| }, | |
| { | |
| "epoch": 0.656, | |
| "grad_norm": 5.945812709802962, | |
| "learning_rate": 4.788643784698316e-06, | |
| "logits/chosen": -0.8415448069572449, | |
| "logits/rejected": -0.8126734495162964, | |
| "logps/chosen": -1.1990188360214233, | |
| "logps/rejected": -1.1923567056655884, | |
| "loss": 1.1919, | |
| "odds_ratio_loss": 0.7749138474464417, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": -0.05995094031095505, | |
| "rewards/margins": -0.0003331051266286522, | |
| "rewards/rejected": -0.05961783602833748, | |
| "sft_loss": 1.1990188360214233, | |
| "step": 205 | |
| }, | |
| { | |
| "epoch": 0.672, | |
| "grad_norm": 6.175015091799187, | |
| "learning_rate": 4.769478473325908e-06, | |
| "logits/chosen": -0.9676336050033569, | |
| "logits/rejected": -1.0954523086547852, | |
| "logps/chosen": -1.0937362909317017, | |
| "logps/rejected": -1.169339895248413, | |
| "loss": 1.2244, | |
| "odds_ratio_loss": 0.7255644798278809, | |
| "rewards/accuracies": 0.550000011920929, | |
| "rewards/chosen": -0.054686807096004486, | |
| "rewards/margins": 0.003780178725719452, | |
| "rewards/rejected": -0.058466989547014236, | |
| "sft_loss": 1.0937362909317017, | |
| "step": 210 | |
| }, | |
| { | |
| "epoch": 0.688, | |
| "grad_norm": 4.1222127125265935, | |
| "learning_rate": 4.7495233402871336e-06, | |
| "logits/chosen": -0.7574236392974854, | |
| "logits/rejected": -0.9107638597488403, | |
| "logps/chosen": -1.0908468961715698, | |
| "logps/rejected": -1.3144160509109497, | |
| "loss": 1.2612, | |
| "odds_ratio_loss": 0.6122642159461975, | |
| "rewards/accuracies": 0.675000011920929, | |
| "rewards/chosen": -0.05454235151410103, | |
| "rewards/margins": 0.011178453452885151, | |
| "rewards/rejected": -0.0657208114862442, | |
| "sft_loss": 1.0908468961715698, | |
| "step": 215 | |
| }, | |
| { | |
| "epoch": 0.704, | |
| "grad_norm": 6.629592523219295, | |
| "learning_rate": 4.728785330347771e-06, | |
| "logits/chosen": -1.0126041173934937, | |
| "logits/rejected": -1.1902836561203003, | |
| "logps/chosen": -1.0830891132354736, | |
| "logps/rejected": -1.2832982540130615, | |
| "loss": 1.1849, | |
| "odds_ratio_loss": 0.6457486748695374, | |
| "rewards/accuracies": 0.675000011920929, | |
| "rewards/chosen": -0.05415446311235428, | |
| "rewards/margins": 0.010010452009737492, | |
| "rewards/rejected": -0.0641649141907692, | |
| "sft_loss": 1.0830891132354736, | |
| "step": 220 | |
| }, | |
| { | |
| "epoch": 0.72, | |
| "grad_norm": 4.769856705836216, | |
| "learning_rate": 4.70727166072964e-06, | |
| "logits/chosen": -0.9934102296829224, | |
| "logits/rejected": -1.0441877841949463, | |
| "logps/chosen": -1.1385221481323242, | |
| "logps/rejected": -1.3962942361831665, | |
| "loss": 1.2004, | |
| "odds_ratio_loss": 0.631730318069458, | |
| "rewards/accuracies": 0.574999988079071, | |
| "rewards/chosen": -0.05692611262202263, | |
| "rewards/margins": 0.012888607569038868, | |
| "rewards/rejected": -0.06981472671031952, | |
| "sft_loss": 1.1385221481323242, | |
| "step": 225 | |
| }, | |
| { | |
| "epoch": 0.736, | |
| "grad_norm": 9.602754215145033, | |
| "learning_rate": 4.684989818598887e-06, | |
| "logits/chosen": -0.8686431050300598, | |
| "logits/rejected": -0.9307680130004883, | |
| "logps/chosen": -1.2431721687316895, | |
| "logps/rejected": -1.3505405187606812, | |
| "loss": 1.1679, | |
| "odds_ratio_loss": 0.7377614974975586, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": -0.06215860694646835, | |
| "rewards/margins": 0.005368414800614119, | |
| "rewards/rejected": -0.06752702593803406, | |
| "sft_loss": 1.2431721687316895, | |
| "step": 230 | |
| }, | |
| { | |
| "epoch": 0.752, | |
| "grad_norm": 4.991520073276977, | |
| "learning_rate": 4.661947558460296e-06, | |
| "logits/chosen": -0.893735408782959, | |
| "logits/rejected": -1.037488579750061, | |
| "logps/chosen": -1.2366321086883545, | |
| "logps/rejected": -1.422572135925293, | |
| "loss": 1.1988, | |
| "odds_ratio_loss": 0.6442903280258179, | |
| "rewards/accuracies": 0.6499999761581421, | |
| "rewards/chosen": -0.06183161213994026, | |
| "rewards/margins": 0.009297001175582409, | |
| "rewards/rejected": -0.07112861424684525, | |
| "sft_loss": 1.2366321086883545, | |
| "step": 235 | |
| }, | |
| { | |
| "epoch": 0.768, | |
| "grad_norm": 5.765863831511136, | |
| "learning_rate": 4.63815289945858e-06, | |
| "logits/chosen": -0.8988453149795532, | |
| "logits/rejected": -1.1190879344940186, | |
| "logps/chosen": -1.1124114990234375, | |
| "logps/rejected": -1.2776150703430176, | |
| "loss": 1.2519, | |
| "odds_ratio_loss": 0.697569727897644, | |
| "rewards/accuracies": 0.6000000238418579, | |
| "rewards/chosen": -0.055620573461055756, | |
| "rewards/margins": 0.008260180242359638, | |
| "rewards/rejected": -0.06388075649738312, | |
| "sft_loss": 1.1124114990234375, | |
| "step": 240 | |
| }, | |
| { | |
| "epoch": 0.784, | |
| "grad_norm": 4.2480178226924465, | |
| "learning_rate": 4.613614122587563e-06, | |
| "logits/chosen": -0.8890430331230164, | |
| "logits/rejected": -0.993693470954895, | |
| "logps/chosen": -1.1632936000823975, | |
| "logps/rejected": -1.2893292903900146, | |
| "loss": 1.2055, | |
| "odds_ratio_loss": 0.7878369092941284, | |
| "rewards/accuracies": 0.5249999761581421, | |
| "rewards/chosen": -0.058164678514003754, | |
| "rewards/margins": 0.0063017881475389, | |
| "rewards/rejected": -0.0644664615392685, | |
| "sft_loss": 1.1632936000823975, | |
| "step": 245 | |
| }, | |
| { | |
| "epoch": 0.8, | |
| "grad_norm": 4.066953723548615, | |
| "learning_rate": 4.5883397678082385e-06, | |
| "logits/chosen": -0.8833068013191223, | |
| "logits/rejected": -1.0138885974884033, | |
| "logps/chosen": -1.2117881774902344, | |
| "logps/rejected": -1.1593796014785767, | |
| "loss": 1.1903, | |
| "odds_ratio_loss": 0.7823435664176941, | |
| "rewards/accuracies": 0.4749999940395355, | |
| "rewards/chosen": -0.06058941036462784, | |
| "rewards/margins": -0.002620431361719966, | |
| "rewards/rejected": -0.05796898156404495, | |
| "sft_loss": 1.2117881774902344, | |
| "step": 250 | |
| }, | |
| { | |
| "epoch": 0.816, | |
| "grad_norm": 5.662450247054175, | |
| "learning_rate": 4.562338631076703e-06, | |
| "logits/chosen": -1.0948327779769897, | |
| "logits/rejected": -1.1347147226333618, | |
| "logps/chosen": -1.0046937465667725, | |
| "logps/rejected": -1.2093970775604248, | |
| "loss": 1.1825, | |
| "odds_ratio_loss": 0.6374476552009583, | |
| "rewards/accuracies": 0.6499999761581421, | |
| "rewards/chosen": -0.05023468658328056, | |
| "rewards/margins": 0.010235178284347057, | |
| "rewards/rejected": -0.060469865798950195, | |
| "sft_loss": 1.0046937465667725, | |
| "step": 255 | |
| }, | |
| { | |
| "epoch": 0.832, | |
| "grad_norm": 5.116135672128333, | |
| "learning_rate": 4.535619761282989e-06, | |
| "logits/chosen": -0.9458154439926147, | |
| "logits/rejected": -1.224017858505249, | |
| "logps/chosen": -1.202074646949768, | |
| "logps/rejected": -1.3588545322418213, | |
| "loss": 1.23, | |
| "odds_ratio_loss": 0.668530285358429, | |
| "rewards/accuracies": 0.675000011920929, | |
| "rewards/chosen": -0.06010373681783676, | |
| "rewards/margins": 0.007838994264602661, | |
| "rewards/rejected": -0.06794272363185883, | |
| "sft_loss": 1.202074646949768, | |
| "step": 260 | |
| }, | |
| { | |
| "epoch": 0.848, | |
| "grad_norm": 8.619415579204428, | |
| "learning_rate": 4.508192457101886e-06, | |
| "logits/chosen": -1.0687763690948486, | |
| "logits/rejected": -1.213895320892334, | |
| "logps/chosen": -1.2021548748016357, | |
| "logps/rejected": -1.541288137435913, | |
| "loss": 1.2482, | |
| "odds_ratio_loss": 0.658495306968689, | |
| "rewards/accuracies": 0.5249999761581421, | |
| "rewards/chosen": -0.060107748955488205, | |
| "rewards/margins": 0.016956666484475136, | |
| "rewards/rejected": -0.0770644098520279, | |
| "sft_loss": 1.2021548748016357, | |
| "step": 265 | |
| }, | |
| { | |
| "epoch": 0.864, | |
| "grad_norm": 5.000713262682842, | |
| "learning_rate": 4.480066263756821e-06, | |
| "logits/chosen": -0.9636927843093872, | |
| "logits/rejected": -1.1979575157165527, | |
| "logps/chosen": -1.220902442932129, | |
| "logps/rejected": -1.5320258140563965, | |
| "loss": 1.1889, | |
| "odds_ratio_loss": 0.5880663990974426, | |
| "rewards/accuracies": 0.7250000238418579, | |
| "rewards/chosen": -0.06104512885212898, | |
| "rewards/margins": 0.01555616408586502, | |
| "rewards/rejected": -0.0766012892127037, | |
| "sft_loss": 1.220902442932129, | |
| "step": 270 | |
| }, | |
| { | |
| "epoch": 0.88, | |
| "grad_norm": 6.295390583446683, | |
| "learning_rate": 4.451250969697944e-06, | |
| "logits/chosen": -0.9099109768867493, | |
| "logits/rejected": -1.1926413774490356, | |
| "logps/chosen": -1.1539406776428223, | |
| "logps/rejected": -1.4602700471878052, | |
| "loss": 1.2078, | |
| "odds_ratio_loss": 0.5688539743423462, | |
| "rewards/accuracies": 0.675000011920929, | |
| "rewards/chosen": -0.05769703909754753, | |
| "rewards/margins": 0.015316471457481384, | |
| "rewards/rejected": -0.07301349937915802, | |
| "sft_loss": 1.1539406776428223, | |
| "step": 275 | |
| }, | |
| { | |
| "epoch": 0.896, | |
| "grad_norm": 3.602560879317352, | |
| "learning_rate": 4.42175660319555e-06, | |
| "logits/chosen": -1.159501314163208, | |
| "logits/rejected": -1.2406375408172607, | |
| "logps/chosen": -1.192801594734192, | |
| "logps/rejected": -1.3772008419036865, | |
| "loss": 1.1898, | |
| "odds_ratio_loss": 0.7453117966651917, | |
| "rewards/accuracies": 0.5249999761581421, | |
| "rewards/chosen": -0.059640079736709595, | |
| "rewards/margins": 0.00921996496617794, | |
| "rewards/rejected": -0.06886004656553268, | |
| "sft_loss": 1.192801594734192, | |
| "step": 280 | |
| }, | |
| { | |
| "epoch": 0.912, | |
| "grad_norm": 8.783987785350908, | |
| "learning_rate": 4.391593428850069e-06, | |
| "logits/chosen": -0.9893490076065063, | |
| "logits/rejected": -1.1613223552703857, | |
| "logps/chosen": -1.0789204835891724, | |
| "logps/rejected": -1.1841206550598145, | |
| "loss": 1.2078, | |
| "odds_ratio_loss": 0.7538634538650513, | |
| "rewards/accuracies": 0.550000011920929, | |
| "rewards/chosen": -0.05394602566957474, | |
| "rewards/margins": 0.005260012112557888, | |
| "rewards/rejected": -0.0592060312628746, | |
| "sft_loss": 1.0789204835891724, | |
| "step": 285 | |
| }, | |
| { | |
| "epoch": 0.928, | |
| "grad_norm": 6.736280205252853, | |
| "learning_rate": 4.360771944019767e-06, | |
| "logits/chosen": -1.0662925243377686, | |
| "logits/rejected": -1.1013010740280151, | |
| "logps/chosen": -1.2803829908370972, | |
| "logps/rejected": -1.6290887594223022, | |
| "loss": 1.2225, | |
| "odds_ratio_loss": 0.6528395414352417, | |
| "rewards/accuracies": 0.675000011920929, | |
| "rewards/chosen": -0.06401915848255157, | |
| "rewards/margins": 0.017435286194086075, | |
| "rewards/rejected": -0.08145444095134735, | |
| "sft_loss": 1.2803829908370972, | |
| "step": 290 | |
| }, | |
| { | |
| "epoch": 0.944, | |
| "grad_norm": 7.2196796373916365, | |
| "learning_rate": 4.329302875167486e-06, | |
| "logits/chosen": -1.0543363094329834, | |
| "logits/rejected": -1.2899835109710693, | |
| "logps/chosen": -1.1794860363006592, | |
| "logps/rejected": -1.4053813219070435, | |
| "loss": 1.2094, | |
| "odds_ratio_loss": 0.6699637174606323, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": -0.05897430330514908, | |
| "rewards/margins": 0.011294771917164326, | |
| "rewards/rejected": -0.07026907801628113, | |
| "sft_loss": 1.1794860363006592, | |
| "step": 295 | |
| }, | |
| { | |
| "epoch": 0.96, | |
| "grad_norm": 5.962697902517566, | |
| "learning_rate": 4.297197174127619e-06, | |
| "logits/chosen": -1.0228321552276611, | |
| "logits/rejected": -1.2345274686813354, | |
| "logps/chosen": -1.0752114057540894, | |
| "logps/rejected": -1.2755658626556396, | |
| "loss": 1.212, | |
| "odds_ratio_loss": 0.6680848002433777, | |
| "rewards/accuracies": 0.550000011920929, | |
| "rewards/chosen": -0.05376056954264641, | |
| "rewards/margins": 0.01001772377640009, | |
| "rewards/rejected": -0.06377829611301422, | |
| "sft_loss": 1.0752114057540894, | |
| "step": 300 | |
| }, | |
| { | |
| "epoch": 0.976, | |
| "grad_norm": 3.4092575585850917, | |
| "learning_rate": 4.2644660142946685e-06, | |
| "logits/chosen": -0.9500937461853027, | |
| "logits/rejected": -1.0103486776351929, | |
| "logps/chosen": -1.0463281869888306, | |
| "logps/rejected": -1.4567468166351318, | |
| "loss": 1.2082, | |
| "odds_ratio_loss": 0.5394900441169739, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": -0.05231640487909317, | |
| "rewards/margins": 0.020520929247140884, | |
| "rewards/rejected": -0.07283733785152435, | |
| "sft_loss": 1.0463281869888306, | |
| "step": 305 | |
| }, | |
| { | |
| "epoch": 0.992, | |
| "grad_norm": 4.944690721660475, | |
| "learning_rate": 4.231120786734689e-06, | |
| "logits/chosen": -1.0554213523864746, | |
| "logits/rejected": -1.2123026847839355, | |
| "logps/chosen": -1.0677235126495361, | |
| "logps/rejected": -1.427178978919983, | |
| "loss": 1.1548, | |
| "odds_ratio_loss": 0.5765484571456909, | |
| "rewards/accuracies": 0.675000011920929, | |
| "rewards/chosen": -0.05338617414236069, | |
| "rewards/margins": 0.01797277294099331, | |
| "rewards/rejected": -0.07135894149541855, | |
| "sft_loss": 1.0677235126495361, | |
| "step": 310 | |
| }, | |
| { | |
| "epoch": 1.008, | |
| "grad_norm": 3.9331589062338344, | |
| "learning_rate": 4.197173096220983e-06, | |
| "logits/chosen": -1.0465214252471924, | |
| "logits/rejected": -1.0950441360473633, | |
| "logps/chosen": -1.0325384140014648, | |
| "logps/rejected": -1.4658787250518799, | |
| "loss": 1.1007, | |
| "odds_ratio_loss": 0.5475068688392639, | |
| "rewards/accuracies": 0.7250000238418579, | |
| "rewards/chosen": -0.05162692070007324, | |
| "rewards/margins": 0.02166702225804329, | |
| "rewards/rejected": -0.07329393923282623, | |
| "sft_loss": 1.0325384140014648, | |
| "step": 315 | |
| }, | |
| { | |
| "epoch": 1.024, | |
| "grad_norm": 2.810636096476599, | |
| "learning_rate": 4.162634757195418e-06, | |
| "logits/chosen": -0.9247652888298035, | |
| "logits/rejected": -1.031118392944336, | |
| "logps/chosen": -0.9556388854980469, | |
| "logps/rejected": -1.2451565265655518, | |
| "loss": 0.9386, | |
| "odds_ratio_loss": 0.594650149345398, | |
| "rewards/accuracies": 0.574999988079071, | |
| "rewards/chosen": -0.047781944274902344, | |
| "rewards/margins": 0.014475872740149498, | |
| "rewards/rejected": -0.06225781887769699, | |
| "sft_loss": 0.9556388854980469, | |
| "step": 320 | |
| }, | |
| { | |
| "epoch": 1.04, | |
| "grad_norm": 3.0448051628925454, | |
| "learning_rate": 4.127517789656772e-06, | |
| "logits/chosen": -0.9287153482437134, | |
| "logits/rejected": -1.0283979177474976, | |
| "logps/chosen": -1.0202507972717285, | |
| "logps/rejected": -1.2847683429718018, | |
| "loss": 1.0353, | |
| "odds_ratio_loss": 0.5959427952766418, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": -0.051012538373470306, | |
| "rewards/margins": 0.01322587113827467, | |
| "rewards/rejected": -0.06423841416835785, | |
| "sft_loss": 1.0202507972717285, | |
| "step": 325 | |
| }, | |
| { | |
| "epoch": 1.056, | |
| "grad_norm": 4.1771500190151825, | |
| "learning_rate": 4.091834414977556e-06, | |
| "logits/chosen": -1.0129518508911133, | |
| "logits/rejected": -1.271144986152649, | |
| "logps/chosen": -1.018913984298706, | |
| "logps/rejected": -1.554570198059082, | |
| "loss": 0.9511, | |
| "odds_ratio_loss": 0.47632694244384766, | |
| "rewards/accuracies": 0.824999988079071, | |
| "rewards/chosen": -0.0509456992149353, | |
| "rewards/margins": 0.026782814413309097, | |
| "rewards/rejected": -0.0777285099029541, | |
| "sft_loss": 1.018913984298706, | |
| "step": 330 | |
| }, | |
| { | |
| "epoch": 1.072, | |
| "grad_norm": 6.50448197903886, | |
| "learning_rate": 4.055597051650731e-06, | |
| "logits/chosen": -1.0193812847137451, | |
| "logits/rejected": -1.1772067546844482, | |
| "logps/chosen": -0.9281299710273743, | |
| "logps/rejected": -1.3907569646835327, | |
| "loss": 0.9662, | |
| "odds_ratio_loss": 0.5411213040351868, | |
| "rewards/accuracies": 0.7749999761581421, | |
| "rewards/chosen": -0.046406496316194534, | |
| "rewards/margins": 0.023131350055336952, | |
| "rewards/rejected": -0.06953784823417664, | |
| "sft_loss": 0.9281299710273743, | |
| "step": 335 | |
| }, | |
| { | |
| "epoch": 1.088, | |
| "grad_norm": 5.600060917092442, | |
| "learning_rate": 4.018818310967843e-06, | |
| "logits/chosen": -1.1713335514068604, | |
| "logits/rejected": -1.2236130237579346, | |
| "logps/chosen": -1.0068949460983276, | |
| "logps/rejected": -1.1553858518600464, | |
| "loss": 0.9817, | |
| "odds_ratio_loss": 0.7148981094360352, | |
| "rewards/accuracies": 0.574999988079071, | |
| "rewards/chosen": -0.05034474655985832, | |
| "rewards/margins": 0.007424544543027878, | |
| "rewards/rejected": -0.0577692911028862, | |
| "sft_loss": 1.0068949460983276, | |
| "step": 340 | |
| }, | |
| { | |
| "epoch": 1.104, | |
| "grad_norm": 5.092643762749835, | |
| "learning_rate": 3.981510992630055e-06, | |
| "logits/chosen": -0.9599639177322388, | |
| "logits/rejected": -1.189029574394226, | |
| "logps/chosen": -0.9695978164672852, | |
| "logps/rejected": -1.4548832178115845, | |
| "loss": 0.9953, | |
| "odds_ratio_loss": 0.5450643301010132, | |
| "rewards/accuracies": 0.675000011920929, | |
| "rewards/chosen": -0.048479896038770676, | |
| "rewards/margins": 0.024264268577098846, | |
| "rewards/rejected": -0.07274416834115982, | |
| "sft_loss": 0.9695978164672852, | |
| "step": 345 | |
| }, | |
| { | |
| "epoch": 1.12, | |
| "grad_norm": 4.234304914475125, | |
| "learning_rate": 3.943688080293607e-06, | |
| "logits/chosen": -0.9754691123962402, | |
| "logits/rejected": -1.1199839115142822, | |
| "logps/chosen": -0.894666314125061, | |
| "logps/rejected": -1.577906847000122, | |
| "loss": 0.9707, | |
| "odds_ratio_loss": 0.45880669355392456, | |
| "rewards/accuracies": 0.8500000238418579, | |
| "rewards/chosen": -0.044733308255672455, | |
| "rewards/margins": 0.03416203707456589, | |
| "rewards/rejected": -0.07889535278081894, | |
| "sft_loss": 0.894666314125061, | |
| "step": 350 | |
| }, | |
| { | |
| "epoch": 1.1360000000000001, | |
| "grad_norm": 4.426257939120987, | |
| "learning_rate": 3.905362737051252e-06, | |
| "logits/chosen": -1.0120149850845337, | |
| "logits/rejected": -1.1284050941467285, | |
| "logps/chosen": -0.8394268751144409, | |
| "logps/rejected": -1.3487886190414429, | |
| "loss": 0.9606, | |
| "odds_ratio_loss": 0.5031847357749939, | |
| "rewards/accuracies": 0.7749999761581421, | |
| "rewards/chosen": -0.041971348226070404, | |
| "rewards/margins": 0.02546808496117592, | |
| "rewards/rejected": -0.06743943691253662, | |
| "sft_loss": 0.8394268751144409, | |
| "step": 355 | |
| }, | |
| { | |
| "epoch": 1.152, | |
| "grad_norm": 6.846072733066629, | |
| "learning_rate": 3.866548300851254e-06, | |
| "logits/chosen": -1.0661356449127197, | |
| "logits/rejected": -1.1536469459533691, | |
| "logps/chosen": -0.9550708532333374, | |
| "logps/rejected": -1.400836706161499, | |
| "loss": 0.9989, | |
| "odds_ratio_loss": 0.5404989123344421, | |
| "rewards/accuracies": 0.6499999761581421, | |
| "rewards/chosen": -0.04775355011224747, | |
| "rewards/margins": 0.02228829450905323, | |
| "rewards/rejected": -0.07004183530807495, | |
| "sft_loss": 0.9550708532333374, | |
| "step": 360 | |
| }, | |
| { | |
| "epoch": 1.168, | |
| "grad_norm": 4.238159230888541, | |
| "learning_rate": 3.827258279855527e-06, | |
| "logits/chosen": -1.1802518367767334, | |
| "logits/rejected": -1.206186056137085, | |
| "logps/chosen": -1.0155624151229858, | |
| "logps/rejected": -1.3900431394577026, | |
| "loss": 0.9456, | |
| "odds_ratio_loss": 0.6698687672615051, | |
| "rewards/accuracies": 0.6499999761581421, | |
| "rewards/chosen": -0.05077812820672989, | |
| "rewards/margins": 0.01872403547167778, | |
| "rewards/rejected": -0.06950215995311737, | |
| "sft_loss": 1.0155624151229858, | |
| "step": 365 | |
| }, | |
| { | |
| "epoch": 1.184, | |
| "grad_norm": 3.7353385077354337, | |
| "learning_rate": 3.787506347738538e-06, | |
| "logits/chosen": -0.9678732752799988, | |
| "logits/rejected": -1.1043659448623657, | |
| "logps/chosen": -0.9759182929992676, | |
| "logps/rejected": -1.3616999387741089, | |
| "loss": 0.9721, | |
| "odds_ratio_loss": 0.6270595788955688, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": -0.0487959161400795, | |
| "rewards/margins": 0.019289087504148483, | |
| "rewards/rejected": -0.06808499991893768, | |
| "sft_loss": 0.9759182929992676, | |
| "step": 370 | |
| }, | |
| { | |
| "epoch": 1.2, | |
| "grad_norm": 4.7540211314956204, | |
| "learning_rate": 3.747306338928609e-06, | |
| "logits/chosen": -1.049630045890808, | |
| "logits/rejected": -1.2382904291152954, | |
| "logps/chosen": -0.938238263130188, | |
| "logps/rejected": -1.2308955192565918, | |
| "loss": 1.0158, | |
| "odds_ratio_loss": 0.5973559021949768, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": -0.04691191390156746, | |
| "rewards/margins": 0.01463286578655243, | |
| "rewards/rejected": -0.06154477596282959, | |
| "sft_loss": 0.938238263130188, | |
| "step": 375 | |
| }, | |
| { | |
| "epoch": 1.216, | |
| "grad_norm": 5.565360284878935, | |
| "learning_rate": 3.706672243793271e-06, | |
| "logits/chosen": -0.9532748460769653, | |
| "logits/rejected": -1.03403902053833, | |
| "logps/chosen": -0.9681148529052734, | |
| "logps/rejected": -1.2792482376098633, | |
| "loss": 0.9464, | |
| "odds_ratio_loss": 0.5441717505455017, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": -0.04840574413537979, | |
| "rewards/margins": 0.015556666068732738, | |
| "rewards/rejected": -0.0639624148607254, | |
| "sft_loss": 0.9681148529052734, | |
| "step": 380 | |
| }, | |
| { | |
| "epoch": 1.232, | |
| "grad_norm": 3.447185504214593, | |
| "learning_rate": 3.665618203770352e-06, | |
| "logits/chosen": -1.0210788249969482, | |
| "logits/rejected": -1.1396093368530273, | |
| "logps/chosen": -0.8457318544387817, | |
| "logps/rejected": -1.2301608324050903, | |
| "loss": 0.9478, | |
| "odds_ratio_loss": 0.5166088342666626, | |
| "rewards/accuracies": 0.7250000238418579, | |
| "rewards/chosen": -0.04228659346699715, | |
| "rewards/margins": 0.01922144927084446, | |
| "rewards/rejected": -0.06150804087519646, | |
| "sft_loss": 0.8457318544387817, | |
| "step": 385 | |
| }, | |
| { | |
| "epoch": 1.248, | |
| "grad_norm": 3.3338981917851047, | |
| "learning_rate": 3.6241585064464846e-06, | |
| "logits/chosen": -1.2911275625228882, | |
| "logits/rejected": -1.1232960224151611, | |
| "logps/chosen": -0.8720508813858032, | |
| "logps/rejected": -1.23488187789917, | |
| "loss": 0.9435, | |
| "odds_ratio_loss": 0.5066360831260681, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": -0.04360254481434822, | |
| "rewards/margins": 0.01814154163002968, | |
| "rewards/rejected": -0.0617440864443779, | |
| "sft_loss": 0.8720508813858032, | |
| "step": 390 | |
| }, | |
| { | |
| "epoch": 1.264, | |
| "grad_norm": 3.567835515660478, | |
| "learning_rate": 3.582307580584759e-06, | |
| "logits/chosen": -0.959577739238739, | |
| "logits/rejected": -1.2563647031784058, | |
| "logps/chosen": -0.8968993425369263, | |
| "logps/rejected": -1.2443525791168213, | |
| "loss": 0.9669, | |
| "odds_ratio_loss": 0.521682620048523, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": -0.04484497010707855, | |
| "rewards/margins": 0.01737266220152378, | |
| "rewards/rejected": -0.062217630445957184, | |
| "sft_loss": 0.8968993425369263, | |
| "step": 395 | |
| }, | |
| { | |
| "epoch": 1.28, | |
| "grad_norm": 3.9755484861044956, | |
| "learning_rate": 3.5400799911032357e-06, | |
| "logits/chosen": -1.054391860961914, | |
| "logits/rejected": -1.017757534980774, | |
| "logps/chosen": -0.8325561285018921, | |
| "logps/rejected": -1.2529734373092651, | |
| "loss": 0.9323, | |
| "odds_ratio_loss": 0.4909485876560211, | |
| "rewards/accuracies": 0.824999988079071, | |
| "rewards/chosen": -0.041627805680036545, | |
| "rewards/margins": 0.021020859479904175, | |
| "rewards/rejected": -0.06264867633581161, | |
| "sft_loss": 0.8325561285018921, | |
| "step": 400 | |
| }, | |
| { | |
| "epoch": 1.296, | |
| "grad_norm": 3.4685607506272476, | |
| "learning_rate": 3.4974904340060756e-06, | |
| "logits/chosen": -1.004164218902588, | |
| "logits/rejected": -1.320536732673645, | |
| "logps/chosen": -1.0670472383499146, | |
| "logps/rejected": -1.2909588813781738, | |
| "loss": 0.9978, | |
| "odds_ratio_loss": 0.6580184102058411, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": -0.05335236340761185, | |
| "rewards/margins": 0.011195586062967777, | |
| "rewards/rejected": -0.06454795598983765, | |
| "sft_loss": 1.0670472383499146, | |
| "step": 405 | |
| }, | |
| { | |
| "epoch": 1.312, | |
| "grad_norm": 4.3019598153539516, | |
| "learning_rate": 3.4545537312690565e-06, | |
| "logits/chosen": -1.1336114406585693, | |
| "logits/rejected": -1.1372759342193604, | |
| "logps/chosen": -0.9065093994140625, | |
| "logps/rejected": -1.415405511856079, | |
| "loss": 0.9679, | |
| "odds_ratio_loss": 0.4388805031776428, | |
| "rewards/accuracies": 0.8500000238418579, | |
| "rewards/chosen": -0.045325469225645065, | |
| "rewards/margins": 0.02544480562210083, | |
| "rewards/rejected": -0.0707702785730362, | |
| "sft_loss": 0.9065093994140625, | |
| "step": 410 | |
| }, | |
| { | |
| "epoch": 1.328, | |
| "grad_norm": 3.490824264107295, | |
| "learning_rate": 3.4112848256812374e-06, | |
| "logits/chosen": -1.0414103269577026, | |
| "logits/rejected": -1.164736032485962, | |
| "logps/chosen": -0.8454300761222839, | |
| "logps/rejected": -1.311577558517456, | |
| "loss": 0.9274, | |
| "odds_ratio_loss": 0.503560483455658, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": -0.04227150231599808, | |
| "rewards/margins": 0.02330738492310047, | |
| "rewards/rejected": -0.0655788853764534, | |
| "sft_loss": 0.8454300761222839, | |
| "step": 415 | |
| }, | |
| { | |
| "epoch": 1.3439999999999999, | |
| "grad_norm": 8.789242453958117, | |
| "learning_rate": 3.3676987756445894e-06, | |
| "logits/chosen": -0.935681164264679, | |
| "logits/rejected": -1.1432523727416992, | |
| "logps/chosen": -0.9042528867721558, | |
| "logps/rejected": -1.587890386581421, | |
| "loss": 0.9137, | |
| "odds_ratio_loss": 0.47100192308425903, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": -0.04521264508366585, | |
| "rewards/margins": 0.0341818742454052, | |
| "rewards/rejected": -0.07939452677965164, | |
| "sft_loss": 0.9042528867721558, | |
| "step": 420 | |
| }, | |
| { | |
| "epoch": 1.3599999999999999, | |
| "grad_norm": 4.930177739224116, | |
| "learning_rate": 3.323810749933381e-06, | |
| "logits/chosen": -1.0698130130767822, | |
| "logits/rejected": -1.226738691329956, | |
| "logps/chosen": -0.8717530369758606, | |
| "logps/rejected": -1.4678089618682861, | |
| "loss": 0.9124, | |
| "odds_ratio_loss": 0.4427013397216797, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": -0.04358765110373497, | |
| "rewards/margins": 0.02980278991162777, | |
| "rewards/rejected": -0.07339043915271759, | |
| "sft_loss": 0.8717530369758606, | |
| "step": 425 | |
| }, | |
| { | |
| "epoch": 1.376, | |
| "grad_norm": 7.570978097521412, | |
| "learning_rate": 3.2796360224151587e-06, | |
| "logits/chosen": -1.2224234342575073, | |
| "logits/rejected": -1.3439141511917114, | |
| "logps/chosen": -0.9194822311401367, | |
| "logps/rejected": -1.3092845678329468, | |
| "loss": 0.9594, | |
| "odds_ratio_loss": 0.5272402763366699, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": -0.045974113047122955, | |
| "rewards/margins": 0.019490113481879234, | |
| "rewards/rejected": -0.06546422094106674, | |
| "sft_loss": 0.9194822311401367, | |
| "step": 430 | |
| }, | |
| { | |
| "epoch": 1.392, | |
| "grad_norm": 4.982061148370284, | |
| "learning_rate": 3.235189966735148e-06, | |
| "logits/chosen": -1.2072560787200928, | |
| "logits/rejected": -1.1339049339294434, | |
| "logps/chosen": -0.805392861366272, | |
| "logps/rejected": -1.594913125038147, | |
| "loss": 0.9259, | |
| "odds_ratio_loss": 0.406252384185791, | |
| "rewards/accuracies": 0.875, | |
| "rewards/chosen": -0.0402696393430233, | |
| "rewards/margins": 0.03947601094841957, | |
| "rewards/rejected": -0.07974565029144287, | |
| "sft_loss": 0.805392861366272, | |
| "step": 435 | |
| }, | |
| { | |
| "epoch": 1.408, | |
| "grad_norm": 7.692151482525084, | |
| "learning_rate": 3.1904880509659397e-06, | |
| "logits/chosen": -1.0890783071517944, | |
| "logits/rejected": -1.177278757095337, | |
| "logps/chosen": -0.9858812093734741, | |
| "logps/rejected": -1.204777479171753, | |
| "loss": 0.9667, | |
| "odds_ratio_loss": 0.637945830821991, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": -0.049294065684080124, | |
| "rewards/margins": 0.010944806039333344, | |
| "rewards/rejected": -0.060238875448703766, | |
| "sft_loss": 0.9858812093734741, | |
| "step": 440 | |
| }, | |
| { | |
| "epoch": 1.424, | |
| "grad_norm": 4.492542501706703, | |
| "learning_rate": 3.1455458322242943e-06, | |
| "logits/chosen": -1.004298448562622, | |
| "logits/rejected": -1.1840426921844482, | |
| "logps/chosen": -0.9921313524246216, | |
| "logps/rejected": -1.2852861881256104, | |
| "loss": 0.9938, | |
| "odds_ratio_loss": 0.6350023150444031, | |
| "rewards/accuracies": 0.6000000238418579, | |
| "rewards/chosen": -0.0496065691113472, | |
| "rewards/margins": 0.014657738618552685, | |
| "rewards/rejected": -0.06426431238651276, | |
| "sft_loss": 0.9921313524246216, | |
| "step": 445 | |
| }, | |
| { | |
| "epoch": 1.44, | |
| "grad_norm": 4.069388708509856, | |
| "learning_rate": 3.100378951256981e-06, | |
| "logits/chosen": -1.2047039270401, | |
| "logits/rejected": -1.2526731491088867, | |
| "logps/chosen": -0.8970128297805786, | |
| "logps/rejected": -1.087720274925232, | |
| "loss": 0.9649, | |
| "odds_ratio_loss": 0.6119378805160522, | |
| "rewards/accuracies": 0.6499999761581421, | |
| "rewards/chosen": -0.04485064372420311, | |
| "rewards/margins": 0.009535368531942368, | |
| "rewards/rejected": -0.05438600853085518, | |
| "sft_loss": 0.8970128297805786, | |
| "step": 450 | |
| }, | |
| { | |
| "epoch": 1.456, | |
| "grad_norm": 4.1796047584641745, | |
| "learning_rate": 3.055003126997495e-06, | |
| "logits/chosen": -1.100693941116333, | |
| "logits/rejected": -1.1093151569366455, | |
| "logps/chosen": -0.9937971234321594, | |
| "logps/rejected": -1.3282194137573242, | |
| "loss": 0.9642, | |
| "odds_ratio_loss": 0.5750898122787476, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": -0.04968985915184021, | |
| "rewards/margins": 0.01672111637890339, | |
| "rewards/rejected": -0.06641098111867905, | |
| "sft_loss": 0.9937971234321594, | |
| "step": 455 | |
| }, | |
| { | |
| "epoch": 1.472, | |
| "grad_norm": 4.0507990154789075, | |
| "learning_rate": 3.0094341510955697e-06, | |
| "logits/chosen": -1.1363385915756226, | |
| "logits/rejected": -1.1738272905349731, | |
| "logps/chosen": -0.877064049243927, | |
| "logps/rejected": -1.4007574319839478, | |
| "loss": 0.959, | |
| "odds_ratio_loss": 0.5275009870529175, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": -0.04385320469737053, | |
| "rewards/margins": 0.026184672489762306, | |
| "rewards/rejected": -0.07003787159919739, | |
| "sft_loss": 0.877064049243927, | |
| "step": 460 | |
| }, | |
| { | |
| "epoch": 1.488, | |
| "grad_norm": 5.088249905544024, | |
| "learning_rate": 2.963687882421383e-06, | |
| "logits/chosen": -1.096651554107666, | |
| "logits/rejected": -1.1972352266311646, | |
| "logps/chosen": -0.9147326350212097, | |
| "logps/rejected": -1.253187894821167, | |
| "loss": 0.9591, | |
| "odds_ratio_loss": 0.5869191288948059, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": -0.04573662951588631, | |
| "rewards/margins": 0.01692276820540428, | |
| "rewards/rejected": -0.06265939772129059, | |
| "sft_loss": 0.9147326350212097, | |
| "step": 465 | |
| }, | |
| { | |
| "epoch": 1.504, | |
| "grad_norm": 4.206721558657063, | |
| "learning_rate": 2.9177802415463714e-06, | |
| "logits/chosen": -0.9876214265823364, | |
| "logits/rejected": -1.073717713356018, | |
| "logps/chosen": -0.7661324143409729, | |
| "logps/rejected": -1.2461497783660889, | |
| "loss": 0.9619, | |
| "odds_ratio_loss": 0.45304179191589355, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": -0.038306623697280884, | |
| "rewards/margins": 0.0240008607506752, | |
| "rewards/rejected": -0.062307484447956085, | |
| "sft_loss": 0.7661324143409729, | |
| "step": 470 | |
| }, | |
| { | |
| "epoch": 1.52, | |
| "grad_norm": 3.67250528125703, | |
| "learning_rate": 2.871727205202563e-06, | |
| "logits/chosen": -1.0482373237609863, | |
| "logits/rejected": -1.1120043992996216, | |
| "logps/chosen": -0.950273871421814, | |
| "logps/rejected": -1.2787425518035889, | |
| "loss": 0.9498, | |
| "odds_ratio_loss": 0.599994421005249, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": -0.047513701021671295, | |
| "rewards/margins": 0.016423430293798447, | |
| "rewards/rejected": -0.06393712759017944, | |
| "sft_loss": 0.950273871421814, | |
| "step": 475 | |
| }, | |
| { | |
| "epoch": 1.536, | |
| "grad_norm": 4.735321963888987, | |
| "learning_rate": 2.825544800722376e-06, | |
| "logits/chosen": -1.1098222732543945, | |
| "logits/rejected": -1.0207080841064453, | |
| "logps/chosen": -0.9046823382377625, | |
| "logps/rejected": -1.3911302089691162, | |
| "loss": 0.9629, | |
| "odds_ratio_loss": 0.5132350921630859, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": -0.04523410648107529, | |
| "rewards/margins": 0.0243223924189806, | |
| "rewards/rejected": -0.06955650448799133, | |
| "sft_loss": 0.9046823382377625, | |
| "step": 480 | |
| }, | |
| { | |
| "epoch": 1.552, | |
| "grad_norm": 3.159767853907868, | |
| "learning_rate": 2.7792491004607984e-06, | |
| "logits/chosen": -1.0096070766448975, | |
| "logits/rejected": -1.0770386457443237, | |
| "logps/chosen": -0.9277147054672241, | |
| "logps/rejected": -1.344773530960083, | |
| "loss": 0.9485, | |
| "odds_ratio_loss": 0.5044125318527222, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": -0.046385735273361206, | |
| "rewards/margins": 0.020852940157055855, | |
| "rewards/rejected": -0.06723867356777191, | |
| "sft_loss": 0.9277147054672241, | |
| "step": 485 | |
| }, | |
| { | |
| "epoch": 1.568, | |
| "grad_norm": 4.013671913402214, | |
| "learning_rate": 2.732856216201906e-06, | |
| "logits/chosen": -1.063245415687561, | |
| "logits/rejected": -1.1233893632888794, | |
| "logps/chosen": -0.9281940460205078, | |
| "logps/rejected": -1.3444063663482666, | |
| "loss": 0.9529, | |
| "odds_ratio_loss": 0.489690363407135, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": -0.04640969634056091, | |
| "rewards/margins": 0.020810618996620178, | |
| "rewards/rejected": -0.06722031533718109, | |
| "sft_loss": 0.9281940460205078, | |
| "step": 490 | |
| }, | |
| { | |
| "epoch": 1.584, | |
| "grad_norm": 4.31314553790671, | |
| "learning_rate": 2.6863822935516546e-06, | |
| "logits/chosen": -1.0234626531600952, | |
| "logits/rejected": -1.1978650093078613, | |
| "logps/chosen": -0.9632134437561035, | |
| "logps/rejected": -1.206189751625061, | |
| "loss": 0.9733, | |
| "odds_ratio_loss": 0.5884974002838135, | |
| "rewards/accuracies": 0.675000011920929, | |
| "rewards/chosen": -0.048160675913095474, | |
| "rewards/margins": 0.012148816138505936, | |
| "rewards/rejected": -0.06030949205160141, | |
| "sft_loss": 0.9632134437561035, | |
| "step": 495 | |
| }, | |
| { | |
| "epoch": 1.6, | |
| "grad_norm": 3.3125411606876445, | |
| "learning_rate": 2.639843506318899e-06, | |
| "logits/chosen": -0.9256556630134583, | |
| "logits/rejected": -1.1392202377319336, | |
| "logps/chosen": -0.8793741464614868, | |
| "logps/rejected": -1.3441039323806763, | |
| "loss": 0.9562, | |
| "odds_ratio_loss": 0.510412335395813, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": -0.04396871104836464, | |
| "rewards/margins": 0.023236487060785294, | |
| "rewards/rejected": -0.06720519065856934, | |
| "sft_loss": 0.8793741464614868, | |
| "step": 500 | |
| }, | |
| { | |
| "epoch": 1.616, | |
| "grad_norm": 5.866811891887456, | |
| "learning_rate": 2.593256050886603e-06, | |
| "logits/chosen": -0.8647163510322571, | |
| "logits/rejected": -1.0798470973968506, | |
| "logps/chosen": -0.9747751951217651, | |
| "logps/rejected": -1.2537850141525269, | |
| "loss": 0.9531, | |
| "odds_ratio_loss": 0.5470961332321167, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": -0.0487387590110302, | |
| "rewards/margins": 0.01395049411803484, | |
| "rewards/rejected": -0.06268925964832306, | |
| "sft_loss": 0.9747751951217651, | |
| "step": 505 | |
| }, | |
| { | |
| "epoch": 1.6320000000000001, | |
| "grad_norm": 4.074172203592654, | |
| "learning_rate": 2.5466361405751914e-06, | |
| "logits/chosen": -0.9186518788337708, | |
| "logits/rejected": -1.1423577070236206, | |
| "logps/chosen": -0.9840022921562195, | |
| "logps/rejected": -1.3202232122421265, | |
| "loss": 0.949, | |
| "odds_ratio_loss": 0.564594566822052, | |
| "rewards/accuracies": 0.7250000238418579, | |
| "rewards/chosen": -0.049200110137462616, | |
| "rewards/margins": 0.01681104302406311, | |
| "rewards/rejected": -0.06601114571094513, | |
| "sft_loss": 0.9840022921562195, | |
| "step": 510 | |
| }, | |
| { | |
| "epoch": 1.6480000000000001, | |
| "grad_norm": 5.002204843627367, | |
| "learning_rate": 2.5e-06, | |
| "logits/chosen": -0.8605943918228149, | |
| "logits/rejected": -0.9710782766342163, | |
| "logps/chosen": -0.8691279292106628, | |
| "logps/rejected": -1.2410802841186523, | |
| "loss": 0.9795, | |
| "odds_ratio_loss": 0.5254562497138977, | |
| "rewards/accuracies": 0.7250000238418579, | |
| "rewards/chosen": -0.04345639795064926, | |
| "rewards/margins": 0.018597617745399475, | |
| "rewards/rejected": -0.06205401569604874, | |
| "sft_loss": 0.8691279292106628, | |
| "step": 515 | |
| }, | |
| { | |
| "epoch": 1.6640000000000001, | |
| "grad_norm": 8.858448372423531, | |
| "learning_rate": 2.4533638594248094e-06, | |
| "logits/chosen": -0.9867407083511353, | |
| "logits/rejected": -1.0888783931732178, | |
| "logps/chosen": -0.9163697957992554, | |
| "logps/rejected": -1.1703190803527832, | |
| "loss": 0.9221, | |
| "odds_ratio_loss": 0.5501306056976318, | |
| "rewards/accuracies": 0.6499999761581421, | |
| "rewards/chosen": -0.045818496495485306, | |
| "rewards/margins": 0.012697461061179638, | |
| "rewards/rejected": -0.05851595476269722, | |
| "sft_loss": 0.9163697957992554, | |
| "step": 520 | |
| }, | |
| { | |
| "epoch": 1.6800000000000002, | |
| "grad_norm": 4.445755031880296, | |
| "learning_rate": 2.406743949113397e-06, | |
| "logits/chosen": -0.9282479286193848, | |
| "logits/rejected": -1.1179710626602173, | |
| "logps/chosen": -0.872515082359314, | |
| "logps/rejected": -1.3933911323547363, | |
| "loss": 0.9773, | |
| "odds_ratio_loss": 0.5091356039047241, | |
| "rewards/accuracies": 0.675000011920929, | |
| "rewards/chosen": -0.04362575337290764, | |
| "rewards/margins": 0.026043808087706566, | |
| "rewards/rejected": -0.06966955959796906, | |
| "sft_loss": 0.872515082359314, | |
| "step": 525 | |
| }, | |
| { | |
| "epoch": 1.696, | |
| "grad_norm": 3.6413598914526486, | |
| "learning_rate": 2.360156493681102e-06, | |
| "logits/chosen": -0.910436749458313, | |
| "logits/rejected": -1.0247265100479126, | |
| "logps/chosen": -0.9496439695358276, | |
| "logps/rejected": -1.3968040943145752, | |
| "loss": 0.9491, | |
| "odds_ratio_loss": 0.48868894577026367, | |
| "rewards/accuracies": 0.824999988079071, | |
| "rewards/chosen": -0.0474822036921978, | |
| "rewards/margins": 0.0223580040037632, | |
| "rewards/rejected": -0.069840207695961, | |
| "sft_loss": 0.9496439695358276, | |
| "step": 530 | |
| }, | |
| { | |
| "epoch": 1.712, | |
| "grad_norm": 4.294695857238959, | |
| "learning_rate": 2.3136177064483462e-06, | |
| "logits/chosen": -1.0878788232803345, | |
| "logits/rejected": -1.283992052078247, | |
| "logps/chosen": -1.003148078918457, | |
| "logps/rejected": -1.4421515464782715, | |
| "loss": 0.9586, | |
| "odds_ratio_loss": 0.49956098198890686, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": -0.05015740543603897, | |
| "rewards/margins": 0.021950174123048782, | |
| "rewards/rejected": -0.07210757583379745, | |
| "sft_loss": 1.003148078918457, | |
| "step": 535 | |
| }, | |
| { | |
| "epoch": 1.728, | |
| "grad_norm": 4.183928338953188, | |
| "learning_rate": 2.2671437837980943e-06, | |
| "logits/chosen": -1.0223979949951172, | |
| "logits/rejected": -0.9681426286697388, | |
| "logps/chosen": -0.8269376754760742, | |
| "logps/rejected": -1.141745924949646, | |
| "loss": 0.8999, | |
| "odds_ratio_loss": 0.5333009958267212, | |
| "rewards/accuracies": 0.6499999761581421, | |
| "rewards/chosen": -0.04134688898921013, | |
| "rewards/margins": 0.0157404113560915, | |
| "rewards/rejected": -0.05708730220794678, | |
| "sft_loss": 0.8269376754760742, | |
| "step": 540 | |
| }, | |
| { | |
| "epoch": 1.744, | |
| "grad_norm": 4.65432905249252, | |
| "learning_rate": 2.2207508995392024e-06, | |
| "logits/chosen": -0.9679327011108398, | |
| "logits/rejected": -1.0990087985992432, | |
| "logps/chosen": -0.9268666505813599, | |
| "logps/rejected": -1.2932831048965454, | |
| "loss": 0.9613, | |
| "odds_ratio_loss": 0.5248704552650452, | |
| "rewards/accuracies": 0.7250000238418579, | |
| "rewards/chosen": -0.04634333401918411, | |
| "rewards/margins": 0.018320811912417412, | |
| "rewards/rejected": -0.06466414779424667, | |
| "sft_loss": 0.9268666505813599, | |
| "step": 545 | |
| }, | |
| { | |
| "epoch": 1.76, | |
| "grad_norm": 3.9586982529561205, | |
| "learning_rate": 2.1744551992776247e-06, | |
| "logits/chosen": -1.0059576034545898, | |
| "logits/rejected": -1.0268685817718506, | |
| "logps/chosen": -0.9964353442192078, | |
| "logps/rejected": -1.6557565927505493, | |
| "loss": 0.9815, | |
| "odds_ratio_loss": 0.49777206778526306, | |
| "rewards/accuracies": 0.8500000238418579, | |
| "rewards/chosen": -0.04982176423072815, | |
| "rewards/margins": 0.032966069877147675, | |
| "rewards/rejected": -0.08278782665729523, | |
| "sft_loss": 0.9964353442192078, | |
| "step": 550 | |
| }, | |
| { | |
| "epoch": 1.776, | |
| "grad_norm": 9.562472852188419, | |
| "learning_rate": 2.1282727947974373e-06, | |
| "logits/chosen": -0.9853806495666504, | |
| "logits/rejected": -1.1351251602172852, | |
| "logps/chosen": -0.8706343770027161, | |
| "logps/rejected": -1.4279866218566895, | |
| "loss": 0.9807, | |
| "odds_ratio_loss": 0.4323902130126953, | |
| "rewards/accuracies": 0.824999988079071, | |
| "rewards/chosen": -0.043531715869903564, | |
| "rewards/margins": 0.02786761149764061, | |
| "rewards/rejected": -0.07139933109283447, | |
| "sft_loss": 0.8706343770027161, | |
| "step": 555 | |
| }, | |
| { | |
| "epoch": 1.792, | |
| "grad_norm": 4.3083896592948845, | |
| "learning_rate": 2.082219758453629e-06, | |
| "logits/chosen": -0.8532622456550598, | |
| "logits/rejected": -1.0556373596191406, | |
| "logps/chosen": -0.9265901446342468, | |
| "logps/rejected": -1.6611661911010742, | |
| "loss": 0.9808, | |
| "odds_ratio_loss": 0.47050419449806213, | |
| "rewards/accuracies": 0.7250000238418579, | |
| "rewards/chosen": -0.04632951319217682, | |
| "rewards/margins": 0.03672880306839943, | |
| "rewards/rejected": -0.08305831253528595, | |
| "sft_loss": 0.9265901446342468, | |
| "step": 560 | |
| }, | |
| { | |
| "epoch": 1.808, | |
| "grad_norm": 4.988596811372773, | |
| "learning_rate": 2.036312117578617e-06, | |
| "logits/chosen": -0.8543407320976257, | |
| "logits/rejected": -1.035388708114624, | |
| "logps/chosen": -0.9570035934448242, | |
| "logps/rejected": -1.4836640357971191, | |
| "loss": 0.9573, | |
| "odds_ratio_loss": 0.4832231104373932, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": -0.04785018041729927, | |
| "rewards/margins": 0.02633301541209221, | |
| "rewards/rejected": -0.07418319582939148, | |
| "sft_loss": 0.9570035934448242, | |
| "step": 565 | |
| }, | |
| { | |
| "epoch": 1.8239999999999998, | |
| "grad_norm": 3.4762921171914463, | |
| "learning_rate": 1.990565848904431e-06, | |
| "logits/chosen": -0.8208405375480652, | |
| "logits/rejected": -0.940590500831604, | |
| "logps/chosen": -0.8431119918823242, | |
| "logps/rejected": -1.3455002307891846, | |
| "loss": 0.9561, | |
| "odds_ratio_loss": 0.4739263653755188, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": -0.04215560108423233, | |
| "rewards/margins": 0.025119412690401077, | |
| "rewards/rejected": -0.06727501004934311, | |
| "sft_loss": 0.8431119918823242, | |
| "step": 570 | |
| }, | |
| { | |
| "epoch": 1.8399999999999999, | |
| "grad_norm": 4.955237689773966, | |
| "learning_rate": 1.9449968730025055e-06, | |
| "logits/chosen": -0.9368654489517212, | |
| "logits/rejected": -1.018002986907959, | |
| "logps/chosen": -1.0393065214157104, | |
| "logps/rejected": -1.3489172458648682, | |
| "loss": 0.9946, | |
| "odds_ratio_loss": 0.5891560316085815, | |
| "rewards/accuracies": 0.574999988079071, | |
| "rewards/chosen": -0.05196532607078552, | |
| "rewards/margins": 0.01548053603619337, | |
| "rewards/rejected": -0.06744585931301117, | |
| "sft_loss": 1.0393065214157104, | |
| "step": 575 | |
| }, | |
| { | |
| "epoch": 1.8559999999999999, | |
| "grad_norm": 4.122518133723736, | |
| "learning_rate": 1.899621048743019e-06, | |
| "logits/chosen": -0.8472858667373657, | |
| "logits/rejected": -0.9123756289482117, | |
| "logps/chosen": -0.8696669340133667, | |
| "logps/rejected": -1.3730722665786743, | |
| "loss": 0.9332, | |
| "odds_ratio_loss": 0.504239559173584, | |
| "rewards/accuracies": 0.7250000238418579, | |
| "rewards/chosen": -0.043483346700668335, | |
| "rewards/margins": 0.025170262902975082, | |
| "rewards/rejected": -0.06865362077951431, | |
| "sft_loss": 0.8696669340133667, | |
| "step": 580 | |
| }, | |
| { | |
| "epoch": 1.8719999999999999, | |
| "grad_norm": 5.090413419223807, | |
| "learning_rate": 1.854454167775706e-06, | |
| "logits/chosen": -0.8803795576095581, | |
| "logits/rejected": -1.011498212814331, | |
| "logps/chosen": -0.8195604085922241, | |
| "logps/rejected": -1.5066629648208618, | |
| "loss": 0.9053, | |
| "odds_ratio_loss": 0.4221072793006897, | |
| "rewards/accuracies": 0.7749999761581421, | |
| "rewards/chosen": -0.040978025645017624, | |
| "rewards/margins": 0.03435512259602547, | |
| "rewards/rejected": -0.07533314824104309, | |
| "sft_loss": 0.8195604085922241, | |
| "step": 585 | |
| }, | |
| { | |
| "epoch": 1.888, | |
| "grad_norm": 3.962979618406243, | |
| "learning_rate": 1.8095119490340618e-06, | |
| "logits/chosen": -0.9934213757514954, | |
| "logits/rejected": -1.028438925743103, | |
| "logps/chosen": -0.9060677289962769, | |
| "logps/rejected": -1.4857866764068604, | |
| "loss": 0.9753, | |
| "odds_ratio_loss": 0.6394127011299133, | |
| "rewards/accuracies": 0.675000011920929, | |
| "rewards/chosen": -0.04530338943004608, | |
| "rewards/margins": 0.028985943645238876, | |
| "rewards/rejected": -0.07428933680057526, | |
| "sft_loss": 0.9060677289962769, | |
| "step": 590 | |
| }, | |
| { | |
| "epoch": 1.904, | |
| "grad_norm": 4.384891130460857, | |
| "learning_rate": 1.764810033264852e-06, | |
| "logits/chosen": -0.907455563545227, | |
| "logits/rejected": -1.0589368343353271, | |
| "logps/chosen": -1.0206191539764404, | |
| "logps/rejected": -1.4559767246246338, | |
| "loss": 0.9771, | |
| "odds_ratio_loss": 0.5931872129440308, | |
| "rewards/accuracies": 0.7250000238418579, | |
| "rewards/chosen": -0.0510309636592865, | |
| "rewards/margins": 0.02176786959171295, | |
| "rewards/rejected": -0.07279883325099945, | |
| "sft_loss": 1.0206191539764404, | |
| "step": 595 | |
| }, | |
| { | |
| "epoch": 1.92, | |
| "grad_norm": 3.4134689622305383, | |
| "learning_rate": 1.7203639775848423e-06, | |
| "logits/chosen": -1.0145207643508911, | |
| "logits/rejected": -0.9972650408744812, | |
| "logps/chosen": -0.956381618976593, | |
| "logps/rejected": -1.3823316097259521, | |
| "loss": 0.9631, | |
| "odds_ratio_loss": 0.5243911743164062, | |
| "rewards/accuracies": 0.675000011920929, | |
| "rewards/chosen": -0.04781908541917801, | |
| "rewards/margins": 0.021297505125403404, | |
| "rewards/rejected": -0.06911659240722656, | |
| "sft_loss": 0.956381618976593, | |
| "step": 600 | |
| }, | |
| { | |
| "epoch": 1.936, | |
| "grad_norm": 4.505148742615646, | |
| "learning_rate": 1.6761892500666195e-06, | |
| "logits/chosen": -1.023581624031067, | |
| "logits/rejected": -1.027942419052124, | |
| "logps/chosen": -0.933574378490448, | |
| "logps/rejected": -1.2963403463363647, | |
| "loss": 0.9786, | |
| "odds_ratio_loss": 0.5001336336135864, | |
| "rewards/accuracies": 0.7749999761581421, | |
| "rewards/chosen": -0.04667872190475464, | |
| "rewards/margins": 0.018138296902179718, | |
| "rewards/rejected": -0.06481701880693436, | |
| "sft_loss": 0.933574378490448, | |
| "step": 605 | |
| }, | |
| { | |
| "epoch": 1.952, | |
| "grad_norm": 4.83889942986056, | |
| "learning_rate": 1.632301224355411e-06, | |
| "logits/chosen": -0.8887473940849304, | |
| "logits/rejected": -0.9829289317131042, | |
| "logps/chosen": -1.0025461912155151, | |
| "logps/rejected": -1.2573713064193726, | |
| "loss": 0.9546, | |
| "odds_ratio_loss": 0.6103852987289429, | |
| "rewards/accuracies": 0.6499999761581421, | |
| "rewards/chosen": -0.0501273088157177, | |
| "rewards/margins": 0.012741250917315483, | |
| "rewards/rejected": -0.06286855787038803, | |
| "sft_loss": 1.0025461912155151, | |
| "step": 610 | |
| }, | |
| { | |
| "epoch": 1.968, | |
| "grad_norm": 4.517273923883564, | |
| "learning_rate": 1.5887151743187634e-06, | |
| "logits/chosen": -0.9073827862739563, | |
| "logits/rejected": -1.0887978076934814, | |
| "logps/chosen": -0.9604595899581909, | |
| "logps/rejected": -1.4289729595184326, | |
| "loss": 0.9811, | |
| "odds_ratio_loss": 0.545965850353241, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": -0.048022981733083725, | |
| "rewards/margins": 0.023425664752721786, | |
| "rewards/rejected": -0.07144864648580551, | |
| "sft_loss": 0.9604595899581909, | |
| "step": 615 | |
| }, | |
| { | |
| "epoch": 1.984, | |
| "grad_norm": 3.1757477627169908, | |
| "learning_rate": 1.5454462687309445e-06, | |
| "logits/chosen": -1.015500783920288, | |
| "logits/rejected": -1.0785764455795288, | |
| "logps/chosen": -0.9297005534172058, | |
| "logps/rejected": -1.5386453866958618, | |
| "loss": 0.9563, | |
| "odds_ratio_loss": 0.542974054813385, | |
| "rewards/accuracies": 0.7250000238418579, | |
| "rewards/chosen": -0.04648502543568611, | |
| "rewards/margins": 0.03044723905622959, | |
| "rewards/rejected": -0.07693226635456085, | |
| "sft_loss": 0.9297005534172058, | |
| "step": 620 | |
| }, | |
| { | |
| "epoch": 2.0, | |
| "grad_norm": 4.242978905390128, | |
| "learning_rate": 1.502509565993925e-06, | |
| "logits/chosen": -0.9173771142959595, | |
| "logits/rejected": -1.0122209787368774, | |
| "logps/chosen": -0.9267832040786743, | |
| "logps/rejected": -1.5153340101242065, | |
| "loss": 0.8898, | |
| "odds_ratio_loss": 0.45579010248184204, | |
| "rewards/accuracies": 0.824999988079071, | |
| "rewards/chosen": -0.04633915796875954, | |
| "rewards/margins": 0.02942754328250885, | |
| "rewards/rejected": -0.07576669752597809, | |
| "sft_loss": 0.9267832040786743, | |
| "step": 625 | |
| }, | |
| { | |
| "epoch": 2.016, | |
| "grad_norm": 3.553798975041404, | |
| "learning_rate": 1.4599200088967652e-06, | |
| "logits/chosen": -0.8619723320007324, | |
| "logits/rejected": -1.0356481075286865, | |
| "logps/chosen": -0.8099665641784668, | |
| "logps/rejected": -1.3428175449371338, | |
| "loss": 0.8054, | |
| "odds_ratio_loss": 0.42676934599876404, | |
| "rewards/accuracies": 0.7749999761581421, | |
| "rewards/chosen": -0.04049833118915558, | |
| "rewards/margins": 0.02664254978299141, | |
| "rewards/rejected": -0.06714087724685669, | |
| "sft_loss": 0.8099665641784668, | |
| "step": 630 | |
| }, | |
| { | |
| "epoch": 2.032, | |
| "grad_norm": 4.569902596370669, | |
| "learning_rate": 1.4176924194152405e-06, | |
| "logits/chosen": -0.9947516322135925, | |
| "logits/rejected": -1.0760139226913452, | |
| "logps/chosen": -0.822666347026825, | |
| "logps/rejected": -1.4580928087234497, | |
| "loss": 0.8194, | |
| "odds_ratio_loss": 0.408845990896225, | |
| "rewards/accuracies": 0.824999988079071, | |
| "rewards/chosen": -0.041133321821689606, | |
| "rewards/margins": 0.03177132084965706, | |
| "rewards/rejected": -0.07290463894605637, | |
| "sft_loss": 0.822666347026825, | |
| "step": 635 | |
| }, | |
| { | |
| "epoch": 2.048, | |
| "grad_norm": 3.58850101082042, | |
| "learning_rate": 1.3758414935535147e-06, | |
| "logits/chosen": -0.8080762624740601, | |
| "logits/rejected": -0.8731328845024109, | |
| "logps/chosen": -0.6912453174591064, | |
| "logps/rejected": -1.2947412729263306, | |
| "loss": 0.7991, | |
| "odds_ratio_loss": 0.39434918761253357, | |
| "rewards/accuracies": 0.875, | |
| "rewards/chosen": -0.03456226736307144, | |
| "rewards/margins": 0.030174797400832176, | |
| "rewards/rejected": -0.06473706662654877, | |
| "sft_loss": 0.6912453174591064, | |
| "step": 640 | |
| }, | |
| { | |
| "epoch": 2.064, | |
| "grad_norm": 3.1095303302424204, | |
| "learning_rate": 1.3343817962296485e-06, | |
| "logits/chosen": -0.9426826238632202, | |
| "logits/rejected": -0.9611290097236633, | |
| "logps/chosen": -0.7455964088439941, | |
| "logps/rejected": -1.3733656406402588, | |
| "loss": 0.8219, | |
| "odds_ratio_loss": 0.400754451751709, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": -0.037279821932315826, | |
| "rewards/margins": 0.03138846904039383, | |
| "rewards/rejected": -0.06866829097270966, | |
| "sft_loss": 0.7455964088439941, | |
| "step": 645 | |
| }, | |
| { | |
| "epoch": 2.08, | |
| "grad_norm": 3.045256483819498, | |
| "learning_rate": 1.293327756206729e-06, | |
| "logits/chosen": -0.8983833193778992, | |
| "logits/rejected": -0.9661597013473511, | |
| "logps/chosen": -0.6603522300720215, | |
| "logps/rejected": -1.5160930156707764, | |
| "loss": 0.7579, | |
| "odds_ratio_loss": 0.3125734031200409, | |
| "rewards/accuracies": 0.8999999761581421, | |
| "rewards/chosen": -0.033017612993717194, | |
| "rewards/margins": 0.042787034064531326, | |
| "rewards/rejected": -0.07580464333295822, | |
| "sft_loss": 0.6603522300720215, | |
| "step": 650 | |
| }, | |
| { | |
| "epoch": 2.096, | |
| "grad_norm": 3.1717779800652592, | |
| "learning_rate": 1.252693661071391e-06, | |
| "logits/chosen": -1.157251238822937, | |
| "logits/rejected": -1.0623748302459717, | |
| "logps/chosen": -0.7086284160614014, | |
| "logps/rejected": -1.514593243598938, | |
| "loss": 0.7865, | |
| "odds_ratio_loss": 0.33622443675994873, | |
| "rewards/accuracies": 0.8500000238418579, | |
| "rewards/chosen": -0.035431426018476486, | |
| "rewards/margins": 0.04029824212193489, | |
| "rewards/rejected": -0.07572966068983078, | |
| "sft_loss": 0.7086284160614014, | |
| "step": 655 | |
| }, | |
| { | |
| "epoch": 2.112, | |
| "grad_norm": 2.7205852811983093, | |
| "learning_rate": 1.2124936522614622e-06, | |
| "logits/chosen": -1.0289185047149658, | |
| "logits/rejected": -1.098934292793274, | |
| "logps/chosen": -0.7381170988082886, | |
| "logps/rejected": -1.4106049537658691, | |
| "loss": 0.7903, | |
| "odds_ratio_loss": 0.45428943634033203, | |
| "rewards/accuracies": 0.7250000238418579, | |
| "rewards/chosen": -0.03690585121512413, | |
| "rewards/margins": 0.033624399453401566, | |
| "rewards/rejected": -0.0705302506685257, | |
| "sft_loss": 0.7381170988082886, | |
| "step": 660 | |
| }, | |
| { | |
| "epoch": 2.128, | |
| "grad_norm": 2.8528967143088666, | |
| "learning_rate": 1.1727417201444735e-06, | |
| "logits/chosen": -1.0716520547866821, | |
| "logits/rejected": -1.0720608234405518, | |
| "logps/chosen": -0.7356234788894653, | |
| "logps/rejected": -1.2771320343017578, | |
| "loss": 0.797, | |
| "odds_ratio_loss": 0.41821280121803284, | |
| "rewards/accuracies": 0.7749999761581421, | |
| "rewards/chosen": -0.036781176924705505, | |
| "rewards/margins": 0.027075421065092087, | |
| "rewards/rejected": -0.0638565942645073, | |
| "sft_loss": 0.7356234788894653, | |
| "step": 665 | |
| }, | |
| { | |
| "epoch": 2.144, | |
| "grad_norm": 3.7706399394026615, | |
| "learning_rate": 1.1334516991487473e-06, | |
| "logits/chosen": -0.7660879492759705, | |
| "logits/rejected": -1.0078188180923462, | |
| "logps/chosen": -0.7483173608779907, | |
| "logps/rejected": -1.2434841394424438, | |
| "loss": 0.7863, | |
| "odds_ratio_loss": 0.3927895426750183, | |
| "rewards/accuracies": 0.8999999761581421, | |
| "rewards/chosen": -0.037415869534015656, | |
| "rewards/margins": 0.02475833147764206, | |
| "rewards/rejected": -0.062174201011657715, | |
| "sft_loss": 0.7483173608779907, | |
| "step": 670 | |
| }, | |
| { | |
| "epoch": 2.16, | |
| "grad_norm": 3.5703943571070482, | |
| "learning_rate": 1.094637262948749e-06, | |
| "logits/chosen": -0.9008601307868958, | |
| "logits/rejected": -1.0412832498550415, | |
| "logps/chosen": -0.821381688117981, | |
| "logps/rejected": -1.3886059522628784, | |
| "loss": 0.7675, | |
| "odds_ratio_loss": 0.4320302903652191, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": -0.04106908664107323, | |
| "rewards/margins": 0.02836121991276741, | |
| "rewards/rejected": -0.06943030655384064, | |
| "sft_loss": 0.821381688117981, | |
| "step": 675 | |
| }, | |
| { | |
| "epoch": 2.176, | |
| "grad_norm": 2.9837572151294065, | |
| "learning_rate": 1.0563119197063934e-06, | |
| "logits/chosen": -1.058257818222046, | |
| "logits/rejected": -1.0895675420761108, | |
| "logps/chosen": -0.914572536945343, | |
| "logps/rejected": -1.3927881717681885, | |
| "loss": 0.7969, | |
| "odds_ratio_loss": 0.46075814962387085, | |
| "rewards/accuracies": 0.7749999761581421, | |
| "rewards/chosen": -0.04572862759232521, | |
| "rewards/margins": 0.023910781368613243, | |
| "rewards/rejected": -0.0696394145488739, | |
| "sft_loss": 0.914572536945343, | |
| "step": 680 | |
| }, | |
| { | |
| "epoch": 2.192, | |
| "grad_norm": 3.2497337072742414, | |
| "learning_rate": 1.018489007369945e-06, | |
| "logits/chosen": -0.8853160738945007, | |
| "logits/rejected": -0.8999375104904175, | |
| "logps/chosen": -0.7777100801467896, | |
| "logps/rejected": -1.448999047279358, | |
| "loss": 0.804, | |
| "odds_ratio_loss": 0.39251843094825745, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": -0.03888550028204918, | |
| "rewards/margins": 0.03356444090604782, | |
| "rewards/rejected": -0.0724499449133873, | |
| "sft_loss": 0.7777100801467896, | |
| "step": 685 | |
| }, | |
| { | |
| "epoch": 2.208, | |
| "grad_norm": 3.029373367507165, | |
| "learning_rate": 9.81181689032158e-07, | |
| "logits/chosen": -0.8918782472610474, | |
| "logits/rejected": -0.8753920793533325, | |
| "logps/chosen": -0.6909304857254028, | |
| "logps/rejected": -1.2901936769485474, | |
| "loss": 0.7751, | |
| "odds_ratio_loss": 0.4143063426017761, | |
| "rewards/accuracies": 0.7749999761581421, | |
| "rewards/chosen": -0.03454653173685074, | |
| "rewards/margins": 0.029963159933686256, | |
| "rewards/rejected": -0.06450968980789185, | |
| "sft_loss": 0.6909304857254028, | |
| "step": 690 | |
| }, | |
| { | |
| "epoch": 2.224, | |
| "grad_norm": 4.5746404005970405, | |
| "learning_rate": 9.444029483492703e-07, | |
| "logits/chosen": -1.0063061714172363, | |
| "logits/rejected": -1.0239533185958862, | |
| "logps/chosen": -0.6867491006851196, | |
| "logps/rejected": -1.245091199874878, | |
| "loss": 0.7732, | |
| "odds_ratio_loss": 0.4617982804775238, | |
| "rewards/accuracies": 0.7749999761581421, | |
| "rewards/chosen": -0.03433745354413986, | |
| "rewards/margins": 0.027917101979255676, | |
| "rewards/rejected": -0.06225455552339554, | |
| "sft_loss": 0.6867491006851196, | |
| "step": 695 | |
| }, | |
| { | |
| "epoch": 2.24, | |
| "grad_norm": 3.694255891644154, | |
| "learning_rate": 9.081655850224449e-07, | |
| "logits/chosen": -1.0207760334014893, | |
| "logits/rejected": -1.1230969429016113, | |
| "logps/chosen": -0.8350968360900879, | |
| "logps/rejected": -1.5500547885894775, | |
| "loss": 0.7583, | |
| "odds_ratio_loss": 0.42910847067832947, | |
| "rewards/accuracies": 0.8500000238418579, | |
| "rewards/chosen": -0.041754838079214096, | |
| "rewards/margins": 0.03574790060520172, | |
| "rewards/rejected": -0.07750274240970612, | |
| "sft_loss": 0.8350968360900879, | |
| "step": 700 | |
| }, | |
| { | |
| "epoch": 2.2560000000000002, | |
| "grad_norm": 3.994487822092531, | |
| "learning_rate": 8.724822103432282e-07, | |
| "logits/chosen": -0.8827289342880249, | |
| "logits/rejected": -0.9654695391654968, | |
| "logps/chosen": -0.7088466286659241, | |
| "logps/rejected": -1.281082034111023, | |
| "loss": 0.7482, | |
| "odds_ratio_loss": 0.3990407884120941, | |
| "rewards/accuracies": 0.875, | |
| "rewards/chosen": -0.035442329943180084, | |
| "rewards/margins": 0.028611773625016212, | |
| "rewards/rejected": -0.06405410915613174, | |
| "sft_loss": 0.7088466286659241, | |
| "step": 705 | |
| }, | |
| { | |
| "epoch": 2.2720000000000002, | |
| "grad_norm": 3.3050191320581166, | |
| "learning_rate": 8.373652428045831e-07, | |
| "logits/chosen": -1.0878264904022217, | |
| "logits/rejected": -1.0405784845352173, | |
| "logps/chosen": -0.65968257188797, | |
| "logps/rejected": -1.3744075298309326, | |
| "loss": 0.7722, | |
| "odds_ratio_loss": 0.3181590139865875, | |
| "rewards/accuracies": 0.949999988079071, | |
| "rewards/chosen": -0.03298413008451462, | |
| "rewards/margins": 0.03573625162243843, | |
| "rewards/rejected": -0.06872038543224335, | |
| "sft_loss": 0.65968257188797, | |
| "step": 710 | |
| }, | |
| { | |
| "epoch": 2.288, | |
| "grad_norm": 3.8117555833594943, | |
| "learning_rate": 8.028269037790171e-07, | |
| "logits/chosen": -0.8991669416427612, | |
| "logits/rejected": -0.9795140027999878, | |
| "logps/chosen": -0.8547693490982056, | |
| "logps/rejected": -1.5010745525360107, | |
| "loss": 0.7688, | |
| "odds_ratio_loss": 0.42176875472068787, | |
| "rewards/accuracies": 0.7749999761581421, | |
| "rewards/chosen": -0.04273846372961998, | |
| "rewards/margins": 0.03231526166200638, | |
| "rewards/rejected": -0.07505372911691666, | |
| "sft_loss": 0.8547693490982056, | |
| "step": 715 | |
| }, | |
| { | |
| "epoch": 2.304, | |
| "grad_norm": 6.060881007391729, | |
| "learning_rate": 7.688792132653111e-07, | |
| "logits/chosen": -0.8518077731132507, | |
| "logits/rejected": -0.9652704000473022, | |
| "logps/chosen": -0.7316969633102417, | |
| "logps/rejected": -1.5251853466033936, | |
| "loss": 0.7621, | |
| "odds_ratio_loss": 0.3367399573326111, | |
| "rewards/accuracies": 0.8500000238418579, | |
| "rewards/chosen": -0.03658485412597656, | |
| "rewards/margins": 0.03967442363500595, | |
| "rewards/rejected": -0.07625927031040192, | |
| "sft_loss": 0.7316969633102417, | |
| "step": 720 | |
| }, | |
| { | |
| "epoch": 2.32, | |
| "grad_norm": 2.9281573250934363, | |
| "learning_rate": 7.355339857053318e-07, | |
| "logits/chosen": -0.8477774858474731, | |
| "logits/rejected": -1.018747329711914, | |
| "logps/chosen": -0.7693859934806824, | |
| "logps/rejected": -1.3998419046401978, | |
| "loss": 0.7714, | |
| "odds_ratio_loss": 0.3711521625518799, | |
| "rewards/accuracies": 0.875, | |
| "rewards/chosen": -0.03846929594874382, | |
| "rewards/margins": 0.031522803008556366, | |
| "rewards/rejected": -0.06999210268259048, | |
| "sft_loss": 0.7693859934806824, | |
| "step": 725 | |
| }, | |
| { | |
| "epoch": 2.336, | |
| "grad_norm": 3.3575685063670213, | |
| "learning_rate": 7.028028258723818e-07, | |
| "logits/chosen": -0.9588859677314758, | |
| "logits/rejected": -1.0327568054199219, | |
| "logps/chosen": -0.7077163457870483, | |
| "logps/rejected": -1.3171709775924683, | |
| "loss": 0.7932, | |
| "odds_ratio_loss": 0.3643357455730438, | |
| "rewards/accuracies": 0.8500000238418579, | |
| "rewards/chosen": -0.03538581356406212, | |
| "rewards/margins": 0.030472736805677414, | |
| "rewards/rejected": -0.06585855782032013, | |
| "sft_loss": 0.7077163457870483, | |
| "step": 730 | |
| }, | |
| { | |
| "epoch": 2.352, | |
| "grad_norm": 3.07955633424655, | |
| "learning_rate": 6.706971248325151e-07, | |
| "logits/chosen": -0.8883659243583679, | |
| "logits/rejected": -1.0441168546676636, | |
| "logps/chosen": -0.8074674606323242, | |
| "logps/rejected": -1.4699620008468628, | |
| "loss": 0.776, | |
| "odds_ratio_loss": 0.39879804849624634, | |
| "rewards/accuracies": 0.824999988079071, | |
| "rewards/chosen": -0.04037337377667427, | |
| "rewards/margins": 0.033124733716249466, | |
| "rewards/rejected": -0.07349809259176254, | |
| "sft_loss": 0.8074674606323242, | |
| "step": 735 | |
| }, | |
| { | |
| "epoch": 2.368, | |
| "grad_norm": 3.6213614434043593, | |
| "learning_rate": 6.392280559802341e-07, | |
| "logits/chosen": -0.9162646532058716, | |
| "logits/rejected": -0.9279802441596985, | |
| "logps/chosen": -0.7218812704086304, | |
| "logps/rejected": -1.4897241592407227, | |
| "loss": 0.7637, | |
| "odds_ratio_loss": 0.411416620016098, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": -0.0360940620303154, | |
| "rewards/margins": 0.03839214891195297, | |
| "rewards/rejected": -0.07448621094226837, | |
| "sft_loss": 0.7218812704086304, | |
| "step": 740 | |
| }, | |
| { | |
| "epoch": 2.384, | |
| "grad_norm": 4.1017905530653245, | |
| "learning_rate": 6.084065711499326e-07, | |
| "logits/chosen": -0.84368896484375, | |
| "logits/rejected": -0.9179951548576355, | |
| "logps/chosen": -0.6008902192115784, | |
| "logps/rejected": -1.4554692506790161, | |
| "loss": 0.7279, | |
| "odds_ratio_loss": 0.3157138526439667, | |
| "rewards/accuracies": 0.925000011920929, | |
| "rewards/chosen": -0.030044514685869217, | |
| "rewards/margins": 0.04272894933819771, | |
| "rewards/rejected": -0.07277346402406693, | |
| "sft_loss": 0.6008902192115784, | |
| "step": 745 | |
| }, | |
| { | |
| "epoch": 2.4, | |
| "grad_norm": 3.8487814271047758, | |
| "learning_rate": 5.782433968044495e-07, | |
| "logits/chosen": -0.8399477005004883, | |
| "logits/rejected": -0.9413140416145325, | |
| "logps/chosen": -0.8139437437057495, | |
| "logps/rejected": -1.5214980840682983, | |
| "loss": 0.7662, | |
| "odds_ratio_loss": 0.5000792741775513, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": -0.040697190910577774, | |
| "rewards/margins": 0.03537772223353386, | |
| "rewards/rejected": -0.07607491314411163, | |
| "sft_loss": 0.8139437437057495, | |
| "step": 750 | |
| }, | |
| { | |
| "epoch": 2.416, | |
| "grad_norm": 3.305488495552487, | |
| "learning_rate": 5.487490303020576e-07, | |
| "logits/chosen": -0.9991554021835327, | |
| "logits/rejected": -0.944214940071106, | |
| "logps/chosen": -0.7551851272583008, | |
| "logps/rejected": -1.2736141681671143, | |
| "loss": 0.7806, | |
| "odds_ratio_loss": 0.4188924729824066, | |
| "rewards/accuracies": 0.8500000238418579, | |
| "rewards/chosen": -0.03775925561785698, | |
| "rewards/margins": 0.025921454653143883, | |
| "rewards/rejected": -0.06368071585893631, | |
| "sft_loss": 0.7551851272583008, | |
| "step": 755 | |
| }, | |
| { | |
| "epoch": 2.432, | |
| "grad_norm": 3.6380522040854566, | |
| "learning_rate": 5.199337362431792e-07, | |
| "logits/chosen": -0.8715981245040894, | |
| "logits/rejected": -1.0095371007919312, | |
| "logps/chosen": -0.716741681098938, | |
| "logps/rejected": -1.7061433792114258, | |
| "loss": 0.7998, | |
| "odds_ratio_loss": 0.40209442377090454, | |
| "rewards/accuracies": 0.8500000238418579, | |
| "rewards/chosen": -0.0358370803296566, | |
| "rewards/margins": 0.04947008565068245, | |
| "rewards/rejected": -0.08530716598033905, | |
| "sft_loss": 0.716741681098938, | |
| "step": 760 | |
| }, | |
| { | |
| "epoch": 2.448, | |
| "grad_norm": 4.799248860716653, | |
| "learning_rate": 4.918075428981148e-07, | |
| "logits/chosen": -0.9698484539985657, | |
| "logits/rejected": -1.0252515077590942, | |
| "logps/chosen": -0.7276674509048462, | |
| "logps/rejected": -1.2364145517349243, | |
| "loss": 0.7519, | |
| "odds_ratio_loss": 0.4512792229652405, | |
| "rewards/accuracies": 0.7250000238418579, | |
| "rewards/chosen": -0.03638336807489395, | |
| "rewards/margins": 0.025437360629439354, | |
| "rewards/rejected": -0.06182073429226875, | |
| "sft_loss": 0.7276674509048462, | |
| "step": 765 | |
| }, | |
| { | |
| "epoch": 2.464, | |
| "grad_norm": 3.975888652914653, | |
| "learning_rate": 4.643802387170118e-07, | |
| "logits/chosen": -0.784801185131073, | |
| "logits/rejected": -0.8751401901245117, | |
| "logps/chosen": -0.736864447593689, | |
| "logps/rejected": -1.4113752841949463, | |
| "loss": 0.7689, | |
| "odds_ratio_loss": 0.41703367233276367, | |
| "rewards/accuracies": 0.875, | |
| "rewards/chosen": -0.03684321790933609, | |
| "rewards/margins": 0.0337255522608757, | |
| "rewards/rejected": -0.07056877762079239, | |
| "sft_loss": 0.736864447593689, | |
| "step": 770 | |
| }, | |
| { | |
| "epoch": 2.48, | |
| "grad_norm": 3.7017186080610873, | |
| "learning_rate": 4.376613689232978e-07, | |
| "logits/chosen": -0.9608728289604187, | |
| "logits/rejected": -1.000018835067749, | |
| "logps/chosen": -0.8199416399002075, | |
| "logps/rejected": -1.469847321510315, | |
| "loss": 0.7588, | |
| "odds_ratio_loss": 0.43195098638534546, | |
| "rewards/accuracies": 0.7749999761581421, | |
| "rewards/chosen": -0.04099708050489426, | |
| "rewards/margins": 0.03249528631567955, | |
| "rewards/rejected": -0.0734923705458641, | |
| "sft_loss": 0.8199416399002075, | |
| "step": 775 | |
| }, | |
| { | |
| "epoch": 2.496, | |
| "grad_norm": 3.1981116471330764, | |
| "learning_rate": 4.1166023219176176e-07, | |
| "logits/chosen": -0.8377124667167664, | |
| "logits/rejected": -1.0116181373596191, | |
| "logps/chosen": -0.7824350595474243, | |
| "logps/rejected": -1.6547685861587524, | |
| "loss": 0.7523, | |
| "odds_ratio_loss": 0.32834020256996155, | |
| "rewards/accuracies": 0.824999988079071, | |
| "rewards/chosen": -0.039121754467487335, | |
| "rewards/margins": 0.043616678565740585, | |
| "rewards/rejected": -0.08273844420909882, | |
| "sft_loss": 0.7824350595474243, | |
| "step": 780 | |
| }, | |
| { | |
| "epoch": 2.512, | |
| "grad_norm": 3.7082914287151945, | |
| "learning_rate": 3.863858774124385e-07, | |
| "logits/chosen": -1.0026713609695435, | |
| "logits/rejected": -1.0367608070373535, | |
| "logps/chosen": -0.6959080696105957, | |
| "logps/rejected": -1.4358288049697876, | |
| "loss": 0.7563, | |
| "odds_ratio_loss": 0.3291171193122864, | |
| "rewards/accuracies": 0.925000011920929, | |
| "rewards/chosen": -0.03479539602994919, | |
| "rewards/margins": 0.036996036767959595, | |
| "rewards/rejected": -0.07179144024848938, | |
| "sft_loss": 0.6959080696105957, | |
| "step": 785 | |
| }, | |
| { | |
| "epoch": 2.528, | |
| "grad_norm": 2.9819183882511004, | |
| "learning_rate": 3.618471005414215e-07, | |
| "logits/chosen": -1.1593316793441772, | |
| "logits/rejected": -1.1186132431030273, | |
| "logps/chosen": -0.5521804094314575, | |
| "logps/rejected": -1.3308180570602417, | |
| "loss": 0.7382, | |
| "odds_ratio_loss": 0.40159231424331665, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": -0.027609020471572876, | |
| "rewards/margins": 0.03893188014626503, | |
| "rewards/rejected": -0.0665409117937088, | |
| "sft_loss": 0.5521804094314575, | |
| "step": 790 | |
| }, | |
| { | |
| "epoch": 2.544, | |
| "grad_norm": 3.3567110818810755, | |
| "learning_rate": 3.380524415397049e-07, | |
| "logits/chosen": -0.8402732014656067, | |
| "logits/rejected": -1.0374484062194824, | |
| "logps/chosen": -0.803017795085907, | |
| "logps/rejected": -1.2986352443695068, | |
| "loss": 0.7742, | |
| "odds_ratio_loss": 0.46145597100257874, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": -0.04015089198946953, | |
| "rewards/margins": 0.0247808750718832, | |
| "rewards/rejected": -0.06493176519870758, | |
| "sft_loss": 0.803017795085907, | |
| "step": 795 | |
| }, | |
| { | |
| "epoch": 2.56, | |
| "grad_norm": 3.3420577148113924, | |
| "learning_rate": 3.150101814011136e-07, | |
| "logits/chosen": -1.031750202178955, | |
| "logits/rejected": -1.02663254737854, | |
| "logps/chosen": -0.7611913681030273, | |
| "logps/rejected": -1.1853117942810059, | |
| "loss": 0.7785, | |
| "odds_ratio_loss": 0.5039734244346619, | |
| "rewards/accuracies": 0.7749999761581421, | |
| "rewards/chosen": -0.03805956989526749, | |
| "rewards/margins": 0.021206015720963478, | |
| "rewards/rejected": -0.05926559120416641, | |
| "sft_loss": 0.7611913681030273, | |
| "step": 800 | |
| }, | |
| { | |
| "epoch": 2.576, | |
| "grad_norm": 3.246898163699306, | |
| "learning_rate": 2.927283392703606e-07, | |
| "logits/chosen": -0.8511770367622375, | |
| "logits/rejected": -0.9470258951187134, | |
| "logps/chosen": -0.7670743465423584, | |
| "logps/rejected": -1.1585172414779663, | |
| "loss": 0.7977, | |
| "odds_ratio_loss": 0.4781486988067627, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": -0.03835371881723404, | |
| "rewards/margins": 0.019572149962186813, | |
| "rewards/rejected": -0.057925861328840256, | |
| "sft_loss": 0.7670743465423584, | |
| "step": 805 | |
| }, | |
| { | |
| "epoch": 2.592, | |
| "grad_norm": 3.9992460548702438, | |
| "learning_rate": 2.712146696522305e-07, | |
| "logits/chosen": -1.022859811782837, | |
| "logits/rejected": -1.129640817642212, | |
| "logps/chosen": -0.716964840888977, | |
| "logps/rejected": -1.246201992034912, | |
| "loss": 0.7552, | |
| "odds_ratio_loss": 0.4771757125854492, | |
| "rewards/accuracies": 0.824999988079071, | |
| "rewards/chosen": -0.03584824129939079, | |
| "rewards/margins": 0.026461860164999962, | |
| "rewards/rejected": -0.062310099601745605, | |
| "sft_loss": 0.716964840888977, | |
| "step": 810 | |
| }, | |
| { | |
| "epoch": 2.608, | |
| "grad_norm": 3.5945402118846403, | |
| "learning_rate": 2.504766597128666e-07, | |
| "logits/chosen": -0.9580585360527039, | |
| "logits/rejected": -0.9554399251937866, | |
| "logps/chosen": -0.8108547925949097, | |
| "logps/rejected": -1.315863847732544, | |
| "loss": 0.7575, | |
| "odds_ratio_loss": 0.44800853729248047, | |
| "rewards/accuracies": 0.7749999761581421, | |
| "rewards/chosen": -0.040542736649513245, | |
| "rewards/margins": 0.025250453501939774, | |
| "rewards/rejected": -0.06579320132732391, | |
| "sft_loss": 0.8108547925949097, | |
| "step": 815 | |
| }, | |
| { | |
| "epoch": 2.624, | |
| "grad_norm": 3.288445101598415, | |
| "learning_rate": 2.3052152667409289e-07, | |
| "logits/chosen": -0.9488247036933899, | |
| "logits/rejected": -0.8991857767105103, | |
| "logps/chosen": -0.7179723978042603, | |
| "logps/rejected": -1.3861563205718994, | |
| "loss": 0.7752, | |
| "odds_ratio_loss": 0.3779214322566986, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": -0.03589861840009689, | |
| "rewards/margins": 0.03340919688344002, | |
| "rewards/rejected": -0.06930781900882721, | |
| "sft_loss": 0.7179723978042603, | |
| "step": 820 | |
| }, | |
| { | |
| "epoch": 2.64, | |
| "grad_norm": 4.333903778456222, | |
| "learning_rate": 2.1135621530168488e-07, | |
| "logits/chosen": -1.028704285621643, | |
| "logits/rejected": -0.9637089967727661, | |
| "logps/chosen": -0.6309131383895874, | |
| "logps/rejected": -1.2894703149795532, | |
| "loss": 0.7935, | |
| "odds_ratio_loss": 0.3709859549999237, | |
| "rewards/accuracies": 0.875, | |
| "rewards/chosen": -0.03154565393924713, | |
| "rewards/margins": 0.03292786702513695, | |
| "rewards/rejected": -0.06447352468967438, | |
| "sft_loss": 0.6309131383895874, | |
| "step": 825 | |
| }, | |
| { | |
| "epoch": 2.656, | |
| "grad_norm": 3.1671393797255414, | |
| "learning_rate": 1.9298739548845813e-07, | |
| "logits/chosen": -0.9645661115646362, | |
| "logits/rejected": -0.9816401600837708, | |
| "logps/chosen": -0.8160637021064758, | |
| "logps/rejected": -1.374236822128296, | |
| "loss": 0.8091, | |
| "odds_ratio_loss": 0.4034562110900879, | |
| "rewards/accuracies": 0.8500000238418579, | |
| "rewards/chosen": -0.04080318659543991, | |
| "rewards/margins": 0.027908671647310257, | |
| "rewards/rejected": -0.06871185451745987, | |
| "sft_loss": 0.8160637021064758, | |
| "step": 830 | |
| }, | |
| { | |
| "epoch": 2.672, | |
| "grad_norm": 3.093675577889326, | |
| "learning_rate": 1.7542145993301896e-07, | |
| "logits/chosen": -0.9770193099975586, | |
| "logits/rejected": -1.0430421829223633, | |
| "logps/chosen": -0.6873968839645386, | |
| "logps/rejected": -1.183712124824524, | |
| "loss": 0.7636, | |
| "odds_ratio_loss": 0.42723220586776733, | |
| "rewards/accuracies": 0.824999988079071, | |
| "rewards/chosen": -0.03436984866857529, | |
| "rewards/margins": 0.02481575682759285, | |
| "rewards/rejected": -0.05918560549616814, | |
| "sft_loss": 0.6873968839645386, | |
| "step": 835 | |
| }, | |
| { | |
| "epoch": 2.6879999999999997, | |
| "grad_norm": 3.044985818298562, | |
| "learning_rate": 1.5866452191498488e-07, | |
| "logits/chosen": -0.8661419153213501, | |
| "logits/rejected": -1.0225311517715454, | |
| "logps/chosen": -0.7636631727218628, | |
| "logps/rejected": -1.324355959892273, | |
| "loss": 0.7742, | |
| "odds_ratio_loss": 0.3950451910495758, | |
| "rewards/accuracies": 0.8500000238418579, | |
| "rewards/chosen": -0.03818316012620926, | |
| "rewards/margins": 0.028034633025527, | |
| "rewards/rejected": -0.06621779501438141, | |
| "sft_loss": 0.7636631727218628, | |
| "step": 840 | |
| }, | |
| { | |
| "epoch": 2.7039999999999997, | |
| "grad_norm": 3.49139680447691, | |
| "learning_rate": 1.4272241316744456e-07, | |
| "logits/chosen": -0.9768983125686646, | |
| "logits/rejected": -1.0676887035369873, | |
| "logps/chosen": -0.8691689372062683, | |
| "logps/rejected": -1.5023279190063477, | |
| "loss": 0.7386, | |
| "odds_ratio_loss": 0.36283236742019653, | |
| "rewards/accuracies": 0.925000011920929, | |
| "rewards/chosen": -0.043458450585603714, | |
| "rewards/margins": 0.03165794909000397, | |
| "rewards/rejected": -0.07511639595031738, | |
| "sft_loss": 0.8691689372062683, | |
| "step": 845 | |
| }, | |
| { | |
| "epoch": 2.7199999999999998, | |
| "grad_norm": 3.4065842852697674, | |
| "learning_rate": 1.2760068184740597e-07, | |
| "logits/chosen": -0.9277766942977905, | |
| "logits/rejected": -0.9539043307304382, | |
| "logps/chosen": -0.6972528100013733, | |
| "logps/rejected": -1.4623371362686157, | |
| "loss": 0.711, | |
| "odds_ratio_loss": 0.32903754711151123, | |
| "rewards/accuracies": 0.9750000238418579, | |
| "rewards/chosen": -0.03486264497041702, | |
| "rewards/margins": 0.03825421258807182, | |
| "rewards/rejected": -0.07311685383319855, | |
| "sft_loss": 0.6972528100013733, | |
| "step": 850 | |
| }, | |
| { | |
| "epoch": 2.7359999999999998, | |
| "grad_norm": 4.258492525068464, | |
| "learning_rate": 1.133045906049321e-07, | |
| "logits/chosen": -0.9535223245620728, | |
| "logits/rejected": -0.967192530632019, | |
| "logps/chosen": -0.7654585242271423, | |
| "logps/rejected": -1.2852718830108643, | |
| "loss": 0.7165, | |
| "odds_ratio_loss": 0.4885888695716858, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": -0.038272932171821594, | |
| "rewards/margins": 0.025990664958953857, | |
| "rewards/rejected": -0.06426359713077545, | |
| "sft_loss": 0.7654585242271423, | |
| "step": 855 | |
| }, | |
| { | |
| "epoch": 2.752, | |
| "grad_norm": 3.0541533378277403, | |
| "learning_rate": 9.983911475163727e-08, | |
| "logits/chosen": -0.9215459823608398, | |
| "logits/rejected": -1.025499939918518, | |
| "logps/chosen": -0.6536693572998047, | |
| "logps/rejected": -1.464961051940918, | |
| "loss": 0.7491, | |
| "odds_ratio_loss": 0.32264453172683716, | |
| "rewards/accuracies": 0.925000011920929, | |
| "rewards/chosen": -0.032683465629816055, | |
| "rewards/margins": 0.04056458920240402, | |
| "rewards/rejected": -0.07324805110692978, | |
| "sft_loss": 0.6536693572998047, | |
| "step": 860 | |
| }, | |
| { | |
| "epoch": 2.768, | |
| "grad_norm": 3.5262579401831413, | |
| "learning_rate": 8.720894052918566e-08, | |
| "logits/chosen": -0.9398951530456543, | |
| "logits/rejected": -1.0038102865219116, | |
| "logps/chosen": -0.6747088432312012, | |
| "logps/rejected": -1.5684466361999512, | |
| "loss": 0.7404, | |
| "odds_ratio_loss": 0.35855698585510254, | |
| "rewards/accuracies": 0.8500000238418579, | |
| "rewards/chosen": -0.03373543918132782, | |
| "rewards/margins": 0.044686902314424515, | |
| "rewards/rejected": -0.07842233031988144, | |
| "sft_loss": 0.6747088432312012, | |
| "step": 865 | |
| }, | |
| { | |
| "epoch": 2.784, | |
| "grad_norm": 4.66998911832725, | |
| "learning_rate": 7.541846347838915e-08, | |
| "logits/chosen": -0.9948604702949524, | |
| "logits/rejected": -0.9804432988166809, | |
| "logps/chosen": -0.7318006753921509, | |
| "logps/rejected": -1.360406517982483, | |
| "loss": 0.7496, | |
| "odds_ratio_loss": 0.4134978652000427, | |
| "rewards/accuracies": 0.8500000238418579, | |
| "rewards/chosen": -0.036590032279491425, | |
| "rewards/margins": 0.03143029659986496, | |
| "rewards/rejected": -0.06802032887935638, | |
| "sft_loss": 0.7318006753921509, | |
| "step": 870 | |
| }, | |
| { | |
| "epoch": 2.8, | |
| "grad_norm": 3.657269423224737, | |
| "learning_rate": 6.447178690947492e-08, | |
| "logits/chosen": -0.8408433794975281, | |
| "logits/rejected": -1.0504786968231201, | |
| "logps/chosen": -0.6820616126060486, | |
| "logps/rejected": -1.6608028411865234, | |
| "loss": 0.7865, | |
| "odds_ratio_loss": 0.2992148697376251, | |
| "rewards/accuracies": 0.8999999761581421, | |
| "rewards/chosen": -0.03410308063030243, | |
| "rewards/margins": 0.04893706366419792, | |
| "rewards/rejected": -0.08304014801979065, | |
| "sft_loss": 0.6820616126060486, | |
| "step": 875 | |
| }, | |
| { | |
| "epoch": 2.816, | |
| "grad_norm": 3.377129562618715, | |
| "learning_rate": 5.437272047405712e-08, | |
| "logits/chosen": -0.8684722781181335, | |
| "logits/rejected": -0.871597945690155, | |
| "logps/chosen": -0.7219066619873047, | |
| "logps/rejected": -1.350375771522522, | |
| "loss": 0.7529, | |
| "odds_ratio_loss": 0.3972117006778717, | |
| "rewards/accuracies": 0.925000011920929, | |
| "rewards/chosen": -0.036095328629016876, | |
| "rewards/margins": 0.031423456966876984, | |
| "rewards/rejected": -0.06751878559589386, | |
| "sft_loss": 0.7219066619873047, | |
| "step": 880 | |
| }, | |
| { | |
| "epoch": 2.832, | |
| "grad_norm": 3.1663560374696633, | |
| "learning_rate": 4.512477883930527e-08, | |
| "logits/chosen": -0.9551021456718445, | |
| "logits/rejected": -1.0277870893478394, | |
| "logps/chosen": -0.7899004817008972, | |
| "logps/rejected": -1.5054200887680054, | |
| "loss": 0.8151, | |
| "odds_ratio_loss": 0.44721174240112305, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": -0.03949502483010292, | |
| "rewards/margins": 0.03577598184347153, | |
| "rewards/rejected": -0.07527101784944534, | |
| "sft_loss": 0.7899004817008972, | |
| "step": 885 | |
| }, | |
| { | |
| "epoch": 2.848, | |
| "grad_norm": 4.553277973933315, | |
| "learning_rate": 3.673118046477159e-08, | |
| "logits/chosen": -0.9583977460861206, | |
| "logits/rejected": -0.9128350019454956, | |
| "logps/chosen": -0.7262309789657593, | |
| "logps/rejected": -1.2458374500274658, | |
| "loss": 0.7505, | |
| "odds_ratio_loss": 0.4251146912574768, | |
| "rewards/accuracies": 0.8500000238418579, | |
| "rewards/chosen": -0.0363115519285202, | |
| "rewards/margins": 0.025980323553085327, | |
| "rewards/rejected": -0.06229187175631523, | |
| "sft_loss": 0.7262309789657593, | |
| "step": 890 | |
| }, | |
| { | |
| "epoch": 2.864, | |
| "grad_norm": 3.38523058441021, | |
| "learning_rate": 2.9194846482308026e-08, | |
| "logits/chosen": -0.9198935627937317, | |
| "logits/rejected": -1.0665795803070068, | |
| "logps/chosen": -0.7491655349731445, | |
| "logps/rejected": -1.439427137374878, | |
| "loss": 0.7622, | |
| "odds_ratio_loss": 0.3993758261203766, | |
| "rewards/accuracies": 0.824999988079071, | |
| "rewards/chosen": -0.037458278238773346, | |
| "rewards/margins": 0.03451308235526085, | |
| "rewards/rejected": -0.0719713643193245, | |
| "sft_loss": 0.7491655349731445, | |
| "step": 895 | |
| }, | |
| { | |
| "epoch": 2.88, | |
| "grad_norm": 5.053660983515992, | |
| "learning_rate": 2.251839967945535e-08, | |
| "logits/chosen": -0.9375904202461243, | |
| "logits/rejected": -1.060706377029419, | |
| "logps/chosen": -0.8767011761665344, | |
| "logps/rejected": -1.344328761100769, | |
| "loss": 0.766, | |
| "odds_ratio_loss": 0.5356343984603882, | |
| "rewards/accuracies": 0.8500000238418579, | |
| "rewards/chosen": -0.04383506253361702, | |
| "rewards/margins": 0.02338137850165367, | |
| "rewards/rejected": -0.06721644103527069, | |
| "sft_loss": 0.8767011761665344, | |
| "step": 900 | |
| }, | |
| { | |
| "epoch": 2.896, | |
| "grad_norm": 4.052840811239487, | |
| "learning_rate": 1.6704163586663825e-08, | |
| "logits/chosen": -0.9922626614570618, | |
| "logits/rejected": -0.952767014503479, | |
| "logps/chosen": -0.7710371017456055, | |
| "logps/rejected": -1.4237831830978394, | |
| "loss": 0.8119, | |
| "odds_ratio_loss": 0.37999871373176575, | |
| "rewards/accuracies": 0.8500000238418579, | |
| "rewards/chosen": -0.03855185955762863, | |
| "rewards/margins": 0.032637301832437515, | |
| "rewards/rejected": -0.07118916511535645, | |
| "sft_loss": 0.7710371017456055, | |
| "step": 905 | |
| }, | |
| { | |
| "epoch": 2.912, | |
| "grad_norm": 3.5836749421401146, | |
| "learning_rate": 1.1754161668660612e-08, | |
| "logits/chosen": -0.8529298901557922, | |
| "logits/rejected": -0.905608057975769, | |
| "logps/chosen": -0.7730112671852112, | |
| "logps/rejected": -1.4924733638763428, | |
| "loss": 0.7818, | |
| "odds_ratio_loss": 0.40126991271972656, | |
| "rewards/accuracies": 0.875, | |
| "rewards/chosen": -0.03865056484937668, | |
| "rewards/margins": 0.035973113030195236, | |
| "rewards/rejected": -0.07462368160486221, | |
| "sft_loss": 0.7730112671852112, | |
| "step": 910 | |
| }, | |
| { | |
| "epoch": 2.928, | |
| "grad_norm": 4.257912492562235, | |
| "learning_rate": 7.670116620245304e-09, | |
| "logits/chosen": -0.9177722930908203, | |
| "logits/rejected": -1.0528053045272827, | |
| "logps/chosen": -0.7789991497993469, | |
| "logps/rejected": -1.4093081951141357, | |
| "loss": 0.7862, | |
| "odds_ratio_loss": 0.41386303305625916, | |
| "rewards/accuracies": 0.824999988079071, | |
| "rewards/chosen": -0.03894995525479317, | |
| "rewards/margins": 0.0315154567360878, | |
| "rewards/rejected": -0.07046540826559067, | |
| "sft_loss": 0.7789991497993469, | |
| "step": 915 | |
| }, | |
| { | |
| "epoch": 2.944, | |
| "grad_norm": 4.5782153540538575, | |
| "learning_rate": 4.453449766758933e-09, | |
| "logits/chosen": -1.0013253688812256, | |
| "logits/rejected": -0.9696464538574219, | |
| "logps/chosen": -0.7406843900680542, | |
| "logps/rejected": -1.4851348400115967, | |
| "loss": 0.7435, | |
| "odds_ratio_loss": 0.3701634407043457, | |
| "rewards/accuracies": 0.875, | |
| "rewards/chosen": -0.037034209817647934, | |
| "rewards/margins": 0.03722252696752548, | |
| "rewards/rejected": -0.07425674051046371, | |
| "sft_loss": 0.7406843900680542, | |
| "step": 920 | |
| }, | |
| { | |
| "epoch": 2.96, | |
| "grad_norm": 3.8405956220288906, | |
| "learning_rate": 2.1052805694365964e-09, | |
| "logits/chosen": -0.9112046360969543, | |
| "logits/rejected": -0.953746497631073, | |
| "logps/chosen": -0.9097617864608765, | |
| "logps/rejected": -1.2347737550735474, | |
| "loss": 0.787, | |
| "odds_ratio_loss": 0.5556334853172302, | |
| "rewards/accuracies": 0.6499999761581421, | |
| "rewards/chosen": -0.04548809304833412, | |
| "rewards/margins": 0.016250593587756157, | |
| "rewards/rejected": -0.061738692224025726, | |
| "sft_loss": 0.9097617864608765, | |
| "step": 925 | |
| }, | |
| { | |
| "epoch": 2.976, | |
| "grad_norm": 3.3132646219123676, | |
| "learning_rate": 6.264262358129936e-10, | |
| "logits/chosen": -0.8271144032478333, | |
| "logits/rejected": -1.076704502105713, | |
| "logps/chosen": -0.8304189443588257, | |
| "logps/rejected": -1.3862361907958984, | |
| "loss": 0.7539, | |
| "odds_ratio_loss": 0.4186479449272156, | |
| "rewards/accuracies": 0.7749999761581421, | |
| "rewards/chosen": -0.04152094945311546, | |
| "rewards/margins": 0.027790868654847145, | |
| "rewards/rejected": -0.06931181252002716, | |
| "sft_loss": 0.8304189443588257, | |
| "step": 930 | |
| }, | |
| { | |
| "epoch": 2.992, | |
| "grad_norm": 3.846388656199648, | |
| "learning_rate": 1.7401435318531444e-11, | |
| "logits/chosen": -1.0561763048171997, | |
| "logits/rejected": -1.0017073154449463, | |
| "logps/chosen": -0.6843768358230591, | |
| "logps/rejected": -1.4440100193023682, | |
| "loss": 0.7977, | |
| "odds_ratio_loss": 0.4135671555995941, | |
| "rewards/accuracies": 0.8500000238418579, | |
| "rewards/chosen": -0.034218840301036835, | |
| "rewards/margins": 0.03798165172338486, | |
| "rewards/rejected": -0.07220049947500229, | |
| "sft_loss": 0.6843768358230591, | |
| "step": 935 | |
| }, | |
| { | |
| "epoch": 2.9952, | |
| "step": 936, | |
| "total_flos": 172904605286400.0, | |
| "train_loss": 0.9994224820636276, | |
| "train_runtime": 29817.6595, | |
| "train_samples_per_second": 2.012, | |
| "train_steps_per_second": 0.031 | |
| } | |
| ], | |
| "logging_steps": 5, | |
| "max_steps": 936, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 3, | |
| "save_steps": 100.0, | |
| "total_flos": 172904605286400.0, | |
| "train_batch_size": 2, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |