diff --git "a/.ipynb_checkpoints/trainer_state-checkpoint.json" "b/.ipynb_checkpoints/trainer_state-checkpoint.json" new file mode 100644--- /dev/null +++ "b/.ipynb_checkpoints/trainer_state-checkpoint.json" @@ -0,0 +1,2898 @@ +{ + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.9997382884061764, + "eval_steps": 500, + "global_step": 1910, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.005234231876472127, + "grad_norm": 39.5, + "learning_rate": 1.0416666666666667e-06, + "logits/chosen": 0.6487364768981934, + "logits/rejected": 0.7462336421012878, + "logps/chosen": -331.4830322265625, + "logps/rejected": -289.36822509765625, + "loss": 0.6928, + "rewards/accuracies": 0.3968749940395355, + "rewards/chosen": 0.0010190431494265795, + "rewards/margins": 0.0010569205041974783, + "rewards/rejected": -3.78774493583478e-05, + "step": 10 + }, + { + "epoch": 0.010468463752944255, + "grad_norm": 36.25, + "learning_rate": 2.0833333333333334e-06, + "logits/chosen": 0.7357114553451538, + "logits/rejected": 0.7462834715843201, + "logps/chosen": -358.1224670410156, + "logps/rejected": -318.0662536621094, + "loss": 0.6909, + "rewards/accuracies": 0.512499988079071, + "rewards/chosen": 0.0003305533900856972, + "rewards/margins": 0.005077774170786142, + "rewards/rejected": -0.00474722171202302, + "step": 20 + }, + { + "epoch": 0.015702695629416383, + "grad_norm": 39.75, + "learning_rate": 3.125e-06, + "logits/chosen": 0.7648854851722717, + "logits/rejected": 0.8517886400222778, + "logps/chosen": -351.1484375, + "logps/rejected": -318.3990783691406, + "loss": 0.6907, + "rewards/accuracies": 0.518750011920929, + "rewards/chosen": 0.0017980957636609674, + "rewards/margins": 0.005581674166023731, + "rewards/rejected": -0.0037835785187780857, + "step": 30 + }, + { + "epoch": 0.02093692750588851, + "grad_norm": 37.25, + "learning_rate": 4.166666666666667e-06, + "logits/chosen": 0.7295006513595581, + "logits/rejected": 0.7714008688926697, + "logps/chosen": -319.7439880371094, + "logps/rejected": -278.13323974609375, + "loss": 0.6823, + "rewards/accuracies": 0.5687500238418579, + "rewards/chosen": 0.010593170300126076, + "rewards/margins": 0.023843348026275635, + "rewards/rejected": -0.013250177726149559, + "step": 40 + }, + { + "epoch": 0.02617115938236064, + "grad_norm": 32.75, + "learning_rate": 5.208333333333334e-06, + "logits/chosen": 0.685738205909729, + "logits/rejected": 0.7062253952026367, + "logps/chosen": -320.6109313964844, + "logps/rejected": -284.9264831542969, + "loss": 0.6737, + "rewards/accuracies": 0.628125011920929, + "rewards/chosen": 0.0017792358994483948, + "rewards/margins": 0.04503456503152847, + "rewards/rejected": -0.04325533285737038, + "step": 50 + }, + { + "epoch": 0.031405391258832765, + "grad_norm": 36.5, + "learning_rate": 6.25e-06, + "logits/chosen": 0.6452481746673584, + "logits/rejected": 0.7145572304725647, + "logps/chosen": -337.2304382324219, + "logps/rejected": -300.0358581542969, + "loss": 0.666, + "rewards/accuracies": 0.612500011920929, + "rewards/chosen": -0.002510682214051485, + "rewards/margins": 0.07026001065969467, + "rewards/rejected": -0.07277069240808487, + "step": 60 + }, + { + "epoch": 0.036639623135304895, + "grad_norm": 33.25, + "learning_rate": 7.291666666666667e-06, + "logits/chosen": 0.8339419364929199, + "logits/rejected": 0.8587329983711243, + "logps/chosen": -325.0736999511719, + "logps/rejected": -313.72161865234375, + "loss": 0.6569, + "rewards/accuracies": 0.609375, + "rewards/chosen": -0.022171273827552795, + "rewards/margins": 0.10362186282873154, + "rewards/rejected": -0.12579312920570374, + "step": 70 + }, + { + "epoch": 0.04187385501177702, + "grad_norm": 35.25, + "learning_rate": 8.333333333333334e-06, + "logits/chosen": 0.7884281873703003, + "logits/rejected": 0.9599426984786987, + "logps/chosen": -309.9776306152344, + "logps/rejected": -280.77960205078125, + "loss": 0.6485, + "rewards/accuracies": 0.637499988079071, + "rewards/chosen": -0.03900856524705887, + "rewards/margins": 0.13111677765846252, + "rewards/rejected": -0.1701253354549408, + "step": 80 + }, + { + "epoch": 0.04710808688824915, + "grad_norm": 36.5, + "learning_rate": 9.375000000000001e-06, + "logits/chosen": 0.7847155928611755, + "logits/rejected": 0.7387144565582275, + "logps/chosen": -330.1953430175781, + "logps/rejected": -277.9568176269531, + "loss": 0.6419, + "rewards/accuracies": 0.640625, + "rewards/chosen": -0.05813335254788399, + "rewards/margins": 0.1728575974702835, + "rewards/rejected": -0.230990931391716, + "step": 90 + }, + { + "epoch": 0.05234231876472128, + "grad_norm": 29.25, + "learning_rate": 9.999880027023295e-06, + "logits/chosen": 0.4318479001522064, + "logits/rejected": 0.5929848551750183, + "logps/chosen": -327.99481201171875, + "logps/rejected": -298.76483154296875, + "loss": 0.6061, + "rewards/accuracies": 0.6656249761581421, + "rewards/chosen": -0.12621721625328064, + "rewards/margins": 0.2853361964225769, + "rewards/rejected": -0.41155338287353516, + "step": 100 + }, + { + "epoch": 0.05757655064119341, + "grad_norm": 30.5, + "learning_rate": 9.998530397154684e-06, + "logits/chosen": 0.4563142657279968, + "logits/rejected": 0.5782237648963928, + "logps/chosen": -325.25836181640625, + "logps/rejected": -314.2166748046875, + "loss": 0.6221, + "rewards/accuracies": 0.640625, + "rewards/chosen": -0.18687985837459564, + "rewards/margins": 0.2569567561149597, + "rewards/rejected": -0.44383659958839417, + "step": 110 + }, + { + "epoch": 0.06281078251766553, + "grad_norm": 36.0, + "learning_rate": 9.995681577335256e-06, + "logits/chosen": 0.3232671320438385, + "logits/rejected": 0.4357617497444153, + "logps/chosen": -340.3876037597656, + "logps/rejected": -319.13226318359375, + "loss": 0.6172, + "rewards/accuracies": 0.6468750238418579, + "rewards/chosen": -0.3543888330459595, + "rewards/margins": 0.3290286958217621, + "rewards/rejected": -0.6834174990653992, + "step": 120 + }, + { + "epoch": 0.06804501439413765, + "grad_norm": 56.5, + "learning_rate": 9.99133442200056e-06, + "logits/chosen": 0.2823497951030731, + "logits/rejected": 0.4065135419368744, + "logps/chosen": -354.84442138671875, + "logps/rejected": -311.0687561035156, + "loss": 0.6218, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.40330594778060913, + "rewards/margins": 0.3464129567146301, + "rewards/rejected": -0.749718964099884, + "step": 130 + }, + { + "epoch": 0.07327924627060979, + "grad_norm": 37.5, + "learning_rate": 9.985490234976132e-06, + "logits/chosen": 0.4105094075202942, + "logits/rejected": 0.4262070059776306, + "logps/chosen": -345.6540222167969, + "logps/rejected": -291.80731201171875, + "loss": 0.5984, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.41038641333580017, + "rewards/margins": 0.41181617975234985, + "rewards/rejected": -0.8222025632858276, + "step": 140 + }, + { + "epoch": 0.07851347814708191, + "grad_norm": 33.25, + "learning_rate": 9.978150769086457e-06, + "logits/chosen": 0.3375703692436218, + "logits/rejected": 0.4788056015968323, + "logps/chosen": -340.313232421875, + "logps/rejected": -304.1383361816406, + "loss": 0.6335, + "rewards/accuracies": 0.643750011920929, + "rewards/chosen": -0.42093929648399353, + "rewards/margins": 0.3101397156715393, + "rewards/rejected": -0.7310789823532104, + "step": 150 + }, + { + "epoch": 0.08374771002355404, + "grad_norm": 36.75, + "learning_rate": 9.96931822562924e-06, + "logits/chosen": 0.2715611755847931, + "logits/rejected": 0.3587436079978943, + "logps/chosen": -352.6816101074219, + "logps/rejected": -338.83575439453125, + "loss": 0.6146, + "rewards/accuracies": 0.671875, + "rewards/chosen": -0.332112193107605, + "rewards/margins": 0.29170310497283936, + "rewards/rejected": -0.6238153576850891, + "step": 160 + }, + { + "epoch": 0.08898194190002617, + "grad_norm": 34.0, + "learning_rate": 9.958995253715193e-06, + "logits/chosen": 0.3311766982078552, + "logits/rejected": 0.3224788010120392, + "logps/chosen": -357.66912841796875, + "logps/rejected": -316.62017822265625, + "loss": 0.6098, + "rewards/accuracies": 0.668749988079071, + "rewards/chosen": -0.31313422322273254, + "rewards/margins": 0.3176955580711365, + "rewards/rejected": -0.6308297514915466, + "step": 170 + }, + { + "epoch": 0.0942161737764983, + "grad_norm": 38.25, + "learning_rate": 9.947184949473478e-06, + "logits/chosen": 0.23619632422924042, + "logits/rejected": 0.2766094505786896, + "logps/chosen": -343.662109375, + "logps/rejected": -298.932373046875, + "loss": 0.5899, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.472648948431015, + "rewards/margins": 0.3989109396934509, + "rewards/rejected": -0.8715597987174988, + "step": 180 + }, + { + "epoch": 0.09945040565297043, + "grad_norm": 41.5, + "learning_rate": 9.933890855123114e-06, + "logits/chosen": 0.1247895210981369, + "logits/rejected": 0.18231646716594696, + "logps/chosen": -373.7274169921875, + "logps/rejected": -356.18646240234375, + "loss": 0.6329, + "rewards/accuracies": 0.6468750238418579, + "rewards/chosen": -0.8531802892684937, + "rewards/margins": 0.3459845781326294, + "rewards/rejected": -1.199164867401123, + "step": 190 + }, + { + "epoch": 0.10468463752944256, + "grad_norm": 43.75, + "learning_rate": 9.919116957910566e-06, + "logits/chosen": 0.10570982843637466, + "logits/rejected": 0.11052794754505157, + "logps/chosen": -349.4450988769531, + "logps/rejected": -290.8612365722656, + "loss": 0.5914, + "rewards/accuracies": 0.659375011920929, + "rewards/chosen": -0.6416983604431152, + "rewards/margins": 0.45599108934402466, + "rewards/rejected": -1.0976893901824951, + "step": 200 + }, + { + "epoch": 0.10991886940591468, + "grad_norm": 38.5, + "learning_rate": 9.902867688913869e-06, + "logits/chosen": 0.3035932779312134, + "logits/rejected": 0.3483582139015198, + "logps/chosen": -353.8571472167969, + "logps/rejected": -306.32904052734375, + "loss": 0.5826, + "rewards/accuracies": 0.6781250238418579, + "rewards/chosen": -0.2641884386539459, + "rewards/margins": 0.4089083671569824, + "rewards/rejected": -0.6730968356132507, + "step": 210 + }, + { + "epoch": 0.11515310128238682, + "grad_norm": 41.0, + "learning_rate": 9.885147921713621e-06, + "logits/chosen": 0.12661756575107574, + "logits/rejected": 0.20708250999450684, + "logps/chosen": -325.804443359375, + "logps/rejected": -309.0091247558594, + "loss": 0.5871, + "rewards/accuracies": 0.684374988079071, + "rewards/chosen": -0.2193823605775833, + "rewards/margins": 0.4421941637992859, + "rewards/rejected": -0.6615765690803528, + "step": 220 + }, + { + "epoch": 0.12038733315885894, + "grad_norm": 30.625, + "learning_rate": 9.865962970931287e-06, + "logits/chosen": 0.3643207848072052, + "logits/rejected": 0.4040307402610779, + "logps/chosen": -358.22705078125, + "logps/rejected": -312.9072570800781, + "loss": 0.5905, + "rewards/accuracies": 0.690625011920929, + "rewards/chosen": -0.3125769793987274, + "rewards/margins": 0.4290243983268738, + "rewards/rejected": -0.7416013479232788, + "step": 230 + }, + { + "epoch": 0.12562156503533106, + "grad_norm": 35.0, + "learning_rate": 9.845318590635186e-06, + "logits/chosen": 0.44897300004959106, + "logits/rejected": 0.6014218330383301, + "logps/chosen": -352.2749328613281, + "logps/rejected": -309.62176513671875, + "loss": 0.5765, + "rewards/accuracies": 0.690625011920929, + "rewards/chosen": -0.27151674032211304, + "rewards/margins": 0.4776983857154846, + "rewards/rejected": -0.7492151260375977, + "step": 240 + }, + { + "epoch": 0.13085579691180318, + "grad_norm": 35.25, + "learning_rate": 9.823220972614712e-06, + "logits/chosen": 0.3287668228149414, + "logits/rejected": 0.40800076723098755, + "logps/chosen": -367.3862609863281, + "logps/rejected": -299.0010986328125, + "loss": 0.5782, + "rewards/accuracies": 0.6625000238418579, + "rewards/chosen": -0.4212459921836853, + "rewards/margins": 0.4778348505496979, + "rewards/rejected": -0.8990808725357056, + "step": 250 + }, + { + "epoch": 0.1360900287882753, + "grad_norm": 39.75, + "learning_rate": 9.79967674452324e-06, + "logits/chosen": 0.2960013747215271, + "logits/rejected": 0.3838788866996765, + "logps/chosen": -350.2323303222656, + "logps/rejected": -340.3704528808594, + "loss": 0.5902, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.5522688627243042, + "rewards/margins": 0.5364810228347778, + "rewards/rejected": -1.0887497663497925, + "step": 260 + }, + { + "epoch": 0.14132426066474746, + "grad_norm": 36.25, + "learning_rate": 9.774692967890332e-06, + "logits/chosen": 0.13149850070476532, + "logits/rejected": 0.1432052105665207, + "logps/chosen": -356.21356201171875, + "logps/rejected": -323.2649230957031, + "loss": 0.5835, + "rewards/accuracies": 0.715624988079071, + "rewards/chosen": -0.3824268877506256, + "rewards/margins": 0.4610356390476227, + "rewards/rejected": -0.8434625864028931, + "step": 270 + }, + { + "epoch": 0.14655849254121958, + "grad_norm": 35.75, + "learning_rate": 9.74827713600379e-06, + "logits/chosen": 0.245396226644516, + "logits/rejected": 0.37196552753448486, + "logps/chosen": -308.9268798828125, + "logps/rejected": -277.8331298828125, + "loss": 0.6087, + "rewards/accuracies": 0.6781250238418579, + "rewards/chosen": -0.23228350281715393, + "rewards/margins": 0.37543928623199463, + "rewards/rejected": -0.6077227592468262, + "step": 280 + }, + { + "epoch": 0.1517927244176917, + "grad_norm": 30.875, + "learning_rate": 9.720437171662232e-06, + "logits/chosen": 0.3768264651298523, + "logits/rejected": 0.4466184675693512, + "logps/chosen": -336.4764099121094, + "logps/rejected": -312.9964294433594, + "loss": 0.5865, + "rewards/accuracies": 0.706250011920929, + "rewards/chosen": -0.21132774651050568, + "rewards/margins": 0.4387349486351013, + "rewards/rejected": -0.6500626802444458, + "step": 290 + }, + { + "epoch": 0.15702695629416383, + "grad_norm": 33.5, + "learning_rate": 9.691181424798825e-06, + "logits/chosen": 0.27258244156837463, + "logits/rejected": 0.24488012492656708, + "logps/chosen": -327.12249755859375, + "logps/rejected": -303.77685546875, + "loss": 0.5972, + "rewards/accuracies": 0.6656249761581421, + "rewards/chosen": -0.48688727617263794, + "rewards/margins": 0.43369269371032715, + "rewards/rejected": -0.9205799102783203, + "step": 300 + }, + { + "epoch": 0.16226118817063595, + "grad_norm": 38.25, + "learning_rate": 9.660518669976936e-06, + "logits/chosen": 0.22490909695625305, + "logits/rejected": 0.3626439869403839, + "logps/chosen": -359.6585998535156, + "logps/rejected": -312.7129821777344, + "loss": 0.6088, + "rewards/accuracies": 0.671875, + "rewards/chosen": -0.6881123781204224, + "rewards/margins": 0.4073651432991028, + "rewards/rejected": -1.0954773426055908, + "step": 310 + }, + { + "epoch": 0.16749542004710807, + "grad_norm": 33.75, + "learning_rate": 9.628458103758403e-06, + "logits/chosen": 0.18019357323646545, + "logits/rejected": 0.30855241417884827, + "logps/chosen": -371.63665771484375, + "logps/rejected": -335.8221435546875, + "loss": 0.5503, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.6700717210769653, + "rewards/margins": 0.5360826253890991, + "rewards/rejected": -1.2061543464660645, + "step": 320 + }, + { + "epoch": 0.17272965192358022, + "grad_norm": 39.25, + "learning_rate": 9.595009341945246e-06, + "logits/chosen": 0.13094857335090637, + "logits/rejected": 0.19792452454566956, + "logps/chosen": -334.1517639160156, + "logps/rejected": -320.565673828125, + "loss": 0.6307, + "rewards/accuracies": 0.6656249761581421, + "rewards/chosen": -0.8788414001464844, + "rewards/margins": 0.45465344190597534, + "rewards/rejected": -1.3334949016571045, + "step": 330 + }, + { + "epoch": 0.17796388380005235, + "grad_norm": 34.5, + "learning_rate": 9.560182416695639e-06, + "logits/chosen": 0.21438567340373993, + "logits/rejected": 0.22045159339904785, + "logps/chosen": -331.2416687011719, + "logps/rejected": -329.4930419921875, + "loss": 0.5683, + "rewards/accuracies": 0.7281249761581421, + "rewards/chosen": -0.6694966554641724, + "rewards/margins": 0.5601910352706909, + "rewards/rejected": -1.2296875715255737, + "step": 340 + }, + { + "epoch": 0.18319811567652447, + "grad_norm": 39.0, + "learning_rate": 9.523987773514999e-06, + "logits/chosen": 0.14380326867103577, + "logits/rejected": 0.2659078538417816, + "logps/chosen": -337.06488037109375, + "logps/rejected": -299.07940673828125, + "loss": 0.6124, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.4915216565132141, + "rewards/margins": 0.37540850043296814, + "rewards/rejected": -0.8669301271438599, + "step": 350 + }, + { + "epoch": 0.1884323475529966, + "grad_norm": 40.5, + "learning_rate": 9.486436268123112e-06, + "logits/chosen": 0.1830969899892807, + "logits/rejected": 0.2517342269420624, + "logps/chosen": -365.96990966796875, + "logps/rejected": -339.3229675292969, + "loss": 0.5989, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.35854586958885193, + "rewards/margins": 0.44347500801086426, + "rewards/rejected": -0.8020208477973938, + "step": 360 + }, + { + "epoch": 0.19366657942946872, + "grad_norm": 40.5, + "learning_rate": 9.447539163198218e-06, + "logits/chosen": 0.34863370656967163, + "logits/rejected": 0.34325408935546875, + "logps/chosen": -344.86480712890625, + "logps/rejected": -307.914306640625, + "loss": 0.599, + "rewards/accuracies": 0.671875, + "rewards/chosen": -0.4770238399505615, + "rewards/margins": 0.5033739805221558, + "rewards/rejected": -0.9803977012634277, + "step": 370 + }, + { + "epoch": 0.19890081130594087, + "grad_norm": 24.75, + "learning_rate": 9.407308124999031e-06, + "logits/chosen": 0.5095074772834778, + "logits/rejected": 0.5166248083114624, + "logps/chosen": -355.6305847167969, + "logps/rejected": -334.64788818359375, + "loss": 0.6052, + "rewards/accuracies": 0.671875, + "rewards/chosen": -0.5633832812309265, + "rewards/margins": 0.515869677066803, + "rewards/rejected": -1.0792529582977295, + "step": 380 + }, + { + "epoch": 0.204135043182413, + "grad_norm": 30.5, + "learning_rate": 9.365755219865733e-06, + "logits/chosen": 0.5360152125358582, + "logits/rejected": 0.6312729716300964, + "logps/chosen": -351.32574462890625, + "logps/rejected": -332.18890380859375, + "loss": 0.5507, + "rewards/accuracies": 0.721875011920929, + "rewards/chosen": -0.3784104585647583, + "rewards/margins": 0.6517626643180847, + "rewards/rejected": -1.0301730632781982, + "step": 390 + }, + { + "epoch": 0.2093692750588851, + "grad_norm": 40.0, + "learning_rate": 9.322892910600959e-06, + "logits/chosen": 0.5068603754043579, + "logits/rejected": 0.7184394598007202, + "logps/chosen": -324.0625, + "logps/rejected": -290.60235595703125, + "loss": 0.6159, + "rewards/accuracies": 0.659375011920929, + "rewards/chosen": -0.39290112257003784, + "rewards/margins": 0.38739123940467834, + "rewards/rejected": -0.7802923917770386, + "step": 400 + }, + { + "epoch": 0.21460350693535724, + "grad_norm": 55.75, + "learning_rate": 9.278734052731876e-06, + "logits/chosen": 0.47780805826187134, + "logits/rejected": 0.48742303252220154, + "logps/chosen": -347.030517578125, + "logps/rejected": -322.89068603515625, + "loss": 0.5722, + "rewards/accuracies": 0.721875011920929, + "rewards/chosen": -0.47796815633773804, + "rewards/margins": 0.547213613986969, + "rewards/rejected": -1.025181770324707, + "step": 410 + }, + { + "epoch": 0.21983773881182936, + "grad_norm": 42.0, + "learning_rate": 9.233291890654477e-06, + "logits/chosen": 0.08945097029209137, + "logits/rejected": 0.17587144672870636, + "logps/chosen": -353.0492248535156, + "logps/rejected": -310.22900390625, + "loss": 0.5454, + "rewards/accuracies": 0.6968749761581421, + "rewards/chosen": -0.6215865612030029, + "rewards/margins": 0.6318700313568115, + "rewards/rejected": -1.253456711769104, + "step": 420 + }, + { + "epoch": 0.22507197068830148, + "grad_norm": 39.5, + "learning_rate": 9.186580053661238e-06, + "logits/chosen": 0.19737955927848816, + "logits/rejected": 0.20681536197662354, + "logps/chosen": -342.05816650390625, + "logps/rejected": -354.335693359375, + "loss": 0.6142, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.4559093117713928, + "rewards/margins": 0.5013066530227661, + "rewards/rejected": -0.9572159051895142, + "step": 430 + }, + { + "epoch": 0.23030620256477363, + "grad_norm": 32.0, + "learning_rate": 9.138612551853334e-06, + "logits/chosen": 0.1024736613035202, + "logits/rejected": 0.2752048373222351, + "logps/chosen": -355.54364013671875, + "logps/rejected": -307.53631591796875, + "loss": 0.5634, + "rewards/accuracies": 0.715624988079071, + "rewards/chosen": -0.20539434254169464, + "rewards/margins": 0.5175925493240356, + "rewards/rejected": -0.7229868769645691, + "step": 440 + }, + { + "epoch": 0.23554043444124576, + "grad_norm": 35.5, + "learning_rate": 9.089403771938651e-06, + "logits/chosen": 0.1647811233997345, + "logits/rejected": 0.2729242742061615, + "logps/chosen": -343.55419921875, + "logps/rejected": -314.5325012207031, + "loss": 0.5835, + "rewards/accuracies": 0.715624988079071, + "rewards/chosen": -0.2670280635356903, + "rewards/margins": 0.5155874490737915, + "rewards/rejected": -0.7826155424118042, + "step": 450 + }, + { + "epoch": 0.24077466631771788, + "grad_norm": 36.75, + "learning_rate": 9.038968472916831e-06, + "logits/chosen": 0.14953358471393585, + "logits/rejected": 0.24258370697498322, + "logps/chosen": -373.007080078125, + "logps/rejected": -365.58245849609375, + "loss": 0.5612, + "rewards/accuracies": 0.721875011920929, + "rewards/chosen": -0.530977725982666, + "rewards/margins": 0.6304374933242798, + "rewards/rejected": -1.1614153385162354, + "step": 460 + }, + { + "epoch": 0.24600889819419, + "grad_norm": 35.75, + "learning_rate": 8.987321781652663e-06, + "logits/chosen": 0.20829911530017853, + "logits/rejected": 0.25458788871765137, + "logps/chosen": -337.02142333984375, + "logps/rejected": -305.6877746582031, + "loss": 0.5678, + "rewards/accuracies": 0.6968749761581421, + "rewards/chosen": -0.9067142605781555, + "rewards/margins": 0.6314893960952759, + "rewards/rejected": -1.538203477859497, + "step": 470 + }, + { + "epoch": 0.2512431300706621, + "grad_norm": 41.75, + "learning_rate": 8.93447918833914e-06, + "logits/chosen": 0.18115252256393433, + "logits/rejected": 0.17303326725959778, + "logps/chosen": -371.98431396484375, + "logps/rejected": -319.89947509765625, + "loss": 0.5904, + "rewards/accuracies": 0.6781250238418579, + "rewards/chosen": -0.8123010396957397, + "rewards/margins": 0.5886048078536987, + "rewards/rejected": -1.400905966758728, + "step": 480 + }, + { + "epoch": 0.2564773619471343, + "grad_norm": 36.0, + "learning_rate": 8.880456541851544e-06, + "logits/chosen": 0.12780669331550598, + "logits/rejected": 0.21940919756889343, + "logps/chosen": -397.94189453125, + "logps/rejected": -341.4785461425781, + "loss": 0.54, + "rewards/accuracies": 0.7406250238418579, + "rewards/chosen": -0.6672419309616089, + "rewards/margins": 0.6856431365013123, + "rewards/rejected": -1.3528850078582764, + "step": 490 + }, + { + "epoch": 0.26171159382360637, + "grad_norm": 45.75, + "learning_rate": 8.825270044993963e-06, + "logits/chosen": 0.20649901032447815, + "logits/rejected": 0.31849536299705505, + "logps/chosen": -320.14593505859375, + "logps/rejected": -326.33624267578125, + "loss": 0.5853, + "rewards/accuracies": 0.668749988079071, + "rewards/chosen": -0.5974608659744263, + "rewards/margins": 0.5049938559532166, + "rewards/rejected": -1.1024547815322876, + "step": 500 + }, + { + "epoch": 0.2669458257000785, + "grad_norm": 42.75, + "learning_rate": 8.768936249639632e-06, + "logits/chosen": 0.046025604009628296, + "logits/rejected": 0.18477210402488708, + "logps/chosen": -335.65899658203125, + "logps/rejected": -325.29730224609375, + "loss": 0.5911, + "rewards/accuracies": 0.6781250238418579, + "rewards/chosen": -0.6880438923835754, + "rewards/margins": 0.4891757071018219, + "rewards/rejected": -1.1772195100784302, + "step": 510 + }, + { + "epoch": 0.2721800575765506, + "grad_norm": 38.75, + "learning_rate": 8.711472051766606e-06, + "logits/chosen": 0.12576794624328613, + "logits/rejected": 0.19818930327892303, + "logps/chosen": -358.3614196777344, + "logps/rejected": -336.26629638671875, + "loss": 0.536, + "rewards/accuracies": 0.6937500238418579, + "rewards/chosen": -0.5878896713256836, + "rewards/margins": 0.6127403974533081, + "rewards/rejected": -1.2006300687789917, + "step": 520 + }, + { + "epoch": 0.27741428945302277, + "grad_norm": 38.5, + "learning_rate": 8.652894686390205e-06, + "logits/chosen": 0.18315266072750092, + "logits/rejected": 0.24707746505737305, + "logps/chosen": -356.6893005371094, + "logps/rejected": -326.3449401855469, + "loss": 0.5773, + "rewards/accuracies": 0.690625011920929, + "rewards/chosen": -0.5467976331710815, + "rewards/margins": 0.6014063954353333, + "rewards/rejected": -1.1482040882110596, + "step": 530 + }, + { + "epoch": 0.2826485213294949, + "grad_norm": 45.25, + "learning_rate": 8.593221722393789e-06, + "logits/chosen": 0.1029457077383995, + "logits/rejected": 0.20355193316936493, + "logps/chosen": -357.2725524902344, + "logps/rejected": -325.27288818359375, + "loss": 0.5543, + "rewards/accuracies": 0.7093750238418579, + "rewards/chosen": -0.8227931261062622, + "rewards/margins": 0.6153770685195923, + "rewards/rejected": -1.4381701946258545, + "step": 540 + }, + { + "epoch": 0.287882753205967, + "grad_norm": 41.5, + "learning_rate": 8.53247105725939e-06, + "logits/chosen": 0.20170524716377258, + "logits/rejected": 0.1971486508846283, + "logps/chosen": -320.24566650390625, + "logps/rejected": -294.46917724609375, + "loss": 0.5441, + "rewards/accuracies": 0.71875, + "rewards/chosen": -0.39367663860321045, + "rewards/margins": 0.6428011059761047, + "rewards/rejected": -1.0364776849746704, + "step": 550 + }, + { + "epoch": 0.29311698508243916, + "grad_norm": 42.0, + "learning_rate": 8.470660911699783e-06, + "logits/chosen": 0.049773894250392914, + "logits/rejected": 0.12189406156539917, + "logps/chosen": -332.68487548828125, + "logps/rejected": -287.7740173339844, + "loss": 0.5803, + "rewards/accuracies": 0.6937500238418579, + "rewards/chosen": -0.35799601674079895, + "rewards/margins": 0.5440070629119873, + "rewards/rejected": -0.9020029902458191, + "step": 560 + }, + { + "epoch": 0.29835121695891126, + "grad_norm": 36.5, + "learning_rate": 8.407809824193624e-06, + "logits/chosen": 0.03806518390774727, + "logits/rejected": 0.16943983733654022, + "logps/chosen": -371.38580322265625, + "logps/rejected": -338.66558837890625, + "loss": 0.5838, + "rewards/accuracies": 0.7124999761581421, + "rewards/chosen": -0.5821398496627808, + "rewards/margins": 0.6142541170120239, + "rewards/rejected": -1.1963939666748047, + "step": 570 + }, + { + "epoch": 0.3035854488353834, + "grad_norm": 35.25, + "learning_rate": 8.343936645425277e-06, + "logits/chosen": 0.2618806064128876, + "logits/rejected": 0.3286944329738617, + "logps/chosen": -325.019287109375, + "logps/rejected": -308.45611572265625, + "loss": 0.5144, + "rewards/accuracies": 0.7406250238418579, + "rewards/chosen": -0.5636202692985535, + "rewards/margins": 0.7985345721244812, + "rewards/rejected": -1.3621549606323242, + "step": 580 + }, + { + "epoch": 0.30881968071185556, + "grad_norm": 34.75, + "learning_rate": 8.279060532630991e-06, + "logits/chosen": 0.21167007088661194, + "logits/rejected": 0.33110693097114563, + "logps/chosen": -361.67559814453125, + "logps/rejected": -336.69476318359375, + "loss": 0.5956, + "rewards/accuracies": 0.684374988079071, + "rewards/chosen": -0.8580425977706909, + "rewards/margins": 0.647322416305542, + "rewards/rejected": -1.5053651332855225, + "step": 590 + }, + { + "epoch": 0.31405391258832765, + "grad_norm": 43.0, + "learning_rate": 8.21320094385316e-06, + "logits/chosen": 0.257051020860672, + "logits/rejected": 0.3034771680831909, + "logps/chosen": -373.9206848144531, + "logps/rejected": -343.5843200683594, + "loss": 0.6014, + "rewards/accuracies": 0.65625, + "rewards/chosen": -0.7790796756744385, + "rewards/margins": 0.5478149056434631, + "rewards/rejected": -1.3268945217132568, + "step": 600 + }, + { + "epoch": 0.3192881444647998, + "grad_norm": 41.25, + "learning_rate": 8.146377632104328e-06, + "logits/chosen": 0.17596812546253204, + "logits/rejected": 0.34751996397972107, + "logps/chosen": -384.0326843261719, + "logps/rejected": -325.1558532714844, + "loss": 0.5376, + "rewards/accuracies": 0.6937500238418579, + "rewards/chosen": -0.5697289705276489, + "rewards/margins": 0.7126566171646118, + "rewards/rejected": -1.2823854684829712, + "step": 610 + }, + { + "epoch": 0.3245223763412719, + "grad_norm": 38.75, + "learning_rate": 8.078610639442761e-06, + "logits/chosen": 0.187199205160141, + "logits/rejected": 0.22407758235931396, + "logps/chosen": -365.73095703125, + "logps/rejected": -313.1622619628906, + "loss": 0.563, + "rewards/accuracies": 0.7124999761581421, + "rewards/chosen": -0.44766363501548767, + "rewards/margins": 0.5626498460769653, + "rewards/rejected": -1.0103133916854858, + "step": 620 + }, + { + "epoch": 0.32975660821774405, + "grad_norm": 43.25, + "learning_rate": 8.009920290961302e-06, + "logits/chosen": 0.0821579247713089, + "logits/rejected": 0.02954455092549324, + "logps/chosen": -342.8247375488281, + "logps/rejected": -329.66412353515625, + "loss": 0.5461, + "rewards/accuracies": 0.734375, + "rewards/chosen": -0.33613795042037964, + "rewards/margins": 0.6381479501724243, + "rewards/rejected": -0.9742859601974487, + "step": 630 + }, + { + "epoch": 0.33499084009421615, + "grad_norm": 41.5, + "learning_rate": 7.94032718869134e-06, + "logits/chosen": 0.05233108997344971, + "logits/rejected": 0.01644396223127842, + "logps/chosen": -358.94122314453125, + "logps/rejected": -318.9773864746094, + "loss": 0.5484, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.31456658244132996, + "rewards/margins": 0.5918359160423279, + "rewards/rejected": -0.906402587890625, + "step": 640 + }, + { + "epoch": 0.3402250719706883, + "grad_norm": 33.75, + "learning_rate": 7.869852205423738e-06, + "logits/chosen": 0.009163993410766125, + "logits/rejected": -0.007742050103843212, + "logps/chosen": -342.1306457519531, + "logps/rejected": -305.15631103515625, + "loss": 0.5761, + "rewards/accuracies": 0.715624988079071, + "rewards/chosen": -0.31523770093917847, + "rewards/margins": 0.5241441130638123, + "rewards/rejected": -0.8393818140029907, + "step": 650 + }, + { + "epoch": 0.34545930384716045, + "grad_norm": 40.75, + "learning_rate": 7.798516478448514e-06, + "logits/chosen": -0.01641426980495453, + "logits/rejected": 0.01689126156270504, + "logps/chosen": -364.1932067871094, + "logps/rejected": -316.0845031738281, + "loss": 0.5663, + "rewards/accuracies": 0.6812499761581421, + "rewards/chosen": -0.4109156131744385, + "rewards/margins": 0.5688261389732361, + "rewards/rejected": -0.9797417521476746, + "step": 660 + }, + { + "epoch": 0.35069353572363254, + "grad_norm": 31.0, + "learning_rate": 7.726341403215237e-06, + "logits/chosen": 0.09717679023742676, + "logits/rejected": 0.10045663267374039, + "logps/chosen": -344.9303894042969, + "logps/rejected": -297.6980895996094, + "loss": 0.5746, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.5955493450164795, + "rewards/margins": 0.6834920644760132, + "rewards/rejected": -1.2790412902832031, + "step": 670 + }, + { + "epoch": 0.3559277676001047, + "grad_norm": 33.75, + "learning_rate": 7.653348626915957e-06, + "logits/chosen": 0.15363553166389465, + "logits/rejected": 0.20421214401721954, + "logps/chosen": -342.5677795410156, + "logps/rejected": -321.9688415527344, + "loss": 0.5366, + "rewards/accuracies": 0.753125011920929, + "rewards/chosen": -0.6576627492904663, + "rewards/margins": 0.7189033627510071, + "rewards/rejected": -1.3765661716461182, + "step": 680 + }, + { + "epoch": 0.3611619994765768, + "grad_norm": 32.75, + "learning_rate": 7.5795600419926595e-06, + "logits/chosen": 0.2883613705635071, + "logits/rejected": 0.2674182951450348, + "logps/chosen": -353.3540344238281, + "logps/rejected": -311.6170959472656, + "loss": 0.561, + "rewards/accuracies": 0.690625011920929, + "rewards/chosen": -0.47711777687072754, + "rewards/margins": 0.6064115762710571, + "rewards/rejected": -1.0835294723510742, + "step": 690 + }, + { + "epoch": 0.36639623135304894, + "grad_norm": 31.5, + "learning_rate": 7.504997779571134e-06, + "logits/chosen": 0.3076362907886505, + "logits/rejected": 0.3693523108959198, + "logps/chosen": -341.3320007324219, + "logps/rejected": -311.7213439941406, + "loss": 0.5929, + "rewards/accuracies": 0.71875, + "rewards/chosen": -0.3323260247707367, + "rewards/margins": 0.46539098024368286, + "rewards/rejected": -0.7977169752120972, + "step": 700 + }, + { + "epoch": 0.3716304632295211, + "grad_norm": 41.75, + "learning_rate": 7.429684202823284e-06, + "logits/chosen": 0.24711641669273376, + "logits/rejected": 0.15847380459308624, + "logps/chosen": -364.9874572753906, + "logps/rejected": -315.2619934082031, + "loss": 0.5373, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.3125119209289551, + "rewards/margins": 0.6405046582221985, + "rewards/rejected": -0.9530165791511536, + "step": 710 + }, + { + "epoch": 0.3768646951059932, + "grad_norm": 38.75, + "learning_rate": 7.353641900259823e-06, + "logits/chosen": 0.26350438594818115, + "logits/rejected": 0.20197024941444397, + "logps/chosen": -346.3494567871094, + "logps/rejected": -314.46673583984375, + "loss": 0.5653, + "rewards/accuracies": 0.7124999761581421, + "rewards/chosen": -0.5180891752243042, + "rewards/margins": 0.6086361408233643, + "rewards/rejected": -1.1267253160476685, + "step": 720 + }, + { + "epoch": 0.38209892698246534, + "grad_norm": 43.25, + "learning_rate": 7.276893678955387e-06, + "logits/chosen": 0.06701471656560898, + "logits/rejected": 0.2811892330646515, + "logps/chosen": -367.32171630859375, + "logps/rejected": -336.2926330566406, + "loss": 0.5983, + "rewards/accuracies": 0.715624988079071, + "rewards/chosen": -0.7037767171859741, + "rewards/margins": 0.647846519947052, + "rewards/rejected": -1.3516231775283813, + "step": 730 + }, + { + "epoch": 0.38733315885893743, + "grad_norm": 31.0, + "learning_rate": 7.199462557708098e-06, + "logits/chosen": 0.059072770178318024, + "logits/rejected": 0.15490247309207916, + "logps/chosen": -310.3693542480469, + "logps/rejected": -300.7638244628906, + "loss": 0.5609, + "rewards/accuracies": 0.671875, + "rewards/chosen": -0.6676520705223083, + "rewards/margins": 0.6050979495048523, + "rewards/rejected": -1.2727501392364502, + "step": 740 + }, + { + "epoch": 0.3925673907354096, + "grad_norm": 37.25, + "learning_rate": 7.1213717601356245e-06, + "logits/chosen": 0.04896034672856331, + "logits/rejected": 0.07459446787834167, + "logps/chosen": -362.3270568847656, + "logps/rejected": -325.6581115722656, + "loss": 0.5211, + "rewards/accuracies": 0.762499988079071, + "rewards/chosen": -0.5440580248832703, + "rewards/margins": 0.6718584299087524, + "rewards/rejected": -1.2159165143966675, + "step": 750 + }, + { + "epoch": 0.39780162261188173, + "grad_norm": 35.75, + "learning_rate": 7.042644707709816e-06, + "logits/chosen": 0.028451938182115555, + "logits/rejected": 0.1025298684835434, + "logps/chosen": -352.1075134277344, + "logps/rejected": -334.69586181640625, + "loss": 0.575, + "rewards/accuracies": 0.731249988079071, + "rewards/chosen": -0.5233157873153687, + "rewards/margins": 0.5710444450378418, + "rewards/rejected": -1.0943602323532104, + "step": 760 + }, + { + "epoch": 0.40303585448835383, + "grad_norm": 48.5, + "learning_rate": 6.963305012731984e-06, + "logits/chosen": 0.03199886158108711, + "logits/rejected": 0.04575660824775696, + "logps/chosen": -304.0777893066406, + "logps/rejected": -298.5992126464844, + "loss": 0.5943, + "rewards/accuracies": 0.6781250238418579, + "rewards/chosen": -0.4543026089668274, + "rewards/margins": 0.5623122453689575, + "rewards/rejected": -1.0166150331497192, + "step": 770 + }, + { + "epoch": 0.408270086364826, + "grad_norm": 44.25, + "learning_rate": 6.8833764712509554e-06, + "logits/chosen": 0.07288331538438797, + "logits/rejected": 0.12292104959487915, + "logps/chosen": -316.65643310546875, + "logps/rejected": -302.11065673828125, + "loss": 0.5738, + "rewards/accuracies": 0.7124999761581421, + "rewards/chosen": -0.31171008944511414, + "rewards/margins": 0.5113183259963989, + "rewards/rejected": -0.8230283856391907, + "step": 780 + }, + { + "epoch": 0.4135043182412981, + "grad_norm": 40.5, + "learning_rate": 6.802883055926026e-06, + "logits/chosen": 0.010061124339699745, + "logits/rejected": 0.09118668735027313, + "logps/chosen": -333.7193603515625, + "logps/rejected": -297.7068786621094, + "loss": 0.5423, + "rewards/accuracies": 0.7093750238418579, + "rewards/chosen": -0.29549041390419006, + "rewards/margins": 0.7173663973808289, + "rewards/rejected": -1.0128569602966309, + "step": 790 + }, + { + "epoch": 0.4187385501177702, + "grad_norm": 44.0, + "learning_rate": 6.721848908836921e-06, + "logits/chosen": 0.02005094103515148, + "logits/rejected": 0.028864353895187378, + "logps/chosen": -380.8367614746094, + "logps/rejected": -323.69317626953125, + "loss": 0.5161, + "rewards/accuracies": 0.721875011920929, + "rewards/chosen": -0.44863682985305786, + "rewards/margins": 0.7399032115936279, + "rewards/rejected": -1.188539981842041, + "step": 800 + }, + { + "epoch": 0.4239727819942423, + "grad_norm": 33.5, + "learning_rate": 6.640298334242959e-06, + "logits/chosen": -0.040520161390304565, + "logits/rejected": 0.05559789016842842, + "logps/chosen": -326.5780029296875, + "logps/rejected": -323.2212219238281, + "loss": 0.5382, + "rewards/accuracies": 0.753125011920929, + "rewards/chosen": -0.7126988172531128, + "rewards/margins": 0.7181462049484253, + "rewards/rejected": -1.430845022201538, + "step": 810 + }, + { + "epoch": 0.42920701387071447, + "grad_norm": 43.75, + "learning_rate": 6.558255791293572e-06, + "logits/chosen": -0.013416662812232971, + "logits/rejected": 0.10103078186511993, + "logps/chosen": -359.6937255859375, + "logps/rejected": -331.66778564453125, + "loss": 0.5828, + "rewards/accuracies": 0.7093750238418579, + "rewards/chosen": -0.7645487189292908, + "rewards/margins": 0.6964177489280701, + "rewards/rejected": -1.4609664678573608, + "step": 820 + }, + { + "epoch": 0.4344412457471866, + "grad_norm": 34.25, + "learning_rate": 6.475745886692361e-06, + "logits/chosen": 0.08711127936840057, + "logits/rejected": 0.07562930881977081, + "logps/chosen": -356.294189453125, + "logps/rejected": -345.10723876953125, + "loss": 0.5504, + "rewards/accuracies": 0.715624988079071, + "rewards/chosen": -0.7678012251853943, + "rewards/margins": 0.7699213027954102, + "rewards/rejected": -1.5377224683761597, + "step": 830 + }, + { + "epoch": 0.4396754776236587, + "grad_norm": 43.75, + "learning_rate": 6.392793367316905e-06, + "logits/chosen": -0.004989707376807928, + "logits/rejected": 0.02837694250047207, + "logps/chosen": -347.4369812011719, + "logps/rejected": -331.71673583984375, + "loss": 0.5274, + "rewards/accuracies": 0.765625, + "rewards/chosen": -0.7738672494888306, + "rewards/margins": 0.6914864778518677, + "rewards/rejected": -1.4653537273406982, + "step": 840 + }, + { + "epoch": 0.44490970950013087, + "grad_norm": 38.5, + "learning_rate": 6.309423112796529e-06, + "logits/chosen": 0.01806572452187538, + "logits/rejected": 0.247653529047966, + "logps/chosen": -328.3951110839844, + "logps/rejected": -334.0901794433594, + "loss": 0.5665, + "rewards/accuracies": 0.7281249761581421, + "rewards/chosen": -0.7129162549972534, + "rewards/margins": 0.6553431749343872, + "rewards/rejected": -1.3682594299316406, + "step": 850 + }, + { + "epoch": 0.45014394137660296, + "grad_norm": 50.25, + "learning_rate": 6.225660128050248e-06, + "logits/chosen": 0.07560832798480988, + "logits/rejected": 0.13501077890396118, + "logps/chosen": -341.61297607421875, + "logps/rejected": -323.997802734375, + "loss": 0.5584, + "rewards/accuracies": 0.7124999761581421, + "rewards/chosen": -0.7762230038642883, + "rewards/margins": 0.6634593605995178, + "rewards/rejected": -1.4396824836730957, + "step": 860 + }, + { + "epoch": 0.4553781732530751, + "grad_norm": 42.5, + "learning_rate": 6.141529535787139e-06, + "logits/chosen": 0.18502117693424225, + "logits/rejected": 0.2866303622722626, + "logps/chosen": -381.30023193359375, + "logps/rejected": -345.3155517578125, + "loss": 0.5162, + "rewards/accuracies": 0.7437499761581421, + "rewards/chosen": -0.7158933877944946, + "rewards/margins": 0.7875123023986816, + "rewards/rejected": -1.5034056901931763, + "step": 870 + }, + { + "epoch": 0.46061240512954726, + "grad_norm": 39.25, + "learning_rate": 6.057056568971383e-06, + "logits/chosen": 0.06999083608388901, + "logits/rejected": 0.14929188787937164, + "logps/chosen": -363.5868225097656, + "logps/rejected": -335.22454833984375, + "loss": 0.5283, + "rewards/accuracies": 0.7093750238418579, + "rewards/chosen": -0.768639326095581, + "rewards/margins": 0.8374959230422974, + "rewards/rejected": -1.6061351299285889, + "step": 880 + }, + { + "epoch": 0.46584663700601936, + "grad_norm": 36.5, + "learning_rate": 5.972266563254246e-06, + "logits/chosen": 0.27822867035865784, + "logits/rejected": 0.2106626331806183, + "logps/chosen": -391.5980224609375, + "logps/rejected": -348.16497802734375, + "loss": 0.5582, + "rewards/accuracies": 0.7093750238418579, + "rewards/chosen": -0.8508477210998535, + "rewards/margins": 0.7439194917678833, + "rewards/rejected": -1.5947673320770264, + "step": 890 + }, + { + "epoch": 0.4710808688824915, + "grad_norm": 33.25, + "learning_rate": 5.887184949375242e-06, + "logits/chosen": 0.13532599806785583, + "logits/rejected": 0.22985470294952393, + "logps/chosen": -346.69805908203125, + "logps/rejected": -307.9049072265625, + "loss": 0.5271, + "rewards/accuracies": 0.746874988079071, + "rewards/chosen": -0.6407856345176697, + "rewards/margins": 0.7918336391448975, + "rewards/rejected": -1.432619333267212, + "step": 900 + }, + { + "epoch": 0.4763151007589636, + "grad_norm": 36.75, + "learning_rate": 5.8018372455348e-06, + "logits/chosen": 0.20308074355125427, + "logits/rejected": 0.22425875067710876, + "logps/chosen": -364.8111572265625, + "logps/rejected": -323.373291015625, + "loss": 0.5568, + "rewards/accuracies": 0.703125, + "rewards/chosen": -0.7124744057655334, + "rewards/margins": 0.7094903588294983, + "rewards/rejected": -1.4219647645950317, + "step": 910 + }, + { + "epoch": 0.48154933263543576, + "grad_norm": 33.0, + "learning_rate": 5.71624904974069e-06, + "logits/chosen": 0.08423123508691788, + "logits/rejected": 0.2612914443016052, + "logps/chosen": -365.93475341796875, + "logps/rejected": -344.0113830566406, + "loss": 0.5509, + "rewards/accuracies": 0.703125, + "rewards/chosen": -0.7027541399002075, + "rewards/margins": 0.7311244606971741, + "rewards/rejected": -1.4338784217834473, + "step": 920 + }, + { + "epoch": 0.48678356451190785, + "grad_norm": 28.75, + "learning_rate": 5.630446032130498e-06, + "logits/chosen": 0.08106087893247604, + "logits/rejected": 0.20746131241321564, + "logps/chosen": -348.0682678222656, + "logps/rejected": -333.0579833984375, + "loss": 0.5412, + "rewards/accuracies": 0.715624988079071, + "rewards/chosen": -0.5800120234489441, + "rewards/margins": 0.6838679313659668, + "rewards/rejected": -1.2638797760009766, + "step": 930 + }, + { + "epoch": 0.49201779638838, + "grad_norm": 41.25, + "learning_rate": 5.5444539272724925e-06, + "logits/chosen": 0.10578608512878418, + "logits/rejected": 0.28211677074432373, + "logps/chosen": -348.66021728515625, + "logps/rejected": -336.518798828125, + "loss": 0.5391, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.6300241351127625, + "rewards/margins": 0.74030601978302, + "rewards/rejected": -1.3703300952911377, + "step": 940 + }, + { + "epoch": 0.49725202826485215, + "grad_norm": 46.5, + "learning_rate": 5.458298526447155e-06, + "logits/chosen": 0.2096971720457077, + "logits/rejected": 0.2496938705444336, + "logps/chosen": -347.2626037597656, + "logps/rejected": -311.06072998046875, + "loss": 0.6027, + "rewards/accuracies": 0.6968749761581421, + "rewards/chosen": -0.5817705392837524, + "rewards/margins": 0.5896257162094116, + "rewards/rejected": -1.171396255493164, + "step": 950 + }, + { + "epoch": 0.5024862601413242, + "grad_norm": 46.0, + "learning_rate": 5.372005669911694e-06, + "logits/chosen": 0.0424225851893425, + "logits/rejected": 0.15929560363292694, + "logps/chosen": -323.9616394042969, + "logps/rejected": -302.2296142578125, + "loss": 0.6337, + "rewards/accuracies": 0.621874988079071, + "rewards/chosen": -0.5780489444732666, + "rewards/margins": 0.45929765701293945, + "rewards/rejected": -1.037346601486206, + "step": 960 + }, + { + "epoch": 0.5077204920177963, + "grad_norm": 37.75, + "learning_rate": 5.285601239149875e-06, + "logits/chosen": 0.13957878947257996, + "logits/rejected": 0.06276361644268036, + "logps/chosen": -363.7620849609375, + "logps/rejected": -332.8384704589844, + "loss": 0.5881, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.4892084002494812, + "rewards/margins": 0.5942028760910034, + "rewards/rejected": -1.0834112167358398, + "step": 970 + }, + { + "epoch": 0.5129547238942685, + "grad_norm": 43.0, + "learning_rate": 5.199111149109498e-06, + "logits/chosen": 0.02179548144340515, + "logits/rejected": 0.19822855293750763, + "logps/chosen": -311.2777404785156, + "logps/rejected": -302.4115295410156, + "loss": 0.5712, + "rewards/accuracies": 0.6812499761581421, + "rewards/chosen": -0.624075174331665, + "rewards/margins": 0.6447365880012512, + "rewards/rejected": -1.2688117027282715, + "step": 980 + }, + { + "epoch": 0.5181889557707406, + "grad_norm": 41.25, + "learning_rate": 5.112561340429817e-06, + "logits/chosen": 0.18798553943634033, + "logits/rejected": 0.09373607486486435, + "logps/chosen": -341.14532470703125, + "logps/rejected": -302.0753479003906, + "loss": 0.5605, + "rewards/accuracies": 0.6968749761581421, + "rewards/chosen": -0.6896688342094421, + "rewards/margins": 0.6189961433410645, + "rewards/rejected": -1.3086649179458618, + "step": 990 + }, + { + "epoch": 0.5234231876472127, + "grad_norm": 38.5, + "learning_rate": 5.0259777716612665e-06, + "logits/chosen": 0.12692488729953766, + "logits/rejected": 0.19669848680496216, + "logps/chosen": -368.73553466796875, + "logps/rejected": -342.2942810058594, + "loss": 0.5928, + "rewards/accuracies": 0.7093750238418579, + "rewards/chosen": -0.6040617823600769, + "rewards/margins": 0.5547485947608948, + "rewards/rejected": -1.1588103771209717, + "step": 1000 + }, + { + "epoch": 0.528657419523685, + "grad_norm": 44.5, + "learning_rate": 4.939386411479814e-06, + "logits/chosen": 0.17332792282104492, + "logits/rejected": 0.23241786658763885, + "logps/chosen": -363.03436279296875, + "logps/rejected": -353.3703918457031, + "loss": 0.566, + "rewards/accuracies": 0.715624988079071, + "rewards/chosen": -0.49267879128456116, + "rewards/margins": 0.6268835067749023, + "rewards/rejected": -1.1195623874664307, + "step": 1010 + }, + { + "epoch": 0.533891651400157, + "grad_norm": 45.25, + "learning_rate": 4.85281323089828e-06, + "logits/chosen": 0.21631035208702087, + "logits/rejected": 0.18023517727851868, + "logps/chosen": -378.2434387207031, + "logps/rejected": -343.40557861328125, + "loss": 0.6151, + "rewards/accuracies": 0.640625, + "rewards/chosen": -0.60394287109375, + "rewards/margins": 0.5025305151939392, + "rewards/rejected": -1.106473445892334, + "step": 1020 + }, + { + "epoch": 0.5391258832766291, + "grad_norm": 39.25, + "learning_rate": 4.766284195476943e-06, + "logits/chosen": 0.26781684160232544, + "logits/rejected": 0.3123559355735779, + "logps/chosen": -358.23638916015625, + "logps/rejected": -327.5035705566406, + "loss": 0.5133, + "rewards/accuracies": 0.734375, + "rewards/chosen": -0.4111200273036957, + "rewards/margins": 0.7634793519973755, + "rewards/rejected": -1.1745994091033936, + "step": 1030 + }, + { + "epoch": 0.5443601151531012, + "grad_norm": 35.5, + "learning_rate": 4.679825257535795e-06, + "logits/chosen": 0.20860572159290314, + "logits/rejected": 0.17548814415931702, + "logps/chosen": -352.4187316894531, + "logps/rejected": -312.301513671875, + "loss": 0.5343, + "rewards/accuracies": 0.7437499761581421, + "rewards/chosen": -0.4872250556945801, + "rewards/margins": 0.6681305170059204, + "rewards/rejected": -1.1553555727005005, + "step": 1040 + }, + { + "epoch": 0.5495943470295734, + "grad_norm": 39.75, + "learning_rate": 4.593462348370759e-06, + "logits/chosen": 0.18573978543281555, + "logits/rejected": 0.301888644695282, + "logps/chosen": -346.60955810546875, + "logps/rejected": -321.0350036621094, + "loss": 0.5507, + "rewards/accuracies": 0.6968749761581421, + "rewards/chosen": -0.5732883214950562, + "rewards/margins": 0.6586001515388489, + "rewards/rejected": -1.2318884134292603, + "step": 1050 + }, + { + "epoch": 0.5548285789060455, + "grad_norm": 34.5, + "learning_rate": 4.507221370476223e-06, + "logits/chosen": 0.27575668692588806, + "logits/rejected": 0.2642131447792053, + "logps/chosen": -355.8514099121094, + "logps/rejected": -338.43194580078125, + "loss": 0.5667, + "rewards/accuracies": 0.7093750238418579, + "rewards/chosen": -0.603151798248291, + "rewards/margins": 0.6535589694976807, + "rewards/rejected": -1.2567107677459717, + "step": 1060 + }, + { + "epoch": 0.5600628107825176, + "grad_norm": 27.625, + "learning_rate": 4.421128189776195e-06, + "logits/chosen": 0.17187847197055817, + "logits/rejected": 0.30097633600234985, + "logps/chosen": -317.4275817871094, + "logps/rejected": -276.91058349609375, + "loss": 0.5341, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": -0.6077526807785034, + "rewards/margins": 0.6645117402076721, + "rewards/rejected": -1.2722644805908203, + "step": 1070 + }, + { + "epoch": 0.5652970426589898, + "grad_norm": 29.625, + "learning_rate": 4.335208627866438e-06, + "logits/chosen": 0.3107313811779022, + "logits/rejected": 0.3381400406360626, + "logps/chosen": -345.51873779296875, + "logps/rejected": -302.8568420410156, + "loss": 0.5109, + "rewards/accuracies": 0.7562500238418579, + "rewards/chosen": -0.5785449743270874, + "rewards/margins": 0.72198086977005, + "rewards/rejected": -1.3005257844924927, + "step": 1080 + }, + { + "epoch": 0.5705312745354619, + "grad_norm": 41.0, + "learning_rate": 4.249488454269908e-06, + "logits/chosen": 0.2692057490348816, + "logits/rejected": 0.36852699518203735, + "logps/chosen": -359.2057189941406, + "logps/rejected": -338.135986328125, + "loss": 0.5348, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.66764897108078, + "rewards/margins": 0.6854020953178406, + "rewards/rejected": -1.3530510663986206, + "step": 1090 + }, + { + "epoch": 0.575765506411934, + "grad_norm": 38.75, + "learning_rate": 4.163993378707786e-06, + "logits/chosen": 0.23850250244140625, + "logits/rejected": 0.32512611150741577, + "logps/chosen": -323.40374755859375, + "logps/rejected": -301.8957214355469, + "loss": 0.5664, + "rewards/accuracies": 0.7093750238418579, + "rewards/chosen": -0.5826855897903442, + "rewards/margins": 0.6959084272384644, + "rewards/rejected": -1.278593897819519, + "step": 1100 + }, + { + "epoch": 0.5809997382884062, + "grad_norm": 45.0, + "learning_rate": 4.0787490433884685e-06, + "logits/chosen": 0.25609683990478516, + "logits/rejected": 0.2937456965446472, + "logps/chosen": -329.973388671875, + "logps/rejected": -304.82904052734375, + "loss": 0.5641, + "rewards/accuracies": 0.6812499761581421, + "rewards/chosen": -0.6114211082458496, + "rewards/margins": 0.586783766746521, + "rewards/rejected": -1.1982048749923706, + "step": 1110 + }, + { + "epoch": 0.5862339701648783, + "grad_norm": 41.25, + "learning_rate": 3.993781015316802e-06, + "logits/chosen": 0.2453668862581253, + "logits/rejected": 0.2688624858856201, + "logps/chosen": -379.10552978515625, + "logps/rejected": -333.52215576171875, + "loss": 0.5727, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.6501410007476807, + "rewards/margins": 0.6753184199333191, + "rewards/rejected": -1.3254594802856445, + "step": 1120 + }, + { + "epoch": 0.5914682020413504, + "grad_norm": 45.25, + "learning_rate": 3.909114778625861e-06, + "logits/chosen": 0.22569675743579865, + "logits/rejected": 0.20706626772880554, + "logps/chosen": -384.211181640625, + "logps/rejected": -316.38995361328125, + "loss": 0.4917, + "rewards/accuracies": 0.7593749761581421, + "rewards/chosen": -0.5756205320358276, + "rewards/margins": 0.8201796412467957, + "rewards/rejected": -1.3958003520965576, + "step": 1130 + }, + { + "epoch": 0.5967024339178225, + "grad_norm": 41.25, + "learning_rate": 3.824775726933596e-06, + "logits/chosen": 0.28916609287261963, + "logits/rejected": 0.35677972435951233, + "logps/chosen": -349.084716796875, + "logps/rejected": -304.26617431640625, + "loss": 0.5363, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.662204384803772, + "rewards/margins": 0.72789466381073, + "rewards/rejected": -1.3900991678237915, + "step": 1140 + }, + { + "epoch": 0.6019366657942947, + "grad_norm": 56.0, + "learning_rate": 3.7407891557266242e-06, + "logits/chosen": 0.2205003947019577, + "logits/rejected": 0.27626657485961914, + "logps/chosen": -340.46710205078125, + "logps/rejected": -329.74786376953125, + "loss": 0.5708, + "rewards/accuracies": 0.7093750238418579, + "rewards/chosen": -0.8766458630561829, + "rewards/margins": 0.6808016896247864, + "rewards/rejected": -1.5574474334716797, + "step": 1150 + }, + { + "epoch": 0.6071708976707668, + "grad_norm": 43.0, + "learning_rate": 3.6571802547734457e-06, + "logits/chosen": 0.19416388869285583, + "logits/rejected": 0.3043220043182373, + "logps/chosen": -349.0753479003906, + "logps/rejected": -327.81964111328125, + "loss": 0.5312, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.8548088073730469, + "rewards/margins": 0.7565596103668213, + "rewards/rejected": -1.6113685369491577, + "step": 1160 + }, + { + "epoch": 0.6124051295472389, + "grad_norm": 38.75, + "learning_rate": 3.5739741005693807e-06, + "logits/chosen": 0.26571187376976013, + "logits/rejected": 0.403964102268219, + "logps/chosen": -375.9601135253906, + "logps/rejected": -345.172119140625, + "loss": 0.5493, + "rewards/accuracies": 0.715624988079071, + "rewards/chosen": -0.7909923195838928, + "rewards/margins": 0.7638007998466492, + "rewards/rejected": -1.5547930002212524, + "step": 1170 + }, + { + "epoch": 0.6176393614237111, + "grad_norm": 34.0, + "learning_rate": 3.4911956488154696e-06, + "logits/chosen": 0.22815632820129395, + "logits/rejected": 0.1856929063796997, + "logps/chosen": -343.6809997558594, + "logps/rejected": -315.6387939453125, + "loss": 0.6059, + "rewards/accuracies": 0.668749988079071, + "rewards/chosen": -0.8829870223999023, + "rewards/margins": 0.6338706016540527, + "rewards/rejected": -1.5168575048446655, + "step": 1180 + }, + { + "epoch": 0.6228735933001832, + "grad_norm": 33.5, + "learning_rate": 3.4088697269336045e-06, + "logits/chosen": 0.27643901109695435, + "logits/rejected": 0.3212670385837555, + "logps/chosen": -365.35595703125, + "logps/rejected": -312.13323974609375, + "loss": 0.4889, + "rewards/accuracies": 0.7593749761581421, + "rewards/chosen": -0.6881390810012817, + "rewards/margins": 0.8743413686752319, + "rewards/rejected": -1.5624804496765137, + "step": 1190 + }, + { + "epoch": 0.6281078251766553, + "grad_norm": 51.0, + "learning_rate": 3.3270210266201373e-06, + "logits/chosen": 0.29524120688438416, + "logits/rejected": 0.3692258298397064, + "logps/chosen": -351.9238586425781, + "logps/rejected": -327.85919189453125, + "loss": 0.587, + "rewards/accuracies": 0.7093750238418579, + "rewards/chosen": -0.8268892168998718, + "rewards/margins": 0.6596170663833618, + "rewards/rejected": -1.4865062236785889, + "step": 1200 + }, + { + "epoch": 0.6333420570531274, + "grad_norm": 32.0, + "learning_rate": 3.2456740964401977e-06, + "logits/chosen": 0.2849295735359192, + "logits/rejected": 0.4959246516227722, + "logps/chosen": -354.4574279785156, + "logps/rejected": -339.2894287109375, + "loss": 0.5613, + "rewards/accuracies": 0.7093750238418579, + "rewards/chosen": -0.8398796319961548, + "rewards/margins": 0.6982079744338989, + "rewards/rejected": -1.5380874872207642, + "step": 1210 + }, + { + "epoch": 0.6385762889295996, + "grad_norm": 41.5, + "learning_rate": 3.1648533344649303e-06, + "logits/chosen": 0.19813226163387299, + "logits/rejected": 0.44143158197402954, + "logps/chosen": -340.7344970703125, + "logps/rejected": -349.23114013671875, + "loss": 0.5171, + "rewards/accuracies": 0.737500011920929, + "rewards/chosen": -0.8335161209106445, + "rewards/margins": 0.7508944869041443, + "rewards/rejected": -1.5844106674194336, + "step": 1220 + }, + { + "epoch": 0.6438105208060717, + "grad_norm": 51.5, + "learning_rate": 3.084582980953881e-06, + "logits/chosen": 0.2836536169052124, + "logits/rejected": 0.2980581223964691, + "logps/chosen": -388.7774963378906, + "logps/rejected": -311.330322265625, + "loss": 0.5422, + "rewards/accuracies": 0.734375, + "rewards/chosen": -0.7870951294898987, + "rewards/margins": 0.7385523915290833, + "rewards/rejected": -1.525647521018982, + "step": 1230 + }, + { + "epoch": 0.6490447526825438, + "grad_norm": 37.0, + "learning_rate": 3.0048871110847043e-06, + "logits/chosen": 0.25779467821121216, + "logits/rejected": 0.23647575080394745, + "logps/chosen": -365.7320861816406, + "logps/rejected": -324.42779541015625, + "loss": 0.5212, + "rewards/accuracies": 0.7406250238418579, + "rewards/chosen": -0.7111884951591492, + "rewards/margins": 0.8184769749641418, + "rewards/rejected": -1.529665470123291, + "step": 1240 + }, + { + "epoch": 0.654278984559016, + "grad_norm": 37.75, + "learning_rate": 2.925789627732395e-06, + "logits/chosen": 0.2080594003200531, + "logits/rejected": 0.24620768427848816, + "logps/chosen": -357.9142150878906, + "logps/rejected": -327.7608947753906, + "loss": 0.5338, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.7053016424179077, + "rewards/margins": 0.8197522163391113, + "rewards/rejected": -1.5250537395477295, + "step": 1250 + }, + { + "epoch": 0.6595132164354881, + "grad_norm": 32.75, + "learning_rate": 2.8473142543001818e-06, + "logits/chosen": 0.2071513682603836, + "logits/rejected": 0.3076412081718445, + "logps/chosen": -320.2906188964844, + "logps/rejected": -300.67083740234375, + "loss": 0.5539, + "rewards/accuracies": 0.715624988079071, + "rewards/chosen": -0.808573842048645, + "rewards/margins": 0.7147833108901978, + "rewards/rejected": -1.5233570337295532, + "step": 1260 + }, + { + "epoch": 0.6647474483119602, + "grad_norm": 40.5, + "learning_rate": 2.7694845276042714e-06, + "logits/chosen": 0.19621708989143372, + "logits/rejected": 0.25396865606307983, + "logps/chosen": -360.89154052734375, + "logps/rejected": -332.4544677734375, + "loss": 0.5323, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.7095572352409363, + "rewards/margins": 0.8451035618782043, + "rewards/rejected": -1.5546607971191406, + "step": 1270 + }, + { + "epoch": 0.6699816801884323, + "grad_norm": 42.0, + "learning_rate": 2.6923237908145227e-06, + "logits/chosen": 0.26621609926223755, + "logits/rejected": 0.26851433515548706, + "logps/chosen": -325.02984619140625, + "logps/rejected": -332.8418884277344, + "loss": 0.5279, + "rewards/accuracies": 0.734375, + "rewards/chosen": -0.6964584589004517, + "rewards/margins": 0.7985239028930664, + "rewards/rejected": -1.4949824810028076, + "step": 1280 + }, + { + "epoch": 0.6752159120649045, + "grad_norm": 33.5, + "learning_rate": 2.615855186453241e-06, + "logits/chosen": 0.2269335240125656, + "logits/rejected": 0.3090178966522217, + "logps/chosen": -358.5539855957031, + "logps/rejected": -346.3840026855469, + "loss": 0.515, + "rewards/accuracies": 0.746874988079071, + "rewards/chosen": -0.6508886814117432, + "rewards/margins": 0.8423940539360046, + "rewards/rejected": -1.4932825565338135, + "step": 1290 + }, + { + "epoch": 0.6804501439413766, + "grad_norm": 46.25, + "learning_rate": 2.5401016494541193e-06, + "logits/chosen": 0.18353882431983948, + "logits/rejected": 0.3234695494174957, + "logps/chosen": -341.1846923828125, + "logps/rejected": -330.22100830078125, + "loss": 0.5505, + "rewards/accuracies": 0.6968749761581421, + "rewards/chosen": -0.7482860684394836, + "rewards/margins": 0.7044668197631836, + "rewards/rejected": -1.4527528285980225, + "step": 1300 + }, + { + "epoch": 0.6856843758178487, + "grad_norm": 53.0, + "learning_rate": 2.4650859002834465e-06, + "logits/chosen": 0.20721504092216492, + "logits/rejected": 0.4179702699184418, + "logps/chosen": -349.2493591308594, + "logps/rejected": -329.89178466796875, + "loss": 0.5464, + "rewards/accuracies": 0.715624988079071, + "rewards/chosen": -0.7340461015701294, + "rewards/margins": 0.6964015960693359, + "rewards/rejected": -1.4304475784301758, + "step": 1310 + }, + { + "epoch": 0.6909186076943209, + "grad_norm": 41.0, + "learning_rate": 2.390830438125661e-06, + "logits/chosen": 0.12962877750396729, + "logits/rejected": 0.1510620415210724, + "logps/chosen": -353.7647399902344, + "logps/rejected": -325.97857666015625, + "loss": 0.5903, + "rewards/accuracies": 0.7093750238418579, + "rewards/chosen": -0.7643977403640747, + "rewards/margins": 0.6256237626075745, + "rewards/rejected": -1.390021562576294, + "step": 1320 + }, + { + "epoch": 0.696152839570793, + "grad_norm": 35.5, + "learning_rate": 2.3173575341352457e-06, + "logits/chosen": 0.2343304455280304, + "logits/rejected": 0.3055940568447113, + "logps/chosen": -346.0968017578125, + "logps/rejected": -324.8904724121094, + "loss": 0.5805, + "rewards/accuracies": 0.6812499761581421, + "rewards/chosen": -0.664479672908783, + "rewards/margins": 0.6432613134384155, + "rewards/rejected": -1.3077409267425537, + "step": 1330 + }, + { + "epoch": 0.7013870714472651, + "grad_norm": 42.5, + "learning_rate": 2.2446892247570257e-06, + "logits/chosen": 0.28046920895576477, + "logits/rejected": 0.32017889618873596, + "logps/chosen": -351.0575866699219, + "logps/rejected": -331.1203918457031, + "loss": 0.5721, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.7322802543640137, + "rewards/margins": 0.6340840458869934, + "rewards/rejected": -1.3663642406463623, + "step": 1340 + }, + { + "epoch": 0.7066213033237373, + "grad_norm": 33.75, + "learning_rate": 2.172847305116872e-06, + "logits/chosen": 0.21674051880836487, + "logits/rejected": 0.2154543697834015, + "logps/chosen": -347.7149353027344, + "logps/rejected": -322.49041748046875, + "loss": 0.5757, + "rewards/accuracies": 0.6625000238418579, + "rewards/chosen": -0.6023791432380676, + "rewards/margins": 0.6135936975479126, + "rewards/rejected": -1.2159727811813354, + "step": 1350 + }, + { + "epoch": 0.7118555352002094, + "grad_norm": 35.25, + "learning_rate": 2.1018533224847638e-06, + "logits/chosen": 0.25386855006217957, + "logits/rejected": 0.2589409351348877, + "logps/chosen": -377.95098876953125, + "logps/rejected": -333.69635009765625, + "loss": 0.5638, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.6216534376144409, + "rewards/margins": 0.7461624145507812, + "rewards/rejected": -1.3678158521652222, + "step": 1360 + }, + { + "epoch": 0.7170897670766815, + "grad_norm": 34.25, + "learning_rate": 2.0317285698122035e-06, + "logits/chosen": 0.16483676433563232, + "logits/rejected": 0.32940173149108887, + "logps/chosen": -339.90887451171875, + "logps/rejected": -323.4661865234375, + "loss": 0.5358, + "rewards/accuracies": 0.71875, + "rewards/chosen": -0.5953302383422852, + "rewards/margins": 0.7433405518531799, + "rewards/rejected": -1.3386707305908203, + "step": 1370 + }, + { + "epoch": 0.7223239989531536, + "grad_norm": 36.5, + "learning_rate": 1.962494079345906e-06, + "logits/chosen": 0.09827329963445663, + "logits/rejected": 0.15172025561332703, + "logps/chosen": -383.4322509765625, + "logps/rejected": -332.6456604003906, + "loss": 0.5587, + "rewards/accuracies": 0.6625000238418579, + "rewards/chosen": -0.6343564391136169, + "rewards/margins": 0.7060499787330627, + "rewards/rejected": -1.3404064178466797, + "step": 1380 + }, + { + "epoch": 0.7275582308296258, + "grad_norm": 31.5, + "learning_rate": 1.8941706163196676e-06, + "logits/chosen": 0.2607867121696472, + "logits/rejected": 0.40171027183532715, + "logps/chosen": -314.8638610839844, + "logps/rejected": -309.70318603515625, + "loss": 0.5448, + "rewards/accuracies": 0.715624988079071, + "rewards/chosen": -0.6349200010299683, + "rewards/margins": 0.6689399480819702, + "rewards/rejected": -1.3038599491119385, + "step": 1390 + }, + { + "epoch": 0.7327924627060979, + "grad_norm": 32.75, + "learning_rate": 1.8267786727263426e-06, + "logits/chosen": 0.2444543093442917, + "logits/rejected": 0.3561408519744873, + "logps/chosen": -341.54486083984375, + "logps/rejected": -317.6834411621094, + "loss": 0.5475, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.5884548425674438, + "rewards/margins": 0.6964614987373352, + "rewards/rejected": -1.2849162817001343, + "step": 1400 + }, + { + "epoch": 0.73802669458257, + "grad_norm": 29.375, + "learning_rate": 1.760338461171755e-06, + "logits/chosen": 0.21381571888923645, + "logits/rejected": 0.2990735173225403, + "logps/chosen": -327.5777282714844, + "logps/rejected": -322.8582763671875, + "loss": 0.5882, + "rewards/accuracies": 0.6937500238418579, + "rewards/chosen": -0.6773046255111694, + "rewards/margins": 0.6310230493545532, + "rewards/rejected": -1.3083276748657227, + "step": 1410 + }, + { + "epoch": 0.7432609264590422, + "grad_norm": 35.5, + "learning_rate": 1.6948699088123992e-06, + "logits/chosen": 0.21470919251441956, + "logits/rejected": 0.25266513228416443, + "logps/chosen": -333.81622314453125, + "logps/rejected": -307.29010009765625, + "loss": 0.5675, + "rewards/accuracies": 0.684374988079071, + "rewards/chosen": -0.5910285711288452, + "rewards/margins": 0.6486441493034363, + "rewards/rejected": -1.2396726608276367, + "step": 1420 + }, + { + "epoch": 0.7484951583355143, + "grad_norm": 41.5, + "learning_rate": 1.6303926513787821e-06, + "logits/chosen": 0.08108960837125778, + "logits/rejected": 0.06331077218055725, + "logps/chosen": -339.13446044921875, + "logps/rejected": -310.05657958984375, + "loss": 0.5359, + "rewards/accuracies": 0.715624988079071, + "rewards/chosen": -0.6054891347885132, + "rewards/margins": 0.7288047075271606, + "rewards/rejected": -1.3342937231063843, + "step": 1430 + }, + { + "epoch": 0.7537293902119864, + "grad_norm": 35.25, + "learning_rate": 1.5669260272861426e-06, + "logits/chosen": 0.20187553763389587, + "logits/rejected": 0.1850132793188095, + "logps/chosen": -336.2139892578125, + "logps/rejected": -334.64141845703125, + "loss": 0.5212, + "rewards/accuracies": 0.7562500238418579, + "rewards/chosen": -0.5741112232208252, + "rewards/margins": 0.7714685201644897, + "rewards/rejected": -1.3455798625946045, + "step": 1440 + }, + { + "epoch": 0.7589636220884585, + "grad_norm": 40.0, + "learning_rate": 1.5044890718343535e-06, + "logits/chosen": 0.23868337273597717, + "logits/rejected": 0.12508736550807953, + "logps/chosen": -325.13470458984375, + "logps/rejected": -315.5563049316406, + "loss": 0.5758, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.7012775540351868, + "rewards/margins": 0.6083887815475464, + "rewards/rejected": -1.3096662759780884, + "step": 1450 + }, + { + "epoch": 0.7641978539649307, + "grad_norm": 36.0, + "learning_rate": 1.4431005114987485e-06, + "logits/chosen": 0.24810700118541718, + "logits/rejected": 0.24045827984809875, + "logps/chosen": -395.248779296875, + "logps/rejected": -352.9403381347656, + "loss": 0.5406, + "rewards/accuracies": 0.7124999761581421, + "rewards/chosen": -0.5400234460830688, + "rewards/margins": 0.7289952635765076, + "rewards/rejected": -1.2690187692642212, + "step": 1460 + }, + { + "epoch": 0.7694320858414028, + "grad_norm": 37.0, + "learning_rate": 1.3827787583135533e-06, + "logits/chosen": 0.17350813746452332, + "logits/rejected": 0.3169202506542206, + "logps/chosen": -348.0475769042969, + "logps/rejected": -334.5708923339844, + "loss": 0.5937, + "rewards/accuracies": 0.671875, + "rewards/chosen": -0.6126903295516968, + "rewards/margins": 0.6701362133026123, + "rewards/rejected": -1.2828264236450195, + "step": 1470 + }, + { + "epoch": 0.7746663177178749, + "grad_norm": 47.25, + "learning_rate": 1.3235419043496362e-06, + "logits/chosen": 0.31210190057754517, + "logits/rejected": 0.4039413034915924, + "logps/chosen": -340.84112548828125, + "logps/rejected": -317.56768798828125, + "loss": 0.5859, + "rewards/accuracies": 0.690625011920929, + "rewards/chosen": -0.6002423167228699, + "rewards/margins": 0.638708770275116, + "rewards/rejected": -1.2389512062072754, + "step": 1480 + }, + { + "epoch": 0.7799005495943471, + "grad_norm": 36.0, + "learning_rate": 1.2654077162882271e-06, + "logits/chosen": 0.23288901150226593, + "logits/rejected": 0.18572565913200378, + "logps/chosen": -346.56048583984375, + "logps/rejected": -320.6227722167969, + "loss": 0.5603, + "rewards/accuracies": 0.715624988079071, + "rewards/chosen": -0.597626805305481, + "rewards/margins": 0.7199206352233887, + "rewards/rejected": -1.3175475597381592, + "step": 1490 + }, + { + "epoch": 0.7851347814708192, + "grad_norm": 34.0, + "learning_rate": 1.2083936300922238e-06, + "logits/chosen": 0.34366169571876526, + "logits/rejected": 0.3978513181209564, + "logps/chosen": -362.42364501953125, + "logps/rejected": -334.7811279296875, + "loss": 0.5865, + "rewards/accuracies": 0.6968749761581421, + "rewards/chosen": -0.5686925649642944, + "rewards/margins": 0.6736392974853516, + "rewards/rejected": -1.2423319816589355, + "step": 1500 + }, + { + "epoch": 0.7903690133472913, + "grad_norm": 29.875, + "learning_rate": 1.1525167457766856e-06, + "logits/chosen": 0.2138892114162445, + "logits/rejected": 0.18896692991256714, + "logps/chosen": -337.1097412109375, + "logps/rejected": -315.19427490234375, + "loss": 0.5706, + "rewards/accuracies": 0.6781250238418579, + "rewards/chosen": -0.5940120816230774, + "rewards/margins": 0.6726430654525757, + "rewards/rejected": -1.2666552066802979, + "step": 1510 + }, + { + "epoch": 0.7956032452237635, + "grad_norm": 41.5, + "learning_rate": 1.0977938222801004e-06, + "logits/chosen": 0.2619992792606354, + "logits/rejected": 0.3644074499607086, + "logps/chosen": -340.20709228515625, + "logps/rejected": -314.8345031738281, + "loss": 0.5829, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.6193765997886658, + "rewards/margins": 0.6324874758720398, + "rewards/rejected": -1.2518641948699951, + "step": 1520 + }, + { + "epoch": 0.8008374771002356, + "grad_norm": 38.75, + "learning_rate": 1.0442412724379365e-06, + "logits/chosen": 0.19523002207279205, + "logits/rejected": 0.19391083717346191, + "logps/chosen": -346.6025390625, + "logps/rejected": -295.1070861816406, + "loss": 0.5791, + "rewards/accuracies": 0.706250011920929, + "rewards/chosen": -0.6979666948318481, + "rewards/margins": 0.6040533781051636, + "rewards/rejected": -1.3020200729370117, + "step": 1530 + }, + { + "epoch": 0.8060717089767077, + "grad_norm": 35.25, + "learning_rate": 9.9187515806e-07, + "logits/chosen": 0.3515494167804718, + "logits/rejected": 0.3660346567630768, + "logps/chosen": -368.04571533203125, + "logps/rejected": -319.6786193847656, + "loss": 0.562, + "rewards/accuracies": 0.7250000238418579, + "rewards/chosen": -0.5553494095802307, + "rewards/margins": 0.6650460958480835, + "rewards/rejected": -1.220395565032959, + "step": 1540 + }, + { + "epoch": 0.8113059408531798, + "grad_norm": 32.5, + "learning_rate": 9.407111851130879e-07, + "logits/chosen": 0.3301383852958679, + "logits/rejected": 0.277874231338501, + "logps/chosen": -338.82794189453125, + "logps/rejected": -320.33892822265625, + "loss": 0.517, + "rewards/accuracies": 0.731249988079071, + "rewards/chosen": -0.5561447143554688, + "rewards/margins": 0.7759675979614258, + "rewards/rejected": -1.3321123123168945, + "step": 1550 + }, + { + "epoch": 0.816540172729652, + "grad_norm": 36.5, + "learning_rate": 8.907646990103496e-07, + "logits/chosen": 0.2506190538406372, + "logits/rejected": 0.3602852523326874, + "logps/chosen": -330.9645080566406, + "logps/rejected": -305.1396179199219, + "loss": 0.5224, + "rewards/accuracies": 0.7124999761581421, + "rewards/chosen": -0.6034108400344849, + "rewards/margins": 0.7130244970321655, + "rewards/rejected": -1.3164353370666504, + "step": 1560 + }, + { + "epoch": 0.821774404606124, + "grad_norm": 31.0, + "learning_rate": 8.42050680008798e-07, + "logits/chosen": 0.11515147984027863, + "logits/rejected": 0.17567628622055054, + "logps/chosen": -345.45513916015625, + "logps/rejected": -329.71380615234375, + "loss": 0.5606, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.6171659231185913, + "rewards/margins": 0.6560701131820679, + "rewards/rejected": -1.2732360363006592, + "step": 1570 + }, + { + "epoch": 0.8270086364825961, + "grad_norm": 40.25, + "learning_rate": 7.945837387163424e-07, + "logits/chosen": 0.32047078013420105, + "logits/rejected": 0.30428266525268555, + "logps/chosen": -354.98297119140625, + "logps/rejected": -323.7526550292969, + "loss": 0.5838, + "rewards/accuracies": 0.6968749761581421, + "rewards/chosen": -0.6056606769561768, + "rewards/margins": 0.6512014865875244, + "rewards/rejected": -1.2568622827529907, + "step": 1580 + }, + { + "epoch": 0.8322428683590684, + "grad_norm": 46.5, + "learning_rate": 7.483781117096828e-07, + "logits/chosen": 0.3148021996021271, + "logits/rejected": 0.3127804100513458, + "logps/chosen": -372.3219299316406, + "logps/rejected": -344.1582946777344, + "loss": 0.5491, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.6119446754455566, + "rewards/margins": 0.7214257121086121, + "rewards/rejected": -1.3333704471588135, + "step": 1590 + }, + { + "epoch": 0.8374771002355405, + "grad_norm": 46.25, + "learning_rate": 7.034476572643855e-07, + "logits/chosen": 0.30679917335510254, + "logits/rejected": 0.35226863622665405, + "logps/chosen": -350.220703125, + "logps/rejected": -324.85552978515625, + "loss": 0.5779, + "rewards/accuracies": 0.6812499761581421, + "rewards/chosen": -0.6405495405197144, + "rewards/margins": 0.6569965481758118, + "rewards/rejected": -1.297546148300171, + "step": 1600 + }, + { + "epoch": 0.8427113321120125, + "grad_norm": 39.25, + "learning_rate": 6.598058511984307e-07, + "logits/chosen": 0.30778008699417114, + "logits/rejected": 0.32357412576675415, + "logps/chosen": -336.26043701171875, + "logps/rejected": -302.003662109375, + "loss": 0.5563, + "rewards/accuracies": 0.706250011920929, + "rewards/chosen": -0.7136401534080505, + "rewards/margins": 0.7009720802307129, + "rewards/rejected": -1.4146122932434082, + "step": 1610 + }, + { + "epoch": 0.8479455639884846, + "grad_norm": 31.375, + "learning_rate": 6.174657828304543e-07, + "logits/chosen": 0.2360578030347824, + "logits/rejected": 0.269861102104187, + "logps/chosen": -333.8213806152344, + "logps/rejected": -317.9136962890625, + "loss": 0.6052, + "rewards/accuracies": 0.653124988079071, + "rewards/chosen": -0.653602659702301, + "rewards/margins": 0.5380398631095886, + "rewards/rejected": -1.1916425228118896, + "step": 1620 + }, + { + "epoch": 0.8531797958649568, + "grad_norm": 31.0, + "learning_rate": 5.764401510539253e-07, + "logits/chosen": 0.3052513599395752, + "logits/rejected": 0.2237941026687622, + "logps/chosen": -359.7833557128906, + "logps/rejected": -303.59979248046875, + "loss": 0.5544, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.5926748514175415, + "rewards/margins": 0.6909803748130798, + "rewards/rejected": -1.2836551666259766, + "step": 1630 + }, + { + "epoch": 0.8584140277414289, + "grad_norm": 29.375, + "learning_rate": 5.36741260528415e-07, + "logits/chosen": 0.13227298855781555, + "logits/rejected": 0.2516798973083496, + "logps/chosen": -374.15216064453125, + "logps/rejected": -356.4462585449219, + "loss": 0.4841, + "rewards/accuracies": 0.778124988079071, + "rewards/chosen": -0.5345168113708496, + "rewards/margins": 0.8594783544540405, + "rewards/rejected": -1.3939951658248901, + "step": 1640 + }, + { + "epoch": 0.863648259617901, + "grad_norm": 33.25, + "learning_rate": 4.98381017989103e-07, + "logits/chosen": 0.12983646988868713, + "logits/rejected": 0.21437188982963562, + "logps/chosen": -347.4361572265625, + "logps/rejected": -309.8014221191406, + "loss": 0.5053, + "rewards/accuracies": 0.7593749761581421, + "rewards/chosen": -0.5407166481018066, + "rewards/margins": 0.7795384526252747, + "rewards/rejected": -1.3202550411224365, + "step": 1650 + }, + { + "epoch": 0.8688824914943732, + "grad_norm": 34.0, + "learning_rate": 4.6137092867564127e-07, + "logits/chosen": 0.2845227122306824, + "logits/rejected": 0.30273061990737915, + "logps/chosen": -319.15130615234375, + "logps/rejected": -301.53167724609375, + "loss": 0.5385, + "rewards/accuracies": 0.703125, + "rewards/chosen": -0.5724095106124878, + "rewards/margins": 0.7000355124473572, + "rewards/rejected": -1.2724450826644897, + "step": 1660 + }, + { + "epoch": 0.8741167233708453, + "grad_norm": 31.875, + "learning_rate": 4.2572209288143095e-07, + "logits/chosen": 0.24127666652202606, + "logits/rejected": 0.24279674887657166, + "logps/chosen": -349.64788818359375, + "logps/rejected": -320.1210021972656, + "loss": 0.5966, + "rewards/accuracies": 0.7124999761581421, + "rewards/chosen": -0.6160230040550232, + "rewards/margins": 0.5984134674072266, + "rewards/rejected": -1.2144365310668945, + "step": 1670 + }, + { + "epoch": 0.8793509552473174, + "grad_norm": 27.75, + "learning_rate": 3.9144520262435094e-07, + "logits/chosen": 0.2617672085762024, + "logits/rejected": 0.2997414469718933, + "logps/chosen": -375.3115234375, + "logps/rejected": -318.6556701660156, + "loss": 0.4994, + "rewards/accuracies": 0.7437499761581421, + "rewards/chosen": -0.47033149003982544, + "rewards/margins": 0.8409943580627441, + "rewards/rejected": -1.3113257884979248, + "step": 1680 + }, + { + "epoch": 0.8845851871237895, + "grad_norm": 43.0, + "learning_rate": 3.5855053843994625e-07, + "logits/chosen": 0.21160957217216492, + "logits/rejected": 0.2680891454219818, + "logps/chosen": -332.6085205078125, + "logps/rejected": -345.51708984375, + "loss": 0.5922, + "rewards/accuracies": 0.71875, + "rewards/chosen": -0.6225179433822632, + "rewards/margins": 0.6167303323745728, + "rewards/rejected": -1.239248275756836, + "step": 1690 + }, + { + "epoch": 0.8898194190002617, + "grad_norm": 38.5, + "learning_rate": 3.270479662980247e-07, + "logits/chosen": 0.37835609912872314, + "logits/rejected": 0.3830730617046356, + "logps/chosen": -342.5943908691406, + "logps/rejected": -330.6464538574219, + "loss": 0.5673, + "rewards/accuracies": 0.6937500238418579, + "rewards/chosen": -0.6359362602233887, + "rewards/margins": 0.7126865386962891, + "rewards/rejected": -1.3486229181289673, + "step": 1700 + }, + { + "epoch": 0.8950536508767338, + "grad_norm": 40.0, + "learning_rate": 2.9694693464359434e-07, + "logits/chosen": 0.2732701599597931, + "logits/rejected": 0.22347459197044373, + "logps/chosen": -359.74591064453125, + "logps/rejected": -350.09124755859375, + "loss": 0.5584, + "rewards/accuracies": 0.734375, + "rewards/chosen": -0.5950442552566528, + "rewards/margins": 0.7091079950332642, + "rewards/rejected": -1.304152250289917, + "step": 1710 + }, + { + "epoch": 0.9002878827532059, + "grad_norm": 35.75, + "learning_rate": 2.682564715630287e-07, + "logits/chosen": 0.23081643879413605, + "logits/rejected": 0.25710368156433105, + "logps/chosen": -352.5965881347656, + "logps/rejected": -327.4501647949219, + "loss": 0.4938, + "rewards/accuracies": 0.7593749761581421, + "rewards/chosen": -0.5555008053779602, + "rewards/margins": 0.8596852421760559, + "rewards/rejected": -1.4151861667633057, + "step": 1720 + }, + { + "epoch": 0.9055221146296781, + "grad_norm": 47.75, + "learning_rate": 2.4098518207630706e-07, + "logits/chosen": 0.2609720826148987, + "logits/rejected": 0.33673757314682007, + "logps/chosen": -347.4607238769531, + "logps/rejected": -302.12030029296875, + "loss": 0.5831, + "rewards/accuracies": 0.684374988079071, + "rewards/chosen": -0.6414428949356079, + "rewards/margins": 0.6207069158554077, + "rewards/rejected": -1.2621498107910156, + "step": 1730 + }, + { + "epoch": 0.9107563465061502, + "grad_norm": 44.5, + "learning_rate": 2.1514124555614412e-07, + "logits/chosen": 0.12182704359292984, + "logits/rejected": 0.2441065013408661, + "logps/chosen": -375.3338623046875, + "logps/rejected": -339.09783935546875, + "loss": 0.5553, + "rewards/accuracies": 0.7093750238418579, + "rewards/chosen": -0.6452317833900452, + "rewards/margins": 0.691677451133728, + "rewards/rejected": -1.336909294128418, + "step": 1740 + }, + { + "epoch": 0.9159905783826223, + "grad_norm": 35.0, + "learning_rate": 1.9073241327478287e-07, + "logits/chosen": 0.15027470886707306, + "logits/rejected": 0.1401127278804779, + "logps/chosen": -337.196044921875, + "logps/rejected": -298.7091979980469, + "loss": 0.5701, + "rewards/accuracies": 0.675000011920929, + "rewards/chosen": -0.5936441421508789, + "rewards/margins": 0.6106031537055969, + "rewards/rejected": -1.204247236251831, + "step": 1750 + }, + { + "epoch": 0.9212248102590945, + "grad_norm": 39.0, + "learning_rate": 1.677660060791836e-07, + "logits/chosen": 0.28163331747055054, + "logits/rejected": 0.28244131803512573, + "logps/chosen": -352.9237365722656, + "logps/rejected": -314.7523193359375, + "loss": 0.5306, + "rewards/accuracies": 0.7124999761581421, + "rewards/chosen": -0.5691486597061157, + "rewards/margins": 0.7542569041252136, + "rewards/rejected": -1.3234055042266846, + "step": 1760 + }, + { + "epoch": 0.9264590421355666, + "grad_norm": 35.75, + "learning_rate": 1.4624891219531256e-07, + "logits/chosen": 0.22607457637786865, + "logits/rejected": 0.24305304884910583, + "logps/chosen": -352.2106018066406, + "logps/rejected": -320.9057312011719, + "loss": 0.5535, + "rewards/accuracies": 0.7093750238418579, + "rewards/chosen": -0.5814008116722107, + "rewards/margins": 0.6882680058479309, + "rewards/rejected": -1.2696688175201416, + "step": 1770 + }, + { + "epoch": 0.9316932740120387, + "grad_norm": 31.625, + "learning_rate": 1.2618758516218187e-07, + "logits/chosen": 0.2960145175457001, + "logits/rejected": 0.2859836220741272, + "logps/chosen": -311.757080078125, + "logps/rejected": -291.49859619140625, + "loss": 0.5605, + "rewards/accuracies": 0.715624988079071, + "rewards/chosen": -0.5901347398757935, + "rewards/margins": 0.6353054046630859, + "rewards/rejected": -1.225440263748169, + "step": 1780 + }, + { + "epoch": 0.9369275058885108, + "grad_norm": 41.5, + "learning_rate": 1.0758804189626492e-07, + "logits/chosen": 0.2627493441104889, + "logits/rejected": 0.3188014328479767, + "logps/chosen": -339.60894775390625, + "logps/rejected": -312.9742736816406, + "loss": 0.5805, + "rewards/accuracies": 0.6812499761581421, + "rewards/chosen": -0.6211244463920593, + "rewards/margins": 0.610070526599884, + "rewards/rejected": -1.2311948537826538, + "step": 1790 + }, + { + "epoch": 0.942161737764983, + "grad_norm": 44.25, + "learning_rate": 9.045586088686497e-08, + "logits/chosen": 0.24961581826210022, + "logits/rejected": 0.1900576502084732, + "logps/chosen": -364.18084716796875, + "logps/rejected": -320.77935791015625, + "loss": 0.5466, + "rewards/accuracies": 0.7093750238418579, + "rewards/chosen": -0.5827781558036804, + "rewards/margins": 0.7221345901489258, + "rewards/rejected": -1.304912805557251, + "step": 1800 + }, + { + "epoch": 0.9473959696414551, + "grad_norm": 47.25, + "learning_rate": 7.479618052298132e-08, + "logits/chosen": 0.3448382019996643, + "logits/rejected": 0.25041282176971436, + "logps/chosen": -367.3106689453125, + "logps/rejected": -348.275634765625, + "loss": 0.5575, + "rewards/accuracies": 0.721875011920929, + "rewards/chosen": -0.6221089363098145, + "rewards/margins": 0.6926394104957581, + "rewards/rejected": -1.3147482872009277, + "step": 1810 + }, + { + "epoch": 0.9526302015179272, + "grad_norm": 34.25, + "learning_rate": 6.06136975521715e-08, + "logits/chosen": 0.12194709479808807, + "logits/rejected": 0.2333538830280304, + "logps/chosen": -364.0761413574219, + "logps/rejected": -331.1363525390625, + "loss": 0.5335, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.6387330293655396, + "rewards/margins": 0.7657533288002014, + "rewards/rejected": -1.4044864177703857, + "step": 1820 + }, + { + "epoch": 0.9578644333943994, + "grad_norm": 34.25, + "learning_rate": 4.7912665671874246e-08, + "logits/chosen": 0.1737120896577835, + "logits/rejected": 0.22887209057807922, + "logps/chosen": -345.92010498046875, + "logps/rejected": -320.91265869140625, + "loss": 0.5706, + "rewards/accuracies": 0.690625011920929, + "rewards/chosen": -0.6163133978843689, + "rewards/margins": 0.6825416088104248, + "rewards/rejected": -1.2988550662994385, + "step": 1830 + }, + { + "epoch": 0.9630986652708715, + "grad_norm": 40.0, + "learning_rate": 3.669689425361444e-08, + "logits/chosen": 0.23761753737926483, + "logits/rejected": 0.2581033408641815, + "logps/chosen": -318.6600341796875, + "logps/rejected": -311.02447509765625, + "loss": 0.5523, + "rewards/accuracies": 0.706250011920929, + "rewards/chosen": -0.545235276222229, + "rewards/margins": 0.6656547784805298, + "rewards/rejected": -1.2108900547027588, + "step": 1840 + }, + { + "epoch": 0.9683328971473436, + "grad_norm": 47.25, + "learning_rate": 2.6969747200472073e-08, + "logits/chosen": 0.2629963159561157, + "logits/rejected": 0.41755908727645874, + "logps/chosen": -329.5137634277344, + "logps/rejected": -319.69329833984375, + "loss": 0.5877, + "rewards/accuracies": 0.671875, + "rewards/chosen": -0.6437569856643677, + "rewards/margins": 0.6761563420295715, + "rewards/rejected": -1.3199132680892944, + "step": 1850 + }, + { + "epoch": 0.9735671290238157, + "grad_norm": 41.0, + "learning_rate": 1.873414193816092e-08, + "logits/chosen": 0.28942468762397766, + "logits/rejected": 0.29614606499671936, + "logps/chosen": -374.42071533203125, + "logps/rejected": -350.49163818359375, + "loss": 0.5343, + "rewards/accuracies": 0.7281249761581421, + "rewards/chosen": -0.6072831749916077, + "rewards/margins": 0.7554194331169128, + "rewards/rejected": -1.3627026081085205, + "step": 1860 + }, + { + "epoch": 0.9788013609002879, + "grad_norm": 52.5, + "learning_rate": 1.1992548540016858e-08, + "logits/chosen": 0.20397917926311493, + "logits/rejected": 0.19432392716407776, + "logps/chosen": -374.0498962402344, + "logps/rejected": -343.69732666015625, + "loss": 0.5625, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.6645746827125549, + "rewards/margins": 0.6493507027626038, + "rewards/rejected": -1.3139253854751587, + "step": 1870 + }, + { + "epoch": 0.98403559277676, + "grad_norm": 40.0, + "learning_rate": 6.746988986156e-09, + "logits/chosen": 0.26821833848953247, + "logits/rejected": 0.34926992654800415, + "logps/chosen": -333.3935546875, + "logps/rejected": -306.5715637207031, + "loss": 0.543, + "rewards/accuracies": 0.721875011920929, + "rewards/chosen": -0.6133612394332886, + "rewards/margins": 0.7218830585479736, + "rewards/rejected": -1.3352441787719727, + "step": 1880 + }, + { + "epoch": 0.9892698246532321, + "grad_norm": 39.5, + "learning_rate": 2.9990365570314874e-09, + "logits/chosen": 0.2864960730075836, + "logits/rejected": 0.2247813194990158, + "logps/chosen": -365.5644226074219, + "logps/rejected": -339.42352294921875, + "loss": 0.5211, + "rewards/accuracies": 0.7281249761581421, + "rewards/chosen": -0.5904373526573181, + "rewards/margins": 0.726112961769104, + "rewards/rejected": -1.3165501356124878, + "step": 1890 + }, + { + "epoch": 0.9945040565297043, + "grad_norm": 40.75, + "learning_rate": 7.498153615653758e-10, + "logits/chosen": 0.1960332989692688, + "logits/rejected": 0.2011827528476715, + "logps/chosen": -312.5413818359375, + "logps/rejected": -306.91925048828125, + "loss": 0.5729, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.5954924821853638, + "rewards/margins": 0.6465845704078674, + "rewards/rejected": -1.242077112197876, + "step": 1900 + }, + { + "epoch": 0.9997382884061764, + "grad_norm": 38.0, + "learning_rate": 0.0, + "logits/chosen": 0.20826788246631622, + "logits/rejected": 0.1714758276939392, + "logps/chosen": -357.2660217285156, + "logps/rejected": -325.0472412109375, + "loss": 0.5777, + "rewards/accuracies": 0.7124999761581421, + "rewards/chosen": -0.650245726108551, + "rewards/margins": 0.598173975944519, + "rewards/rejected": -1.2484197616577148, + "step": 1910 + } + ], + "logging_steps": 10, + "max_steps": 1910, + "num_input_tokens_seen": 0, + "num_train_epochs": 1, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 0.0, + "train_batch_size": 8, + "trial_name": null, + "trial_params": null +}