| { | |
| "best_metric": null, | |
| "best_model_checkpoint": null, | |
| "epoch": 0.9984, | |
| "eval_steps": 500, | |
| "global_step": 156, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "epoch": 0.01, | |
| "eta": 0.0010000000474974513, | |
| "grad_norm": 21.712186498887842, | |
| "learning_rate": 3.125e-08, | |
| "logits/chosen": -1.4551665782928467, | |
| "logits/rejected": -1.606083869934082, | |
| "logps/chosen": -144.822265625, | |
| "logps/pi_response": -243.71868896484375, | |
| "logps/ref_response": -243.71868896484375, | |
| "logps/rejected": -162.54443359375, | |
| "loss": 0.6931, | |
| "rewards/accuracies": 0.0, | |
| "rewards/chosen": 0.0, | |
| "rewards/margins": 0.0, | |
| "rewards/rejected": 0.0, | |
| "step": 1 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "eta": 0.0010000000474974513, | |
| "grad_norm": 15.364437403667353, | |
| "learning_rate": 3.1249999999999997e-07, | |
| "logits/chosen": -1.8610674142837524, | |
| "logits/rejected": -1.8439589738845825, | |
| "logps/chosen": -181.88343811035156, | |
| "logps/pi_response": -265.68951416015625, | |
| "logps/ref_response": -263.9686584472656, | |
| "logps/rejected": -183.3703155517578, | |
| "loss": 0.693, | |
| "rewards/accuracies": 0.4652777910232544, | |
| "rewards/chosen": -0.005265166517347097, | |
| "rewards/margins": -0.0010041914647445083, | |
| "rewards/rejected": -0.004260974936187267, | |
| "step": 10 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "eta": 0.0010000000474974513, | |
| "grad_norm": 18.345421633966485, | |
| "learning_rate": 4.989935734988097e-07, | |
| "logits/chosen": -1.6329383850097656, | |
| "logits/rejected": -1.6391578912734985, | |
| "logps/chosen": -178.48463439941406, | |
| "logps/pi_response": -288.0892639160156, | |
| "logps/ref_response": -259.4861755371094, | |
| "logps/rejected": -181.0362091064453, | |
| "loss": 0.6907, | |
| "rewards/accuracies": 0.53125, | |
| "rewards/chosen": -0.13044723868370056, | |
| "rewards/margins": 0.01964881829917431, | |
| "rewards/rejected": -0.1500960886478424, | |
| "step": 20 | |
| }, | |
| { | |
| "epoch": 0.19, | |
| "eta": 0.0010000000474974513, | |
| "grad_norm": 17.13627597968646, | |
| "learning_rate": 4.877641290737883e-07, | |
| "logits/chosen": -1.431341290473938, | |
| "logits/rejected": -1.421644687652588, | |
| "logps/chosen": -222.0276641845703, | |
| "logps/pi_response": -336.2035827636719, | |
| "logps/ref_response": -273.8411865234375, | |
| "logps/rejected": -218.7247314453125, | |
| "loss": 0.6957, | |
| "rewards/accuracies": 0.5406249761581421, | |
| "rewards/chosen": -0.4222361147403717, | |
| "rewards/margins": 0.015909332782030106, | |
| "rewards/rejected": -0.43814539909362793, | |
| "step": 30 | |
| }, | |
| { | |
| "epoch": 0.26, | |
| "eta": 0.0010000000474974513, | |
| "grad_norm": 14.253311206515136, | |
| "learning_rate": 4.646121984004665e-07, | |
| "logits/chosen": -1.4934971332550049, | |
| "logits/rejected": -1.447249412536621, | |
| "logps/chosen": -192.37478637695312, | |
| "logps/pi_response": -272.644775390625, | |
| "logps/ref_response": -250.92996215820312, | |
| "logps/rejected": -193.36312866210938, | |
| "loss": 0.6915, | |
| "rewards/accuracies": 0.534375011920929, | |
| "rewards/chosen": -0.19741061329841614, | |
| "rewards/margins": 0.010645559057593346, | |
| "rewards/rejected": -0.20805618166923523, | |
| "step": 40 | |
| }, | |
| { | |
| "epoch": 0.32, | |
| "eta": 0.0010000000474974513, | |
| "grad_norm": 16.796841718291038, | |
| "learning_rate": 4.3069871595684787e-07, | |
| "logits/chosen": -1.7099899053573608, | |
| "logits/rejected": -1.7535244226455688, | |
| "logps/chosen": -194.2742462158203, | |
| "logps/pi_response": -277.76318359375, | |
| "logps/ref_response": -266.2118835449219, | |
| "logps/rejected": -201.6314239501953, | |
| "loss": 0.6895, | |
| "rewards/accuracies": 0.5249999761581421, | |
| "rewards/chosen": -0.14295700192451477, | |
| "rewards/margins": 0.017321351915597916, | |
| "rewards/rejected": -0.1602783501148224, | |
| "step": 50 | |
| }, | |
| { | |
| "epoch": 0.38, | |
| "eta": 0.0010000000474974513, | |
| "grad_norm": 14.319074470267418, | |
| "learning_rate": 3.877242453630256e-07, | |
| "logits/chosen": -1.2345731258392334, | |
| "logits/rejected": -1.2892169952392578, | |
| "logps/chosen": -206.3093719482422, | |
| "logps/pi_response": -289.80902099609375, | |
| "logps/ref_response": -256.5821533203125, | |
| "logps/rejected": -212.19924926757812, | |
| "loss": 0.6896, | |
| "rewards/accuracies": 0.503125011920929, | |
| "rewards/chosen": -0.30671426653862, | |
| "rewards/margins": 0.010099029168486595, | |
| "rewards/rejected": -0.31681329011917114, | |
| "step": 60 | |
| }, | |
| { | |
| "epoch": 0.45, | |
| "eta": 0.0010000000474974513, | |
| "grad_norm": 24.50253677275901, | |
| "learning_rate": 3.378437060203357e-07, | |
| "logits/chosen": -1.03169584274292, | |
| "logits/rejected": -1.021615743637085, | |
| "logps/chosen": -221.443359375, | |
| "logps/pi_response": -319.42242431640625, | |
| "logps/ref_response": -262.21759033203125, | |
| "logps/rejected": -225.0926971435547, | |
| "loss": 0.6847, | |
| "rewards/accuracies": 0.534375011920929, | |
| "rewards/chosen": -0.44103240966796875, | |
| "rewards/margins": 0.028266970068216324, | |
| "rewards/rejected": -0.4692993760108948, | |
| "step": 70 | |
| }, | |
| { | |
| "epoch": 0.51, | |
| "eta": 0.0010000000474974513, | |
| "grad_norm": 19.526479759830394, | |
| "learning_rate": 2.8355831645441387e-07, | |
| "logits/chosen": -0.8754630088806152, | |
| "logits/rejected": -0.9458340406417847, | |
| "logps/chosen": -225.5301971435547, | |
| "logps/pi_response": -346.0572204589844, | |
| "logps/ref_response": -269.8856201171875, | |
| "logps/rejected": -233.25045776367188, | |
| "loss": 0.689, | |
| "rewards/accuracies": 0.518750011920929, | |
| "rewards/chosen": -0.513852596282959, | |
| "rewards/margins": 0.01622053235769272, | |
| "rewards/rejected": -0.5300731658935547, | |
| "step": 80 | |
| }, | |
| { | |
| "epoch": 0.58, | |
| "eta": 0.0010000000474974513, | |
| "grad_norm": 21.868311725000435, | |
| "learning_rate": 2.2759017277414164e-07, | |
| "logits/chosen": -0.814942479133606, | |
| "logits/rejected": -0.8241022825241089, | |
| "logps/chosen": -234.712646484375, | |
| "logps/pi_response": -357.76568603515625, | |
| "logps/ref_response": -260.8216552734375, | |
| "logps/rejected": -250.9972686767578, | |
| "loss": 0.6791, | |
| "rewards/accuracies": 0.5843750238418579, | |
| "rewards/chosen": -0.6007817983627319, | |
| "rewards/margins": 0.07638835906982422, | |
| "rewards/rejected": -0.6771702170372009, | |
| "step": 90 | |
| }, | |
| { | |
| "epoch": 0.64, | |
| "eta": 0.0010000000474974513, | |
| "grad_norm": 20.43299106630643, | |
| "learning_rate": 1.7274575140626315e-07, | |
| "logits/chosen": -0.897548496723175, | |
| "logits/rejected": -0.932096004486084, | |
| "logps/chosen": -236.4069061279297, | |
| "logps/pi_response": -367.8551940917969, | |
| "logps/ref_response": -274.1803283691406, | |
| "logps/rejected": -245.63119506835938, | |
| "loss": 0.6895, | |
| "rewards/accuracies": 0.565625011920929, | |
| "rewards/chosen": -0.5783185958862305, | |
| "rewards/margins": 0.0737391859292984, | |
| "rewards/rejected": -0.6520577669143677, | |
| "step": 100 | |
| }, | |
| { | |
| "epoch": 0.7, | |
| "eta": 0.0010000000474974513, | |
| "grad_norm": 20.07983919936344, | |
| "learning_rate": 1.2177518064852348e-07, | |
| "logits/chosen": -0.7010077834129333, | |
| "logits/rejected": -0.7780792713165283, | |
| "logps/chosen": -239.2875518798828, | |
| "logps/pi_response": -375.39691162109375, | |
| "logps/ref_response": -266.8659362792969, | |
| "logps/rejected": -249.2735595703125, | |
| "loss": 0.6817, | |
| "rewards/accuracies": 0.578125, | |
| "rewards/chosen": -0.6955354809761047, | |
| "rewards/margins": 0.06171605736017227, | |
| "rewards/rejected": -0.7572515606880188, | |
| "step": 110 | |
| }, | |
| { | |
| "epoch": 0.77, | |
| "eta": 0.0010000000474974513, | |
| "grad_norm": 18.440065783036147, | |
| "learning_rate": 7.723433775328384e-08, | |
| "logits/chosen": -0.7951020002365112, | |
| "logits/rejected": -0.8204092979431152, | |
| "logps/chosen": -261.5052795410156, | |
| "logps/pi_response": -384.1082458496094, | |
| "logps/ref_response": -273.2558288574219, | |
| "logps/rejected": -263.2559509277344, | |
| "loss": 0.678, | |
| "rewards/accuracies": 0.5375000238418579, | |
| "rewards/chosen": -0.7765410542488098, | |
| "rewards/margins": 0.03736639395356178, | |
| "rewards/rejected": -0.8139075040817261, | |
| "step": 120 | |
| }, | |
| { | |
| "epoch": 0.83, | |
| "eta": 0.0010000000474974513, | |
| "grad_norm": 20.63429522696927, | |
| "learning_rate": 4.1356686569674335e-08, | |
| "logits/chosen": -0.6491920351982117, | |
| "logits/rejected": -0.6609460711479187, | |
| "logps/chosen": -245.2913055419922, | |
| "logps/pi_response": -372.6864929199219, | |
| "logps/ref_response": -255.30258178710938, | |
| "logps/rejected": -250.04989624023438, | |
| "loss": 0.6865, | |
| "rewards/accuracies": 0.515625, | |
| "rewards/chosen": -0.7680090665817261, | |
| "rewards/margins": 0.01968952640891075, | |
| "rewards/rejected": -0.78769850730896, | |
| "step": 130 | |
| }, | |
| { | |
| "epoch": 0.9, | |
| "eta": 0.0010000000474974513, | |
| "grad_norm": 18.194304704086697, | |
| "learning_rate": 1.5941282340065697e-08, | |
| "logits/chosen": -0.8394004702568054, | |
| "logits/rejected": -0.8077179789543152, | |
| "logps/chosen": -243.73538208007812, | |
| "logps/pi_response": -391.2145690917969, | |
| "logps/ref_response": -279.54559326171875, | |
| "logps/rejected": -255.0926513671875, | |
| "loss": 0.6744, | |
| "rewards/accuracies": 0.578125, | |
| "rewards/chosen": -0.700167179107666, | |
| "rewards/margins": 0.07673807442188263, | |
| "rewards/rejected": -0.7769052386283875, | |
| "step": 140 | |
| }, | |
| { | |
| "epoch": 0.96, | |
| "eta": 0.0010000000474974513, | |
| "grad_norm": 17.724132159919495, | |
| "learning_rate": 2.2625595580163247e-09, | |
| "logits/chosen": -0.7538624405860901, | |
| "logits/rejected": -0.7211672067642212, | |
| "logps/chosen": -247.0625457763672, | |
| "logps/pi_response": -371.68939208984375, | |
| "logps/ref_response": -256.93328857421875, | |
| "logps/rejected": -252.12765502929688, | |
| "loss": 0.6781, | |
| "rewards/accuracies": 0.5093749761581421, | |
| "rewards/chosen": -0.7719146609306335, | |
| "rewards/margins": 0.03903389722108841, | |
| "rewards/rejected": -0.8109486699104309, | |
| "step": 150 | |
| }, | |
| { | |
| "epoch": 1.0, | |
| "step": 156, | |
| "total_flos": 0.0, | |
| "train_loss": 0.6867941472774897, | |
| "train_runtime": 22649.1506, | |
| "train_samples_per_second": 0.883, | |
| "train_steps_per_second": 0.007 | |
| } | |
| ], | |
| "logging_steps": 10, | |
| "max_steps": 156, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 1, | |
| "save_steps": 200, | |
| "total_flos": 0.0, | |
| "train_batch_size": 2, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |