| { | |
| "best_metric": 0.09428545832633972, | |
| "best_model_checkpoint": "saves/Llama-3.1-8B-Instruct/lora/saa-600/checkpoint-250", | |
| "epoch": 9.777777777777779, | |
| "eval_steps": 50, | |
| "global_step": 330, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "epoch": 0.2962962962962963, | |
| "grad_norm": 7.07548189163208, | |
| "learning_rate": 1.5151515151515152e-06, | |
| "logits/chosen": -0.4374169409275055, | |
| "logits/rejected": -0.5023793578147888, | |
| "logps/chosen": -1.741115927696228, | |
| "logps/rejected": -2.1606717109680176, | |
| "loss": 1.7946, | |
| "odds_ratio_loss": 15.69953727722168, | |
| "rewards/accuracies": 0.7749999761581421, | |
| "rewards/chosen": -0.17411158978939056, | |
| "rewards/margins": 0.04195558652281761, | |
| "rewards/rejected": -0.21606719493865967, | |
| "sft_loss": 0.22465327382087708, | |
| "step": 10 | |
| }, | |
| { | |
| "epoch": 0.5925925925925926, | |
| "grad_norm": 7.023080825805664, | |
| "learning_rate": 3.0303030303030305e-06, | |
| "logits/chosen": -0.42782774567604065, | |
| "logits/rejected": -0.48748907446861267, | |
| "logps/chosen": -1.7547874450683594, | |
| "logps/rejected": -2.1007962226867676, | |
| "loss": 1.8133, | |
| "odds_ratio_loss": 15.838772773742676, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": -0.1754787415266037, | |
| "rewards/margins": 0.034600891172885895, | |
| "rewards/rejected": -0.210079625248909, | |
| "sft_loss": 0.22939057648181915, | |
| "step": 20 | |
| }, | |
| { | |
| "epoch": 0.8888888888888888, | |
| "grad_norm": 8.079118728637695, | |
| "learning_rate": 4.5454545454545455e-06, | |
| "logits/chosen": -0.4177670478820801, | |
| "logits/rejected": -0.49106597900390625, | |
| "logps/chosen": -1.6719223260879517, | |
| "logps/rejected": -2.094174861907959, | |
| "loss": 1.725, | |
| "odds_ratio_loss": 15.089022636413574, | |
| "rewards/accuracies": 0.78125, | |
| "rewards/chosen": -0.1671922504901886, | |
| "rewards/margins": 0.042225271463394165, | |
| "rewards/rejected": -0.20941750705242157, | |
| "sft_loss": 0.21611404418945312, | |
| "step": 30 | |
| }, | |
| { | |
| "epoch": 1.1851851851851851, | |
| "grad_norm": 7.101726531982422, | |
| "learning_rate": 4.993149937871306e-06, | |
| "logits/chosen": -0.42014995217323303, | |
| "logits/rejected": -0.4878144860267639, | |
| "logps/chosen": -1.4801180362701416, | |
| "logps/rejected": -1.8868948221206665, | |
| "loss": 1.5344, | |
| "odds_ratio_loss": 13.476564407348633, | |
| "rewards/accuracies": 0.731249988079071, | |
| "rewards/chosen": -0.14801180362701416, | |
| "rewards/margins": 0.040677666664123535, | |
| "rewards/rejected": -0.1886894851922989, | |
| "sft_loss": 0.18675227463245392, | |
| "step": 40 | |
| }, | |
| { | |
| "epoch": 1.4814814814814814, | |
| "grad_norm": 4.9273481369018555, | |
| "learning_rate": 4.959688949822748e-06, | |
| "logits/chosen": -0.4227227568626404, | |
| "logits/rejected": -0.4957185685634613, | |
| "logps/chosen": -1.2785080671310425, | |
| "logps/rejected": -1.6517393589019775, | |
| "loss": 1.3352, | |
| "odds_ratio_loss": 11.81715202331543, | |
| "rewards/accuracies": 0.7250000238418579, | |
| "rewards/chosen": -0.12785081565380096, | |
| "rewards/margins": 0.03732311353087425, | |
| "rewards/rejected": -0.16517391800880432, | |
| "sft_loss": 0.15344038605690002, | |
| "step": 50 | |
| }, | |
| { | |
| "epoch": 1.4814814814814814, | |
| "eval_logits/chosen": -0.40017402172088623, | |
| "eval_logits/rejected": -0.4812173843383789, | |
| "eval_logps/chosen": -0.9889497756958008, | |
| "eval_logps/rejected": -1.5758014917373657, | |
| "eval_loss": 1.0316624641418457, | |
| "eval_odds_ratio_loss": 9.149198532104492, | |
| "eval_rewards/accuracies": 0.8333333134651184, | |
| "eval_rewards/chosen": -0.09889498353004456, | |
| "eval_rewards/margins": 0.058685168623924255, | |
| "eval_rewards/rejected": -0.1575801521539688, | |
| "eval_runtime": 2.3161, | |
| "eval_samples_per_second": 25.906, | |
| "eval_sft_loss": 0.11674254387617111, | |
| "eval_steps_per_second": 12.953, | |
| "step": 50 | |
| }, | |
| { | |
| "epoch": 1.7777777777777777, | |
| "grad_norm": 3.42924427986145, | |
| "learning_rate": 4.8987324340362445e-06, | |
| "logits/chosen": -0.4220319390296936, | |
| "logits/rejected": -0.4980909824371338, | |
| "logps/chosen": -0.89045250415802, | |
| "logps/rejected": -1.3505830764770508, | |
| "loss": 0.9359, | |
| "odds_ratio_loss": 8.349299430847168, | |
| "rewards/accuracies": 0.824999988079071, | |
| "rewards/chosen": -0.08904524892568588, | |
| "rewards/margins": 0.04601306468248367, | |
| "rewards/rejected": -0.13505831360816956, | |
| "sft_loss": 0.10094638913869858, | |
| "step": 60 | |
| }, | |
| { | |
| "epoch": 2.074074074074074, | |
| "grad_norm": 3.1744749546051025, | |
| "learning_rate": 4.810961790316731e-06, | |
| "logits/chosen": -0.4295685291290283, | |
| "logits/rejected": -0.5065377950668335, | |
| "logps/chosen": -0.5825018882751465, | |
| "logps/rejected": -1.0753108263015747, | |
| "loss": 0.625, | |
| "odds_ratio_loss": 5.649188995361328, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": -0.05825018882751465, | |
| "rewards/margins": 0.049280889332294464, | |
| "rewards/rejected": -0.10753107070922852, | |
| "sft_loss": 0.0600874125957489, | |
| "step": 70 | |
| }, | |
| { | |
| "epoch": 2.3703703703703702, | |
| "grad_norm": 1.9260555505752563, | |
| "learning_rate": 4.697358159051549e-06, | |
| "logits/chosen": -0.40925922989845276, | |
| "logits/rejected": -0.4894172251224518, | |
| "logps/chosen": -0.44294339418411255, | |
| "logps/rejected": -0.9772504568099976, | |
| "loss": 0.4878, | |
| "odds_ratio_loss": 4.406769275665283, | |
| "rewards/accuracies": 0.793749988079071, | |
| "rewards/chosen": -0.0442943349480629, | |
| "rewards/margins": 0.053430717438459396, | |
| "rewards/rejected": -0.097725048661232, | |
| "sft_loss": 0.047148533165454865, | |
| "step": 80 | |
| }, | |
| { | |
| "epoch": 2.6666666666666665, | |
| "grad_norm": 2.225752115249634, | |
| "learning_rate": 4.559191453574582e-06, | |
| "logits/chosen": -0.3779674470424652, | |
| "logits/rejected": -0.4604215621948242, | |
| "logps/chosen": -0.28717148303985596, | |
| "logps/rejected": -0.7895299196243286, | |
| "loss": 0.332, | |
| "odds_ratio_loss": 3.0206964015960693, | |
| "rewards/accuracies": 0.7875000238418579, | |
| "rewards/chosen": -0.028717149049043655, | |
| "rewards/margins": 0.050235848873853683, | |
| "rewards/rejected": -0.07895299792289734, | |
| "sft_loss": 0.029953395947813988, | |
| "step": 90 | |
| }, | |
| { | |
| "epoch": 2.962962962962963, | |
| "grad_norm": 1.3990237712860107, | |
| "learning_rate": 4.398006164494358e-06, | |
| "logits/chosen": -0.4050057828426361, | |
| "logits/rejected": -0.4781204164028168, | |
| "logps/chosen": -0.19231440126895905, | |
| "logps/rejected": -0.5829997062683105, | |
| "loss": 0.2371, | |
| "odds_ratio_loss": 2.1702122688293457, | |
| "rewards/accuracies": 0.7875000238418579, | |
| "rewards/chosen": -0.019231440499424934, | |
| "rewards/margins": 0.03906853124499321, | |
| "rewards/rejected": -0.058299969881772995, | |
| "sft_loss": 0.02010512165725231, | |
| "step": 100 | |
| }, | |
| { | |
| "epoch": 2.962962962962963, | |
| "eval_logits/chosen": -0.38126423954963684, | |
| "eval_logits/rejected": -0.455107718706131, | |
| "eval_logps/chosen": -0.13484641909599304, | |
| "eval_logps/rejected": -0.6987236142158508, | |
| "eval_loss": 0.16552023589611053, | |
| "eval_odds_ratio_loss": 1.47817862033844, | |
| "eval_rewards/accuracies": 0.8833333253860474, | |
| "eval_rewards/chosen": -0.013484641909599304, | |
| "eval_rewards/margins": 0.05638771876692772, | |
| "eval_rewards/rejected": -0.06987235695123672, | |
| "eval_runtime": 2.3132, | |
| "eval_samples_per_second": 25.938, | |
| "eval_sft_loss": 0.01770237274467945, | |
| "eval_steps_per_second": 12.969, | |
| "step": 100 | |
| }, | |
| { | |
| "epoch": 3.259259259259259, | |
| "grad_norm": 1.6745034456253052, | |
| "learning_rate": 4.215604094671835e-06, | |
| "logits/chosen": -0.39228641986846924, | |
| "logits/rejected": -0.4650408625602722, | |
| "logps/chosen": -0.14995309710502625, | |
| "logps/rejected": -0.6513184309005737, | |
| "loss": 0.1933, | |
| "odds_ratio_loss": 1.7675580978393555, | |
| "rewards/accuracies": 0.7875000238418579, | |
| "rewards/chosen": -0.014995308592915535, | |
| "rewards/margins": 0.05013653635978699, | |
| "rewards/rejected": -0.06513184309005737, | |
| "sft_loss": 0.016546962782740593, | |
| "step": 110 | |
| }, | |
| { | |
| "epoch": 3.5555555555555554, | |
| "grad_norm": 2.232027053833008, | |
| "learning_rate": 4.014024217844167e-06, | |
| "logits/chosen": -0.3439103364944458, | |
| "logits/rejected": -0.41849011182785034, | |
| "logps/chosen": -0.1279471218585968, | |
| "logps/rejected": -0.5881286859512329, | |
| "loss": 0.1679, | |
| "odds_ratio_loss": 1.5120834112167358, | |
| "rewards/accuracies": 0.856249988079071, | |
| "rewards/chosen": -0.012794713489711285, | |
| "rewards/margins": 0.04601815715432167, | |
| "rewards/rejected": -0.05881286785006523, | |
| "sft_loss": 0.01666136085987091, | |
| "step": 120 | |
| }, | |
| { | |
| "epoch": 3.851851851851852, | |
| "grad_norm": 1.1567457914352417, | |
| "learning_rate": 3.7955198860439892e-06, | |
| "logits/chosen": -0.4037134051322937, | |
| "logits/rejected": -0.4531572461128235, | |
| "logps/chosen": -0.10376612842082977, | |
| "logps/rejected": -0.5740376114845276, | |
| "loss": 0.1434, | |
| "odds_ratio_loss": 1.3169727325439453, | |
| "rewards/accuracies": 0.7749999761581421, | |
| "rewards/chosen": -0.010376612655818462, | |
| "rewards/margins": 0.04702714830636978, | |
| "rewards/rejected": -0.05740376561880112, | |
| "sft_loss": 0.011749515309929848, | |
| "step": 130 | |
| }, | |
| { | |
| "epoch": 4.148148148148148, | |
| "grad_norm": 0.8253294229507446, | |
| "learning_rate": 3.5625336406000752e-06, | |
| "logits/chosen": -0.41028180718421936, | |
| "logits/rejected": -0.46746310591697693, | |
| "logps/chosen": -0.09531185775995255, | |
| "logps/rejected": -0.5239280462265015, | |
| "loss": 0.1289, | |
| "odds_ratio_loss": 1.1706035137176514, | |
| "rewards/accuracies": 0.824999988079071, | |
| "rewards/chosen": -0.00953118596225977, | |
| "rewards/margins": 0.04286161810159683, | |
| "rewards/rejected": -0.05239280313253403, | |
| "sft_loss": 0.011803574860095978, | |
| "step": 140 | |
| }, | |
| { | |
| "epoch": 4.444444444444445, | |
| "grad_norm": 0.9827601313591003, | |
| "learning_rate": 3.3176699082935546e-06, | |
| "logits/chosen": -0.3458485007286072, | |
| "logits/rejected": -0.4066559374332428, | |
| "logps/chosen": -0.10192994773387909, | |
| "logps/rejected": -0.5467253923416138, | |
| "loss": 0.1421, | |
| "odds_ratio_loss": 1.2931029796600342, | |
| "rewards/accuracies": 0.8062499761581421, | |
| "rewards/chosen": -0.010192994959652424, | |
| "rewards/margins": 0.04447954148054123, | |
| "rewards/rejected": -0.05467253923416138, | |
| "sft_loss": 0.01281714253127575, | |
| "step": 150 | |
| }, | |
| { | |
| "epoch": 4.444444444444445, | |
| "eval_logits/chosen": -0.34726279973983765, | |
| "eval_logits/rejected": -0.4106636047363281, | |
| "eval_logps/chosen": -0.0770278051495552, | |
| "eval_logps/rejected": -0.5773364901542664, | |
| "eval_loss": 0.10104309767484665, | |
| "eval_odds_ratio_loss": 0.8869253396987915, | |
| "eval_rewards/accuracies": 0.8833333253860474, | |
| "eval_rewards/chosen": -0.007702780421823263, | |
| "eval_rewards/margins": 0.05003087595105171, | |
| "eval_rewards/rejected": -0.05773365497589111, | |
| "eval_runtime": 2.316, | |
| "eval_samples_per_second": 25.907, | |
| "eval_sft_loss": 0.012350580655038357, | |
| "eval_steps_per_second": 12.953, | |
| "step": 150 | |
| }, | |
| { | |
| "epoch": 4.7407407407407405, | |
| "grad_norm": 1.6250287294387817, | |
| "learning_rate": 3.0636658878845116e-06, | |
| "logits/chosen": -0.38808631896972656, | |
| "logits/rejected": -0.45208558440208435, | |
| "logps/chosen": -0.10056424140930176, | |
| "logps/rejected": -0.5426880717277527, | |
| "loss": 0.1333, | |
| "odds_ratio_loss": 1.2175222635269165, | |
| "rewards/accuracies": 0.824999988079071, | |
| "rewards/chosen": -0.01005642395466566, | |
| "rewards/margins": 0.04421238973736763, | |
| "rewards/rejected": -0.054268814623355865, | |
| "sft_loss": 0.011499151587486267, | |
| "step": 160 | |
| }, | |
| { | |
| "epoch": 5.037037037037037, | |
| "grad_norm": 1.2549740076065063, | |
| "learning_rate": 2.803360952452705e-06, | |
| "logits/chosen": -0.3857024013996124, | |
| "logits/rejected": -0.43612140417099, | |
| "logps/chosen": -0.08464725315570831, | |
| "logps/rejected": -0.4786381125450134, | |
| "loss": 0.1229, | |
| "odds_ratio_loss": 1.1305350065231323, | |
| "rewards/accuracies": 0.7875000238418579, | |
| "rewards/chosen": -0.008464725688099861, | |
| "rewards/margins": 0.03939909487962723, | |
| "rewards/rejected": -0.04786381870508194, | |
| "sft_loss": 0.00985820684581995, | |
| "step": 170 | |
| }, | |
| { | |
| "epoch": 5.333333333333333, | |
| "grad_norm": 2.602710485458374, | |
| "learning_rate": 2.53966490958702e-06, | |
| "logits/chosen": -0.32125982642173767, | |
| "logits/rejected": -0.3869190812110901, | |
| "logps/chosen": -0.0981438010931015, | |
| "logps/rejected": -0.6228185892105103, | |
| "loss": 0.1321, | |
| "odds_ratio_loss": 1.208888053894043, | |
| "rewards/accuracies": 0.84375, | |
| "rewards/chosen": -0.00981437973678112, | |
| "rewards/margins": 0.05246748402714729, | |
| "rewards/rejected": -0.062281858175992966, | |
| "sft_loss": 0.011162296868860722, | |
| "step": 180 | |
| }, | |
| { | |
| "epoch": 5.62962962962963, | |
| "grad_norm": 0.7487705945968628, | |
| "learning_rate": 2.275525474225771e-06, | |
| "logits/chosen": -0.38048022985458374, | |
| "logits/rejected": -0.45359840989112854, | |
| "logps/chosen": -0.08556422591209412, | |
| "logps/rejected": -0.568170428276062, | |
| "loss": 0.118, | |
| "odds_ratio_loss": 1.0685060024261475, | |
| "rewards/accuracies": 0.831250011920929, | |
| "rewards/chosen": -0.008556422777473927, | |
| "rewards/margins": 0.04826062172651291, | |
| "rewards/rejected": -0.05681704729795456, | |
| "sft_loss": 0.011130120605230331, | |
| "step": 190 | |
| }, | |
| { | |
| "epoch": 5.925925925925926, | |
| "grad_norm": 1.8954200744628906, | |
| "learning_rate": 2.013895317751323e-06, | |
| "logits/chosen": -0.3612784445285797, | |
| "logits/rejected": -0.398723840713501, | |
| "logps/chosen": -0.09013709425926208, | |
| "logps/rejected": -0.47434768080711365, | |
| "loss": 0.1291, | |
| "odds_ratio_loss": 1.1944711208343506, | |
| "rewards/accuracies": 0.78125, | |
| "rewards/chosen": -0.009013709612190723, | |
| "rewards/margins": 0.03842105716466904, | |
| "rewards/rejected": -0.047434769570827484, | |
| "sft_loss": 0.00965641625225544, | |
| "step": 200 | |
| }, | |
| { | |
| "epoch": 5.925925925925926, | |
| "eval_logits/chosen": -0.3441879153251648, | |
| "eval_logits/rejected": -0.40659084916114807, | |
| "eval_logps/chosen": -0.07519559562206268, | |
| "eval_logps/rejected": -0.5936176776885986, | |
| "eval_loss": 0.09843841940164566, | |
| "eval_odds_ratio_loss": 0.8613345623016357, | |
| "eval_rewards/accuracies": 0.8833333253860474, | |
| "eval_rewards/chosen": -0.007519559469074011, | |
| "eval_rewards/margins": 0.05184221267700195, | |
| "eval_rewards/rejected": -0.0593617707490921, | |
| "eval_runtime": 2.3134, | |
| "eval_samples_per_second": 25.936, | |
| "eval_sft_loss": 0.012304977513849735, | |
| "eval_steps_per_second": 12.968, | |
| "step": 200 | |
| }, | |
| { | |
| "epoch": 6.222222222222222, | |
| "grad_norm": 2.2815189361572266, | |
| "learning_rate": 1.7576990616793139e-06, | |
| "logits/chosen": -0.3727927803993225, | |
| "logits/rejected": -0.4259300231933594, | |
| "logps/chosen": -0.06345033645629883, | |
| "logps/rejected": -0.5240000486373901, | |
| "loss": 0.0946, | |
| "odds_ratio_loss": 0.8643038868904114, | |
| "rewards/accuracies": 0.856249988079071, | |
| "rewards/chosen": -0.006345034576952457, | |
| "rewards/margins": 0.04605497419834137, | |
| "rewards/rejected": -0.052400004118680954, | |
| "sft_loss": 0.008158734068274498, | |
| "step": 210 | |
| }, | |
| { | |
| "epoch": 6.518518518518518, | |
| "grad_norm": 1.3194066286087036, | |
| "learning_rate": 1.509800584902108e-06, | |
| "logits/chosen": -0.3670283854007721, | |
| "logits/rejected": -0.427605003118515, | |
| "logps/chosen": -0.09667733311653137, | |
| "logps/rejected": -0.5639557838439941, | |
| "loss": 0.1333, | |
| "odds_ratio_loss": 1.2246453762054443, | |
| "rewards/accuracies": 0.824999988079071, | |
| "rewards/chosen": -0.009667733684182167, | |
| "rewards/margins": 0.04672784358263016, | |
| "rewards/rejected": -0.056395579129457474, | |
| "sft_loss": 0.010851002298295498, | |
| "step": 220 | |
| }, | |
| { | |
| "epoch": 6.814814814814815, | |
| "grad_norm": 1.5913020372390747, | |
| "learning_rate": 1.2729710099410802e-06, | |
| "logits/chosen": -0.3422110676765442, | |
| "logits/rejected": -0.41096681356430054, | |
| "logps/chosen": -0.07137643545866013, | |
| "logps/rejected": -0.5844155550003052, | |
| "loss": 0.0972, | |
| "odds_ratio_loss": 0.8859140276908875, | |
| "rewards/accuracies": 0.8687499761581421, | |
| "rewards/chosen": -0.007137644104659557, | |
| "rewards/margins": 0.05130390450358391, | |
| "rewards/rejected": -0.05844154953956604, | |
| "sft_loss": 0.008584940806031227, | |
| "step": 230 | |
| }, | |
| { | |
| "epoch": 7.111111111111111, | |
| "grad_norm": 1.441452980041504, | |
| "learning_rate": 1.049857726072005e-06, | |
| "logits/chosen": -0.37981483340263367, | |
| "logits/rejected": -0.42586684226989746, | |
| "logps/chosen": -0.09972624480724335, | |
| "logps/rejected": -0.5454004406929016, | |
| "loss": 0.1338, | |
| "odds_ratio_loss": 1.2288706302642822, | |
| "rewards/accuracies": 0.831250011920929, | |
| "rewards/chosen": -0.009972624480724335, | |
| "rewards/margins": 0.044567424803972244, | |
| "rewards/rejected": -0.05454004928469658, | |
| "sft_loss": 0.010933582670986652, | |
| "step": 240 | |
| }, | |
| { | |
| "epoch": 7.407407407407407, | |
| "grad_norm": 2.5117592811584473, | |
| "learning_rate": 8.4295479559726e-07, | |
| "logits/chosen": -0.38271045684814453, | |
| "logits/rejected": -0.4315881133079529, | |
| "logps/chosen": -0.0871758908033371, | |
| "logps/rejected": -0.5703214406967163, | |
| "loss": 0.1246, | |
| "odds_ratio_loss": 1.1464191675186157, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": -0.00871758908033371, | |
| "rewards/margins": 0.04831455647945404, | |
| "rewards/rejected": -0.05703214555978775, | |
| "sft_loss": 0.009948917664587498, | |
| "step": 250 | |
| }, | |
| { | |
| "epoch": 7.407407407407407, | |
| "eval_logits/chosen": -0.34323057532310486, | |
| "eval_logits/rejected": -0.4047623574733734, | |
| "eval_logps/chosen": -0.07215116173028946, | |
| "eval_logps/rejected": -0.6233159303665161, | |
| "eval_loss": 0.09428545832633972, | |
| "eval_odds_ratio_loss": 0.8242944478988647, | |
| "eval_rewards/accuracies": 0.8833333253860474, | |
| "eval_rewards/chosen": -0.0072151171043515205, | |
| "eval_rewards/margins": 0.05511648207902908, | |
| "eval_rewards/rejected": -0.06233159825205803, | |
| "eval_runtime": 2.3121, | |
| "eval_samples_per_second": 25.95, | |
| "eval_sft_loss": 0.01185599621385336, | |
| "eval_steps_per_second": 12.975, | |
| "step": 250 | |
| }, | |
| { | |
| "epoch": 7.703703703703704, | |
| "grad_norm": 1.5904881954193115, | |
| "learning_rate": 6.545750740770338e-07, | |
| "logits/chosen": -0.3598392605781555, | |
| "logits/rejected": -0.423635333776474, | |
| "logps/chosen": -0.06470540165901184, | |
| "logps/rejected": -0.5819328427314758, | |
| "loss": 0.0906, | |
| "odds_ratio_loss": 0.8195532560348511, | |
| "rewards/accuracies": 0.84375, | |
| "rewards/chosen": -0.006470539607107639, | |
| "rewards/margins": 0.05172274261713028, | |
| "rewards/rejected": -0.05819328501820564, | |
| "sft_loss": 0.008671595714986324, | |
| "step": 260 | |
| }, | |
| { | |
| "epoch": 8.0, | |
| "grad_norm": 0.6165652871131897, | |
| "learning_rate": 4.868243561723535e-07, | |
| "logits/chosen": -0.34859612584114075, | |
| "logits/rejected": -0.4086515009403229, | |
| "logps/chosen": -0.09018560498952866, | |
| "logps/rejected": -0.587788462638855, | |
| "loss": 0.1201, | |
| "odds_ratio_loss": 1.098928689956665, | |
| "rewards/accuracies": 0.862500011920929, | |
| "rewards/chosen": -0.009018560871481895, | |
| "rewards/margins": 0.04976029321551323, | |
| "rewards/rejected": -0.058778852224349976, | |
| "sft_loss": 0.010236375033855438, | |
| "step": 270 | |
| }, | |
| { | |
| "epoch": 8.296296296296296, | |
| "grad_norm": 1.252172589302063, | |
| "learning_rate": 3.4157783610952263e-07, | |
| "logits/chosen": -0.3684031367301941, | |
| "logits/rejected": -0.4260830283164978, | |
| "logps/chosen": -0.0856148824095726, | |
| "logps/rejected": -0.5833510756492615, | |
| "loss": 0.1153, | |
| "odds_ratio_loss": 1.06239914894104, | |
| "rewards/accuracies": 0.831250011920929, | |
| "rewards/chosen": -0.008561487309634686, | |
| "rewards/margins": 0.049773626029491425, | |
| "rewards/rejected": -0.058335114270448685, | |
| "sft_loss": 0.009099993854761124, | |
| "step": 280 | |
| }, | |
| { | |
| "epoch": 8.592592592592592, | |
| "grad_norm": 1.9929240942001343, | |
| "learning_rate": 2.2045914590165252e-07, | |
| "logits/chosen": -0.4020005166530609, | |
| "logits/rejected": -0.46092405915260315, | |
| "logps/chosen": -0.07739080488681793, | |
| "logps/rejected": -0.62553870677948, | |
| "loss": 0.1022, | |
| "odds_ratio_loss": 0.9326642155647278, | |
| "rewards/accuracies": 0.84375, | |
| "rewards/chosen": -0.007739080581814051, | |
| "rewards/margins": 0.054814793169498444, | |
| "rewards/rejected": -0.06255386769771576, | |
| "sft_loss": 0.008896315470337868, | |
| "step": 290 | |
| }, | |
| { | |
| "epoch": 8.88888888888889, | |
| "grad_norm": 0.9893295764923096, | |
| "learning_rate": 1.2482220564763669e-07, | |
| "logits/chosen": -0.34586095809936523, | |
| "logits/rejected": -0.39634814858436584, | |
| "logps/chosen": -0.07016898691654205, | |
| "logps/rejected": -0.5465742349624634, | |
| "loss": 0.1045, | |
| "odds_ratio_loss": 0.9643081426620483, | |
| "rewards/accuracies": 0.8374999761581421, | |
| "rewards/chosen": -0.0070168995298445225, | |
| "rewards/margins": 0.04764052852988243, | |
| "rewards/rejected": -0.054657429456710815, | |
| "sft_loss": 0.008084597066044807, | |
| "step": 300 | |
| }, | |
| { | |
| "epoch": 8.88888888888889, | |
| "eval_logits/chosen": -0.34323617815971375, | |
| "eval_logits/rejected": -0.40456622838974, | |
| "eval_logps/chosen": -0.07242080569267273, | |
| "eval_logps/rejected": -0.6277292966842651, | |
| "eval_loss": 0.09481088072061539, | |
| "eval_odds_ratio_loss": 0.829154908657074, | |
| "eval_rewards/accuracies": 0.8833333253860474, | |
| "eval_rewards/chosen": -0.007242080755531788, | |
| "eval_rewards/margins": 0.0555308535695076, | |
| "eval_rewards/rejected": -0.06277292966842651, | |
| "eval_runtime": 2.3109, | |
| "eval_samples_per_second": 25.964, | |
| "eval_sft_loss": 0.011895372532308102, | |
| "eval_steps_per_second": 12.982, | |
| "step": 300 | |
| }, | |
| { | |
| "epoch": 9.185185185185185, | |
| "grad_norm": 1.7534313201904297, | |
| "learning_rate": 5.573608879422876e-08, | |
| "logits/chosen": -0.35441476106643677, | |
| "logits/rejected": -0.4039112627506256, | |
| "logps/chosen": -0.08877753466367722, | |
| "logps/rejected": -0.5493366718292236, | |
| "loss": 0.1254, | |
| "odds_ratio_loss": 1.14632248878479, | |
| "rewards/accuracies": 0.8125, | |
| "rewards/chosen": -0.008877754211425781, | |
| "rewards/margins": 0.046055909246206284, | |
| "rewards/rejected": -0.05493366718292236, | |
| "sft_loss": 0.01074306946247816, | |
| "step": 310 | |
| }, | |
| { | |
| "epoch": 9.481481481481481, | |
| "grad_norm": 1.6547372341156006, | |
| "learning_rate": 1.3973071544233219e-08, | |
| "logits/chosen": -0.37551018595695496, | |
| "logits/rejected": -0.4359508454799652, | |
| "logps/chosen": -0.07044418156147003, | |
| "logps/rejected": -0.5796228647232056, | |
| "loss": 0.098, | |
| "odds_ratio_loss": 0.9046875834465027, | |
| "rewards/accuracies": 0.8812500238418579, | |
| "rewards/chosen": -0.007044418249279261, | |
| "rewards/margins": 0.05091787129640579, | |
| "rewards/rejected": -0.05796227976679802, | |
| "sft_loss": 0.007553444243967533, | |
| "step": 320 | |
| }, | |
| { | |
| "epoch": 9.777777777777779, | |
| "grad_norm": 1.2167924642562866, | |
| "learning_rate": 0.0, | |
| "logits/chosen": -0.36357811093330383, | |
| "logits/rejected": -0.4220617711544037, | |
| "logps/chosen": -0.0854811817407608, | |
| "logps/rejected": -0.5617056488990784, | |
| "loss": 0.1206, | |
| "odds_ratio_loss": 1.0994065999984741, | |
| "rewards/accuracies": 0.824999988079071, | |
| "rewards/chosen": -0.008548117242753506, | |
| "rewards/margins": 0.047622449696063995, | |
| "rewards/rejected": -0.05617056414484978, | |
| "sft_loss": 0.010644225403666496, | |
| "step": 330 | |
| }, | |
| { | |
| "epoch": 9.777777777777779, | |
| "step": 330, | |
| "total_flos": 5.97337003547689e+16, | |
| "train_loss": 0.41489303653890436, | |
| "train_runtime": 721.1538, | |
| "train_samples_per_second": 7.488, | |
| "train_steps_per_second": 0.458 | |
| } | |
| ], | |
| "logging_steps": 10, | |
| "max_steps": 330, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 10, | |
| "save_steps": 50, | |
| "stateful_callbacks": { | |
| "TrainerControl": { | |
| "args": { | |
| "should_epoch_stop": false, | |
| "should_evaluate": false, | |
| "should_log": false, | |
| "should_save": true, | |
| "should_training_stop": true | |
| }, | |
| "attributes": {} | |
| } | |
| }, | |
| "total_flos": 5.97337003547689e+16, | |
| "train_batch_size": 2, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |