| { | |
| "best_metric": 3.1000759601593018, | |
| "best_model_checkpoint": "/home/p318482/babyLM_controlled/models_trained/de_mlm/wikipedia_13/checkpoint-76000", | |
| "epoch": 55.698057896665446, | |
| "eval_steps": 2000, | |
| "global_step": 76000, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "epoch": 1.4657383657017222, | |
| "eval_loss": 8.087221145629883, | |
| "eval_runtime": 2.215, | |
| "eval_samples_per_second": 1035.232, | |
| "eval_steps_per_second": 65.012, | |
| "step": 2000 | |
| }, | |
| { | |
| "epoch": 2.9314767314034444, | |
| "grad_norm": 0.9454871416091919, | |
| "learning_rate": 1e-05, | |
| "loss": 8.1326, | |
| "step": 4000 | |
| }, | |
| { | |
| "epoch": 2.9314767314034444, | |
| "eval_loss": 7.397463798522949, | |
| "eval_runtime": 2.2812, | |
| "eval_samples_per_second": 1005.158, | |
| "eval_steps_per_second": 63.124, | |
| "step": 4000 | |
| }, | |
| { | |
| "epoch": 4.397215097105167, | |
| "eval_loss": 7.2718353271484375, | |
| "eval_runtime": 2.3459, | |
| "eval_samples_per_second": 977.446, | |
| "eval_steps_per_second": 61.383, | |
| "step": 6000 | |
| }, | |
| { | |
| "epoch": 5.862953462806889, | |
| "grad_norm": 1.4085547924041748, | |
| "learning_rate": 2e-05, | |
| "loss": 7.2846, | |
| "step": 8000 | |
| }, | |
| { | |
| "epoch": 5.862953462806889, | |
| "eval_loss": 7.183506965637207, | |
| "eval_runtime": 2.2787, | |
| "eval_samples_per_second": 1006.271, | |
| "eval_steps_per_second": 63.194, | |
| "step": 8000 | |
| }, | |
| { | |
| "epoch": 7.328691828508611, | |
| "eval_loss": 7.09923791885376, | |
| "eval_runtime": 2.1833, | |
| "eval_samples_per_second": 1050.26, | |
| "eval_steps_per_second": 65.956, | |
| "step": 10000 | |
| }, | |
| { | |
| "epoch": 8.794430194210333, | |
| "grad_norm": 1.4775667190551758, | |
| "learning_rate": 2.9995e-05, | |
| "loss": 7.1078, | |
| "step": 12000 | |
| }, | |
| { | |
| "epoch": 8.794430194210333, | |
| "eval_loss": 7.033052444458008, | |
| "eval_runtime": 2.2436, | |
| "eval_samples_per_second": 1022.003, | |
| "eval_steps_per_second": 64.182, | |
| "step": 12000 | |
| }, | |
| { | |
| "epoch": 10.260168559912056, | |
| "eval_loss": 6.949419975280762, | |
| "eval_runtime": 2.3034, | |
| "eval_samples_per_second": 995.501, | |
| "eval_steps_per_second": 62.517, | |
| "step": 14000 | |
| }, | |
| { | |
| "epoch": 11.725906925613778, | |
| "grad_norm": 2.364133358001709, | |
| "learning_rate": 3.999e-05, | |
| "loss": 6.942, | |
| "step": 16000 | |
| }, | |
| { | |
| "epoch": 11.725906925613778, | |
| "eval_loss": 6.889880180358887, | |
| "eval_runtime": 2.3275, | |
| "eval_samples_per_second": 985.162, | |
| "eval_steps_per_second": 61.868, | |
| "step": 16000 | |
| }, | |
| { | |
| "epoch": 13.191645291315501, | |
| "eval_loss": 6.782171726226807, | |
| "eval_runtime": 2.2331, | |
| "eval_samples_per_second": 1026.812, | |
| "eval_steps_per_second": 64.484, | |
| "step": 18000 | |
| }, | |
| { | |
| "epoch": 14.657383657017222, | |
| "grad_norm": 2.5763492584228516, | |
| "learning_rate": 4.9985e-05, | |
| "loss": 6.7676, | |
| "step": 20000 | |
| }, | |
| { | |
| "epoch": 14.657383657017222, | |
| "eval_loss": 6.718458652496338, | |
| "eval_runtime": 2.2092, | |
| "eval_samples_per_second": 1037.91, | |
| "eval_steps_per_second": 65.181, | |
| "step": 20000 | |
| }, | |
| { | |
| "epoch": 16.123122022718945, | |
| "eval_loss": 6.653624057769775, | |
| "eval_runtime": 2.33, | |
| "eval_samples_per_second": 984.135, | |
| "eval_steps_per_second": 61.804, | |
| "step": 22000 | |
| }, | |
| { | |
| "epoch": 17.588860388420667, | |
| "grad_norm": 4.001818656921387, | |
| "learning_rate": 5.9980000000000005e-05, | |
| "loss": 6.5959, | |
| "step": 24000 | |
| }, | |
| { | |
| "epoch": 17.588860388420667, | |
| "eval_loss": 6.543092727661133, | |
| "eval_runtime": 2.3092, | |
| "eval_samples_per_second": 992.981, | |
| "eval_steps_per_second": 62.359, | |
| "step": 24000 | |
| }, | |
| { | |
| "epoch": 19.054598754122388, | |
| "eval_loss": 6.392500400543213, | |
| "eval_runtime": 2.4925, | |
| "eval_samples_per_second": 919.956, | |
| "eval_steps_per_second": 57.773, | |
| "step": 26000 | |
| }, | |
| { | |
| "epoch": 20.520337119824113, | |
| "grad_norm": 4.359928607940674, | |
| "learning_rate": 6.99775e-05, | |
| "loss": 6.3624, | |
| "step": 28000 | |
| }, | |
| { | |
| "epoch": 20.520337119824113, | |
| "eval_loss": 6.211884021759033, | |
| "eval_runtime": 2.3499, | |
| "eval_samples_per_second": 975.797, | |
| "eval_steps_per_second": 61.28, | |
| "step": 28000 | |
| }, | |
| { | |
| "epoch": 21.986075485525834, | |
| "eval_loss": 5.952591896057129, | |
| "eval_runtime": 2.3365, | |
| "eval_samples_per_second": 981.37, | |
| "eval_steps_per_second": 61.63, | |
| "step": 30000 | |
| }, | |
| { | |
| "epoch": 23.451813851227556, | |
| "grad_norm": 5.47481632232666, | |
| "learning_rate": 7.997250000000001e-05, | |
| "loss": 5.9309, | |
| "step": 32000 | |
| }, | |
| { | |
| "epoch": 23.451813851227556, | |
| "eval_loss": 5.4161834716796875, | |
| "eval_runtime": 2.2165, | |
| "eval_samples_per_second": 1034.515, | |
| "eval_steps_per_second": 64.967, | |
| "step": 32000 | |
| }, | |
| { | |
| "epoch": 24.917552216929277, | |
| "eval_loss": 5.0255045890808105, | |
| "eval_runtime": 2.3492, | |
| "eval_samples_per_second": 976.095, | |
| "eval_steps_per_second": 61.299, | |
| "step": 34000 | |
| }, | |
| { | |
| "epoch": 26.383290582631002, | |
| "grad_norm": 5.85042667388916, | |
| "learning_rate": 8.996750000000001e-05, | |
| "loss": 5.0575, | |
| "step": 36000 | |
| }, | |
| { | |
| "epoch": 26.383290582631002, | |
| "eval_loss": 4.76804256439209, | |
| "eval_runtime": 2.2482, | |
| "eval_samples_per_second": 1019.914, | |
| "eval_steps_per_second": 64.05, | |
| "step": 36000 | |
| }, | |
| { | |
| "epoch": 27.849028948332723, | |
| "eval_loss": 4.502030372619629, | |
| "eval_runtime": 2.3066, | |
| "eval_samples_per_second": 994.084, | |
| "eval_steps_per_second": 62.428, | |
| "step": 38000 | |
| }, | |
| { | |
| "epoch": 29.314767314034444, | |
| "grad_norm": 4.6283698081970215, | |
| "learning_rate": 9.99625e-05, | |
| "loss": 4.5282, | |
| "step": 40000 | |
| }, | |
| { | |
| "epoch": 29.314767314034444, | |
| "eval_loss": 4.321413993835449, | |
| "eval_runtime": 2.2564, | |
| "eval_samples_per_second": 1016.223, | |
| "eval_steps_per_second": 63.819, | |
| "step": 40000 | |
| }, | |
| { | |
| "epoch": 30.780505679736166, | |
| "eval_loss": 4.131194114685059, | |
| "eval_runtime": 2.27, | |
| "eval_samples_per_second": 1010.131, | |
| "eval_steps_per_second": 63.436, | |
| "step": 42000 | |
| }, | |
| { | |
| "epoch": 32.24624404543789, | |
| "grad_norm": 4.427154064178467, | |
| "learning_rate": 9.336166666666667e-05, | |
| "loss": 4.1335, | |
| "step": 44000 | |
| }, | |
| { | |
| "epoch": 32.24624404543789, | |
| "eval_loss": 3.9708411693573, | |
| "eval_runtime": 2.1314, | |
| "eval_samples_per_second": 1075.829, | |
| "eval_steps_per_second": 67.562, | |
| "step": 44000 | |
| }, | |
| { | |
| "epoch": 33.71198241113961, | |
| "eval_loss": 3.861647605895996, | |
| "eval_runtime": 2.2216, | |
| "eval_samples_per_second": 1032.141, | |
| "eval_steps_per_second": 64.818, | |
| "step": 46000 | |
| }, | |
| { | |
| "epoch": 35.17772077684133, | |
| "grad_norm": 4.964674949645996, | |
| "learning_rate": 8.669666666666667e-05, | |
| "loss": 3.8339, | |
| "step": 48000 | |
| }, | |
| { | |
| "epoch": 35.17772077684133, | |
| "eval_loss": 3.763975143432617, | |
| "eval_runtime": 2.2678, | |
| "eval_samples_per_second": 1011.117, | |
| "eval_steps_per_second": 63.498, | |
| "step": 48000 | |
| }, | |
| { | |
| "epoch": 36.64345914254306, | |
| "eval_loss": 3.707411527633667, | |
| "eval_runtime": 2.1845, | |
| "eval_samples_per_second": 1049.673, | |
| "eval_steps_per_second": 65.919, | |
| "step": 50000 | |
| }, | |
| { | |
| "epoch": 38.109197508244776, | |
| "grad_norm": 5.166478633880615, | |
| "learning_rate": 8.003333333333333e-05, | |
| "loss": 3.6042, | |
| "step": 52000 | |
| }, | |
| { | |
| "epoch": 38.109197508244776, | |
| "eval_loss": 3.636035680770874, | |
| "eval_runtime": 2.279, | |
| "eval_samples_per_second": 1006.125, | |
| "eval_steps_per_second": 63.184, | |
| "step": 52000 | |
| }, | |
| { | |
| "epoch": 39.5749358739465, | |
| "eval_loss": 3.520251750946045, | |
| "eval_runtime": 2.3313, | |
| "eval_samples_per_second": 983.562, | |
| "eval_steps_per_second": 61.768, | |
| "step": 54000 | |
| }, | |
| { | |
| "epoch": 41.040674239648226, | |
| "grad_norm": 5.001477241516113, | |
| "learning_rate": 7.337000000000001e-05, | |
| "loss": 3.4291, | |
| "step": 56000 | |
| }, | |
| { | |
| "epoch": 41.040674239648226, | |
| "eval_loss": 3.4423604011535645, | |
| "eval_runtime": 2.4163, | |
| "eval_samples_per_second": 948.983, | |
| "eval_steps_per_second": 59.596, | |
| "step": 56000 | |
| }, | |
| { | |
| "epoch": 42.506412605349944, | |
| "eval_loss": 3.427623987197876, | |
| "eval_runtime": 2.1948, | |
| "eval_samples_per_second": 1044.73, | |
| "eval_steps_per_second": 65.609, | |
| "step": 58000 | |
| }, | |
| { | |
| "epoch": 43.97215097105167, | |
| "grad_norm": 5.074405670166016, | |
| "learning_rate": 6.670666666666668e-05, | |
| "loss": 3.286, | |
| "step": 60000 | |
| }, | |
| { | |
| "epoch": 43.97215097105167, | |
| "eval_loss": 3.3796520233154297, | |
| "eval_runtime": 2.2145, | |
| "eval_samples_per_second": 1035.457, | |
| "eval_steps_per_second": 65.027, | |
| "step": 60000 | |
| }, | |
| { | |
| "epoch": 45.437889336753386, | |
| "eval_loss": 3.327679395675659, | |
| "eval_runtime": 2.4352, | |
| "eval_samples_per_second": 941.608, | |
| "eval_steps_per_second": 59.133, | |
| "step": 62000 | |
| }, | |
| { | |
| "epoch": 46.90362770245511, | |
| "grad_norm": 4.627829551696777, | |
| "learning_rate": 6.004333333333334e-05, | |
| "loss": 3.1748, | |
| "step": 64000 | |
| }, | |
| { | |
| "epoch": 46.90362770245511, | |
| "eval_loss": 3.292200803756714, | |
| "eval_runtime": 2.358, | |
| "eval_samples_per_second": 972.453, | |
| "eval_steps_per_second": 61.07, | |
| "step": 64000 | |
| }, | |
| { | |
| "epoch": 48.369366068156836, | |
| "eval_loss": 3.2361395359039307, | |
| "eval_runtime": 2.2526, | |
| "eval_samples_per_second": 1017.92, | |
| "eval_steps_per_second": 63.925, | |
| "step": 66000 | |
| }, | |
| { | |
| "epoch": 49.835104433858554, | |
| "grad_norm": 4.962168216705322, | |
| "learning_rate": 5.338000000000001e-05, | |
| "loss": 3.0842, | |
| "step": 68000 | |
| }, | |
| { | |
| "epoch": 49.835104433858554, | |
| "eval_loss": 3.204303741455078, | |
| "eval_runtime": 2.2836, | |
| "eval_samples_per_second": 1004.135, | |
| "eval_steps_per_second": 63.059, | |
| "step": 68000 | |
| }, | |
| { | |
| "epoch": 51.30084279956028, | |
| "eval_loss": 3.187018871307373, | |
| "eval_runtime": 2.2063, | |
| "eval_samples_per_second": 1039.279, | |
| "eval_steps_per_second": 65.267, | |
| "step": 70000 | |
| }, | |
| { | |
| "epoch": 52.766581165262004, | |
| "grad_norm": 5.1038713455200195, | |
| "learning_rate": 4.671666666666667e-05, | |
| "loss": 3.0082, | |
| "step": 72000 | |
| }, | |
| { | |
| "epoch": 52.766581165262004, | |
| "eval_loss": 3.148749589920044, | |
| "eval_runtime": 2.1633, | |
| "eval_samples_per_second": 1059.95, | |
| "eval_steps_per_second": 66.565, | |
| "step": 72000 | |
| }, | |
| { | |
| "epoch": 54.23231953096372, | |
| "eval_loss": 3.1257400512695312, | |
| "eval_runtime": 2.188, | |
| "eval_samples_per_second": 1048.007, | |
| "eval_steps_per_second": 65.815, | |
| "step": 74000 | |
| }, | |
| { | |
| "epoch": 55.698057896665446, | |
| "grad_norm": 4.805927753448486, | |
| "learning_rate": 4.005333333333334e-05, | |
| "loss": 2.9483, | |
| "step": 76000 | |
| }, | |
| { | |
| "epoch": 55.698057896665446, | |
| "eval_loss": 3.1000759601593018, | |
| "eval_runtime": 2.3388, | |
| "eval_samples_per_second": 980.424, | |
| "eval_steps_per_second": 61.57, | |
| "step": 76000 | |
| } | |
| ], | |
| "logging_steps": 4000, | |
| "max_steps": 100000, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 74, | |
| "save_steps": 4000, | |
| "stateful_callbacks": { | |
| "TrainerControl": { | |
| "args": { | |
| "should_epoch_stop": false, | |
| "should_evaluate": false, | |
| "should_log": false, | |
| "should_save": true, | |
| "should_training_stop": false | |
| }, | |
| "attributes": {} | |
| } | |
| }, | |
| "total_flos": 2.370397082812416e+16, | |
| "train_batch_size": 16, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |