sam1120's picture
Training in progress, step 80
ea5466d
{
"best_metric": 0.020909180864691734,
"best_model_checkpoint": "/robodata/smodak/Projects/nspl/scripts/terrainseg/training/models/parking-utcustom-train-SF-RGBD-b0_6/checkpoint-80",
"epoch": 80.0,
"global_step": 80,
"is_hyper_param_search": false,
"is_local_process_zero": true,
"is_world_process_zero": true,
"log_history": [
{
"epoch": 1.0,
"learning_rate": 5e-05,
"loss": 1.1298,
"step": 1
},
{
"epoch": 2.0,
"learning_rate": 0.0001,
"loss": 1.1302,
"step": 2
},
{
"epoch": 3.0,
"learning_rate": 0.00015000000000000001,
"loss": 1.1043,
"step": 3
},
{
"epoch": 4.0,
"learning_rate": 0.0002,
"loss": 1.0838,
"step": 4
},
{
"epoch": 5.0,
"learning_rate": 0.00025,
"loss": 1.0484,
"step": 5
},
{
"epoch": 6.0,
"learning_rate": 0.00030000000000000003,
"loss": 1.0053,
"step": 6
},
{
"epoch": 7.0,
"learning_rate": 0.00035,
"loss": 0.9466,
"step": 7
},
{
"epoch": 8.0,
"learning_rate": 0.0004,
"loss": 0.9073,
"step": 8
},
{
"epoch": 9.0,
"learning_rate": 0.0003971830985915493,
"loss": 0.8271,
"step": 9
},
{
"epoch": 10.0,
"learning_rate": 0.0003943661971830986,
"loss": 0.7486,
"step": 10
},
{
"epoch": 11.0,
"learning_rate": 0.0003915492957746479,
"loss": 0.6685,
"step": 11
},
{
"epoch": 12.0,
"learning_rate": 0.00038873239436619723,
"loss": 0.646,
"step": 12
},
{
"epoch": 13.0,
"learning_rate": 0.00038591549295774653,
"loss": 0.5879,
"step": 13
},
{
"epoch": 14.0,
"learning_rate": 0.0003830985915492958,
"loss": 0.5739,
"step": 14
},
{
"epoch": 15.0,
"learning_rate": 0.0003802816901408451,
"loss": 0.4959,
"step": 15
},
{
"epoch": 16.0,
"learning_rate": 0.0003774647887323944,
"loss": 0.4776,
"step": 16
},
{
"epoch": 17.0,
"learning_rate": 0.0003746478873239437,
"loss": 0.4419,
"step": 17
},
{
"epoch": 18.0,
"learning_rate": 0.000371830985915493,
"loss": 0.4183,
"step": 18
},
{
"epoch": 19.0,
"learning_rate": 0.0003690140845070423,
"loss": 0.3886,
"step": 19
},
{
"epoch": 20.0,
"learning_rate": 0.0003661971830985916,
"loss": 0.3631,
"step": 20
},
{
"epoch": 20.0,
"eval_accuracy_parking": NaN,
"eval_accuracy_unlabeled": NaN,
"eval_accuracy_unparking": 1.0,
"eval_iou_parking": NaN,
"eval_iou_unlabeled": NaN,
"eval_iou_unparking": 1.0,
"eval_loss": 0.3717308044433594,
"eval_mean_accuracy": 1.0,
"eval_mean_iou": 1.0,
"eval_overall_accuracy": 1.0,
"eval_runtime": 0.6855,
"eval_samples_per_second": 1.459,
"eval_steps_per_second": 1.459,
"step": 20
},
{
"epoch": 21.0,
"learning_rate": 0.0003633802816901409,
"loss": 0.3486,
"step": 21
},
{
"epoch": 22.0,
"learning_rate": 0.00036056338028169014,
"loss": 0.3251,
"step": 22
},
{
"epoch": 23.0,
"learning_rate": 0.00035774647887323944,
"loss": 0.3112,
"step": 23
},
{
"epoch": 24.0,
"learning_rate": 0.00035492957746478874,
"loss": 0.2953,
"step": 24
},
{
"epoch": 25.0,
"learning_rate": 0.00035211267605633805,
"loss": 0.2822,
"step": 25
},
{
"epoch": 26.0,
"learning_rate": 0.0003492957746478873,
"loss": 0.2697,
"step": 26
},
{
"epoch": 27.0,
"learning_rate": 0.0003464788732394366,
"loss": 0.2631,
"step": 27
},
{
"epoch": 28.0,
"learning_rate": 0.0003436619718309859,
"loss": 0.2537,
"step": 28
},
{
"epoch": 29.0,
"learning_rate": 0.0003408450704225352,
"loss": 0.2388,
"step": 29
},
{
"epoch": 30.0,
"learning_rate": 0.0003380281690140845,
"loss": 0.2289,
"step": 30
},
{
"epoch": 31.0,
"learning_rate": 0.0003352112676056338,
"loss": 0.2221,
"step": 31
},
{
"epoch": 32.0,
"learning_rate": 0.0003323943661971831,
"loss": 0.2142,
"step": 32
},
{
"epoch": 33.0,
"learning_rate": 0.0003295774647887324,
"loss": 0.2013,
"step": 33
},
{
"epoch": 34.0,
"learning_rate": 0.0003267605633802817,
"loss": 0.1996,
"step": 34
},
{
"epoch": 35.0,
"learning_rate": 0.000323943661971831,
"loss": 0.1904,
"step": 35
},
{
"epoch": 36.0,
"learning_rate": 0.0003211267605633803,
"loss": 0.185,
"step": 36
},
{
"epoch": 37.0,
"learning_rate": 0.0003183098591549296,
"loss": 0.1769,
"step": 37
},
{
"epoch": 38.0,
"learning_rate": 0.00031549295774647887,
"loss": 0.175,
"step": 38
},
{
"epoch": 39.0,
"learning_rate": 0.00031267605633802817,
"loss": 0.1644,
"step": 39
},
{
"epoch": 40.0,
"learning_rate": 0.00030985915492957747,
"loss": 0.1606,
"step": 40
},
{
"epoch": 40.0,
"eval_accuracy_parking": NaN,
"eval_accuracy_unlabeled": NaN,
"eval_accuracy_unparking": 1.0,
"eval_iou_parking": NaN,
"eval_iou_unlabeled": NaN,
"eval_iou_unparking": 1.0,
"eval_loss": 0.08129558712244034,
"eval_mean_accuracy": 1.0,
"eval_mean_iou": 1.0,
"eval_overall_accuracy": 1.0,
"eval_runtime": 0.7101,
"eval_samples_per_second": 1.408,
"eval_steps_per_second": 1.408,
"step": 40
},
{
"epoch": 41.0,
"learning_rate": 0.00030704225352112677,
"loss": 0.1524,
"step": 41
},
{
"epoch": 42.0,
"learning_rate": 0.0003042253521126761,
"loss": 0.1482,
"step": 42
},
{
"epoch": 43.0,
"learning_rate": 0.0003014084507042254,
"loss": 0.1396,
"step": 43
},
{
"epoch": 44.0,
"learning_rate": 0.0002985915492957747,
"loss": 0.1362,
"step": 44
},
{
"epoch": 45.0,
"learning_rate": 0.000295774647887324,
"loss": 0.1344,
"step": 45
},
{
"epoch": 46.0,
"learning_rate": 0.0002929577464788733,
"loss": 0.1322,
"step": 46
},
{
"epoch": 47.0,
"learning_rate": 0.0002901408450704226,
"loss": 0.1295,
"step": 47
},
{
"epoch": 48.0,
"learning_rate": 0.0002873239436619719,
"loss": 0.1271,
"step": 48
},
{
"epoch": 49.0,
"learning_rate": 0.00028450704225352114,
"loss": 0.1282,
"step": 49
},
{
"epoch": 50.0,
"learning_rate": 0.00028169014084507044,
"loss": 0.1177,
"step": 50
},
{
"epoch": 51.0,
"learning_rate": 0.00027887323943661974,
"loss": 0.1216,
"step": 51
},
{
"epoch": 52.0,
"learning_rate": 0.00027605633802816904,
"loss": 0.1207,
"step": 52
},
{
"epoch": 53.0,
"learning_rate": 0.0002732394366197183,
"loss": 0.1127,
"step": 53
},
{
"epoch": 54.0,
"learning_rate": 0.0002704225352112676,
"loss": 0.1161,
"step": 54
},
{
"epoch": 55.0,
"learning_rate": 0.0002676056338028169,
"loss": 0.108,
"step": 55
},
{
"epoch": 56.0,
"learning_rate": 0.0002647887323943662,
"loss": 0.1071,
"step": 56
},
{
"epoch": 57.0,
"learning_rate": 0.0002619718309859155,
"loss": 0.1057,
"step": 57
},
{
"epoch": 58.0,
"learning_rate": 0.0002591549295774648,
"loss": 0.103,
"step": 58
},
{
"epoch": 59.0,
"learning_rate": 0.0002563380281690141,
"loss": 0.1038,
"step": 59
},
{
"epoch": 60.0,
"learning_rate": 0.0002535211267605634,
"loss": 0.0988,
"step": 60
},
{
"epoch": 60.0,
"eval_accuracy_parking": NaN,
"eval_accuracy_unlabeled": NaN,
"eval_accuracy_unparking": 1.0,
"eval_iou_parking": NaN,
"eval_iou_unlabeled": NaN,
"eval_iou_unparking": 1.0,
"eval_loss": 0.02334783226251602,
"eval_mean_accuracy": 1.0,
"eval_mean_iou": 1.0,
"eval_overall_accuracy": 1.0,
"eval_runtime": 0.6675,
"eval_samples_per_second": 1.498,
"eval_steps_per_second": 1.498,
"step": 60
},
{
"epoch": 61.0,
"learning_rate": 0.0002507042253521127,
"loss": 0.099,
"step": 61
},
{
"epoch": 62.0,
"learning_rate": 0.00024788732394366195,
"loss": 0.0983,
"step": 62
},
{
"epoch": 63.0,
"learning_rate": 0.00024507042253521126,
"loss": 0.0955,
"step": 63
},
{
"epoch": 64.0,
"learning_rate": 0.00024225352112676056,
"loss": 0.0982,
"step": 64
},
{
"epoch": 65.0,
"learning_rate": 0.00023943661971830986,
"loss": 0.0913,
"step": 65
},
{
"epoch": 66.0,
"learning_rate": 0.00023661971830985916,
"loss": 0.0983,
"step": 66
},
{
"epoch": 67.0,
"learning_rate": 0.00023380281690140847,
"loss": 0.0981,
"step": 67
},
{
"epoch": 68.0,
"learning_rate": 0.00023098591549295777,
"loss": 0.0913,
"step": 68
},
{
"epoch": 69.0,
"learning_rate": 0.00022816901408450707,
"loss": 0.0913,
"step": 69
},
{
"epoch": 70.0,
"learning_rate": 0.00022535211267605637,
"loss": 0.0892,
"step": 70
},
{
"epoch": 71.0,
"learning_rate": 0.00022253521126760565,
"loss": 0.0865,
"step": 71
},
{
"epoch": 72.0,
"learning_rate": 0.00021971830985915495,
"loss": 0.0871,
"step": 72
},
{
"epoch": 73.0,
"learning_rate": 0.00021690140845070425,
"loss": 0.0873,
"step": 73
},
{
"epoch": 74.0,
"learning_rate": 0.00021408450704225353,
"loss": 0.0857,
"step": 74
},
{
"epoch": 75.0,
"learning_rate": 0.0002112676056338028,
"loss": 0.0817,
"step": 75
},
{
"epoch": 76.0,
"learning_rate": 0.0002084507042253521,
"loss": 0.0857,
"step": 76
},
{
"epoch": 77.0,
"learning_rate": 0.0002056338028169014,
"loss": 0.0837,
"step": 77
},
{
"epoch": 78.0,
"learning_rate": 0.0002028169014084507,
"loss": 0.0834,
"step": 78
},
{
"epoch": 79.0,
"learning_rate": 0.0002,
"loss": 0.0787,
"step": 79
},
{
"epoch": 80.0,
"learning_rate": 0.0001971830985915493,
"loss": 0.08,
"step": 80
},
{
"epoch": 80.0,
"eval_accuracy_parking": NaN,
"eval_accuracy_unlabeled": NaN,
"eval_accuracy_unparking": 1.0,
"eval_iou_parking": NaN,
"eval_iou_unlabeled": NaN,
"eval_iou_unparking": 1.0,
"eval_loss": 0.020909180864691734,
"eval_mean_accuracy": 1.0,
"eval_mean_iou": 1.0,
"eval_overall_accuracy": 1.0,
"eval_runtime": 0.6545,
"eval_samples_per_second": 1.528,
"eval_steps_per_second": 1.528,
"step": 80
}
],
"max_steps": 150,
"num_train_epochs": 150,
"total_flos": 4.493149300850688e+16,
"trial_name": null,
"trial_params": null
}