Training in progress, epoch 3, checkpoint

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:91f65e969be4028fe272c5483c861732ab6037b3ce2126b309ae32a493ba3d05
+oid sha256:196cc2cfcc2664c4fd5cf36c41954ce899b20038eb4dc555ce085b5683692307
 size 166496880

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:94bf5a4b75c7e86d1548a14506e57038d254b2ba030b41984e3d46327faaaab7
+oid sha256:33f735862ccde6d0b31db8e86029731e0868fe7eade6541461c4ecdf8a55753b
 size 330495866

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:cff47de40d69968bf56457151c7d483f89f49db23f02a8d0ea32914d1d258248
+oid sha256:33f5213c99792362a6ad6f5e9bebafef268fa11b8180f752f261508437c870dc
 size 14244

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e4033ac437c9bdc023a8e8010339474a4e6dc76e6f5e0758831867a9a3f523e9
+oid sha256:f0568bedf04967b808836c597b9b41e2761cceefa383ed9c5db4174e2423ab9f
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": 1.1410857439041138,
   "best_model_checkpoint": "joe611/chickens-60-epoch-1000-images-aug/checkpoint-1000",
-  "epoch": 2.0,
+  "epoch": 3.0,
   "eval_steps": 500,
-  "global_step": 1000,
+  "global_step": 1500,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -290,6 +290,151 @@
       "eval_samples_per_second": 20.183,
       "eval_steps_per_second": 2.624,
       "step": 1000
+    },
+    {
+      "epoch": 2.04,
+      "grad_norm": 69.85189056396484,
+      "learning_rate": 9.971503951984996e-06,
+      "loss": 1.1616,
+      "step": 1020
+    },
+    {
+      "epoch": 2.1,
+      "grad_norm": 37.134403228759766,
+      "learning_rate": 9.9698047772759e-06,
+      "loss": 1.0393,
+      "step": 1050
+    },
+    {
+      "epoch": 2.16,
+      "grad_norm": 52.74504852294922,
+      "learning_rate": 9.968056552600043e-06,
+      "loss": 1.1147,
+      "step": 1080
+    },
+    {
+      "epoch": 2.22,
+      "grad_norm": 70.72219848632812,
+      "learning_rate": 9.966259295211698e-06,
+      "loss": 1.0797,
+      "step": 1110
+    },
+    {
+      "epoch": 2.2800000000000002,
+      "grad_norm": 104.32415771484375,
+      "learning_rate": 9.964413022849069e-06,
+      "loss": 0.9984,
+      "step": 1140
+    },
+    {
+      "epoch": 2.34,
+      "grad_norm": 65.52819061279297,
+      "learning_rate": 9.96251775373412e-06,
+      "loss": 1.036,
+      "step": 1170
+    },
+    {
+      "epoch": 2.4,
+      "grad_norm": 43.15259552001953,
+      "learning_rate": 9.960573506572391e-06,
+      "loss": 1.3154,
+      "step": 1200
+    },
+    {
+      "epoch": 2.46,
+      "grad_norm": 76.51600646972656,
+      "learning_rate": 9.958580300552816e-06,
+      "loss": 1.1412,
+      "step": 1230
+    },
+    {
+      "epoch": 2.52,
+      "grad_norm": 0.6177086234092712,
+      "learning_rate": 9.956538155347534e-06,
+      "loss": 1.2674,
+      "step": 1260
+    },
+    {
+      "epoch": 2.58,
+      "grad_norm": 53.222103118896484,
+      "learning_rate": 9.954447091111695e-06,
+      "loss": 1.1138,
+      "step": 1290
+    },
+    {
+      "epoch": 2.64,
+      "grad_norm": 94.70368194580078,
+      "learning_rate": 9.952307128483257e-06,
+      "loss": 1.0613,
+      "step": 1320
+    },
+    {
+      "epoch": 2.7,
+      "grad_norm": 89.58301544189453,
+      "learning_rate": 9.95011828858279e-06,
+      "loss": 1.0912,
+      "step": 1350
+    },
+    {
+      "epoch": 2.76,
+      "grad_norm": 43.07695388793945,
+      "learning_rate": 9.947880593013256e-06,
+      "loss": 1.1115,
+      "step": 1380
+    },
+    {
+      "epoch": 2.82,
+      "grad_norm": 52.474220275878906,
+      "learning_rate": 9.94559406385981e-06,
+      "loss": 0.9757,
+      "step": 1410
+    },
+    {
+      "epoch": 2.88,
+      "grad_norm": 46.698448181152344,
+      "learning_rate": 9.94325872368957e-06,
+      "loss": 1.1364,
+      "step": 1440
+    },
+    {
+      "epoch": 2.94,
+      "grad_norm": 49.13819885253906,
+      "learning_rate": 9.940874595551403e-06,
+      "loss": 0.9772,
+      "step": 1470
+    },
+    {
+      "epoch": 3.0,
+      "grad_norm": 101.07115173339844,
+      "learning_rate": 9.938441702975689e-06,
+      "loss": 1.06,
+      "step": 1500
+    },
+    {
+      "epoch": 3.0,
+      "eval_loss": 1.5299323797225952,
+      "eval_map": 0.1865,
+      "eval_map_50": 0.2571,
+      "eval_map_75": 0.2063,
+      "eval_map_chicken": 0.0,
+      "eval_map_duck": 0.0,
+      "eval_map_large": 0.6155,
+      "eval_map_medium": 0.1497,
+      "eval_map_plant": 0.5595,
+      "eval_map_small": 0.0172,
+      "eval_mar_1": 0.0695,
+      "eval_mar_10": 0.2154,
+      "eval_mar_100": 0.2187,
+      "eval_mar_100_chicken": 0.0,
+      "eval_mar_100_duck": 0.0,
+      "eval_mar_100_plant": 0.6562,
+      "eval_mar_large": 0.6971,
+      "eval_mar_medium": 0.199,
+      "eval_mar_small": 0.0583,
+      "eval_runtime": 4.9962,
+      "eval_samples_per_second": 20.015,
+      "eval_steps_per_second": 2.602,
+      "step": 1500
     }
   ],
   "logging_steps": 30,
@@ -309,7 +454,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 3.440231129088e+17,
+  "total_flos": 5.160346693632e+17,
   "train_batch_size": 2,
   "trial_name": null,
   "trial_params": null