Training in progress, epoch 43, checkpoint

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:848953d25d5a9f4f9b7b663d111bb668c6455ace866e6c0d8c2d2253dcd1b6f9
+oid sha256:4d69a853ce44d2a4c6a1f289809b194fbf5a858060c6c022ed50371f82355daf
 size 166496880

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7fa989f187882ac7dd49a61086937b636de2892613dde482a88e0faf39426ed5
+oid sha256:d8897b9557334ef700d8217bf78bf859bb2374469468874a040168b0470cc76f
 size 330495866

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e0d931d489a6ecaf38c7d11c5f9d1ed6d115bfa7679ca5bd4cd5a652e7c55db7
+oid sha256:85e4133488da63327fc9ea031838bb6fd69fb43cc377eec164eb44278dfb4b45
 size 14244

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:91946c64eb5ced4862f474458840a35ce7039f1ebfd0264eda91b7e2f3af5260
+oid sha256:545d510515f5179456e33cb07c6feb588b2ff2a682f4164e86878f83fb45a36d
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
-  "best_metric": 0.30137816071510315,
-  "best_model_checkpoint": "joe611/chickens-60-epoch-1000-images-aug/checkpoint-21000",
-  "epoch": 42.0,
+  "best_metric": 0.29740211367607117,
+  "best_model_checkpoint": "joe611/chickens-60-epoch-1000-images-aug/checkpoint-21500",
+  "epoch": 43.0,
   "eval_steps": 500,
-  "global_step": 21000,
+  "global_step": 21500,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -5999,6 +5999,144 @@
       "eval_samples_per_second": 19.708,
       "eval_steps_per_second": 2.562,
       "step": 21000
+    },
+    {
+      "epoch": 42.06,
+      "grad_norm": 41.583213806152344,
+      "learning_rate": 2.0483802532185286e-06,
+      "loss": 0.4022,
+      "step": 21030
+    },
+    {
+      "epoch": 42.12,
+      "grad_norm": 320.62518310546875,
+      "learning_rate": 2.035715899194704e-06,
+      "loss": 0.4408,
+      "step": 21060
+    },
+    {
+      "epoch": 42.18,
+      "grad_norm": 42.51158142089844,
+      "learning_rate": 2.0230808014582265e-06,
+      "loss": 0.4566,
+      "step": 21090
+    },
+    {
+      "epoch": 42.24,
+      "grad_norm": 0.00034786213655024767,
+      "learning_rate": 2.0104750847124075e-06,
+      "loss": 0.5319,
+      "step": 21120
+    },
+    {
+      "epoch": 42.3,
+      "grad_norm": 93.6717529296875,
+      "learning_rate": 1.9978988733705807e-06,
+      "loss": 0.4083,
+      "step": 21150
+    },
+    {
+      "epoch": 42.36,
+      "grad_norm": 52.52644348144531,
+      "learning_rate": 1.9853522915548777e-06,
+      "loss": 0.4354,
+      "step": 21180
+    },
+    {
+      "epoch": 42.42,
+      "grad_norm": 36.161460876464844,
+      "learning_rate": 1.9728354630949935e-06,
+      "loss": 0.4591,
+      "step": 21210
+    },
+    {
+      "epoch": 42.48,
+      "grad_norm": 60.393306732177734,
+      "learning_rate": 1.9603485115269743e-06,
+      "loss": 0.5063,
+      "step": 21240
+    },
+    {
+      "epoch": 42.54,
+      "grad_norm": 223.328857421875,
+      "learning_rate": 1.9478915600919877e-06,
+      "loss": 0.4404,
+      "step": 21270
+    },
+    {
+      "epoch": 42.6,
+      "grad_norm": 42.48649215698242,
+      "learning_rate": 1.9354647317351187e-06,
+      "loss": 0.4508,
+      "step": 21300
+    },
+    {
+      "epoch": 42.66,
+      "grad_norm": 46.062191009521484,
+      "learning_rate": 1.9230681491041425e-06,
+      "loss": 0.4275,
+      "step": 21330
+    },
+    {
+      "epoch": 42.72,
+      "grad_norm": 0.0008085273439064622,
+      "learning_rate": 1.910701934548329e-06,
+      "loss": 0.4588,
+      "step": 21360
+    },
+    {
+      "epoch": 42.78,
+      "grad_norm": 44.36118698120117,
+      "learning_rate": 1.8983662101172217e-06,
+      "loss": 0.4563,
+      "step": 21390
+    },
+    {
+      "epoch": 42.84,
+      "grad_norm": 139.61676025390625,
+      "learning_rate": 1.8860610975594384e-06,
+      "loss": 0.3606,
+      "step": 21420
+    },
+    {
+      "epoch": 42.9,
+      "grad_norm": 37.38964080810547,
+      "learning_rate": 1.873786718321476e-06,
+      "loss": 0.4464,
+      "step": 21450
+    },
+    {
+      "epoch": 42.96,
+      "grad_norm": 72.3614730834961,
+      "learning_rate": 1.8615431935464984e-06,
+      "loss": 0.4322,
+      "step": 21480
+    },
+    {
+      "epoch": 43.0,
+      "eval_loss": 0.29740211367607117,
+      "eval_map": 0.7974,
+      "eval_map_50": 0.9595,
+      "eval_map_75": 0.9169,
+      "eval_map_chicken": 0.8005,
+      "eval_map_duck": 0.7586,
+      "eval_map_large": 0.8951,
+      "eval_map_medium": 0.7927,
+      "eval_map_plant": 0.8331,
+      "eval_map_small": 0.1879,
+      "eval_mar_1": 0.3078,
+      "eval_mar_10": 0.8296,
+      "eval_mar_100": 0.834,
+      "eval_mar_100_chicken": 0.8369,
+      "eval_mar_100_duck": 0.7876,
+      "eval_mar_100_plant": 0.8775,
+      "eval_mar_large": 0.931,
+      "eval_mar_medium": 0.8411,
+      "eval_mar_small": 0.3085,
+      "eval_runtime": 5.6419,
+      "eval_samples_per_second": 17.724,
+      "eval_steps_per_second": 2.304,
+      "step": 21500
     }
   ],
   "logging_steps": 30,
@@ -6018,7 +6156,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 7.2244853710848e+18,
+  "total_flos": 7.3964969275392e+18,
   "train_batch_size": 2,
   "trial_name": null,
   "trial_params": null