Training in progress, epoch 24, checkpoint

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b0b155b6c159aad5d499dbbfcd000f172ecfda66b201d4940a5f789194a7ca81
+oid sha256:31fa9c03b236f3326abdb4196194072fbc949d245254e9cdd13231fc0ff9b3a1
 size 166496880

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:93dc21f8623c3f6028f166852f07242a27af2a9e42ec647d8950119cb47b32d3
+oid sha256:8e06dc7f754a56ad9ba987b678b0f2d0c44a55075024d1d86ef926192c38101b
 size 330495866

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:afe5f3e38edc53d84c568132d5a1fe6617c029725cb5ed6465b95e27267304ae
+oid sha256:83af2978e1223e5ba59e9c2a394b19ba33d5e55d1f204b990d525feddec0613f
 size 14244

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:685c336afeba3f61098971b07e0888920b1e8afabd8b33c977a10c5f12a34b13
+oid sha256:7a639bc5a01160d5eb2cc6041c59f4ef1d6b5b5f12eeaae15618a65a0588847a
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
-  "best_metric": 0.41423293948173523,
-  "best_model_checkpoint": "joe611/chickens-60-epoch-1000-images-aug/checkpoint-11000",
-  "epoch": 23.0,
+  "best_metric": 0.4076879024505615,
+  "best_model_checkpoint": "joe611/chickens-60-epoch-1000-images-aug/checkpoint-12000",
+  "epoch": 24.0,
   "eval_steps": 500,
-  "global_step": 11500,
+  "global_step": 12000,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -3286,6 +3286,151 @@
       "eval_samples_per_second": 19.71,
       "eval_steps_per_second": 2.562,
       "step": 11500
+    },
+    {
+      "epoch": 23.04,
+      "grad_norm": 42.534297943115234,
+      "learning_rate": 6.782059393566254e-06,
+      "loss": 0.4992,
+      "step": 11520
+    },
+    {
+      "epoch": 23.1,
+      "grad_norm": 0.002314250450581312,
+      "learning_rate": 6.767374218896286e-06,
+      "loss": 0.474,
+      "step": 11550
+    },
+    {
+      "epoch": 23.16,
+      "grad_norm": 33.95319366455078,
+      "learning_rate": 6.752671600956295e-06,
+      "loss": 0.5098,
+      "step": 11580
+    },
+    {
+      "epoch": 23.22,
+      "grad_norm": 65.58226013183594,
+      "learning_rate": 6.737951684855185e-06,
+      "loss": 0.6279,
+      "step": 11610
+    },
+    {
+      "epoch": 23.28,
+      "grad_norm": 38.85221481323242,
+      "learning_rate": 6.723214615872585e-06,
+      "loss": 0.5288,
+      "step": 11640
+    },
+    {
+      "epoch": 23.34,
+      "grad_norm": 31.03350067138672,
+      "learning_rate": 6.708460539457418e-06,
+      "loss": 0.5167,
+      "step": 11670
+    },
+    {
+      "epoch": 23.4,
+      "grad_norm": 74.7662353515625,
+      "learning_rate": 6.693689601226458e-06,
+      "loss": 0.5782,
+      "step": 11700
+    },
+    {
+      "epoch": 23.46,
+      "grad_norm": 30.249208450317383,
+      "learning_rate": 6.6789019469629034e-06,
+      "loss": 0.4759,
+      "step": 11730
+    },
+    {
+      "epoch": 23.52,
+      "grad_norm": 32.10087966918945,
+      "learning_rate": 6.664097722614934e-06,
+      "loss": 0.4923,
+      "step": 11760
+    },
+    {
+      "epoch": 23.58,
+      "grad_norm": 129.60997009277344,
+      "learning_rate": 6.649277074294265e-06,
+      "loss": 0.5149,
+      "step": 11790
+    },
+    {
+      "epoch": 23.64,
+      "grad_norm": 74.41838073730469,
+      "learning_rate": 6.634440148274712e-06,
+      "loss": 0.5063,
+      "step": 11820
+    },
+    {
+      "epoch": 23.7,
+      "grad_norm": 37.93177032470703,
+      "learning_rate": 6.619587090990748e-06,
+      "loss": 0.4632,
+      "step": 11850
+    },
+    {
+      "epoch": 23.76,
+      "grad_norm": 40.77086639404297,
+      "learning_rate": 6.604718049036047e-06,
+      "loss": 0.5137,
+      "step": 11880
+    },
+    {
+      "epoch": 23.82,
+      "grad_norm": 29.192161560058594,
+      "learning_rate": 6.589833169162055e-06,
+      "loss": 0.5527,
+      "step": 11910
+    },
+    {
+      "epoch": 23.88,
+      "grad_norm": 86.26991271972656,
+      "learning_rate": 6.574932598276524e-06,
+      "loss": 0.4662,
+      "step": 11940
+    },
+    {
+      "epoch": 23.94,
+      "grad_norm": 35.82132339477539,
+      "learning_rate": 6.5600164834420754e-06,
+      "loss": 0.5318,
+      "step": 11970
+    },
+    {
+      "epoch": 24.0,
+      "grad_norm": 53.413780212402344,
+      "learning_rate": 6.545084971874738e-06,
+      "loss": 0.5184,
+      "step": 12000
+    },
+    {
+      "epoch": 24.0,
+      "eval_loss": 0.4076879024505615,
+      "eval_map": 0.7097,
+      "eval_map_50": 0.9464,
+      "eval_map_75": 0.854,
+      "eval_map_chicken": 0.7126,
+      "eval_map_duck": 0.6338,
+      "eval_map_large": 0.8335,
+      "eval_map_medium": 0.7156,
+      "eval_map_plant": 0.7828,
+      "eval_map_small": 0.1197,
+      "eval_mar_1": 0.2741,
+      "eval_mar_10": 0.757,
+      "eval_mar_100": 0.7607,
+      "eval_mar_100_chicken": 0.7667,
+      "eval_mar_100_duck": 0.6784,
+      "eval_mar_100_plant": 0.8372,
+      "eval_mar_large": 0.8828,
+      "eval_mar_medium": 0.7738,
+      "eval_mar_small": 0.2553,
+      "eval_runtime": 4.9733,
+      "eval_samples_per_second": 20.107,
+      "eval_steps_per_second": 2.614,
+      "step": 12000
     }
   ],
   "logging_steps": 30,
@@ -3305,7 +3450,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 3.9562657984512e+18,
+  "total_flos": 4.1282773549056e+18,
   "train_batch_size": 2,
   "trial_name": null,
   "trial_params": null