Training in progress, epoch 44, checkpoint

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4d69a853ce44d2a4c6a1f289809b194fbf5a858060c6c022ed50371f82355daf
+oid sha256:bf72785f9e61f422066d5a55b0560c4c8a69464b26be32f4db1debc616942689
 size 166496880

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d8897b9557334ef700d8217bf78bf859bb2374469468874a040168b0470cc76f
+oid sha256:1a41339ed91a0fc5aebfb27479aad8c07d748b799c7840c36fbe1be25b37e1ab
 size 330495866

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:85e4133488da63327fc9ea031838bb6fd69fb43cc377eec164eb44278dfb4b45
+oid sha256:ef8c417a427a4d61d5632cb068a6069c159668c42d4b0b7a6a15b8a00b11eecc
 size 14244

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:545d510515f5179456e33cb07c6feb588b2ff2a682f4164e86878f83fb45a36d
+oid sha256:0fa74708df9c1325cdd60797f5ccc467462cf147fdacd71b5f8d7ef24d6392cd
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": 0.29740211367607117,
   "best_model_checkpoint": "joe611/chickens-60-epoch-1000-images-aug/checkpoint-21500",
-  "epoch": 43.0,
+  "epoch": 44.0,
   "eval_steps": 500,
-  "global_step": 21500,
+  "global_step": 22000,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -6137,6 +6137,151 @@
       "eval_samples_per_second": 17.724,
       "eval_steps_per_second": 2.304,
       "step": 21500
+    },
+    {
+      "epoch": 43.02,
+      "grad_norm": 44.15309143066406,
+      "learning_rate": 1.8493306440731557e-06,
+      "loss": 0.4896,
+      "step": 21510
+    },
+    {
+      "epoch": 43.08,
+      "grad_norm": 35.48102951049805,
+      "learning_rate": 1.837149190434378e-06,
+      "loss": 0.5013,
+      "step": 21540
+    },
+    {
+      "epoch": 43.14,
+      "grad_norm": 35.87041473388672,
+      "learning_rate": 1.824998952856198e-06,
+      "loss": 0.4439,
+      "step": 21570
+    },
+    {
+      "epoch": 43.2,
+      "grad_norm": 26.650522232055664,
+      "learning_rate": 1.8128800512565514e-06,
+      "loss": 0.4186,
+      "step": 21600
+    },
+    {
+      "epoch": 43.26,
+      "grad_norm": 48.30433654785156,
+      "learning_rate": 1.800792605244109e-06,
+      "loss": 0.4345,
+      "step": 21630
+    },
+    {
+      "epoch": 43.32,
+      "grad_norm": 70.63233184814453,
+      "learning_rate": 1.7887367341170781e-06,
+      "loss": 0.4639,
+      "step": 21660
+    },
+    {
+      "epoch": 43.38,
+      "grad_norm": 29.950109481811523,
+      "learning_rate": 1.7767125568620442e-06,
+      "loss": 0.3931,
+      "step": 21690
+    },
+    {
+      "epoch": 43.44,
+      "grad_norm": 40.98532485961914,
+      "learning_rate": 1.7647201921527802e-06,
+      "loss": 0.3728,
+      "step": 21720
+    },
+    {
+      "epoch": 43.5,
+      "grad_norm": 107.21430969238281,
+      "learning_rate": 1.7527597583490825e-06,
+      "loss": 0.3679,
+      "step": 21750
+    },
+    {
+      "epoch": 43.56,
+      "grad_norm": 41.53556442260742,
+      "learning_rate": 1.7408313734956074e-06,
+      "loss": 0.4278,
+      "step": 21780
+    },
+    {
+      "epoch": 43.62,
+      "grad_norm": 30.475116729736328,
+      "learning_rate": 1.7289351553206952e-06,
+      "loss": 0.4995,
+      "step": 21810
+    },
+    {
+      "epoch": 43.68,
+      "grad_norm": 65.69325256347656,
+      "learning_rate": 1.7170712212352187e-06,
+      "loss": 0.4236,
+      "step": 21840
+    },
+    {
+      "epoch": 43.74,
+      "grad_norm": 83.24858856201172,
+      "learning_rate": 1.7052396883314154e-06,
+      "loss": 0.4865,
+      "step": 21870
+    },
+    {
+      "epoch": 43.8,
+      "grad_norm": 49.75985336303711,
+      "learning_rate": 1.6934406733817417e-06,
+      "loss": 0.4061,
+      "step": 21900
+    },
+    {
+      "epoch": 43.86,
+      "grad_norm": 0.0012297651264816523,
+      "learning_rate": 1.6816742928377072e-06,
+      "loss": 0.4391,
+      "step": 21930
+    },
+    {
+      "epoch": 43.92,
+      "grad_norm": 107.39283752441406,
+      "learning_rate": 1.6699406628287423e-06,
+      "loss": 0.4027,
+      "step": 21960
+    },
+    {
+      "epoch": 43.98,
+      "grad_norm": 53.299110412597656,
+      "learning_rate": 1.658239899161036e-06,
+      "loss": 0.7057,
+      "step": 21990
+    },
+    {
+      "epoch": 44.0,
+      "eval_loss": 0.3092344403266907,
+      "eval_map": 0.7822,
+      "eval_map_50": 0.9563,
+      "eval_map_75": 0.9171,
+      "eval_map_chicken": 0.7941,
+      "eval_map_duck": 0.7348,
+      "eval_map_large": 0.8688,
+      "eval_map_medium": 0.7813,
+      "eval_map_plant": 0.8177,
+      "eval_map_small": 0.1985,
+      "eval_mar_1": 0.3003,
+      "eval_mar_10": 0.8165,
+      "eval_mar_100": 0.821,
+      "eval_mar_100_chicken": 0.8307,
+      "eval_mar_100_duck": 0.766,
+      "eval_mar_100_plant": 0.8663,
+      "eval_mar_large": 0.9117,
+      "eval_mar_medium": 0.8292,
+      "eval_mar_small": 0.3663,
+      "eval_runtime": 5.0773,
+      "eval_samples_per_second": 19.696,
+      "eval_steps_per_second": 2.56,
+      "step": 22000
     }
   ],
   "logging_steps": 30,
@@ -6156,7 +6301,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 7.3964969275392e+18,
+  "total_flos": 7.5685084839936e+18,
   "train_batch_size": 2,
   "trial_name": null,
   "trial_params": null