Training in progress, epoch 25, checkpoint

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:31fa9c03b236f3326abdb4196194072fbc949d245254e9cdd13231fc0ff9b3a1
+oid sha256:afd97dd8765dc1f4b908a232e8ef49fc36e02ae099f595ec2c827bde8c262164
 size 166496880

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8e06dc7f754a56ad9ba987b678b0f2d0c44a55075024d1d86ef926192c38101b
+oid sha256:bf6542aefd316db02c22779974ccf65e3583e7b60d2968b36d4e35421d0930b8
 size 330495866

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:83af2978e1223e5ba59e9c2a394b19ba33d5e55d1f204b990d525feddec0613f
+oid sha256:b45ae187a6921fbe09ae0badebce148b0a1efba8d5e43dc914da848aff37f805
 size 14244

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7a639bc5a01160d5eb2cc6041c59f4ef1d6b5b5f12eeaae15618a65a0588847a
+oid sha256:8ea6dd56bd7b979b6581387262242dcc226985f82ef204d6b0a2805bc6677327
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
-  "best_metric": 0.4076879024505615,
-  "best_model_checkpoint": "joe611/chickens-60-epoch-1000-images-aug/checkpoint-12000",
-  "epoch": 24.0,
+  "best_metric": 0.4043169915676117,
+  "best_model_checkpoint": "joe611/chickens-60-epoch-1000-images-aug/checkpoint-12500",
+  "epoch": 25.0,
   "eval_steps": 500,
-  "global_step": 12000,
+  "global_step": 12500,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -3431,6 +3431,144 @@
       "eval_samples_per_second": 20.107,
       "eval_steps_per_second": 2.614,
       "step": 12000
+    },
+    {
+      "epoch": 24.06,
+      "grad_norm": 52.72776412963867,
+      "learning_rate": 6.530138210942505e-06,
+      "loss": 0.5589,
+      "step": 12030
+    },
+    {
+      "epoch": 24.12,
+      "grad_norm": 29.438343048095703,
+      "learning_rate": 6.5151763481638705e-06,
+      "loss": 0.5368,
+      "step": 12060
+    },
+    {
+      "epoch": 24.18,
+      "grad_norm": 54.32796859741211,
+      "learning_rate": 6.500199531206381e-06,
+      "loss": 0.5092,
+      "step": 12090
+    },
+    {
+      "epoch": 24.24,
+      "grad_norm": 69.30450439453125,
+      "learning_rate": 6.485207907885175e-06,
+      "loss": 0.4992,
+      "step": 12120
+    },
+    {
+      "epoch": 24.3,
+      "grad_norm": 42.193634033203125,
+      "learning_rate": 6.47020162616152e-06,
+      "loss": 0.4091,
+      "step": 12150
+    },
+    {
+      "epoch": 24.36,
+      "grad_norm": 30.120336532592773,
+      "learning_rate": 6.455180834141359e-06,
+      "loss": 0.471,
+      "step": 12180
+    },
+    {
+      "epoch": 24.42,
+      "grad_norm": 30.613067626953125,
+      "learning_rate": 6.440145680073847e-06,
+      "loss": 0.5791,
+      "step": 12210
+    },
+    {
+      "epoch": 24.48,
+      "grad_norm": 38.11183166503906,
+      "learning_rate": 6.425096312349881e-06,
+      "loss": 0.4348,
+      "step": 12240
+    },
+    {
+      "epoch": 24.54,
+      "grad_norm": 46.21797180175781,
+      "learning_rate": 6.410032879500647e-06,
+      "loss": 0.5827,
+      "step": 12270
+    },
+    {
+      "epoch": 24.6,
+      "grad_norm": 39.75735855102539,
+      "learning_rate": 6.3949555301961474e-06,
+      "loss": 0.4915,
+      "step": 12300
+    },
+    {
+      "epoch": 24.66,
+      "grad_norm": 55.170345306396484,
+      "learning_rate": 6.3798644132437304e-06,
+      "loss": 0.4253,
+      "step": 12330
+    },
+    {
+      "epoch": 24.72,
+      "grad_norm": 30.651851654052734,
+      "learning_rate": 6.364759677586627e-06,
+      "loss": 0.5383,
+      "step": 12360
+    },
+    {
+      "epoch": 24.78,
+      "grad_norm": 110.5732421875,
+      "learning_rate": 6.349641472302484e-06,
+      "loss": 0.4379,
+      "step": 12390
+    },
+    {
+      "epoch": 24.84,
+      "grad_norm": 37.65018081665039,
+      "learning_rate": 6.334509946601879e-06,
+      "loss": 0.4374,
+      "step": 12420
+    },
+    {
+      "epoch": 24.9,
+      "grad_norm": 48.68293762207031,
+      "learning_rate": 6.3193652498268656e-06,
+      "loss": 0.5241,
+      "step": 12450
+    },
+    {
+      "epoch": 24.96,
+      "grad_norm": 131.17486572265625,
+      "learning_rate": 6.304207531449486e-06,
+      "loss": 0.4849,
+      "step": 12480
+    },
+    {
+      "epoch": 25.0,
+      "eval_loss": 0.4043169915676117,
+      "eval_map": 0.7096,
+      "eval_map_50": 0.949,
+      "eval_map_75": 0.8366,
+      "eval_map_chicken": 0.7483,
+      "eval_map_duck": 0.596,
+      "eval_map_large": 0.8412,
+      "eval_map_medium": 0.7084,
+      "eval_map_plant": 0.7843,
+      "eval_map_small": 0.1234,
+      "eval_mar_1": 0.2739,
+      "eval_mar_10": 0.7538,
+      "eval_mar_100": 0.7611,
+      "eval_mar_100_chicken": 0.7902,
+      "eval_mar_100_duck": 0.6546,
+      "eval_mar_100_plant": 0.8383,
+      "eval_mar_large": 0.8891,
+      "eval_mar_medium": 0.7703,
+      "eval_mar_small": 0.258,
+      "eval_runtime": 5.051,
+      "eval_samples_per_second": 19.798,
+      "eval_steps_per_second": 2.574,
+      "step": 12500
     }
   ],
   "logging_steps": 30,
@@ -3450,7 +3588,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 4.1282773549056e+18,
+  "total_flos": 4.30028891136e+18,
   "train_batch_size": 2,
   "trial_name": null,
   "trial_params": null