Training in progress, epoch 47, checkpoint

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:87ad9da164f90ea744048b81b6e35a735052df0bf62c64a7beaadf896e199c1a
+oid sha256:6d5ce355db46607a95aec28c8b78c3595e784c5a4d7b633fba4e3a9ee15eb38c
 size 166496880

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1d08a40bb963af55260213aa96db9795ec46c8ced8e1e958c67197b7a6e73bdd
+oid sha256:6ba6ec41c0dc6f0e349a76e70dac0de7bab1ad5a9835e860fb1ac13bdcf45e24
 size 330495866

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7f82f9668f7fedf5f119dd94c496b08d68f0842eece575c8646e2c5421dad242
+oid sha256:02ad003c0a9f67fd2fbb27db20929ec58a84ae23e961e581a1f533296d899603
 size 14244

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:23d8e99991fb0f832a4848719eeed140cdeda876cf038df0ed1de594bd36ed59
+oid sha256:aa3fe3dee209f5f42e53fe323f65577065b213541550cd1190559073b9095da7
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
-  "best_metric": 0.29740211367607117,
-  "best_model_checkpoint": "joe611/chickens-60-epoch-1000-images-aug/checkpoint-21500",
-  "epoch": 46.0,
+  "best_metric": 0.29506856203079224,
+  "best_model_checkpoint": "joe611/chickens-60-epoch-1000-images-aug/checkpoint-23500",
+  "epoch": 47.0,
   "eval_steps": 500,
-  "global_step": 23000,
+  "global_step": 23500,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -6565,6 +6565,151 @@
       "eval_samples_per_second": 19.89,
       "eval_steps_per_second": 2.586,
       "step": 23000
+    },
+    {
+      "epoch": 46.02,
+      "grad_norm": 82.24079132080078,
+      "learning_rate": 1.2807743509648745e-06,
+      "loss": 0.459,
+      "step": 23010
+    },
+    {
+      "epoch": 46.08,
+      "grad_norm": 35.353336334228516,
+      "learning_rate": 1.2702942728790897e-06,
+      "loss": 0.4215,
+      "step": 23040
+    },
+    {
+      "epoch": 46.14,
+      "grad_norm": 38.322479248046875,
+      "learning_rate": 1.2598510054830888e-06,
+      "loss": 0.3925,
+      "step": 23070
+    },
+    {
+      "epoch": 46.2,
+      "grad_norm": 53.787445068359375,
+      "learning_rate": 1.2494446518477022e-06,
+      "loss": 0.349,
+      "step": 23100
+    },
+    {
+      "epoch": 46.26,
+      "grad_norm": 37.92025375366211,
+      "learning_rate": 1.2390753146794438e-06,
+      "loss": 0.3578,
+      "step": 23130
+    },
+    {
+      "epoch": 46.32,
+      "grad_norm": 68.19427490234375,
+      "learning_rate": 1.2287430963194807e-06,
+      "loss": 0.4342,
+      "step": 23160
+    },
+    {
+      "epoch": 46.38,
+      "grad_norm": 108.89007568359375,
+      "learning_rate": 1.218448098742641e-06,
+      "loss": 0.4607,
+      "step": 23190
+    },
+    {
+      "epoch": 46.44,
+      "grad_norm": 83.83390808105469,
+      "learning_rate": 1.2081904235563908e-06,
+      "loss": 0.4169,
+      "step": 23220
+    },
+    {
+      "epoch": 46.5,
+      "grad_norm": 46.47811508178711,
+      "learning_rate": 1.1979701719998454e-06,
+      "loss": 0.3965,
+      "step": 23250
+    },
+    {
+      "epoch": 46.56,
+      "grad_norm": 83.35205841064453,
+      "learning_rate": 1.18778744494276e-06,
+      "loss": 0.685,
+      "step": 23280
+    },
+    {
+      "epoch": 46.62,
+      "grad_norm": 61.895530700683594,
+      "learning_rate": 1.1776423428845423e-06,
+      "loss": 0.3723,
+      "step": 23310
+    },
+    {
+      "epoch": 46.68,
+      "grad_norm": 68.6615982055664,
+      "learning_rate": 1.1675349659532514e-06,
+      "loss": 0.4277,
+      "step": 23340
+    },
+    {
+      "epoch": 46.74,
+      "grad_norm": 34.00867462158203,
+      "learning_rate": 1.1574654139046171e-06,
+      "loss": 0.4321,
+      "step": 23370
+    },
+    {
+      "epoch": 46.8,
+      "grad_norm": 40.27052688598633,
+      "learning_rate": 1.1474337861210543e-06,
+      "loss": 0.3798,
+      "step": 23400
+    },
+    {
+      "epoch": 46.86,
+      "grad_norm": 58.309085845947266,
+      "learning_rate": 1.1374401816106778e-06,
+      "loss": 0.4083,
+      "step": 23430
+    },
+    {
+      "epoch": 46.92,
+      "grad_norm": 43.79540252685547,
+      "learning_rate": 1.1274846990063314e-06,
+      "loss": 0.3961,
+      "step": 23460
+    },
+    {
+      "epoch": 46.98,
+      "grad_norm": 48.5699462890625,
+      "learning_rate": 1.1175674365646067e-06,
+      "loss": 0.4257,
+      "step": 23490
+    },
+    {
+      "epoch": 47.0,
+      "eval_loss": 0.29506856203079224,
+      "eval_map": 0.792,
+      "eval_map_50": 0.9625,
+      "eval_map_75": 0.9172,
+      "eval_map_chicken": 0.7961,
+      "eval_map_duck": 0.7572,
+      "eval_map_large": 0.8802,
+      "eval_map_medium": 0.7855,
+      "eval_map_plant": 0.8226,
+      "eval_map_small": 0.2075,
+      "eval_mar_1": 0.3067,
+      "eval_mar_10": 0.8262,
+      "eval_mar_100": 0.8309,
+      "eval_mar_100_chicken": 0.8338,
+      "eval_mar_100_duck": 0.7907,
+      "eval_mar_100_plant": 0.8683,
+      "eval_mar_large": 0.9197,
+      "eval_mar_medium": 0.836,
+      "eval_mar_small": 0.3468,
+      "eval_runtime": 5.0742,
+      "eval_samples_per_second": 19.708,
+      "eval_steps_per_second": 2.562,
+      "step": 23500
     }
   ],
   "logging_steps": 30,
@@ -6584,7 +6729,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 7.9125315969024e+18,
+  "total_flos": 8.0845431533568e+18,
   "train_batch_size": 2,
   "trial_name": null,
   "trial_params": null