knowgen/ManuBERT-SequenceClassification

README.md

@@ -17,8 +17,8 @@ should probably proofread and complete it, then remove this comment. -->
 
 This model is a fine-tuned version of [microsoft/deberta-v3-base](https://huggingface.co/microsoft/deberta-v3-base) on an unknown dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.0347
-- Accuracy: 0.9921
+- Loss: 0.0378
+- Accuracy: 0.9929
 
 ## Model description
 
@@ -38,12 +38,12 @@ More information needed
 
 The following hyperparameters were used during training:
 - learning_rate: 2e-05
-- train_batch_size: 64
-- eval_batch_size: 32
+- train_batch_size: 32
+- eval_batch_size: 16
 - seed: 42
 - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
 - lr_scheduler_type: linear
-- num_epochs: 2.0
+- num_epochs: 3.0
 
 ### Training results
 

all_results.json

@@ -1,14 +1,14 @@
 {
-    "epoch": 2.0,
-    "eval_accuracy": 0.9921333333333333,
-    "eval_loss": 0.03474695608019829,
-    "eval_runtime": 410.5726,
+    "epoch": 3.0,
+    "eval_accuracy": 0.9928666666666667,
+    "eval_loss": 0.037820588797330856,
+    "eval_runtime": 557.4739,
     "eval_samples": 60000,
-    "eval_samples_per_second": 146.137,
-    "eval_steps_per_second": 4.567,
-    "train_loss": 0.020007819811503093,
-    "train_runtime": 9507.2866,
+    "eval_samples_per_second": 107.628,
+    "eval_steps_per_second": 6.727,
+    "train_loss": 0.0021281812138027613,
+    "train_runtime": 4797.7609,
     "train_samples": 480000,
-    "train_samples_per_second": 100.975,
-    "train_steps_per_second": 1.578
+    "train_samples_per_second": 300.14,
+    "train_steps_per_second": 4.69
 }

eval_results.json

@@ -1,9 +1,9 @@
 {
-    "epoch": 2.0,
-    "eval_accuracy": 0.9921333333333333,
-    "eval_loss": 0.03474695608019829,
-    "eval_runtime": 410.5726,
+    "epoch": 3.0,
+    "eval_accuracy": 0.9928666666666667,
+    "eval_loss": 0.037820588797330856,
+    "eval_runtime": 557.4739,
     "eval_samples": 60000,
-    "eval_samples_per_second": 146.137,
-    "eval_steps_per_second": 4.567
+    "eval_samples_per_second": 107.628,
+    "eval_steps_per_second": 6.727
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b1a84401d5ebf15b9bd0610a1a07447b87d9af5086e8c882f8a7417d063c71f4
+oid sha256:9d5bff70cebd51da34b0021d6a714142ef4f7b02a776fa2b43a8c02a7c589604
 size 737719272

tokenizer.json

@@ -1,7 +1,21 @@
 {
   "version": "1.0",
-  "truncation": null,
-  "padding": null,
+  "truncation": {
+    "direction": "Right",
+    "max_length": 128,
+    "strategy": "LongestFirst",
+    "stride": 0
+  },
+  "padding": {
+    "strategy": {
+      "Fixed": 128
+    },
+    "direction": "Right",
+    "pad_to_multiple_of": null,
+    "pad_id": 0,
+    "pad_type_id": 0,
+    "pad_token": "[PAD]"
+  },
   "added_tokens": [
     {
       "id": 0,

train_results.json

@@ -1,8 +1,8 @@
 {
-    "epoch": 2.0,
-    "train_loss": 0.020007819811503093,
-    "train_runtime": 9507.2866,
+    "epoch": 3.0,
+    "train_loss": 0.0021281812138027613,
+    "train_runtime": 4797.7609,
     "train_samples": 480000,
-    "train_samples_per_second": 100.975,
-    "train_steps_per_second": 1.578
+    "train_samples_per_second": 300.14,
+    "train_steps_per_second": 4.69
 }

trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 2.0,
+  "epoch": 3.0,
   "eval_steps": 500,
-  "global_step": 15000,
+  "global_step": 22500,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -189,21 +189,111 @@
       "step": 15000
     },
     {
-      "epoch": 2.0,
-      "step": 15000,
-      "total_flos": 1.2629557149696e+17,
-      "train_loss": 0.020007819811503093,
-      "train_runtime": 9507.2866,
-      "train_samples_per_second": 100.975,
-      "train_steps_per_second": 1.578
+      "epoch": 2.07,
+      "learning_rate": 6.222222222222223e-06,
+      "loss": 0.0065,
+      "step": 15500
+    },
+    {
+      "epoch": 2.13,
+      "learning_rate": 5.777777777777778e-06,
+      "loss": 0.0075,
+      "step": 16000
+    },
+    {
+      "epoch": 2.2,
+      "learning_rate": 5.333333333333334e-06,
+      "loss": 0.0068,
+      "step": 16500
+    },
+    {
+      "epoch": 2.27,
+      "learning_rate": 4.888888888888889e-06,
+      "loss": 0.008,
+      "step": 17000
+    },
+    {
+      "epoch": 2.33,
+      "learning_rate": 4.444444444444444e-06,
+      "loss": 0.0074,
+      "step": 17500
+    },
+    {
+      "epoch": 2.4,
+      "learning_rate": 4.000000000000001e-06,
+      "loss": 0.0071,
+      "step": 18000
+    },
+    {
+      "epoch": 2.47,
+      "learning_rate": 3.555555555555556e-06,
+      "loss": 0.0076,
+      "step": 18500
+    },
+    {
+      "epoch": 2.53,
+      "learning_rate": 3.1111111111111116e-06,
+      "loss": 0.0057,
+      "step": 19000
+    },
+    {
+      "epoch": 2.6,
+      "learning_rate": 2.666666666666667e-06,
+      "loss": 0.0058,
+      "step": 19500
+    },
+    {
+      "epoch": 2.67,
+      "learning_rate": 2.222222222222222e-06,
+      "loss": 0.0058,
+      "step": 20000
+    },
+    {
+      "epoch": 2.73,
+      "learning_rate": 1.777777777777778e-06,
+      "loss": 0.0066,
+      "step": 20500
+    },
+    {
+      "epoch": 2.8,
+      "learning_rate": 1.3333333333333334e-06,
+      "loss": 0.0063,
+      "step": 21000
+    },
+    {
+      "epoch": 2.87,
+      "learning_rate": 8.88888888888889e-07,
+      "loss": 0.0045,
+      "step": 21500
+    },
+    {
+      "epoch": 2.93,
+      "learning_rate": 4.444444444444445e-07,
+      "loss": 0.0058,
+      "step": 22000
+    },
+    {
+      "epoch": 3.0,
+      "learning_rate": 0.0,
+      "loss": 0.0044,
+      "step": 22500
+    },
+    {
+      "epoch": 3.0,
+      "step": 22500,
+      "total_flos": 1.578694643712e+17,
+      "train_loss": 0.0021281812138027613,
+      "train_runtime": 4797.7609,
+      "train_samples_per_second": 300.14,
+      "train_steps_per_second": 4.69
     }
   ],
   "logging_steps": 500,
-  "max_steps": 15000,
+  "max_steps": 22500,
   "num_input_tokens_seen": 0,
-  "num_train_epochs": 2,
+  "num_train_epochs": 3,
   "save_steps": 500,
-  "total_flos": 1.2629557149696e+17,
+  "total_flos": 1.578694643712e+17,
   "train_batch_size": 64,
   "trial_name": null,
   "trial_params": null

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7401efe3d3c5e93ce21e6db161b08470a4eede364cc4ea93cd79288a998f8de8
+oid sha256:e61d300b2103b9c085700526bd17373cbe7f9d3f0619de44b50575abb0251062
 size 4728