Model save

README.md

@@ -2,13 +2,13 @@
 license: gemma
 library_name: peft
 tags:
-- alignment-handbook
 - trl
 - sft
+- alignment-handbook
 - generated_from_trainer
 base_model: google/gemma-2b
 datasets:
-- llama-duo/synth_summarize_dataset_dedup
+- generator
 model-index:
 - name: gemma2b-summarize-gemini1_5flash-256k
   results: []
@@ -19,9 +19,9 @@ should probably proofread and complete it, then remove this comment. -->
 
 # gemma2b-summarize-gemini1_5flash-256k
 
-This model is a fine-tuned version of [google/gemma-2b](https://huggingface.co/google/gemma-2b) on the llama-duo/synth_summarize_dataset_dedup dataset.
+This model is a fine-tuned version of [google/gemma-2b](https://huggingface.co/google/gemma-2b) on the generator dataset.
 It achieves the following results on the evaluation set:
-- Loss: 2.5669
+- Loss: 2.5681
 
 ## Model description
 
@@ -67,7 +67,7 @@ The following hyperparameters were used during training:
 | 0.8464        | 6.9976 | 1452 | 2.5513          |
 | 0.8353        | 8.0    | 1660 | 2.5615          |
 | 0.8267        | 8.9976 | 1867 | 2.5674          |
-| 0.827         | 9.9759 | 2070 | 2.5669          |
+| 0.8289        | 9.9976 | 2070 | 2.5681          |
 
 
 ### Framework versions

adapter_config.json

@@ -20,13 +20,13 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "up_proj",
     "o_proj",
-    "gate_proj",
     "k_proj",
-    "down_proj",
+    "gate_proj",
     "v_proj",
-    "q_proj"
+    "down_proj",
+    "q_proj",
+    "up_proj"
   ],
   "task_type": "CAUSAL_LM",
   "use_dora": false,

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:465c1b3666bad23b46105166da89ba228655adfb0e551b2809da6f6c1f2df5f3
+oid sha256:778b553eda2c7f1735b85277d6ed5dd9bc1d8df6648e3ab13383249ff9a901a0
 size 78480320

all_results.json

@@ -1,14 +1,14 @@
 {
-    "epoch": 9.975903614457831,
+    "epoch": 9.997590361445782,
     "eval_loss": 2.566892147064209,
     "eval_runtime": 0.495,
     "eval_samples": 25,
     "eval_samples_per_second": 20.203,
     "eval_steps_per_second": 2.02,
     "total_flos": 3.290190024938619e+18,
-    "train_loss": 0.9333097650233099,
-    "train_runtime": 14306.303,
+    "train_loss": 0.02805145143886695,
+    "train_runtime": 534.5593,
     "train_samples": 253412,
-    "train_samples_per_second": 18.532,
-    "train_steps_per_second": 0.145
+    "train_samples_per_second": 495.96,
+    "train_steps_per_second": 3.872
 }

runs/Jun10_15-56-44_6511c8cdb6e6/events.out.tfevents.1718035040.6511c8cdb6e6.3078.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fe2d556b664b2e6d4aeb253ad5f3304790e343fb732889c7fc5c0e31cd2629da
+size 9187

train_results.json

@@ -1,9 +1,9 @@
 {
-    "epoch": 9.975903614457831,
+    "epoch": 9.997590361445782,
     "total_flos": 3.290190024938619e+18,
-    "train_loss": 0.9333097650233099,
-    "train_runtime": 14306.303,
+    "train_loss": 0.02805145143886695,
+    "train_runtime": 534.5593,
     "train_samples": 253412,
-    "train_samples_per_second": 18.532,
-    "train_steps_per_second": 0.145
+    "train_samples_per_second": 495.96,
+    "train_steps_per_second": 3.872
 }

trainer_state.json

@@ -1,7 +1,7 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 9.975903614457831,
+  "epoch": 9.997590361445782,
   "eval_steps": 500,
   "global_step": 2070,
   "is_hyper_param_search": false,
@@ -2888,119 +2888,119 @@
       "step": 2000
     },
     {
-      "epoch": 9.662650602409638,
-      "grad_norm": 0.19921875,
+      "epoch": 9.684337349397591,
+      "grad_norm": 0.2099609375,
       "learning_rate": 6.001169527811268e-07,
-      "loss": 0.8274,
+      "loss": 0.8287,
       "step": 2005
     },
     {
-      "epoch": 9.686746987951807,
+      "epoch": 9.708433734939758,
       "grad_norm": 0.20703125,
       "learning_rate": 5.114180081645214e-07,
-      "loss": 0.8303,
+      "loss": 0.8321,
       "step": 2010
     },
     {
-      "epoch": 9.710843373493976,
-      "grad_norm": 0.212890625,
+      "epoch": 9.732530120481927,
+      "grad_norm": 0.19921875,
       "learning_rate": 4.2979175500050817e-07,
-      "loss": 0.8301,
+      "loss": 0.8297,
       "step": 2015
     },
     {
-      "epoch": 9.734939759036145,
-      "grad_norm": 0.201171875,
+      "epoch": 9.756626506024096,
+      "grad_norm": 0.228515625,
       "learning_rate": 3.552439961389431e-07,
-      "loss": 0.829,
+      "loss": 0.8235,
       "step": 2020
     },
     {
-      "epoch": 9.759036144578314,
-      "grad_norm": 0.212890625,
+      "epoch": 9.780722891566265,
+      "grad_norm": 0.203125,
       "learning_rate": 2.877800312160783e-07,
-      "loss": 0.8267,
+      "loss": 0.8302,
       "step": 2025
     },
     {
-      "epoch": 9.783132530120483,
-      "grad_norm": 0.20703125,
+      "epoch": 9.804819277108434,
+      "grad_norm": 0.2109375,
       "learning_rate": 2.274046562778409e-07,
-      "loss": 0.8285,
+      "loss": 0.8331,
       "step": 2030
     },
     {
-      "epoch": 9.80722891566265,
-      "grad_norm": 0.20703125,
+      "epoch": 9.828915662650603,
+      "grad_norm": 0.2119140625,
       "learning_rate": 1.7412216343885014e-07,
-      "loss": 0.8311,
+      "loss": 0.8281,
       "step": 2035
     },
     {
-      "epoch": 9.831325301204819,
-      "grad_norm": 0.2119140625,
+      "epoch": 9.85301204819277,
+      "grad_norm": 0.2080078125,
       "learning_rate": 1.2793634057732818e-07,
-      "loss": 0.8311,
+      "loss": 0.8321,
       "step": 2040
     },
     {
-      "epoch": 9.855421686746988,
-      "grad_norm": 0.2109375,
+      "epoch": 9.87710843373494,
+      "grad_norm": 0.212890625,
       "learning_rate": 8.885047106578227e-08,
-      "loss": 0.83,
+      "loss": 0.832,
       "step": 2045
     },
     {
-      "epoch": 9.879518072289157,
-      "grad_norm": 0.205078125,
+      "epoch": 9.901204819277108,
+      "grad_norm": 0.2080078125,
       "learning_rate": 5.6867333537580226e-08,
-      "loss": 0.8322,
+      "loss": 0.8308,
       "step": 2050
     },
     {
-      "epoch": 9.903614457831326,
-      "grad_norm": 0.21484375,
+      "epoch": 9.925301204819277,
+      "grad_norm": 0.2001953125,
       "learning_rate": 3.1989201689452967e-08,
-      "loss": 0.8305,
+      "loss": 0.8283,
       "step": 2055
     },
     {
-      "epoch": 9.927710843373493,
+      "epoch": 9.949397590361446,
       "grad_norm": 0.2080078125,
       "learning_rate": 1.4217844119857048e-08,
-      "loss": 0.8293,
+      "loss": 0.8307,
       "step": 2060
     },
     {
-      "epoch": 9.951807228915662,
-      "grad_norm": 0.2041015625,
+      "epoch": 9.973493975903615,
+      "grad_norm": 0.2080078125,
       "learning_rate": 3.554524203175369e-09,
-      "loss": 0.8287,
+      "loss": 0.8252,
       "step": 2065
     },
     {
-      "epoch": 9.975903614457831,
-      "grad_norm": 0.2177734375,
+      "epoch": 9.997590361445782,
+      "grad_norm": 0.203125,
       "learning_rate": 0.0,
-      "loss": 0.827,
+      "loss": 0.8289,
       "step": 2070
     },
     {
-      "epoch": 9.975903614457831,
-      "eval_loss": 2.566892147064209,
-      "eval_runtime": 0.4844,
-      "eval_samples_per_second": 20.643,
-      "eval_steps_per_second": 2.064,
+      "epoch": 9.997590361445782,
+      "eval_loss": 2.5681488513946533,
+      "eval_runtime": 0.4835,
+      "eval_samples_per_second": 20.683,
+      "eval_steps_per_second": 2.068,
       "step": 2070
     },
     {
-      "epoch": 9.975903614457831,
+      "epoch": 9.997590361445782,
       "step": 2070,
       "total_flos": 3.290190024938619e+18,
-      "train_loss": 0.9333097650233099,
-      "train_runtime": 14306.303,
-      "train_samples_per_second": 18.532,
-      "train_steps_per_second": 0.145
+      "train_loss": 0.02805145143886695,
+      "train_runtime": 534.5593,
+      "train_samples_per_second": 495.96,
+      "train_steps_per_second": 3.872
     }
   ],
   "logging_steps": 5,

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3d0e8578086c62d31c0054e61eeb3aad589514780d2ee582bbd68270ce238f7b
+oid sha256:78a14829f27b48698fad520fcdb9f80df4e43a3e7e4532f4e22f6580ed774eb9
 size 5304