Model save

README.md

@@ -34,15 +34,14 @@ More information needed
 
 The following hyperparameters were used during training:
 - learning_rate: 0.0001
-- train_batch_size: 8
+- train_batch_size: 16
 - eval_batch_size: 8
 - seed: 42
 - gradient_accumulation_steps: 16
-- total_train_batch_size: 128
+- total_train_batch_size: 256
 - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
 - lr_scheduler_type: linear
-- lr_scheduler_warmup_steps: 2
-- num_epochs: 2
+- num_epochs: 4
 
 ### Training results
 
@@ -50,7 +49,7 @@ The following hyperparameters were used during training:
 
 ### Framework versions
 
-- PEFT 0.13.2
+- PEFT 0.13.0
 - Transformers 4.45.2
-- Pytorch 2.4.1
-- Tokenizers 0.20.1
+- Pytorch 2.4.0
+- Tokenizers 0.20.0

adapter_config.json

@@ -1,37 +1,34 @@
 {
   "alpha_pattern": {},
-  "auto_mapping": {
-    "base_model_class": "LlamaForCausalLM",
-    "parent_library": "transformers.models.llama.modeling_llama"
-  },
+  "auto_mapping": null,
   "base_model_name_or_path": "meta-llama/Llama-3.2-3B-Instruct",
   "bias": "none",
   "fan_in_fan_out": false,
   "inference_mode": true,
-  "init_lora_weights": "gaussian",
+  "init_lora_weights": true,
   "layer_replication": null,
   "layers_pattern": null,
   "layers_to_transform": null,
   "loftq_config": {},
   "lora_alpha": 32,
-  "lora_dropout": 0.1,
+  "lora_dropout": 0.05,
   "megatron_config": null,
   "megatron_core": "megatron.core",
   "modules_to_save": null,
   "peft_type": "LORA",
-  "r": 8,
+  "r": 16,
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "down_proj",
-    "up_proj",
+    "v_proj",
+    "gate_proj",
     "o_proj",
-    "q_proj",
     "k_proj",
-    "gate_proj",
-    "v_proj"
+    "down_proj",
+    "up_proj",
+    "q_proj"
   ],
-  "task_type": null,
-  "use_dora": true,
+  "task_type": "CAUSAL_LM",
+  "use_dora": false,
   "use_rslora": false
 }

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a4296b9cf510cd2dd41ae8bf4eb415a592e7aba73769a261849b22435972f91f
-size 1639093264
+oid sha256:0edcd8d9a75fca009e240360ca8fd3dcb45f6f3d4f67a449b5e050db25eec1dd
+size 1684597880

trainer_state.json

@@ -1,7 +1,7 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 2.0,
+  "epoch": 4.0,
   "eval_steps": 500,
   "global_step": 2060,
   "is_hyper_param_search": false,
@@ -9,75 +9,75 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 0.24271844660194175,
-      "grad_norm": 2.2880468368530273,
-      "learning_rate": 8.794946550048592e-05,
-      "loss": 0.7378,
+      "epoch": 0.4854368932038835,
+      "grad_norm": 0.5094059705734253,
+      "learning_rate": 8.786407766990292e-05,
+      "loss": 0.7642,
       "step": 250
     },
     {
-      "epoch": 0.4854368932038835,
-      "grad_norm": 1.7398513555526733,
-      "learning_rate": 7.580174927113704e-05,
-      "loss": 0.448,
+      "epoch": 0.970873786407767,
+      "grad_norm": 0.3778647482395172,
+      "learning_rate": 7.572815533980583e-05,
+      "loss": 0.3845,
       "step": 500
     },
     {
-      "epoch": 0.7281553398058253,
-      "grad_norm": 1.7528009414672852,
-      "learning_rate": 6.365403304178815e-05,
-      "loss": 0.4192,
+      "epoch": 1.4563106796116505,
+      "grad_norm": 0.4107059836387634,
+      "learning_rate": 6.359223300970875e-05,
+      "loss": 0.3589,
       "step": 750
     },
     {
-      "epoch": 0.970873786407767,
-      "grad_norm": 1.7135326862335205,
-      "learning_rate": 5.150631681243926e-05,
-      "loss": 0.3959,
+      "epoch": 1.941747572815534,
+      "grad_norm": 0.4316255450248718,
+      "learning_rate": 5.145631067961165e-05,
+      "loss": 0.3445,
       "step": 1000
     },
     {
-      "epoch": 1.2135922330097086,
-      "grad_norm": 2.295848846435547,
-      "learning_rate": 3.9358600583090386e-05,
-      "loss": 0.3666,
+      "epoch": 2.4271844660194173,
+      "grad_norm": 0.4872223436832428,
+      "learning_rate": 3.9320388349514564e-05,
+      "loss": 0.3255,
       "step": 1250
     },
     {
-      "epoch": 1.4563106796116505,
-      "grad_norm": 2.189216136932373,
-      "learning_rate": 2.72108843537415e-05,
-      "loss": 0.3465,
+      "epoch": 2.912621359223301,
+      "grad_norm": 0.6546180248260498,
+      "learning_rate": 2.7184466019417475e-05,
+      "loss": 0.3019,
       "step": 1500
     },
     {
-      "epoch": 1.6990291262135924,
-      "grad_norm": 2.383833885192871,
-      "learning_rate": 1.5063168124392615e-05,
-      "loss": 0.3249,
+      "epoch": 3.3980582524271843,
+      "grad_norm": 0.6317738890647888,
+      "learning_rate": 1.5048543689320387e-05,
+      "loss": 0.2698,
       "step": 1750
     },
     {
-      "epoch": 1.941747572815534,
-      "grad_norm": 2.1898789405822754,
-      "learning_rate": 2.915451895043732e-06,
-      "loss": 0.3096,
+      "epoch": 3.883495145631068,
+      "grad_norm": 0.6270581483840942,
+      "learning_rate": 2.912621359223301e-06,
+      "loss": 0.2537,
       "step": 2000
     },
     {
-      "epoch": 2.0,
+      "epoch": 4.0,
       "step": 2060,
-      "total_flos": 6.440370064857293e+17,
-      "train_loss": 0.4152493615752285,
-      "train_runtime": 78972.7393,
-      "train_samples_per_second": 3.339,
-      "train_steps_per_second": 0.026
+      "total_flos": 1.4514285460762153e+18,
+      "train_loss": 0.3716998405826902,
+      "train_runtime": 16511.4152,
+      "train_samples_per_second": 31.938,
+      "train_steps_per_second": 0.125
     }
   ],
   "logging_steps": 250,
   "max_steps": 2060,
   "num_input_tokens_seen": 0,
-  "num_train_epochs": 2,
+  "num_train_epochs": 4,
   "save_steps": 500,
   "stateful_callbacks": {
     "TrainerControl": {
@@ -91,8 +91,8 @@
       "attributes": {}
     }
   },
-  "total_flos": 6.440370064857293e+17,
-  "train_batch_size": 8,
+  "total_flos": 1.4514285460762153e+18,
+  "train_batch_size": 16,
   "trial_name": null,
   "trial_params": null
 }

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0653084ef74cad66a842993ee891fafc998059dc681e884d2eb0539b717dfcce
-size 5176
+oid sha256:91794dd336216b2a89ff8f8b3a21988f053c3c394a0ec8cbcf0fc4f0977592bb
+size 5240