Training in progress, step 2891

.ipynb_checkpoints/config-checkpoint.yaml

@@ -73,17 +73,17 @@ train:
     # group_by_length: true
     max_seq_length: 2048
     eval_strategy: steps
-    per_device_train_batch_size: 4
-    per_device_eval_batch_size: 8
+    per_device_train_batch_size: 16
+    per_device_eval_batch_size: 32
     gradient_accumulation_steps: 1
     eval_accumulation_steps: 1
 
     optim: paged_adamw_8bit
     bf16: true
     bf16_full_eval: true
-    learning_rate: 0.00002
+    learning_rate: 0.0002
     weight_decay: 0.01
-    num_train_epochs: 1
+    num_train_epochs: 3
     warmup_ratio: 0.005
     max_grad_norm: 2.0
     
@@ -94,5 +94,5 @@ train:
     
     push_to_hub: true
 
-    torch_compile: true
+    # torch_compile: true
 seed: 42

adapter_config.json

@@ -21,11 +21,11 @@
   "revision": null,
   "target_modules": [
     "q_proj",
-    "k_proj",
-    "gate_proj",
-    "v_proj",
     "up_proj",
     "o_proj",
+    "v_proj",
+    "k_proj",
+    "gate_proj",
     "down_proj"
   ],
   "task_type": "CAUSAL_LM",

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:429b256cd598574ebc695ce486072c5c703d6cc4315b96dd20b1ffbe8d8e04eb
+oid sha256:206f133c1a55ddcc354c928242558bab36dd1ea63d300c8cd94f4c8b352a95ec
 size 167832240

config.yaml

@@ -73,17 +73,17 @@ train:
     # group_by_length: true
     max_seq_length: 2048
     eval_strategy: steps
-    per_device_train_batch_size: 4
-    per_device_eval_batch_size: 8
+    per_device_train_batch_size: 16
+    per_device_eval_batch_size: 32
     gradient_accumulation_steps: 1
     eval_accumulation_steps: 1
 
     optim: paged_adamw_8bit
     bf16: true
     bf16_full_eval: true
-    learning_rate: 0.00002
+    learning_rate: 0.0002
     weight_decay: 0.01
-    num_train_epochs: 1
+    num_train_epochs: 3
     warmup_ratio: 0.005
     max_grad_norm: 2.0
     
@@ -94,5 +94,5 @@ train:
     
     push_to_hub: true
 
-    torch_compile: true
+    # torch_compile: true
 seed: 42

tokenizer_config.json

@@ -2052,7 +2052,6 @@
   "bos_token": "<|begin_of_text|>",
   "chat_template": "{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% set loop_messages = messages %}{% for message in loop_messages %}{% set content = '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' %}{% if loop.index0 == 0 %}{% set content = bos_token + content %}{% endif %}{{ content }}{% endfor %}{% if add_generation_prompt %}{{ '<|start_header_id|>assistant<|end_header_id|>\n\n' }}{% endif %}",
   "clean_up_tokenization_spaces": true,
-  "device_map": "auto",
   "eos_token": "<|eot_id|>",
   "model_input_names": [
     "input_ids",

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d14636df665ef9af837b7c0e54de13e25f66ec22c9404e067f74e44d629828fb
+oid sha256:d4be0f582408e2d7d80578b82d0f5d327d6639e4fa47e1a721f251f43f343529
 size 5432