End of training

README.md

@@ -38,7 +38,8 @@ lora_target_modules:
 lora_target_linear: true
 lora_fan_in_fan_out: true  # pythia/GPTNeoX lora specific
 lora_modules_to_save:
-  - embed_tokens
+  - embed_in
+  - embed_out
   - lm_head
 lora_on_cpu: false
 # ReLoRA configuration
@@ -59,7 +60,7 @@ wandb_log_model:
 output_dir: ./outputs/lora-alpaca-pythia-160m-storytelling
 gradient_accumulation_steps: 16
 micro_batch_size: 1
-num_epochs: 1
+num_epochs: 3
 learning_rate: 0.0006
 lr_scheduler: cosine_with_restarts
 #cosine_min_lr_ratio: 0.1
@@ -85,6 +86,9 @@ logging_steps: 1
 save_steps: 200
 save_total_limit: 5
 warmup_steps: 100
+tokens:
+  - "[INST]"
+  - "[/INST]"
 
 ```
 
@@ -94,7 +98,7 @@ warmup_steps: 100
 
 This model is a fine-tuned version of [EleutherAI/pythia-160m-deduped](https://huggingface.co/EleutherAI/pythia-160m-deduped) on the None dataset.
 It achieves the following results on the evaluation set:
-- Loss: 2.8975
+- Loss: 5.0363
 
 ## Model description
 
@@ -122,17 +126,16 @@ The following hyperparameters were used during training:
 - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
 - lr_scheduler_type: cosine_with_restarts
 - lr_scheduler_warmup_steps: 100
-- num_epochs: 1
+- num_epochs: 3
 
 ### Training results
 
 | Training Loss | Epoch  | Step | Validation Loss |
 |:-------------:|:------:|:----:|:---------------:|
-| 5.5185        | 0.0012 | 1    | 4.8333          |
-| 3.7004        | 0.2348 | 200  | 3.2693          |
-| 3.52          | 0.4696 | 400  | 3.3535          |
-| 3.7836        | 0.7043 | 600  | 2.9896          |
-| 3.3058        | 0.9391 | 800  | 2.8975          |
+| 5.5185        | 0.0012 | 1    | 4.8238          |
+| 4.2012        | 0.2348 | 200  | 4.1556          |
+| 4.4185        | 0.4696 | 400  | 4.8159          |
+| 5.0973        | 0.7043 | 600  | 5.0363          |
 
 
 ### Framework versions

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7e37bafe9285ca265c8e3cfa10c5801387ebeaefc77c86451bd9c92159fcc4f5
-size 159266376
+oid sha256:e44ce263e6fd885f50d82ca515b9325375b43ee36ededb75acf161ce88bc2e41
+size 48

config.json

@@ -22,7 +22,7 @@
   "rotary_emb_base": 10000,
   "rotary_pct": 0.25,
   "tie_word_embeddings": false,
-  "torch_dtype": "float16",
+  "torch_dtype": "bfloat16",
   "transformers_version": "4.41.2",
   "use_cache": false,
   "use_parallel_residual": true,

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:727fc3779b3fe223ff519f231c0c6c4a9f9092e972590479a1a23e8a5cb4c7db
+oid sha256:168d7d3a0afe4d5734fa7304faf1fe11e36d3bc39c475402248255244c303708
 size 324696090