Training with 90/10 Spanish dataset, 50 epochs, 3 Batch Size, reduce_lr_on_plateau

README.md

@@ -16,7 +16,7 @@ should probably proofread and complete it, then remove this comment. -->
 
 This model is a fine-tuned version of [mistralai/Mistral-7B-Instruct-v0.2](https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.2) on an unknown dataset.
 It achieves the following results on the evaluation set:
-- Loss: 1.1233
+- Loss: 3.7372
 
 ## Model description
 
@@ -36,29 +36,74 @@ More information needed
 
 The following hyperparameters were used during training:
 - learning_rate: 0.0001
-- train_batch_size: 2
-- eval_batch_size: 2
+- train_batch_size: 3
+- eval_batch_size: 3
 - seed: 42
 - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
-- lr_scheduler_type: constant
-- num_epochs: 5
+- lr_scheduler_type: reduce_lr_on_plateau
+- num_epochs: 50
 - mixed_precision_training: Native AMP
 
 ### Training results
 
-| Training Loss | Epoch  | Step | Validation Loss |
-|:-------------:|:------:|:----:|:---------------:|
-| 1.4939        | 0.9995 | 1847 | 0.9488          |
-| 0.7585        | 1.9989 | 3694 | 0.8857          |
-| 0.4374        | 2.9984 | 5541 | 0.9555          |
-| 0.2946        | 3.9978 | 7388 | 1.0790          |
-| 0.2287        | 4.9973 | 9235 | 1.1233          |
+| Training Loss | Epoch   | Step  | Validation Loss |
+|:-------------:|:-------:|:-----:|:---------------:|
+| 1.4991        | 0.9992  | 1231  | 1.7493          |
+| 1.5973        | 1.9984  | 2462  | 1.5530          |
+| 0.7761        | 2.9976  | 3693  | 1.6627          |
+| 0.4026        | 3.9968  | 4924  | 1.9381          |
+| 0.3181        | 4.9959  | 6155  | 2.1410          |
+| 0.2572        | 5.9951  | 7386  | 2.3047          |
+| 0.2783        | 6.9943  | 8617  | 2.4170          |
+| 0.1911        | 7.9935  | 9848  | 2.5913          |
+| 0.2101        | 8.9927  | 11079 | 2.5669          |
+| 0.1934        | 9.9919  | 12310 | 2.5707          |
+| 0.1641        | 10.9911 | 13541 | 2.5205          |
+| 0.1534        | 11.9903 | 14772 | 2.6706          |
+| 0.1887        | 12.9894 | 16003 | 2.7875          |
+| 0.1146        | 13.9886 | 17234 | 2.9092          |
+| 0.0891        | 14.9878 | 18465 | 3.2176          |
+| 0.0845        | 15.9870 | 19696 | 3.3288          |
+| 0.0901        | 16.9862 | 20927 | 3.4202          |
+| 0.0805        | 17.9854 | 22158 | 3.4854          |
+| 0.0768        | 18.9846 | 23389 | 3.4997          |
+| 0.0788        | 19.9838 | 24620 | 3.5510          |
+| 0.0829        | 20.9830 | 25851 | 3.5782          |
+| 0.0729        | 21.9821 | 27082 | 3.5944          |
+| 0.0747        | 22.9813 | 28313 | 3.6143          |
+| 0.0767        | 23.9805 | 29544 | 3.6171          |
+| 0.0655        | 24.9797 | 30775 | 3.6633          |
+| 0.0695        | 25.9789 | 32006 | 3.6780          |
+| 0.0632        | 26.9781 | 33237 | 3.6896          |
+| 0.0628        | 27.9773 | 34468 | 3.6971          |
+| 0.0626        | 28.9765 | 35699 | 3.7027          |
+| 0.0601        | 29.9756 | 36930 | 3.7070          |
+| 0.0576        | 30.9748 | 38161 | 3.7114          |
+| 0.1134        | 31.9740 | 39392 | 3.7157          |
+| 0.1046        | 32.9732 | 40623 | 3.7186          |
+| 0.1019        | 33.9724 | 41854 | 3.7199          |
+| 0.0935        | 34.9716 | 43085 | 3.7234          |
+| 0.0911        | 35.9708 | 44316 | 3.7252          |
+| 0.0899        | 36.9700 | 45547 | 3.7271          |
+| 0.0919        | 37.9692 | 46778 | 3.7285          |
+| 0.0823        | 38.9683 | 48009 | 3.7299          |
+| 0.0871        | 39.9675 | 49240 | 3.7312          |
+| 0.0824        | 40.9667 | 50471 | 3.7322          |
+| 0.0812        | 41.9659 | 51702 | 3.7332          |
+| 0.0813        | 42.9651 | 52933 | 3.7342          |
+| 0.0802        | 43.9643 | 54164 | 3.7350          |
+| 0.0809        | 44.9635 | 55395 | 3.7359          |
+| 0.0782        | 45.9627 | 56626 | 3.7368          |
+| 0.0765        | 46.9619 | 57857 | 3.7369          |
+| 0.0787        | 47.9610 | 59088 | 3.7370          |
+| 0.076         | 48.9602 | 60319 | 3.7370          |
+| 0.0756        | 49.9594 | 61550 | 3.7372          |
 
 
 ### Framework versions
 
 - PEFT 0.10.0
-- Transformers 4.40.1
+- Transformers 4.40.2
 - Pytorch 2.3.0+cu121
-- Datasets 2.19.0
+- Datasets 2.19.1
 - Tokenizers 0.19.1

adapter_config.json

@@ -11,7 +11,7 @@
   "layers_to_transform": null,
   "loftq_config": {},
   "lora_alpha": 16,
-  "lora_dropout": 0.1,
+  "lora_dropout": 0.0001,
   "megatron_config": null,
   "megatron_core": "megatron.core",
   "modules_to_save": [
@@ -23,12 +23,12 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "gate_proj",
-    "q_proj",
     "v_proj",
-    "up_proj",
     "o_proj",
+    "q_proj",
+    "up_proj",
     "down_proj",
+    "gate_proj",
     "k_proj"
   ],
   "task_type": "CAUSAL_LM",

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9f565c29b31be498510feaf338d5574271b7b15fbdb7604970428180dcf4dda2
+oid sha256:014322293212ffb6e0393d3a94fec34f4e01dc838fdb5213de770b63ffb95aa3
 size 1719726424

special_tokens_map.json

@@ -13,7 +13,7 @@
     "rstrip": false,
     "single_word": false
   },
-  "pad_token": "[PAD]",
+  "pad_token": "</s>",
   "unk_token": {
     "content": "<unk>",
     "lstrip": false,

tokenizer.model

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:dadfd56d766715c61d2ef780a525ab43b8e6da4de6865bda3d95fdef5e134055
-size 493443
+oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347
+size 499723

tokenizer_config.json

@@ -27,16 +27,15 @@
       "special": true
     }
   },
-  "additional_special_tokens": [],
   "bos_token": "<s>",
-  "chat_template": "{{ bos_token }}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if message['role'] == 'user' %}{{ '[INST] ' + message['content'] + ' [/INST]' }}{% elif message['role'] == 'assistant' %}{{ message['content'] + eos_token}}{% else %}{{ raise_exception('Only user and assistant roles are supported!') }}{% endif %}{% endfor %}",
+  "chat_template": "{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% else %}{% set loop_messages = messages %}{% set system_message = false %}{% endif %}{% for message in loop_messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if loop.index0 == 0 and system_message != false %}{% set content = '<<SYS>>\\n' + system_message + '\\n<</SYS>>\\n\\n' + message['content'] %}{% else %}{% set content = message['content'] %}{% endif %}{% if message['role'] == 'user' %}{{ bos_token + '[INST] ' + content.strip() + ' [/INST]' }}{% elif message['role'] == 'assistant' %}{{ ' '  + content.strip() + ' ' + eos_token }}{% endif %}{% endfor %}",
   "clean_up_tokenization_spaces": false,
   "eos_token": "</s>",
-  "legacy": true,
+  "legacy": false,
   "model_max_length": 1000000000000000019884624838656,
-  "pad_token": "[PAD]",
+  "pad_token": "</s>",
+  "padding_side": "right",
   "sp_model_kwargs": {},
-  "spaces_between_special_tokens": false,
   "tokenizer_class": "LlamaTokenizer",
   "unk_token": "<unk>",
   "use_default_system_prompt": false

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:19bb28e5672c55e66a32db22fd44684953525f72d6f8c939f855b9561732a50a
+oid sha256:b2c04dc2d8bfdfe87a7477d14f6eb9822955430937ff5730937bdba6f5c1fc36
 size 5048