mistral-instruct-2-finetuned-clinical

README.md

@@ -5,7 +5,7 @@ tags:
 - trl
 - sft
 - generated_from_trainer
-base_model: mistralai/Mistral-7B-v0.1
+base_model: mistralai/Mistral-7B-Instruct-v0.2
 model-index:
 - name: results
   results: []
@@ -16,9 +16,9 @@ should probably proofread and complete it, then remove this comment. -->
 
 # results
 
-This model is a fine-tuned version of [mistralai/Mistral-7B-v0.1](https://huggingface.co/mistralai/Mistral-7B-v0.1) on an unknown dataset.
+This model is a fine-tuned version of [mistralai/Mistral-7B-Instruct-v0.2](https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.2) on an unknown dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.9724
+- Loss: 0.9955
 
 ## Model description
 
@@ -46,52 +46,24 @@ The following hyperparameters were used during training:
 - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
 - lr_scheduler_type: cosine
 - lr_scheduler_warmup_steps: 0.03
-- num_epochs: 1
+- num_epochs: 2
 
 ### Training results
 
 | Training Loss | Epoch | Step | Validation Loss |
 |:-------------:|:-----:|:----:|:---------------:|
-| 1.0248        | 0.03  | 50   | 1.0145          |
-| 1.0168        | 0.06  | 100  | 1.0078          |
-| 1.008         | 0.09  | 150  | 1.0058          |
-| 1.0082        | 0.12  | 200  | 1.0030          |
-| 0.9846        | 0.14  | 250  | 1.0005          |
-| 0.9807        | 0.17  | 300  | 0.9998          |
-| 0.9968        | 0.2   | 350  | 0.9992          |
-| 0.9834        | 0.23  | 400  | 0.9967          |
-| 1.0267        | 0.26  | 450  | 0.9953          |
-| 1.0119        | 0.29  | 500  | 0.9937          |
-| 0.9759        | 0.32  | 550  | 0.9939          |
-| 0.9978        | 0.35  | 600  | 0.9921          |
-| 1.0145        | 0.38  | 650  | 0.9901          |
-| 1.0064        | 0.4   | 700  | 0.9897          |
-| 0.9949        | 0.43  | 750  | 0.9890          |
-| 0.9936        | 0.46  | 800  | 0.9865          |
-| 0.9944        | 0.49  | 850  | 0.9852          |
-| 0.9819        | 0.52  | 900  | 0.9845          |
-| 0.9991        | 0.55  | 950  | 0.9826          |
-| 0.9874        | 0.58  | 1000 | 0.9812          |
-| 0.981         | 0.61  | 1050 | 0.9798          |
-| 0.9807        | 0.64  | 1100 | 0.9789          |
-| 0.9639        | 0.67  | 1150 | 0.9776          |
-| 0.9645        | 0.69  | 1200 | 0.9767          |
-| 0.9788        | 0.72  | 1250 | 0.9758          |
-| 0.9823        | 0.75  | 1300 | 0.9751          |
-| 0.9906        | 0.78  | 1350 | 0.9745          |
-| 0.9536        | 0.81  | 1400 | 0.9738          |
-| 0.9635        | 0.84  | 1450 | 0.9732          |
-| 0.9754        | 0.87  | 1500 | 0.9729          |
-| 0.9785        | 0.9   | 1550 | 0.9727          |
-| 0.9828        | 0.93  | 1600 | 0.9725          |
-| 0.9951        | 0.95  | 1650 | 0.9724          |
-| 0.983         | 0.98  | 1700 | 0.9724          |
+| 1.0284        | 0.29  | 500  | 1.0100          |
+| 1.0079        | 0.58  | 1000 | 1.0023          |
+| 0.9938        | 0.87  | 1500 | 0.9934          |
+| 0.8683        | 1.16  | 2000 | 1.0001          |
+| 0.8724        | 1.45  | 2500 | 0.9977          |
+| 0.8777        | 1.74  | 3000 | 0.9955          |
 
 
 ### Framework versions
 
-- PEFT 0.9.0
-- Transformers 4.38.2
+- PEFT 0.10.0
+- Transformers 4.39.1
 - Pytorch 2.2.1+cu121
 - Datasets 2.18.0
 - Tokenizers 0.15.2

adapter_config.json

@@ -1,11 +1,12 @@
 {
   "alpha_pattern": {},
   "auto_mapping": null,
-  "base_model_name_or_path": "mistralai/Mistral-7B-v0.1",
+  "base_model_name_or_path": "mistralai/Mistral-7B-Instruct-v0.2",
   "bias": "none",
   "fan_in_fan_out": false,
   "inference_mode": true,
   "init_lora_weights": true,
+  "layer_replication": null,
   "layers_pattern": null,
   "layers_to_transform": null,
   "loftq_config": {},
@@ -19,13 +20,13 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "v_proj",
-    "q_proj",
     "gate_proj",
+    "q_proj",
+    "up_proj",
     "down_proj",
     "o_proj",
-    "up_proj",
-    "k_proj"
+    "k_proj",
+    "v_proj"
   ],
   "task_type": "CAUSAL_LM",
   "use_dora": false,

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a5a2351fb8d9722413f891c353d531fbddecc049984d48bf64c4c063c144c1a7
+oid sha256:659e580c931fb02b0220472c5b34093687e55dfe0ea630fa5ee88cc0595cb3ba
 size 671149168

tokenizer_config.json

@@ -29,6 +29,7 @@
   },
   "additional_special_tokens": [],
   "bos_token": "<s>",
+  "chat_template": "{{ bos_token }}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if message['role'] == 'user' %}{{ '[INST] ' + message['content'] + ' [/INST]' }}{% elif message['role'] == 'assistant' %}{{ message['content'] + eos_token}}{% else %}{{ raise_exception('Only user and assistant roles are supported!') }}{% endif %}{% endfor %}",
   "clean_up_tokenization_spaces": false,
   "eos_token": "</s>",
   "legacy": true,

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e5f0012504e5862af840860bf7a91b6c215c14b0ad783b9adef9fe6669bb8300
-size 4856
+oid sha256:986c5cabedb6dd6f7a8edad563b7a9d396a1e1e440c167b6475c92cd8bf0a2b5
+size 4920