mistral_radiology3.2

README.md

@@ -14,6 +14,8 @@ should probably proofread and complete it, then remove this comment. -->
 # results
 
 This model is a fine-tuned version of [mistralai/Mistral-7B-v0.1](https://huggingface.co/mistralai/Mistral-7B-v0.1) on an unknown dataset.
+It achieves the following results on the evaluation set:
+- Loss: 0.6218
 
 ## Model description
 
@@ -32,25 +34,54 @@ More information needed
 ### Training hyperparameters
 
 The following hyperparameters were used during training:
-- learning_rate: 0.0002
-- train_batch_size: 1
-- eval_batch_size: 8
+- learning_rate: 5e-05
+- train_batch_size: 16
+- eval_batch_size: 16
 - seed: 42
-- gradient_accumulation_steps: 4
-- total_train_batch_size: 4
 - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
-- lr_scheduler_type: constant
-- lr_scheduler_warmup_ratio: 0.03
-- training_steps: 1000
+- lr_scheduler_type: linear
+- lr_scheduler_warmup_steps: 500
+- num_epochs: 2
 - mixed_precision_training: Native AMP
 
 ### Training results
 
+| Training Loss | Epoch | Step  | Validation Loss |
+|:-------------:|:-----:|:-----:|:---------------:|
+| 0.9667        | 0.07  | 500   | 0.8561          |
+| 0.8253        | 0.14  | 1000  | 0.7976          |
+| 0.7771        | 0.2   | 1500  | 0.7676          |
+| 0.7623        | 0.27  | 2000  | 0.7459          |
+| 0.7399        | 0.34  | 2500  | 0.7269          |
+| 0.7253        | 0.41  | 3000  | 0.7166          |
+| 0.7241        | 0.47  | 3500  | 0.7035          |
+| 0.7063        | 0.54  | 4000  | 0.6962          |
+| 0.6857        | 0.61  | 4500  | 0.6883          |
+| 0.6909        | 0.68  | 5000  | 0.6829          |
+| 0.6754        | 0.75  | 5500  | 0.6731          |
+| 0.6803        | 0.81  | 6000  | 0.6657          |
+| 0.6659        | 0.88  | 6500  | 0.6599          |
+| 0.6603        | 0.95  | 7000  | 0.6556          |
+| 0.6249        | 1.02  | 7500  | 0.6610          |
+| 0.53          | 1.09  | 8000  | 0.6583          |
+| 0.5246        | 1.15  | 8500  | 0.6544          |
+| 0.5204        | 1.22  | 9000  | 0.6515          |
+| 0.5135        | 1.29  | 9500  | 0.6498          |
+| 0.5165        | 1.36  | 10000 | 0.6433          |
+| 0.518         | 1.42  | 10500 | 0.6410          |
+| 0.5032        | 1.49  | 11000 | 0.6368          |
+| 0.5091        | 1.56  | 11500 | 0.6335          |
+| 0.5038        | 1.63  | 12000 | 0.6307          |
+| 0.4907        | 1.7   | 12500 | 0.6302          |
+| 0.5006        | 1.76  | 13000 | 0.6262          |
+| 0.4823        | 1.83  | 13500 | 0.6239          |
+| 0.4906        | 1.9   | 14000 | 0.6225          |
+| 0.4905        | 1.97  | 14500 | 0.6218          |
 
 
 ### Framework versions
 
-- Transformers 4.35.0.dev0
-- Pytorch 2.0.1+cu118
-- Datasets 2.14.5
-- Tokenizers 0.14.1
+- Transformers 4.36.0.dev0
+- Pytorch 2.1.0+cu118
+- Datasets 2.15.0
+- Tokenizers 0.15.0

adapter_config.json

@@ -8,19 +8,22 @@
   "init_lora_weights": true,
   "layers_pattern": null,
   "layers_to_transform": null,
-  "lora_alpha": 16,
+  "lora_alpha": 64,
   "lora_dropout": 0.05,
   "modules_to_save": null,
   "peft_type": "LORA",
-  "r": 16,
+  "r": 32,
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "k_proj",
-    "o_proj",
+    "lm_head",
+    "v_proj",
     "gate_proj",
     "q_proj",
-    "v_proj"
+    "up_proj",
+    "down_proj",
+    "o_proj",
+    "k_proj"
   ],
   "task_type": "CAUSAL_LM"
 }

adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7a909ff1155adc29c0b9f13bd5c133e0c4df7a8c7f61f0a7174dd44117ce05a6
+size 340225224

special_tokens_map.json

@@ -1,11 +1,24 @@
 {
-  "additional_special_tokens": [
-    "<unk>",
-    "<s>",
-    "</s>"
-  ],
-  "bos_token": "<s>",
-  "eos_token": "</s>",
-  "pad_token": "</s>",
-  "unk_token": "<unk>"
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "<unk>",
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
 }

tokenizer.json

@@ -2,7 +2,7 @@
   "version": "1.0",
   "truncation": {
     "direction": "Right",
-    "max_length": 1024,
+    "max_length": 512,
     "strategy": "LongestFirst",
     "stride": 0
   },

tokenizer_config.json

@@ -1,4 +1,6 @@
 {
+  "add_bos_token": true,
+  "add_eos_token": true,
   "added_tokens_decoder": {
     "0": {
       "content": "<unk>",
@@ -25,17 +27,15 @@
       "special": true
     }
   },
-  "additional_special_tokens": [
-    "<unk>",
-    "<s>",
-    "</s>"
-  ],
+  "additional_special_tokens": [],
   "bos_token": "<s>",
   "clean_up_tokenization_spaces": false,
   "eos_token": "</s>",
+  "fast_tokenizer": true,
   "legacy": true,
   "model_max_length": 1000000000000000019884624838656,
-  "pad_token": "</s>",
+  "pad_token": "<unk>",
+  "padding_side": "right",
   "sp_model_kwargs": {},
   "spaces_between_special_tokens": false,
   "tokenizer_class": "LlamaTokenizer",

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:726eec208ca45f8472a2a04c9dd82264bab3ba378e66e655924aa2bf40b886fb
-size 4027
+oid sha256:3f794911737b48fa849a15cc946a6c58f35a68f7ebf91035076b8cd3f4dad42d
+size 4600