Training in progress, step 10

adapter_config.json

@@ -16,17 +16,17 @@
   "megatron_core": "megatron.core",
   "modules_to_save": null,
   "peft_type": "LORA",
-  "r": 32,
+  "r": 16,
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "up_proj",
     "o_proj",
-    "q_proj",
+    "up_proj",
     "k_proj",
-    "down_proj",
+    "v_proj",
     "gate_proj",
-    "v_proj"
+    "q_proj",
+    "down_proj"
   ],
   "task_type": "CAUSAL_LM",
   "use_dora": false,

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:84af51dc250669c92d92b05b429eef31fd02f1396e5504973994a79b49c9acf0
-size 60010384
+oid sha256:6a8c26b2c5d8f6ca20b8846fc26b4c55ea31d8ad614db0a006e672950f40910e
+size 30027544

config.json

@@ -12,6 +12,7 @@
   "initializer_range": 0.02,
   "intermediate_size": 5504,
   "max_position_embeddings": 16384,
+  "mlp_bias": false,
   "model_type": "llama",
   "num_attention_heads": 16,
   "num_hidden_layers": 24,
@@ -40,7 +41,7 @@
   "rope_theta": 100000,
   "tie_word_embeddings": false,
   "torch_dtype": "bfloat16",
-  "transformers_version": "4.40.2",
+  "transformers_version": "4.41.1",
   "use_cache": false,
   "vocab_size": 32256
 }

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a6debfef838e651f8575f13b65bb3941024e0ca0d85b1b357df704569e0c5aac
-size 5880
+oid sha256:e16da9b1b8a80a3cdfcd43e7c24a4835abe7dd60daa9ecc48b34fbaba9d72abb
+size 6072