Upload folder using huggingface_hub

config.json

@@ -0,0 +1,32 @@
+{
+  "_name_or_path": "mistralai/Mistral-7B-Instruct-v0.3",
+  "architectures": [
+    "MixtralForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 32768,
+  "model_type": "mixtral",
+  "num_attention_heads": 32,
+  "num_experts_per_tok": 2,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "num_local_experts": 10,
+  "output_router_logits": false,
+  "rms_norm_eps": 1e-05,
+  "rope_theta": 1000000.0,
+  "router_aux_loss_coef": 0.001,
+  "router_jitter_noise": 0.0,
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.46.0.dev0",
+  "use_cache": true,
+  "vocab_size": 32768
+}

mergekit_moe_config.yml

@@ -0,0 +1,16 @@
+base_model: mistralai/Mistral-7B-Instruct-v0.3
+gate_mode: random
+dtype: bfloat16
+experts:
+  - source_model: mistralai/Mistral-7B-Instruct-v0.3
+  - source_model: teknium/OpenHermes-2.5-Mistral-7B
+  - source_model: NousResearch/Hermes-2-Pro-Mistral-7B
+  - source_model: Open-Orca/Mistral-7B-OpenOrca
+  - source_model: Open-Orca/Mistral-7B-OpenOrca
+  - source_model: mistralai/Mistral-7B-Instruct-v0.3
+  - source_model: teknium/OpenHermes-2.5-Mistral-7B
+  - source_model: NousResearch/Hermes-2-Pro-Mistral-7B
+  - source_model: Open-Orca/Mistral-7B-OpenOrca
+  - source_model: Open-Orca/Mistral-7B-OpenOrca
+  
+# and then train the sucker!

model-00001-of-00040.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c8fb9bfa7cfe312f07ceedafc5b26f8f11903fa1e5b99cd015ee2e0ab4f3aa37
+size 2936032888

model-00002-of-00040.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b4ebae345aefe406a2e520a3bbbc4a440c76cc2da4fafb7d76234c5b04f8addf
+size 2902478616

model-00003-of-00040.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7350fd54f5f707807e04ecc13a1d4d455bf17f530f3261bf2d33451170fc9b4c
+size 2902478616

model-00004-of-00040.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f0ea56f45322c4f07acaf5c7c32ef157f5f42f8094075dd0c198d10dad276ad8
+size 2902478616

model-00005-of-00040.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d1b4846b4fb51b75cbf223380058f69ccbed8517bf0a862a57d4a33cba014995
+size 2936016096

model-00006-of-00040.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5f43c922315663ee0511765f6bd9cee63545add7fd7626eddbb7a27fb884a3d9
+size 2902478616

model-00007-of-00040.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0db617bc1608e5ad82e8c3ea6153ed44c56d0a2f5ef45df01392f8c8ed2f5cf1
+size 2902478616

model-00008-of-00040.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:97945d80779f994d831b5147039d2a9ed4c8a14833f2c38d9d5401f9468cf940
+size 2902478616

model-00009-of-00040.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3cfeb605dc7526752e34c73f6aa462dd7891fe3415a841d5a7a7b2033b1dd23b
+size 2902478616

model-00010-of-00040.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5fb8cadb76fc6967ee3884fe19ae5ec1e65929f871f47e38a3ca0012c84461ad
+size 2986356392

model-00011-of-00040.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d69d1a0112394c647de1804a138dabef4f0b1876c1dfb4b7311b7772ea10262e
+size 2969578952

model-00012-of-00040.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:18debd762e827d2f1ca8f672ddd3d844839f256b3ded8103be9a3b06108a5613
+size 2902478616

model-00013-of-00040.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3a268f586b245a3dfed2b5590898108d0946378b89feab0fbb251db3af0364ec
+size 2902478632

model-00014-of-00040.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7241165fe3afb5bd8cad55b07908ed84cd8ace8b95268b344df6500c9f5339da
+size 2902478640

model-00015-of-00040.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cde0608f5defa621df1dfb299f4f3f186937e237dccc602f6199a934d3410929
+size 2902478640

model-00016-of-00040.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:257c2b48ed6e74c22921f4d2950184c14c8597c8fd9a9fcb3f8255a376a1e030
+size 2936016120

model-00017-of-00040.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:09776ab08f52a29dbf99860da975bdf8be916904fc42d363212c8af1259e16b9
+size 2902478640

model-00018-of-00040.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dc2517dc3ec5d1a95adc51222ae44799a9bde467a8bb2906099e9da640d575df
+size 2902478640

model-00019-of-00040.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a41f104b0125ce23bce2e39c4756eab0ef9516d9f4f8a88713feafe4049cfece
+size 2902478640

model-00020-of-00040.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a9f6c167a70bbbb3036338b72a07aa3a191917321add35a9a621df4bc1d5da2
+size 2902478640

model-00021-of-00040.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8b709d9583895f70ff3e07a1a706c194fad2fd843f15005d4b2dbb7b4a1362c7
+size 2936016120

model-00022-of-00040.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ef4a4ab0fc4a936c83204f74732b38cbaa140b27c0154dbcb191d5926c914971
+size 2902478640

model-00023-of-00040.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e6332f913f017cc57c26744b4224295af23f29ff7eaf0119c487ccea9876a8b7
+size 2902478640

model-00024-of-00040.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a85416526c990ef05bbd6f185536a9afcedf90bfb6f11266987e310f48fdb56
+size 2902478640

model-00025-of-00040.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:92eadb591dee4ec70be272002bc4d254a7295cfe108caa1f86b5c2158816faeb
+size 2902478640

model-00026-of-00040.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:122d2ae2d0f2135f4634125d51957e41161eaf41c3b9ebd6212db73a29ade5b4
+size 2936016120

model-00027-of-00040.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dac0fc0e325953112c3c0e16acfb1212f3ec348c5f8ff9c193bef95b5869f1c8
+size 2902478640

model-00028-of-00040.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3211def6df371199a00beccdfb70eb607dac78cc9b30f686f33a967d70e51d95
+size 2902478640

model-00029-of-00040.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8fa99c0daf197ae117fa7e0d9ae5ebde309c4bffba1f5e8fa68b411fc18b0a24
+size 2902478640

model-00030-of-00040.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:04a7f767a1073cb67ff1fd84a5753f34b12c40c5a1bb7663b0aaf150b0c76a66
+size 2902478640

model-00031-of-00040.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b9376ac639def573799b49dd11117dc398493ed967dadb95cca9a92b24208ccb
+size 2936016120

model-00032-of-00040.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:83c61f6fa2cf78656de3167e08585d6628281ba5724e36d5578401a0c65cc016
+size 2902478640

model-00033-of-00040.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5238ccad324113fc4dd7939514dcd304bcc31250d4e2a2615292d373699b23da
+size 2902478640

model-00034-of-00040.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2d982fe7fc921f83814c6d6e99f89419a0142a9965dd13b0f9cff395654c4389
+size 2902478640

model-00035-of-00040.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a7d32b2d324e9f2978c3efeb52f97d28f5c50d51296e5539466cb15710ed5ed2
+size 2902478640

model-00036-of-00040.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e6dff05f8738ce9dd65bc4a52fde19ce1249cbc62f077c9277132a4184624520
+size 2986356424

model-00037-of-00040.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f618bc31dfcb1523c2ef1bed20a7f7e058870d494a608b4c3dc0072075782a0c
+size 2969578984

model-00038-of-00040.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6150a8540cdbe6756cfb4af973a058e5107de83065579ea6c2bfcacecf2d1459
+size 2902478640

model-00039-of-00040.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a775f1654d3778934a38475dd2d3d6b889c2a386efc3d7b37ed88b3dd47dc620
+size 2902478640

model-00040-of-00040.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e2105e7d9b617b28b4dc53094d60f9ceac473a7dc3e2786146c2608cd9f69d92
+size 2267560056

special_tokens_map.json

@@ -0,0 +1,24 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "<s>",
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:37f00374dea48658ee8f5d0f21895b9bc55cb0103939607c8185bfd1c6ca1f89
+size 587404