Upload folder using huggingface_hub

README.md

@@ -1,28 +1,28 @@
 ---
-base_model:
-- sophosympatheia/Midnight-Miqu-70B-v1.5
-- NeverSleep/MiquMaid-v3-70B
-- maywell/miqu-evil-dpo
-- 152334H/miqu-1-70b-sf
+base_model: []
 library_name: transformers
 tags:
 - mergekit
 - merge
-license: other
+
 ---
 # MiquSuperdark-70B-v1
 
-**MiquSuperdark-70B-v1** is a merge of three of the most popular Miqu-derived models, along with Miqu itself. The goal of the merge is to create an strong, well-rounded chat model that picks up desirable traits from its constituent models without sacrificing intelligence.
+This is a merge of pre-trained language models created using [mergekit](https://github.com/cg123/mergekit).
+
+## Merge Details
+### Merge Method
 
-This is a DARE Linear merge with the following composition:
-- [sophosympatheia/Midnight-Miqu-70B-v1.5](https://huggingface.co/sophosympatheia/Midnight-Miqu-70B-v1.5) at weight 0.4
-- [NeverSleep/MiquMaid-v3-70B](https://huggingface.co/NeverSleep/MiquMaid-v3-70B) at weight 0.2
-- [maywell/miqu-evil-dpo](https://huggingface.co/maywell/miqu-evil-dpo) at weight 0.2
-- [152334H/miqu-1-70b-sf](https://huggingface.co/152334H/miqu-1-70b-sf) at weight 0.2 (used as base model)
+This model was merged using the linear [DARE](https://arxiv.org/abs/2311.03099) merge method using /home/dylan/Documents/AI/merge/miqu-1-70b-sf as a base.
 
-DARE Linear was chosen as the merge method based on [this HF discussion](https://huggingface.co/jukofyork/Dark-Miqu-70B/discussions/2), in which the creator of Midnight-Miqu says "*in my own testing I consistently got the best results from using a dare_linear merge when working with miqu models*".
+### Models Merged
 
-## Merge Configuration
+The following models were included in the merge:
+* /home/dylan/Documents/AI/merge/MiquMaid-v3-70B
+* /media/dylan/SanDisk/LLMs/miqu-evil-dpo/
+* /media/dylan/SanDisk/LLMs/Midnight-Miqu-70B-v1.5
+
+### Configuration
 
 The following YAML configuration was used to produce this model:
 
@@ -45,5 +45,3 @@ models:
 dtype: float16
 tokenizer_source: model:/home/dylan/Documents/AI/merge/miqu-1-70b-sf
 ```
-
-The tokenizer is copied from the base model [152334H/miqu-1-70b-sf](https://huggingface.co/152334H/miqu-1-70b-sf).

config.json

@@ -0,0 +1,28 @@
+{
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "hidden_act": "silu",
+  "hidden_size": 8192,
+  "initializer_range": 0.02,
+  "intermediate_size": 28672,
+  "max_position_embeddings": 32764,
+  "model_type": "llama",
+  "num_attention_heads": 64,
+  "num_hidden_layers": 80,
+  "num_key_value_heads": 8,
+  "pad_token_id": 0,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": null,
+  "rope_theta": 1000000,
+  "tie_word_embeddings": false,
+  "torch_dtype": "float16",
+  "transformers_version": "4.36.0",
+  "use_cache": true,
+  "vocab_size": 32000
+}

mergekit_config.yml

@@ -0,0 +1,17 @@
+merge_method: dare_linear
+base_model: /home/dylan/Documents/AI/merge/miqu-1-70b-sf
+models:
+  - model: /media/dylan/SanDisk/LLMs/Midnight-Miqu-70B-v1.5
+    parameters:
+      weight: 0.4
+  - model: /home/dylan/Documents/AI/merge/miqu-1-70b-sf
+    parameters:
+      weight: 0.2
+  - model: /media/dylan/SanDisk/LLMs/miqu-evil-dpo/
+    parameters:
+      weight: 0.2
+  - model: /home/dylan/Documents/AI/merge/MiquMaid-v3-70B
+    parameters:
+      weight: 0.2
+dtype: float16
+tokenizer_source: model:/home/dylan/Documents/AI/merge/miqu-1-70b-sf

model-00001-of-00081.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b0bc44cfc8b0caaa73ca9a20ebc72de37bfda26a3f4dea8f305b25a998e19d10
+size 1988117048

model-00002-of-00081.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4e7090501cae7b0b35ab2f8b37b3263a9c0394d8b0a6f1d35b76b77c03bac663
+size 1711309856

model-00003-of-00081.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:383ba84fba3bef358a09e7c612e7b4e644d3236de5aa57a66f0a7aad53d34b3a
+size 1711309856

model-00004-of-00081.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:68e7f4cce99af7dbf72bc4298773ba0f5436588ea526296ea4f656b3ed1f0b65
+size 1711309864

model-00005-of-00081.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:55a9dcddcc1cae39fa3ac33dbe8ec9239686fd407ad2d9e010ae499f1e42ee68
+size 1711309864

model-00006-of-00081.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ff8c88888900436c2076d9b2aacb17a6030cf297a2489a8f588759f509454bee
+size 1711309864

model-00007-of-00081.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2ce54d1596a6dcf8f9e1d2a134e01123a6cd32d9ca1c6766dda302418919c765
+size 1711309864

model-00008-of-00081.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e999a9d64877e486b37e65388490e76ebdcc2df5b032f039a47c2fa165200710
+size 1711309864

model-00009-of-00081.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0330654c6715a4cec27551583267969ba5e1305b36c9cfadfbd1c82e35d11e50
+size 1711309864

model-00010-of-00081.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:84012fea8f9a1c4b3a45c3a3de932d26e6e1b96a90acd38b64513e0aaf7e9562
+size 1711309864

model-00011-of-00081.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:98664e29bfeaa08b98e0e61b1cc31ea734ab686ee0562d1875ee6ac6052fa048
+size 1711309864

model-00012-of-00081.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0bb5df95096ab6451a76ada113d504d74b9d8388c5cbbbeca4031c5636ad362e
+size 1711309864

model-00013-of-00081.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b55b77aff5d211d4328372bed6167d0268af306f2475b4e76d32252c5281901e
+size 1711309856

model-00014-of-00081.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9faaa7bbf647c01bee079c5d5f9e663834cb276b9178ac2b2d8b8de8eedf9f17
+size 1711309856

model-00015-of-00081.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:146a105b230ba34852af7b33e9edb86f120634b21845bcbd686cd12cbf626d99
+size 1711309864

model-00016-of-00081.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d2db1c7f533c43a9d20c9fc3c97cdd8a8f7ee13bd9ab26b2e3ab80f49615641e
+size 1711309864

model-00017-of-00081.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ae60ff350d4ce02d68f2698b78dd9b3db9860d46dc4e08fdb0bf95a7472eac72
+size 1711309864

model-00018-of-00081.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:35c24049577e17c8ae3441825d4bb51dd8803ddf31f062ab6b77138df1e8f542
+size 1711309864

model-00019-of-00081.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c3ee60d166c9932cef7433a6df20792be146ec1941807a79348204251333a0af
+size 1711309864

model-00020-of-00081.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0aa07a5a8fd60b7c074e585927866b50cf5612e8d4bed73165008fa0e36359c3
+size 1711309864

model-00021-of-00081.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:98ded078f151dc2eac615214d1f563f31596bab89ff0bc1680adb01f9e79eba2
+size 1711309864

model-00022-of-00081.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9bc562141c958ea526ec5e7f301fa7c2968b5c48886a94cc7a6d2b77d72ef18f
+size 1711309864

model-00023-of-00081.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d10317c4bc2a62281b0ae5aad373e8cdd73261ce43d05f3eed61b3240bcf19a4
+size 1711309864

model-00024-of-00081.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d8bb3c7ef05bec8a8d1fba6e3c09778b961a3ab4cd5b2d689e14193fcba73ead
+size 1711309856

model-00025-of-00081.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0a1f406ffa2a2e346c04364aea3f155a7f75212d86e12163bfe4dda320accc9c
+size 1711309856

model-00026-of-00081.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ade9d8dfecc1433fbb19dffb88c400ccd34f42f213bdc99accc757abea979ed0
+size 1711309864

model-00027-of-00081.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c0f5800913053850ee793e8895124187f3d540cf9deb5965d38006680f500468
+size 1711309864

model-00028-of-00081.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:686915f31b6f4c898f20faa2e5d5ccfbf2d209a7848cd048449302ab7d72f329
+size 1711309864

model-00029-of-00081.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:478acd77b84b6ffb3bcc9002910034b578db2a44e8286d405bdc956db82ce4c1
+size 1711309864

model-00030-of-00081.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:46fad18b4b17bba82306de14da6b26825ba9713ccc343531840058458d8a6505
+size 1711309864

model-00031-of-00081.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:219158cf9bbaaa4a4ece380f49015428223cc54198b41cc37f8cb809a66e7d11
+size 1711309864

model-00032-of-00081.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8b85ce2801bf7a10559b2d3617c6e01252e71c1a0fd9a5aa757d4faca91c77ec
+size 1711309864

model-00033-of-00081.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b2a6df7cb60cfb7931b66ba79feb8fa3d26bfca470af68d5105827ed64227047
+size 1711309864

model-00034-of-00081.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:43ffd2f008138380cf55cbcc4740aa7cf28f667ae2dc3bfcd0ef87d0c38f25e0
+size 1711309864

model-00035-of-00081.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:44dca8c133b27ab59ebe6c2bde9b138e538ee66e3e79b6cae1511949e5415730
+size 1711309856

model-00036-of-00081.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:93e3ffe10a51fa17f1d0b31db8924d5554fe4b134542c484ab18f3763f0bbecd
+size 1711309856

model-00037-of-00081.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4650f6dbc9badc41bc5756584b9a084da2a23565bd463e741339f61ad045e759
+size 1711309864

model-00038-of-00081.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a2dbb3250ddc45643b62ae3bf2406206417f68fbf28f435f74addcac36ba5828
+size 1711309864

model-00039-of-00081.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:beb9d845096cb973ff27d48ce148435c5940aeb20337b8d7612e455a3cb3f145
+size 1711309864

model-00040-of-00081.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f7fdaa14f8b6a3f8fccc017215fc57b4aa3375f8e4f29c55bd0496421ee3cbe8
+size 1711309864

model-00041-of-00081.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a94295f40e3a93083ab11756fc0c53a49fec1160d32ecb72fd58ea9b0c848f38
+size 1711309864

model-00042-of-00081.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:33c6217194ea34ef04fad00a2d760833646d8d6507f355b6bd2556123a8059f6
+size 1711309864

model-00043-of-00081.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2a7083d206465589e0b81ac467250588e1ef6bdca905c4cfedf177d6e6a29de0
+size 1711309864

model-00044-of-00081.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c8df90c99310dd115e18b0ca9c09f2265d4d07b31827f7180dddf02620054a34
+size 1711309864

model-00045-of-00081.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d78e1fc039a804cf08ab9fbc9f3fcf5990ed1b1a6523122f56eea372cc7a502d
+size 1711309864

model-00046-of-00081.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:06dddee03932d104e2332258379749b8736b44419447ac24f39783ce9f36c55c
+size 1711309856

model-00047-of-00081.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7f483db92e40ef7b2bdc3ae5275e88cc0633f699cce3f377da33da46d4f96809
+size 1711309856