Upload folder using huggingface_hub

README.md

@@ -0,0 +1,117 @@
+---
+language:
+- en
+license: apache-2.0
+datasets:
+- teknium/OpenHermes-2.5
+base_model: mistral-community/Mixtral-8x22B-v0.1
+tags:
+- exl2
+---
+
+# mixtral-8x22b-instruct-oh - EXL2 7.0bpw
+
+This is a 7.0bpw EXL2 quant of [fireworks-ai/mixtral-8x22b-instruct-oh](https://huggingface.co/fireworks-ai/mixtral-8x22b-instruct-oh)
+
+Details about the model can be found at the above model page.
+
+## EXL2 Version
+
+These quants were made with exllamav2 version 0.0.18. Quants made on this version of EXL2 may not work on older versions of the exllamav2 library.
+
+If you have problems loading these models, please update Text Generation WebUI to the latest version.
+
+## Perplexity Scoring
+
+Below are the perplexity scores for the EXL2 models. A lower score is better. 
+
+_TODO_
+
+### Perplexity Script
+
+This was the script used for perplexity testing.
+
+```bash
+#!/bin/bash
+
+# Activate the conda environment
+source ~/miniconda3/etc/profile.d/conda.sh
+conda activate exllamav2
+
+# Set the model name and bit size
+MODEL_NAME="mixtral-8x22b-instruct-oh"
+BIT_PRECISIONS=(7.0 6.0 5.5 5.0 4.5 4.0 3.5 3.0 2.75 2.5 2.25)
+
+# Print the markdown table header
+echo "| Quant Level | Perplexity Score |"
+echo "|-------------|------------------|"
+
+for BIT_PRECISION in "${BIT_PRECISIONS[@]}"
+do
+  MODEL_DIR="models/${MODEL_NAME}_exl2_${BIT_PRECISION}bpw"
+  if [ -d "$MODEL_DIR" ]; then
+    output=$(python test_inference.py -m "$MODEL_DIR" -gs 44,48 -ed data/wikitext/wikitext-2-v1.parquet)
+    score=$(echo "$output" | grep -oP 'Evaluation perplexity: \K[\d.]+')
+    echo "| $BIT_PRECISION | $score |"
+  fi
+done
+```
+
+
+## Quant Details
+
+This is the script used for quantization.
+
+```bash
+#!/bin/bash
+
+# Activate the conda environment
+source ~/miniconda3/etc/profile.d/conda.sh
+conda activate exllamav2
+
+# Set the model name and bit size
+MODEL_NAME="mixtral-8x22b-instruct-oh"
+
+# Define variables
+MODEL_DIR="models/$MODEL_NAME"
+OUTPUT_DIR="exl2_$MODEL_NAME"
+MEASUREMENT_FILE="measurements/$MODEL_NAME.json"
+
+# Create the measurement file if needed
+if [ ! -f "$MEASUREMENT_FILE" ]; then
+    echo "Creating $MEASUREMENT_FILE"
+    # Create directories
+    if [ -d "$OUTPUT_DIR" ]; then
+        rm -r "$OUTPUT_DIR"
+    fi
+    mkdir "$OUTPUT_DIR"
+
+    python convert.py -i $MODEL_DIR -o $OUTPUT_DIR -nr -om $MEASUREMENT_FILE
+fi
+
+# Choose one of the below. Either create a single quant for testing or a batch of them.
+# BIT_PRECISIONS=(2.25)
+BIT_PRECISIONS=(7.0 6.0 5.5 5.0 4.5 4.0 3.5 3.0 2.75 2.5 2.25)
+
+for BIT_PRECISION in "${BIT_PRECISIONS[@]}"
+do
+    CONVERTED_FOLDER="models/${MODEL_NAME}_exl2_${BIT_PRECISION}bpw"
+
+    # If it doesn't already exist, make the quant
+    if [ ! -d "$CONVERTED_FOLDER" ]; then
+
+        echo "Creating $CONVERTED_FOLDER"
+
+        # Create directories
+        if [ -d "$OUTPUT_DIR" ]; then
+            rm -r "$OUTPUT_DIR"
+        fi
+        mkdir "$OUTPUT_DIR"
+        mkdir "$CONVERTED_FOLDER"
+        
+        # Run conversion commands  
+        python convert.py -i $MODEL_DIR -o $OUTPUT_DIR -nr -m $MEASUREMENT_FILE -b $BIT_PRECISION -cf $CONVERTED_FOLDER
+
+    fi
+done
+```

config.json

@@ -0,0 +1,42 @@
+{
+    "_name_or_path": "mistral-community/Mixtral-8x22B-v0.1",
+    "architectures": [
+        "MixtralForCausalLM"
+    ],
+    "attention_dropout": 0.0,
+    "bos_token_id": 1,
+    "eos_token_id": 2,
+    "hidden_act": "silu",
+    "hidden_size": 6144,
+    "initializer_range": 0.02,
+    "intermediate_size": 16384,
+    "max_position_embeddings": 65536,
+    "model_type": "mixtral",
+    "num_attention_heads": 48,
+    "num_experts_per_tok": 2,
+    "num_hidden_layers": 56,
+    "num_key_value_heads": 8,
+    "num_local_experts": 8,
+    "output_router_logits": false,
+    "rms_norm_eps": 1e-05,
+    "rope_theta": 1000000,
+    "router_aux_loss_coef": 0.001,
+    "router_jitter_noise": 0.0,
+    "sliding_window": null,
+    "tie_word_embeddings": false,
+    "torch_dtype": "float16",
+    "transformers_version": "4.39.3",
+    "use_cache": true,
+    "vocab_size": 32000,
+    "quantization_config": {
+        "quant_method": "exl2",
+        "version": "0.0.18",
+        "bits": 7.0,
+        "head_bits": 6,
+        "calibration": {
+            "rows": 100,
+            "length": 2048,
+            "dataset": "(default)"
+        }
+    }
+}

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "transformers_version": "4.39.3"
+}

output-00001-of-00015.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a7f22d3f29954d1c410719bac8f27c02c53ed1044a56794414a20ed8e03e6105
+size 8580252328

output-00002-of-00015.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a4acca184af86c4c2b7f1451ee5f56ccb28e2687d9d6ebaba027e13cbf78fb1b
+size 8589404656

output-00003-of-00015.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:34f84eec88ea9cfe354dee16eb8cbbec8e90f9376f9f73d5cb60139710b6c106
+size 8531177200

output-00004-of-00015.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:94dd73c4b7e1f7bb7bdb81fce96f54e3c3a0f7909bb8a72bb8e091e67f5a4080
+size 8522260712

output-00005-of-00015.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6e046f441dc80861cc9ed0e35140440aa6240f58dea31ff5bbdaa1f00c418645
+size 8589823896

output-00006-of-00015.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a9b31b504836e76951c0026133bc4455bf8a206836d5ed11040173800c919a72
+size 8551875904

output-00007-of-00015.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:48da7cfb6aa281629d6c0a45e11c914dbc99a16a03807bc82ea09e22377b72f3
+size 8551482560

output-00008-of-00015.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4e6d65f25cfacc9cfe1e800e7c95832f109168e2c163fc3936844881e7d63a2f
+size 8549846248

output-00009-of-00015.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:de1e4fc2b32d96ee239fa9ec46738cb6430fbb74e99707d34703ff590911aae7
+size 8551482560

output-00010-of-00015.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9e6925406a2e830d2af39a47f00b2da1b8c399e053a6e9c7ab5eadccaa1a2c1c
+size 8549846264

output-00011-of-00015.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d75afa3823345a1d9bd82e2a801796357d5d75f915b4ac255a0a9bd7e381c60a
+size 8551482560

output-00012-of-00015.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9a46453676684c9a8ce6f80fda3c7f0e48ce08a0fe8acd7078489057cd58e725
+size 8551482560

output-00013-of-00015.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:07e71d236c478570f487867beb4e69fad51dd67795c9ac7678f586200c4ad4f2
+size 8549846248

output-00014-of-00015.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3d2725ffb8d59925fe535b411375139371000ed5a0d577d749f0ec69832717c0
+size 8551482560

output-00015-of-00015.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:98a528f70b2072e04ecef14b8db01a25e86809982e0a13e21a453c3163e34a6c
+size 3476710520

special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dadfd56d766715c61d2ef780a525ab43b8e6da4de6865bda3d95fdef5e134055
+size 493443

tokenizer_config.json

@@ -0,0 +1,44 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "add_prefix_space": true,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [],
+  "bos_token": "<s>",
+  "chat_template": "{%- set message_roles = ['USER', 'ASSISTANT'] -%}\n{%- set ns = namespace(initial_system_message_handled=false, messages=messages) -%}\n{{ bos_token }}\n{%- for message in ns.messages -%}\n  {%- if message['role'] | upper == 'SYSTEM' and not ns.initial_system_message_handled -%}\n    {%- set ns.initial_system_message_handled = true -%}\n    {{'SYSTEM: ' + message['content'] }}\n  {%- elif message['role'] | upper != 'SYSTEM' -%}\n    {%- if (message['role'] | upper == 'USER') != ((loop.index0 - (1 if ns.initial_system_message_handled else 0)) % 2 == 0) -%}\n      {{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}\n    {%- endif -%}\n    {%- if loop.index0 > 0 or ns.initial_system_message_handled -%}\n      {{ '\\n\\n' }}\n    {%- endif -%}\n    {%- if message['role'] | upper == 'USER' -%}\n      {{ 'USER: ' + message['content'] }}\n    {%- elif message['role'] | upper == 'ASSISTANT' -%}\n      {{ 'ASSISTANT: ' + message['content'] + eos_token}}\n    {%- endif -%}\n  {%- endif -%}\n{%- endfor -%}\n",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "legacy": false,
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<unk>",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}