Upload model and tokenizers

.gitattributes

@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+context_tokenizer/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+text_tokenizer/tokenizer.json filter=lfs diff=lfs merge=lfs -text

config.json

@@ -0,0 +1,131 @@
+{
+  "_name_or_path": "./co_output_model",
+  "architectures": [
+    "CoEncoderForConditionalGeneration"
+  ],
+  "auto_map": {
+    "AutoConfig": "configuration_co_encoder.CoEncoderConfig",
+    "AutoModelForCausalLM": "modeling_co_encoder.CoEncoderForConditionalGeneration"
+  },
+  "begin_of_context_token_id": 128002,
+  "connector_hidden_act": "gelu",
+  "context_config": {
+    "_attn_implementation_autoset": false,
+    "_name_or_path": "Qwen/Qwen2-0.5B",
+    "add_cross_attention": false,
+    "architectures": [
+      "Qwen2ForCausalLM"
+    ],
+    "attention_dropout": 0.0,
+    "bad_words_ids": null,
+    "begin_suppress_tokens": null,
+    "bos_token_id": 151643,
+    "chunk_size_feed_forward": 0,
+    "cross_attention_hidden_size": null,
+    "decoder_start_token_id": null,
+    "diversity_penalty": 0.0,
+    "do_sample": false,
+    "early_stopping": false,
+    "encoder_no_repeat_ngram_size": 0,
+    "eos_token_id": 151643,
+    "exponential_decay_length_penalty": null,
+    "finetuning_task": null,
+    "forced_bos_token_id": null,
+    "forced_eos_token_id": null,
+    "hidden_act": "silu",
+    "hidden_size": 896,
+    "id2label": {
+      "0": "LABEL_0",
+      "1": "LABEL_1"
+    },
+    "initializer_range": 0.02,
+    "intermediate_size": 4864,
+    "is_decoder": false,
+    "is_encoder_decoder": false,
+    "label2id": {
+      "LABEL_0": 0,
+      "LABEL_1": 1
+    },
+    "length_penalty": 1.0,
+    "max_length": 20,
+    "max_position_embeddings": 131072,
+    "max_window_layers": 24,
+    "min_length": 0,
+    "model_type": "qwen2",
+    "no_repeat_ngram_size": 0,
+    "num_attention_heads": 14,
+    "num_beam_groups": 1,
+    "num_beams": 1,
+    "num_hidden_layers": 24,
+    "num_key_value_heads": 2,
+    "num_return_sequences": 1,
+    "output_attentions": false,
+    "output_hidden_states": false,
+    "output_scores": false,
+    "pad_token_id": null,
+    "prefix": null,
+    "problem_type": null,
+    "pruned_heads": {},
+    "remove_invalid_values": false,
+    "repetition_penalty": 1.0,
+    "return_dict": true,
+    "return_dict_in_generate": false,
+    "rms_norm_eps": 1e-06,
+    "rope_scaling": null,
+    "rope_theta": 1000000.0,
+    "sep_token_id": null,
+    "sliding_window": null,
+    "suppress_tokens": null,
+    "task_specific_params": null,
+    "temperature": 1.0,
+    "tf_legacy_loss": false,
+    "tie_encoder_decoder": false,
+    "tie_word_embeddings": true,
+    "tokenizer_class": null,
+    "top_k": 50,
+    "top_p": 1.0,
+    "torch_dtype": "bfloat16",
+    "torchscript": false,
+    "typical_p": 1.0,
+    "use_bfloat16": false,
+    "use_cache": true,
+    "use_sliding_window": false,
+    "vocab_size": 151936
+  },
+  "context_feature_layer": -2,
+  "context_feature_select_strategy": "default",
+  "end_of_context_token_id": 128003,
+  "ignore_index": -100,
+  "model_type": "co_encoder",
+  "projector_hidden_act": "gelu",
+  "text_config": {
+    "_name_or_path": "meta-llama/Llama-3.1-8B-Instruct",
+    "architectures": [
+      "LlamaForCausalLM"
+    ],
+    "bos_token_id": 128000,
+    "eos_token_id": [
+      128001,
+      128008,
+      128009
+    ],
+    "intermediate_size": 14336,
+    "max_position_embeddings": 131072,
+    "model_type": "llama",
+    "num_key_value_heads": 8,
+    "rms_norm_eps": 1e-05,
+    "rope_scaling": {
+      "factor": 8.0,
+      "high_freq_factor": 4.0,
+      "low_freq_factor": 1.0,
+      "original_max_position_embeddings": 8192,
+      "rope_type": "llama3"
+    },
+    "rope_theta": 500000.0,
+    "torch_dtype": "bfloat16",
+    "vocab_size": 128256
+  },
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.47.0.dev0"
+}

configuration_co_encoder.py

@@ -0,0 +1,68 @@
+# coding=utf-8
+"""CoEncoder model configuration"""
+
+from transformers.configuration_utils import PretrainedConfig
+from transformers.utils import logging
+from transformers import CONFIG_MAPPING
+
+logger = logging.get_logger(__name__)
+
+
+class CoEncoderConfig(PretrainedConfig):
+    r"""
+    """
+
+    model_type = "co_encoder"
+
+    def __init__(
+        self,
+        context_config=None,
+        text_config=None,
+        ignore_index=-100,
+        connector_hidden_act="gelu",
+        context_feature_layer=-2,
+        context_feature_select_strategy="default",
+        begin_of_context_token_id=None,
+        end_of_context_token_id=None,
+        tie_word_embeddings=False,
+        **kwargs,
+    ):
+        self.ignore_index = ignore_index
+        self.connector_hidden_act = connector_hidden_act
+        self.context_feature_layer = context_feature_layer
+        self.context_feature_select_strategy = context_feature_select_strategy
+        self.begin_of_context_token_id = begin_of_context_token_id
+        self.end_of_context_token_id = end_of_context_token_id
+
+        if context_feature_select_strategy not in ["default"]:
+            raise ValueError(
+                "context_feature_select_strategy should be one of 'default'."
+                f"Got: {context_feature_select_strategy}"
+            )
+        
+        if isinstance(context_config, dict):
+            context_config["model_type"] = (
+                context_config["model_type"] if "model_type" in context_config else "qwen2"
+            )
+            context_config = CONFIG_MAPPING[context_config["model_type"]](**context_config)
+        
+        self.context_config = context_config
+
+        if isinstance(text_config, dict):
+            text_config["model_type"] = text_config["model_type"] if "model_type" in text_config else "llama"
+            text_config = CONFIG_MAPPING[text_config["model_type"]](**text_config)
+        elif text_config is None:
+            text_config = CONFIG_MAPPING["llama"]()
+
+        self.text_config = text_config
+
+        super().__init__(
+            tie_word_embeddings=tie_word_embeddings, 
+            ignore_index=ignore_index,
+            connector_hidden_act=connector_hidden_act,
+            context_feature_layer=context_feature_layer, 
+            context_feature_select_strategy=context_feature_select_strategy,
+            begin_of_context_token_id=begin_of_context_token_id,
+            end_of_context_token_id=end_of_context_token_id,
+            **kwargs
+        )

context_tokenizer/added_tokens.json

@@ -0,0 +1,5 @@
+{
+  "<|endoftext|>": 151643,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644
+}

context_tokenizer/special_tokens_map.json

@@ -0,0 +1,20 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>"
+  ],
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

context_tokenizer/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bcfe42da0a4497e8b2b172c1f9f4ec423a46dc12907f4349c55025f670422ba9
+size 11418266

context_tokenizer/tokenizer_config.json

@@ -0,0 +1,44 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>"
+  ],
+  "bos_token": null,
+  "chat_template": "{% for message in messages %}{% if loop.first and messages[0]['role'] != 'system' %}{{ '<|im_start|>system\nYou are a helpful assistant<|im_end|>\n' }}{% endif %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "extra_special_tokens": {},
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

generation_config.json

@@ -0,0 +1,10 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 128000,
+  "eos_token_id": [
+    128001,
+    128008,
+    128009
+  ],
+  "transformers_version": "4.47.0.dev0"
+}

model-00001-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1d0de04184a5318130add82db76980315e3cfb153294c11a12cf9e5ca0a12170
+size 4973442402

model-00002-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f13d5f738211d5954cc424baabdf3a4e1001811d63aaf5b33b1ac5453c067ad7
+size 4999804248

model-00003-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7c1cd2162ea622494ba04cd4585cf4b6778740e56326a7f3b07a6cf9632b2327
+size 4915917680

model-00004-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:66d1f965204f37fcd6abf68368fb3a806c1c8cd03089d2a36b4844372187ff6b
+size 2476814440

model.safetensors.index.json

@@ -0,0 +1,602 @@
+{
+  "metadata": {
+    "total_size": 17365900546
+  },
+  "weight_map": {
+    "connector.dynamic_pooling.scale_param": "model-00001-of-00004.safetensors",
+    "connector.dynamic_pooling.size_estimation_attention.k_proj.weight": "model-00001-of-00004.safetensors",
+    "connector.dynamic_pooling.size_estimation_attention.o_proj.weight": "model-00001-of-00004.safetensors",
+    "connector.dynamic_pooling.size_estimation_attention.q_proj.weight": "model-00001-of-00004.safetensors",
+    "connector.dynamic_pooling.size_estimation_attention.v_proj.weight": "model-00001-of-00004.safetensors",
+    "connector.dynamic_pooling.weighted_pooling_attention.k_proj.weight": "model-00001-of-00004.safetensors",
+    "connector.dynamic_pooling.weighted_pooling_attention.o_proj.weight": "model-00001-of-00004.safetensors",
+    "connector.dynamic_pooling.weighted_pooling_attention.q_proj.weight": "model-00001-of-00004.safetensors",
+    "connector.dynamic_pooling.weighted_pooling_attention.v_proj.weight": "model-00001-of-00004.safetensors",
+    "connector.linear_1.bias": "model-00001-of-00004.safetensors",
+    "connector.linear_1.weight": "model-00001-of-00004.safetensors",
+    "connector.linear_2.bias": "model-00001-of-00004.safetensors",
+    "connector.linear_2.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.lm_head.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.embed_tokens.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.0.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.0.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.0.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.0.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.1.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.1.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.1.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.1.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.10.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.10.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.10.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.10.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.10.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.10.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.10.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.10.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.10.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.10.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.10.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.10.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.11.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.11.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.11.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.11.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.11.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.11.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.11.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.11.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.11.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.11.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.11.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.11.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.12.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.12.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.12.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.12.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.12.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.12.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.12.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.12.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.12.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.12.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.12.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.12.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.13.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.13.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.13.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.13.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.13.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.13.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.13.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.13.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.13.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.13.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.13.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.13.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.14.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.14.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.14.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.14.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.14.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.14.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.14.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.14.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.14.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.14.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.14.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.14.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.15.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.15.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.15.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.15.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.15.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.15.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.15.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.15.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.15.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.15.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.15.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.15.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.16.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.16.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.16.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.16.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.16.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.16.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.16.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.16.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.16.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.16.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.16.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.16.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.17.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.17.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.17.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.17.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.17.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.17.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.17.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.17.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.17.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.17.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.17.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.17.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.18.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.18.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.18.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.18.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.18.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.18.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.18.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.18.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.18.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.18.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.18.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.18.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.19.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.19.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.19.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.19.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.19.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.19.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.19.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.19.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.19.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.19.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.19.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.19.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.2.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.2.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.2.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.2.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.20.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.20.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.20.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.20.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.20.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.20.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.20.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.20.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.20.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.20.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.20.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.20.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.21.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.21.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.21.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.21.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.21.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.21.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.21.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.21.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.21.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.21.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.21.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.21.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.22.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.22.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.22.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.22.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.22.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.22.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.22.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.22.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.22.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.22.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.22.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.22.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.23.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.23.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.23.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.23.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.23.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.23.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.23.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.23.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.23.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.23.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.23.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.23.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.3.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.3.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.3.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.3.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.4.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.4.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.4.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.4.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.4.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.4.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.4.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.4.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.4.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.5.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.5.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.5.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.5.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.5.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.5.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.5.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.5.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.5.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.5.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.5.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.5.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.6.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.6.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.6.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.6.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.6.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.6.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.6.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.6.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.6.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.6.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.6.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.6.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.7.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.7.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.7.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.7.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.7.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.7.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.7.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.7.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.7.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.7.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.7.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.7.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.8.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.8.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.8.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.8.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.8.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.8.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.8.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.8.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.8.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.8.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.8.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.8.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.9.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.9.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.9.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.9.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.9.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.9.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.9.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.9.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.9.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.9.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.9.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.layers.9.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "context_tower.tower.model.norm.weight": "model-00001-of-00004.safetensors",
+    "language_model.lm_head.weight": "model-00004-of-00004.safetensors",
+    "language_model.model.embed_tokens.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.0.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.1.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.10.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.10.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.10.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.10.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.10.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.10.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.10.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.10.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.10.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.11.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.11.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.11.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.11.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.11.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.11.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.11.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.11.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.11.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.12.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.12.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.12.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.12.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.12.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.12.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.12.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.12.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.12.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.13.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.13.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.13.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.13.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.13.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.13.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.13.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.13.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.13.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.14.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.14.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.14.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.14.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.14.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.14.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.14.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.14.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.14.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.15.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.15.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.15.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.15.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.15.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.15.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.15.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.15.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.15.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.16.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.16.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.16.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.16.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.16.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.16.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.16.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.16.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.16.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.17.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.17.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.17.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.17.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.17.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.17.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.17.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.17.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.17.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.18.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.18.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.18.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.18.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.18.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.18.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.18.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.18.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.18.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.19.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.19.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.19.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.19.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.19.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.19.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.19.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.19.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.19.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.2.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.20.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.20.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.20.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.20.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.20.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.20.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.20.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.20.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.20.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.21.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.21.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.21.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.21.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.21.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.21.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.21.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.21.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.21.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.22.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.22.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.22.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.22.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.22.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.22.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.22.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.22.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.22.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.23.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.23.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.23.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.23.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.23.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.23.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.23.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.23.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.23.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.24.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.24.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.24.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.24.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.24.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.24.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.24.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.24.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.24.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.25.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.25.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.25.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.25.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.25.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.25.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.25.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.25.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.25.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.26.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.26.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.26.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.26.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.26.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.26.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.26.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.26.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.26.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.27.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.27.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.27.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.27.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.27.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.27.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.27.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.27.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.27.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.28.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "language_model.model.layers.28.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "language_model.model.layers.28.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.28.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.28.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "language_model.model.layers.28.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.28.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.28.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.28.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "language_model.model.layers.29.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "language_model.model.layers.29.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "language_model.model.layers.29.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "language_model.model.layers.29.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "language_model.model.layers.29.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "language_model.model.layers.29.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "language_model.model.layers.29.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "language_model.model.layers.29.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "language_model.model.layers.29.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "language_model.model.layers.3.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.30.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "language_model.model.layers.30.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "language_model.model.layers.30.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "language_model.model.layers.30.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "language_model.model.layers.30.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "language_model.model.layers.30.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "language_model.model.layers.30.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "language_model.model.layers.30.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "language_model.model.layers.30.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "language_model.model.layers.31.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "language_model.model.layers.31.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "language_model.model.layers.31.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "language_model.model.layers.31.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "language_model.model.layers.31.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "language_model.model.layers.31.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "language_model.model.layers.31.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "language_model.model.layers.31.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "language_model.model.layers.31.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "language_model.model.layers.4.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.4.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.4.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.4.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.4.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.4.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.5.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.5.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.5.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.5.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.5.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.5.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.5.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.5.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.5.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "language_model.model.layers.6.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.6.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.6.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.6.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.6.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.6.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.6.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.6.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.6.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.7.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.7.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.7.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.7.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.7.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.7.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.7.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.7.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.7.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.8.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.8.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.8.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.8.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.8.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.8.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.8.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.8.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.8.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.9.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.9.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.9.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.9.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.9.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.9.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.9.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.9.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.layers.9.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "language_model.model.norm.weight": "model-00004-of-00004.safetensors"
+  }
+}

modeling_co_encoder.py

@@ -0,0 +1,761 @@
+# coding=utf-8
+"""PyTorch CoEncoder model."""
+
+import math
+from dataclasses import dataclass
+from typing import List, Optional, Tuple, Union
+
+import numpy as np
+import torch
+import torch.utils.checkpoint
+from torch import nn
+
+from transformers import PreTrainedModel
+from transformers.activations import ACT2FN
+from transformers.image_processing_utils import select_best_resolution
+from transformers.modeling_outputs import ModelOutput
+from transformers.modeling_flash_attention_utils import FlashAttentionKwargs, _flash_attention_forward
+from transformers.utils import (
+    add_start_docstrings,
+    add_start_docstrings_to_model_forward,
+    logging,
+    replace_return_docstrings,
+    is_flash_attn_2_available,
+    is_flash_attn_greater_or_equal_2_10
+)
+from transformers import AutoTokenizer, AutoModel, AutoModelForCausalLM
+from .configuration_co_encoder import CoEncoderConfig
+
+
+logger = logging.get_logger(__name__)
+
+_CONFIG_FOR_DOC = "CoEncoderConfig"
+
+
+def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor:
+    """
+    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
+    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
+    """
+    batch, num_key_value_heads, slen, head_dim = hidden_states.shape
+    if n_rep == 1:
+        return hidden_states
+    hidden_states = hidden_states[:, :, None, :, :].expand(batch, num_key_value_heads, n_rep, slen, head_dim)
+    return hidden_states.reshape(batch, num_key_value_heads * n_rep, slen, head_dim)
+
+
+@dataclass
+class CoEncoderCausalLMOutputWithPast(ModelOutput):
+    """
+    Base class for CoEncoder causal language model (or autoregressive) outputs.
+
+    Args:
+        loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
+            Language modeling loss (for next-token prediction).
+        logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`):
+            Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
+        past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
+            Tuple of `tuple(torch.FloatTensor)` of length `config.context_config.num_layers`, with each tuple having 2 tensors of shape
+            `(batch_size, num_heads, sequence_length, embed_size_per_head)`)
+
+            Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see
+            `past_key_values` input) to speed up sequential decoding.
+        hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
+            Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, +
+            one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`.
+
+            Hidden-states of the model at the output of each layer plus the optional initial embedding outputs.
+        attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
+            Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+            sequence_length)`.
+
+            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
+            heads.
+        context_hidden_states (`torch.FloatTensor`, *optional*):
+            A `torch.FloatTensor` of size (batch_size, sequence_length, hidden_size)`.
+            context_hidden_states of the model produced by the context encoder and after projecting the last hidden state.
+    """
+
+    loss: Optional[torch.FloatTensor] = None
+    logits: torch.FloatTensor = None
+    past_key_values: Optional[List[torch.FloatTensor]] = None
+    hidden_states: Optional[Tuple[torch.FloatTensor]] = None
+    attentions: Optional[Tuple[torch.FloatTensor]] = None
+    context_hidden_states: Optional[torch.FloatTensor] = None
+
+
+class CoEncoderDynamicAttention(nn.Module):
+    """
+    Attention mechanism adapted for dynamic output size based on Mistral's architecture. This attention layer computes
+    the output attention scores which are used to determine the pooling size dynamically.
+    """
+
+    def __init__(self, config: CoEncoderConfig):
+        super().__init__()
+
+        self.hidden_size = config.context_config.hidden_size
+        self.num_heads = config.context_config.num_attention_heads
+        self.head_dim = getattr(config.context_config, "head_dim", self.hidden_size // self.num_heads)
+        self.num_key_value_heads = config.context_config.num_key_value_heads
+        self.num_key_value_groups = self.num_heads // self.num_key_value_heads
+
+        # Query, Key, Value, and Output Projections
+        self.q_proj = nn.Linear(self.hidden_size, self.num_heads * self.head_dim, bias=False)
+        self.k_proj = nn.Linear(self.hidden_size, self.num_key_value_heads * self.head_dim, bias=False)
+        self.v_proj = nn.Linear(self.hidden_size, self.num_key_value_heads * self.head_dim, bias=False)
+        self.o_proj = nn.Linear(self.num_heads * self.head_dim, 1, bias=False)
+
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        attention_mask: Optional[torch.Tensor] = None,
+        output_attentions: bool = False,
+    ):
+        # Get input dimensions
+        bsz, seq_len, hidden_size = hidden_states.size()
+
+        # Query, Key, Value projections
+        query_states = self.q_proj(hidden_states)
+        key_states = self.k_proj(hidden_states)
+        value_states = self.v_proj(hidden_states)
+
+        # Reshape and transpose to [batch_size, num_heads, seq_len, head_dim]
+        query_states = query_states.view(bsz, seq_len, self.num_heads, self.head_dim).transpose(1, 2)
+        key_states = key_states.view(bsz, seq_len, self.num_key_value_heads, self.head_dim).transpose(1, 2)
+        value_states = value_states.view(bsz, seq_len, self.num_key_value_heads, self.head_dim).transpose(1, 2)
+
+        # Repeat key and value states for multi-head attention
+        key_states = repeat_kv(key_states, self.num_key_value_groups)
+        value_states = repeat_kv(value_states, self.num_key_value_groups)
+
+        # Compute attention scores
+        attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) / math.sqrt(self.head_dim)
+
+        if attention_mask is not None:  # no matter the length, we just slice it
+            causal_mask = attention_mask[:, :, :, : key_states.shape[-2]]
+            attn_weights = attn_weights + causal_mask
+
+        # Apply softmax to get attention probabilities
+        attn_weights = nn.functional.softmax(attn_weights, dim=-1)
+
+        # Apply attention to values
+        attn_output = torch.matmul(attn_weights, value_states)
+
+        # Reshape attention output
+        attn_output = attn_output.transpose(1, 2).contiguous()
+        attn_output = attn_output.reshape(bsz, seq_len, -1)
+
+        # Project to output dimension
+        attn_output = self.o_proj(attn_output)
+
+        if not output_attentions:
+            attn_weights = None
+
+        return attn_output, attn_weights
+
+
+class CoEncoderDynamicFlashAttention2(CoEncoderDynamicAttention):
+    def __init__(self, config: CoEncoderConfig):
+        super().__init__(config)
+        self._flash_attn_uses_top_left_mask = not is_flash_attn_greater_or_equal_2_10()
+        self.is_causal = False  # Assuming non-causal attention for this context
+        self.config = config
+
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        attention_mask: Optional[torch.Tensor] = None,
+        output_attentions: bool = False,
+    ):
+        output_attentions = False
+
+        # Get input dimensions
+        bsz, seq_len, hidden_size = hidden_states.size()
+        q_len = seq_len
+
+        query_states = self.q_proj(hidden_states)
+        key_states = self.k_proj(hidden_states)
+        value_states = self.v_proj(hidden_states)
+
+        query_states = query_states.view(bsz, seq_len, self.num_heads, self.head_dim).transpose(1, 2)
+        key_states = key_states.view(bsz, seq_len, self.num_key_value_heads, self.head_dim).transpose(1, 2)
+        value_states = value_states.view(bsz, seq_len, self.num_key_value_heads, self.head_dim).transpose(1, 2)
+
+        # Repeat key and value states for multi-head attention
+        key_states = repeat_kv(key_states, self.num_key_value_groups)
+        value_states = repeat_kv(value_states, self.num_key_value_groups)
+
+        input_dtype = query_states.dtype
+        if input_dtype == torch.float32:
+            if torch.is_autocast_enabled():
+                target_dtype = torch.get_autocast_gpu_dtype()
+            # Handle the case where the model is quantized
+            elif hasattr(self.config, "_pre_quantization_dtype"):
+                target_dtype = self.config._pre_quantization_dtype
+            else:
+                target_dtype = self.q_proj.weight.dtype
+
+            logger.warning_once(
+                f"The input hidden states seem to be silently casted in float32, which might be related to"
+                f" upcasted embedding or layer norm layers in float32. Casting back the input to {target_dtype}."
+            )
+
+            query_states = query_states.to(target_dtype)
+            key_states = key_states.to(target_dtype)
+            value_states = value_states.to(target_dtype)
+
+        # Define attention_mask assuming all positions are valid
+        # because flash_attn does not support custom attention_mask
+        attention_mask = None
+
+        # Define other required variables
+        position_ids = None
+        dropout_rate = getattr(self.config.context_config, "attention_dropout", 0.0)
+
+        attn_output = _flash_attention_forward(
+            query_states,
+            key_states,
+            value_states,
+            attention_mask,
+            q_len,
+            position_ids=position_ids,
+            dropout=dropout_rate,
+            sliding_window=getattr(self, "sliding_window", None),
+            use_top_left_mask=self._flash_attn_uses_top_left_mask,
+            is_causal=self.is_causal,
+        )
+
+        attn_output = attn_output.reshape(bsz, q_len, -1).contiguous()
+        attn_output = self.o_proj(attn_output)
+
+        if not output_attentions:
+            attn_weights = None
+
+        return attn_output, attn_weights
+
+
+class CoEncoderDynamicWeightedAvgPool1d(nn.Module):
+    """
+    A module that dynamically determines the output size based on input
+    and performs weighted average pooling with separate attention mechanisms
+    for output size estimation and weighted pooling.
+    """
+    def __init__(self, config, output_size_min=32, output_size_max=131072):
+        super().__init__()
+        # Attention mechanism for estimating output size
+        self.size_estimation_attention = CoEncoderDynamicFlashAttention2(config)
+        # Attention mechanism for weighted pooling
+        self.weighted_pooling_attention = CoEncoderDynamicFlashAttention2(config)
+        self.output_size_min = output_size_min
+        self.output_size_max = (
+            config.context_config.max_position_embeddings if config.context_config.max_position_embeddings is not None else output_size_max
+        )
+        self.scale_param = nn.Parameter(torch.tensor(0.01))
+
+    def forward(self, hidden_states):
+        """
+        Args:
+            x: Input tensor of shape (batch_size, seq_len, hidden_size)
+
+        Returns:
+            Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
+                - pooled_output: Padded tensor of compressed sequences (batch_size, max_pooled_len, hidden_size)
+                - attention_mask: Binary mask indicating valid tokens (batch_size, max_pooled_len)
+                - dynamic_output_sizes: Dynamic output sizes for each batch (batch_size,)
+        """
+        batch_size, seq_len, hidden_size = hidden_states.size()
+        device = hidden_states.device
+
+        # Estimate output size using attention mechanism
+        # attn_output_size: (batch_size, seq_len, 1)
+        attn_output_size, _ = self.size_estimation_attention(hidden_states)
+
+        # Calculate dynamic output sizes for each batch item
+        # (batch_size, seq_len, 1) -> (batch_size, 1)
+        batch_attn_means = torch.sigmoid(attn_output_size).mean(dim=1)
+        scaled_batch_means = batch_attn_means * self.scale_param.to(batch_attn_means.dtype)
+
+        # Calculate dynamic output sizes (batch_size,)
+        dynamic_output_sizes = (
+            (scaled_batch_means * (self.output_size_max - self.output_size_min)) + self.output_size_min
+        ).int().squeeze(-1)
+
+        # Get the maximum output size across the batch
+        max_pooled_len = dynamic_output_sizes.max().item()
+
+        # Compute attention weights for weighted pooling
+        # attn_output_weights: (batch_size, seq_len, 1)
+        attn_output_weights, _ = self.weighted_pooling_attention(hidden_states)
+        # Normalize with sigmoid function for use as weights
+        # attention_weights: (batch_size, seq_len)
+        attention_weights = torch.sigmoid(attn_output_weights).squeeze(-1)
+
+        # Initialize output tensors
+        # pooled_output: (batch_size, max_pooled_len, hidden_size)
+        pooled_output = torch.zeros(batch_size, max_pooled_len, hidden_size, device=device, dtype=hidden_states.dtype)
+        # attention_mask: (batch_size, max_pooled_len)
+        attention_mask = torch.zeros(batch_size, max_pooled_len, dtype=torch.bool, device=device)
+
+        for batch_idx in range(batch_size):
+            output_size = dynamic_output_sizes[batch_idx].item()
+            item_input = hidden_states[batch_idx]  # Shape: (seq_len, hidden_size)
+            item_weights = attention_weights[batch_idx]  # Shape: (seq_len)
+            # print(f"Sequence lenfth of context: {item_input.size(0)}")
+            # print(f"Output length: {output_size}")
+   
+            # Perform weighted pooling
+            pooled_values = []
+            # Split the sequence evenly
+            intervals = torch.linspace(0, seq_len, steps=output_size + 1).long()
+            for i in range(output_size):
+                start = intervals[i].item()
+                end = intervals[i + 1].item()
+                chunk_input = item_input[start:end]  # Shape: (chunk_size, hidden_size)
+                chunk_weights = item_weights[start:end]  # Shape: (chunk_size)
+                if chunk_weights.sum() == 0:
+                    # If the sum of weights is zero, add a zero vector
+                    pooled_value = torch.zeros(hidden_size, device=device, dtype=hidden_states.dtype)
+                else:
+                    # Calculate weighted average
+                    weighted_input = chunk_input * chunk_weights.unsqueeze(-1)  # Shape: (chunk_size, hidden_size)
+                    pooled_value = weighted_input.sum(dim=0) / (chunk_weights.sum() + 1e-8)  # Shape: (hidden_size)
+                pooled_values.append(pooled_value)
+            # Convert the result to a tensor
+            pooled_values = torch.stack(pooled_values)  # Shape: (output_size, hidden_size)
+            # Store the result
+            pooled_output[batch_idx, -output_size:] = pooled_values.squeeze(0)
+            attention_mask[batch_idx, -output_size:] = True
+
+        return pooled_output, attention_mask, dynamic_output_sizes
+
+
+class CoEncoderContextLanguageConnector(nn.Module):
+    def __init__(self, config: CoEncoderConfig):
+        super().__init__()
+
+        self.dynamic_pooling = CoEncoderDynamicWeightedAvgPool1d(config)
+
+        self.linear_1 = nn.Linear(
+            config.context_config.hidden_size,
+            config.text_config.hidden_size,
+            bias=True
+        )
+        self.act = ACT2FN[config.projector_hidden_act]
+        self.linear_2 = nn.Linear(
+            config.text_config.hidden_size,
+            config.text_config.hidden_size,
+            bias=True
+        )
+
+    def forward(self, context_features):
+        # context_features: [batch_size, seq_len, hidden_size]
+        # Apply dynamic adaptive average pooling with attention
+        pooled_output, attention_mask, dynamic_output_sizes = self.dynamic_pooling(
+            hidden_states=context_features
+        )
+        # pooled_output: [batch_size, max_pooled_len, hidden_size]
+
+        hidden_states = self.linear_1(pooled_output)
+        hidden_states = self.act(hidden_states)
+        hidden_states = self.linear_2(hidden_states)
+        
+        return hidden_states, attention_mask
+
+
+class CoEncoderContextTower(nn.Module):
+    def __init__(self, config: CoEncoderConfig):
+        super().__init__()
+
+        self.tower = AutoModelForCausalLM.from_config(
+            config.context_config,
+            attn_implementation="flash_attention_2" if is_flash_attn_2_available() else "eager"
+        )
+        self.select_layer = config.context_feature_layer
+    
+    def feature_select(self, llm_outputs):
+        hidden_states = llm_outputs.hidden_states
+        return hidden_states[self.select_layer]
+
+    def forward(
+        self,
+        input_ids,
+        inputs_embeds,
+        attention_mask
+    ):
+        outputs = self.tower(
+            input_ids=input_ids,
+            inputs_embeds=inputs_embeds,
+            attention_mask=attention_mask,
+            output_hidden_states=True
+        )
+        
+        features = self.feature_select(outputs)
+        return features
+    
+
+class CoEncoderPreTrainedModel(PreTrainedModel):
+    config_class = CoEncoderConfig
+    base_model_prefix = "model"
+    supports_gradient_checkpointing = True
+    _no_split_modules = [] # ["CoEncoderContextLanguageConnector", "CoEncoderContextTower"]
+    _skip_keys_device_placement = ["past_key_values"]
+    _supports_flash_attn_2 = True
+    _supports_sdpa = True
+    _supports_cache_class = True
+    _supports_quantized_cache = True
+    _supports_static_cache = True
+
+    def _init_weights(self, module):
+        std = (
+            self.config.initializer_range
+            if hasattr(self.config, "initializer_range")
+            else self.config.text_config.initializer_range
+        )
+        if isinstance(module, nn.Linear):
+            module.weight.data.normal_(mean=0.0, std=std)
+            if module.bias is not None:
+                module.bias.data.zero_()
+        elif isinstance(module, nn.Embedding):
+            module.weight.data.normal_(mean=0.0, std=std)
+            if module.padding_idx is not None:
+                module.weight.data[module.padding_idx].zero_()
+
+
+class CoEncoderForConditionalGeneration(CoEncoderPreTrainedModel):
+    def __init__(self, config: CoEncoderConfig):
+        super().__init__(config)
+        self.context_tower = CoEncoderContextTower(config)
+        self.connector = CoEncoderContextLanguageConnector(config)
+
+        self.language_model = AutoModelForCausalLM.from_config(
+            config.text_config,
+            attn_implementation="flash_attention_2" if is_flash_attn_2_available() else "eager"
+        )
+
+        self.vocab_size = config.text_config.vocab_size
+        self.ignore_index = config.ignore_index if hasattr(config, 'ignore_index') else -100
+        self.begin_of_context_token_id = config.begin_of_context_token_id
+        self.end_of_context_token_id = config.end_of_context_token_id
+        self.context_eos_token_id = config.context_config.eos_token_id
+        
+        self.post_init()
+    
+    def get_input_embeddings(self):
+        return self.language_model.get_input_embeddings()
+
+    def get_context_input_embeddings(self):
+        return self.context_tower.tower.get_input_embeddings()
+    
+    def set_input_embeddings(self, value):
+        self.language_model.set_input_embeddings(value)
+
+    def set_context_input_embeddings(self, value):
+        self.context_tower.tower.set_input_embeddings(value)
+
+    def get_output_embeddings(self):
+        return self.language_model.get_output_embeddings()
+    
+    def get_context_output_embeddings(self):
+        return self.context_tower.tower.get_output_embeddings()
+    
+    def set_output_embeddings(self, new_embeddings):
+        self.language_model.set_output_embeddings(new_embeddings)
+
+    def set_context_output_embeddings(self, new_embeddings):
+        self.context_tower.tower.set_output_embeddings(new_embeddings)
+    
+    def set_decoder(self, decoder):
+        self.language_model.set_decoder(decoder)
+
+    def set_context_encoder(self, decoder):
+        self.context_tower.tower.set_decoder(decoder)
+    
+    def get_decoder(self):
+        return self.language_model.get_decoder()
+
+    def get_context_encoder(self):
+        return self.context_tower.tower.get_decoder()
+    
+    def tie_weights(self):
+        return self.language_model.tie_weights()
+    
+    def context_tie_weights(self):
+        return self.context_tower.tower.tie_weights()
+    
+    def resize_token_embeddings(self, new_num_tokens: Optional[int] = None, pad_to_multiple_of=None) -> nn.Embedding:
+        model_embeds = self.language_model.resize_token_embeddings(new_num_tokens, pad_to_multiple_of)
+        # update vocab size
+        self.config.text_config.vocab_size = model_embeds.num_embeddings
+        self.vocab_size = model_embeds.num_embeddings
+        return model_embeds
+    
+    def _merge_context_features(
+        self,
+        context_features = None,
+        inputs_embeds = None,
+        attention_mask = None,
+        context_attention_mask=None,
+        position_ids=None,
+        labels=None,
+    ):
+        if context_features is None:
+            return inputs_embeds, attention_mask, position_ids, labels
+
+        batch_size, seq_length, embed_dim = inputs_embeds.shape
+        context_seq_len = context_features.size(1)
+        
+        # Create embeddings for begin and end of context tokens
+        begin_context_embed = self.get_input_embeddings()(torch.tensor(self.begin_of_context_token_id, device=context_features.device))
+        end_context_embed = self.get_input_embeddings()(torch.tensor(self.end_of_context_token_id, device=context_features.device))
+        
+        # Determine the actual lengths of context sequences (excluding padding)
+        if context_attention_mask is not None:
+            # context_attention_mask: [batch_size, context_seq_len, 1]
+            context_attention_mask = context_attention_mask.squeeze(-1)  # [batch_size, context_seq_len]
+            # Sum over sequence length to get actual lengths
+            context_lengths = context_attention_mask.sum(dim=1).long()  # [batch_size]
+        else:
+            # If no context_attention_mask is provided, assume full length
+            context_lengths = torch.full((batch_size,), context_seq_len, device=context_features.device, dtype=torch.long)
+            context_attention_mask = torch.ones(batch_size, context_seq_len, device=context_features.device, dtype=torch.long)
+        
+        # Rearrange context features to include padding at the beginning
+        # Identify the maximum context length (excluding padding)
+        max_context_length = context_lengths.max().item()
+        # Calculate the amount of padding needed for each sample
+        padding_lengths = context_seq_len - context_lengths  # [batch_size]
+        
+        # Create new context_features with padding at the beginning
+        new_context_features = []
+        for i in range(batch_size):
+            padding_len = padding_lengths[i].item()
+            # Create padding embeddings (zeros)
+            padding_embed = torch.zeros(padding_len, embed_dim, device=context_features.device, dtype=context_features.dtype)
+            # Get actual context features (excluding padding)
+            actual_context = context_features[i, padding_len:context_seq_len]
+            # Concatenate padding, begin token, actual context, end token
+            sample_context = torch.cat([
+                padding_embed,
+                begin_context_embed.unsqueeze(0),
+                actual_context,
+                end_context_embed.unsqueeze(0)
+            ], dim=0)  # [context_seq_len + 2, embed_dim]
+            new_context_features.append(sample_context)
+        # Stack to create [batch_size, new_context_seq_len, embed_dim]
+        context_features = torch.stack(new_context_features, dim=0)
+        new_context_seq_len = context_features.size(1)
+        
+        # Update context_attention_mask accordingly
+        new_context_attention_mask = []
+        for i in range(batch_size):
+            padding_len = padding_lengths[i].item()
+            # Create padding mask (zeros)
+            padding_mask = torch.zeros(padding_len, device=context_features.device, dtype=attention_mask.dtype)
+            # Begin and end token masks
+            begin_attention = torch.ones(1, device=context_features.device, dtype=attention_mask.dtype)
+            end_attention = torch.ones(1, device=context_features.device, dtype=attention_mask.dtype)
+            # Actual context attention mask (excluding padding)
+            actual_mask = context_attention_mask[i, padding_len:context_seq_len]
+            # Concatenate masks
+            sample_mask = torch.cat([
+                padding_mask,
+                begin_attention,
+                actual_mask,
+                end_attention
+            ], dim=0)  # [context_seq_len + 2]
+            new_context_attention_mask.append(sample_mask)
+        # Stack to create [batch_size, new_context_seq_len]
+        context_attention_mask = torch.stack(new_context_attention_mask, dim=0)
+        
+        # Concatenate context features with input embeddings
+        new_inputs_embeds = torch.cat([context_features, inputs_embeds], dim=1)  # [batch_size, total_seq_len, embed_dim]
+        
+        # Concatenate attention masks
+        new_attention_mask = torch.cat([context_attention_mask, attention_mask], dim=1)
+        
+        # Create new position_ids
+        total_seq_len = new_inputs_embeds.size(1)
+        new_position_ids = torch.arange(total_seq_len, device=inputs_embeds.device).unsqueeze(0).expand(batch_size, -1)
+        
+        # Update labels if provided
+        if labels is not None:
+            # Create ignore labels for context (including padding and special tokens)
+            context_labels = torch.full((batch_size, new_context_seq_len), self.ignore_index, device=labels.device, dtype=labels.dtype)
+            new_labels = torch.cat([context_labels, labels], dim=1)
+        else:
+            new_labels = None
+        
+        return new_inputs_embeds, new_attention_mask, new_position_ids, new_labels
+
+
+    @replace_return_docstrings(output_type=CoEncoderCausalLMOutputWithPast, config_class=_CONFIG_FOR_DOC)
+    def forward(
+        self,
+        context_input_ids: torch.LongTensor = None,
+        context_inputs_embeds: Optional[torch.FloatTensor] = None,
+        context_attention_mask: Optional[torch.Tensor] = None,
+        input_ids: torch.LongTensor = None,
+        inputs_embeds: Optional[torch.FloatTensor] = None,
+        attention_mask: Optional[torch.Tensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        past_key_values: Optional[List[torch.FloatTensor]] = None,
+        labels: Optional[torch.LongTensor] = None,
+        use_cache: Optional[bool] = None,
+        output_attentions: Optional[bool] = None,
+        output_hidden_states: Optional[bool] = None,
+        return_dict: Optional[bool] = None,
+    ) -> Union[Tuple, CoEncoderCausalLMOutputWithPast]:
+        """
+        Perform a forward pass through the CoEncoder model, optionally conditioning on context input.
+
+        Args:
+            context_input_ids (`torch.LongTensor` of shape `(batch_size, context_sequence_length)`, *optional*):
+                Token IDs of the context input sequence.
+            context_inputs_embeds (`torch.FloatTensor` of shape `(batch_size, context_sequence_length, hidden_size)`, *optional*):
+                Pre-computed context embeddings. If provided, will not compute embeddings from context_input_ids.
+            context_attention_mask (`torch.Tensor` of shape `(batch_size, context_sequence_length)`, *optional*):
+                Attention mask for context input sequence.
+            input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
+                Token IDs of the input sequence.
+            inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
+                Optionally, instead of passing `input_ids`, you can pass an embedded representation directly.
+            attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
+                Mask to avoid performing attention on padding token indices.
+            position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
+                Indices of positions of each input sequence token.
+            past_key_values (`List[torch.FloatTensor]`, *optional*):
+                Pre-computed hidden-states (key and value tensors) that can be used to speed up sequential decoding.
+            labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
+                Labels for computing the language modeling loss.
+            use_cache (`bool`, *optional*):
+                If `True`, past key values will be used to speed up decoding.
+            output_attentions (`bool`, *optional*):
+                If `True`, return the attention tensors for each layer.
+            output_hidden_states (`bool`, *optional*):
+                If `True`, return the hidden states of all layers.
+            return_dict (`bool`, *optional*):
+                If `True`, return a `CoEncoderCausalLMOutputWithPast` instead of a plain tuple.
+
+        Returns:
+            `Union[Tuple, CoEncoderCausalLMOutputWithPast]`: A tuple containing various model outputs or a `CoEncoderCausalLMOutputWithPast` instance.
+                The CoEncoderCausalLMOutputWithPast contains the following fields:
+                    - loss (`torch.FloatTensor`, *optional*): Language modeling loss if labels provided, None otherwise.
+                    - logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, vocab_size)`): Prediction scores.
+                    - past_key_values (`List[torch.FloatTensor]`, *optional*): Pre-computed hidden states for efficient decoding.
+                    - hidden_states (`Tuple[torch.FloatTensor]`, *optional*): Layer hidden states if output_hidden_states=True.
+                    - attentions (`Tuple[torch.FloatTensor]`, *optional*): Layer attention weights if output_attentions=True.
+                    - context_hidden_states (`torch.FloatTensor`, *optional*): Final hidden states from the context tower.
+        """
+
+        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
+        output_hidden_states = (
+            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+        )
+        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+
+        all_inputs_none = (
+            input_ids is None and 
+            inputs_embeds is None and 
+            context_input_ids is None and 
+            context_inputs_embeds is None
+        )
+        
+        if all_inputs_none:
+            raise ValueError("You must provide either non-empty input_ids/inputs_embeds or context_input_ids/context_inputs_embeds.")
+
+        skip_context = False
+        if context_input_ids is not None:
+            if torch.all(context_input_ids == self.context_eos_token_id):
+                skip_context = True
+
+        if not skip_context and (context_input_ids is not None or context_inputs_embeds is not None):
+            context_features = self.context_tower(
+                input_ids=context_input_ids,
+                inputs_embeds=context_inputs_embeds,
+                attention_mask=context_attention_mask,
+            )
+            context_features, context_attention_mask = self.connector(
+                context_features=context_features
+            )
+        else:
+            context_features = None
+            context_attention_mask = None
+
+
+        if inputs_embeds is None and input_ids is not None:
+            inputs_embeds = self.get_input_embeddings()(input_ids)
+
+        if inputs_embeds is not None:
+            inputs_embeds, attention_mask, position_ids, labels = self._merge_context_features(
+                context_features=context_features,
+                inputs_embeds=inputs_embeds,
+                attention_mask=attention_mask,
+                context_attention_mask=context_attention_mask,
+                position_ids=position_ids,
+                labels=labels,
+            )
+        else:
+            inputs_embeds = context_features
+            attention_mask = context_attention_mask
+
+        outputs = self.language_model(
+            attention_mask=attention_mask,
+            position_ids=position_ids,
+            past_key_values=past_key_values,
+            inputs_embeds=inputs_embeds,
+            use_cache=use_cache,
+            output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
+            return_dict=return_dict,
+        )
+
+        logits = outputs[0]
+
+        loss = None
+        if labels is not None:
+            shift_logits = logits[..., :-1, :].contiguous()
+            shift_labels = labels[..., 1:].contiguous()
+            loss_fct = nn.CrossEntropyLoss(ignore_index=self.ignore_index)
+            loss = loss_fct(shift_logits.view(-1, shift_logits.size(-1)), shift_labels.view(-1))
+
+        if not return_dict:
+            output = (logits,) + outputs[1:]
+            return (loss,) + output if loss is not None else output
+
+        return CoEncoderCausalLMOutputWithPast(
+            loss=loss,
+            logits=logits,
+            past_key_values=outputs.past_key_values,
+            hidden_states=outputs.hidden_states,
+            attentions=outputs.attentions,
+            context_hidden_states=context_features,
+        )
+
+    def prepare_inputs_for_generation(
+        self,
+        input_ids,
+        past_key_values=None,
+        attention_mask=None,
+        inputs_embeds=None,
+        context_features=None,
+        **kwargs
+    ):
+        if past_key_values:
+            input_ids = input_ids[:, -1:]
+
+        # if `inputs_embeds` are passed, we only want to use them in the 1st generation step
+        if inputs_embeds is not None and past_key_values is None:
+            model_inputs = {"inputs_embeds": inputs_embeds}
+        else:
+            model_inputs = {"input_ids": input_ids}
+
+        model_inputs.update(
+            {
+                "past_key_values": past_key_values,
+                "use_cache": kwargs.get("use_cache"),
+                "attention_mask": attention_mask,
+                "context_features": context_features,
+            }
+        )
+        return model_inputs

text_tokenizer/special_tokens_map.json

@@ -0,0 +1,16 @@
+{
+  "bos_token": {
+    "content": "<|begin_of_text|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|eot_id|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

text_tokenizer/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b9e4e7fb171f92fd137b777cc2714bf87d11576700a1dcd7a399e7bbe39537b
+size 17209920

text_tokenizer/tokenizer_config.json

@@ -0,0 +1,2063 @@
+{
+  "added_tokens_decoder": {
+    "128000": {
+      "content": "<|begin_of_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128001": {
+      "content": "<|end_of_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128002": {
+      "content": "<|reserved_special_token_0|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128003": {
+      "content": "<|reserved_special_token_1|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128004": {
+      "content": "<|finetune_right_pad_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128005": {
+      "content": "<|reserved_special_token_2|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128006": {
+      "content": "<|start_header_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128007": {
+      "content": "<|end_header_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128008": {
+      "content": "<|eom_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128009": {
+      "content": "<|eot_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128010": {
+      "content": "<|python_tag|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128011": {
+      "content": "<|reserved_special_token_3|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128012": {
+      "content": "<|reserved_special_token_4|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128013": {
+      "content": "<|reserved_special_token_5|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128014": {
+      "content": "<|reserved_special_token_6|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128015": {
+      "content": "<|reserved_special_token_7|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128016": {
+      "content": "<|reserved_special_token_8|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128017": {
+      "content": "<|reserved_special_token_9|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128018": {
+      "content": "<|reserved_special_token_10|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128019": {
+      "content": "<|reserved_special_token_11|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128020": {
+      "content": "<|reserved_special_token_12|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128021": {
+      "content": "<|reserved_special_token_13|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128022": {
+      "content": "<|reserved_special_token_14|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128023": {
+      "content": "<|reserved_special_token_15|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128024": {
+      "content": "<|reserved_special_token_16|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128025": {
+      "content": "<|reserved_special_token_17|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128026": {
+      "content": "<|reserved_special_token_18|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128027": {
+      "content": "<|reserved_special_token_19|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128028": {
+      "content": "<|reserved_special_token_20|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128029": {
+      "content": "<|reserved_special_token_21|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128030": {
+      "content": "<|reserved_special_token_22|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128031": {
+      "content": "<|reserved_special_token_23|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128032": {
+      "content": "<|reserved_special_token_24|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128033": {
+      "content": "<|reserved_special_token_25|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128034": {
+      "content": "<|reserved_special_token_26|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128035": {
+      "content": "<|reserved_special_token_27|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128036": {
+      "content": "<|reserved_special_token_28|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128037": {
+      "content": "<|reserved_special_token_29|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128038": {
+      "content": "<|reserved_special_token_30|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128039": {
+      "content": "<|reserved_special_token_31|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128040": {
+      "content": "<|reserved_special_token_32|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128041": {
+      "content": "<|reserved_special_token_33|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128042": {
+      "content": "<|reserved_special_token_34|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128043": {
+      "content": "<|reserved_special_token_35|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128044": {
+      "content": "<|reserved_special_token_36|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128045": {
+      "content": "<|reserved_special_token_37|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128046": {
+      "content": "<|reserved_special_token_38|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128047": {
+      "content": "<|reserved_special_token_39|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128048": {
+      "content": "<|reserved_special_token_40|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128049": {
+      "content": "<|reserved_special_token_41|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128050": {
+      "content": "<|reserved_special_token_42|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128051": {
+      "content": "<|reserved_special_token_43|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128052": {
+      "content": "<|reserved_special_token_44|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128053": {
+      "content": "<|reserved_special_token_45|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128054": {
+      "content": "<|reserved_special_token_46|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128055": {
+      "content": "<|reserved_special_token_47|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128056": {
+      "content": "<|reserved_special_token_48|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128057": {
+      "content": "<|reserved_special_token_49|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128058": {
+      "content": "<|reserved_special_token_50|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128059": {
+      "content": "<|reserved_special_token_51|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128060": {
+      "content": "<|reserved_special_token_52|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128061": {
+      "content": "<|reserved_special_token_53|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128062": {
+      "content": "<|reserved_special_token_54|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128063": {
+      "content": "<|reserved_special_token_55|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128064": {
+      "content": "<|reserved_special_token_56|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128065": {
+      "content": "<|reserved_special_token_57|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128066": {
+      "content": "<|reserved_special_token_58|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128067": {
+      "content": "<|reserved_special_token_59|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128068": {
+      "content": "<|reserved_special_token_60|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128069": {
+      "content": "<|reserved_special_token_61|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128070": {
+      "content": "<|reserved_special_token_62|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128071": {
+      "content": "<|reserved_special_token_63|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128072": {
+      "content": "<|reserved_special_token_64|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128073": {
+      "content": "<|reserved_special_token_65|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128074": {
+      "content": "<|reserved_special_token_66|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128075": {
+      "content": "<|reserved_special_token_67|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128076": {
+      "content": "<|reserved_special_token_68|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128077": {
+      "content": "<|reserved_special_token_69|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128078": {
+      "content": "<|reserved_special_token_70|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128079": {
+      "content": "<|reserved_special_token_71|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128080": {
+      "content": "<|reserved_special_token_72|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128081": {
+      "content": "<|reserved_special_token_73|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128082": {
+      "content": "<|reserved_special_token_74|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128083": {
+      "content": "<|reserved_special_token_75|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128084": {
+      "content": "<|reserved_special_token_76|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128085": {
+      "content": "<|reserved_special_token_77|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128086": {
+      "content": "<|reserved_special_token_78|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128087": {
+      "content": "<|reserved_special_token_79|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128088": {
+      "content": "<|reserved_special_token_80|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128089": {
+      "content": "<|reserved_special_token_81|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128090": {
+      "content": "<|reserved_special_token_82|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128091": {
+      "content": "<|reserved_special_token_83|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128092": {
+      "content": "<|reserved_special_token_84|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128093": {
+      "content": "<|reserved_special_token_85|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128094": {
+      "content": "<|reserved_special_token_86|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128095": {
+      "content": "<|reserved_special_token_87|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128096": {
+      "content": "<|reserved_special_token_88|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128097": {
+      "content": "<|reserved_special_token_89|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128098": {
+      "content": "<|reserved_special_token_90|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128099": {
+      "content": "<|reserved_special_token_91|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128100": {
+      "content": "<|reserved_special_token_92|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128101": {
+      "content": "<|reserved_special_token_93|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128102": {
+      "content": "<|reserved_special_token_94|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128103": {
+      "content": "<|reserved_special_token_95|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128104": {
+      "content": "<|reserved_special_token_96|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128105": {
+      "content": "<|reserved_special_token_97|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128106": {
+      "content": "<|reserved_special_token_98|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128107": {
+      "content": "<|reserved_special_token_99|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128108": {
+      "content": "<|reserved_special_token_100|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128109": {
+      "content": "<|reserved_special_token_101|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128110": {
+      "content": "<|reserved_special_token_102|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128111": {
+      "content": "<|reserved_special_token_103|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128112": {
+      "content": "<|reserved_special_token_104|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128113": {
+      "content": "<|reserved_special_token_105|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128114": {
+      "content": "<|reserved_special_token_106|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128115": {
+      "content": "<|reserved_special_token_107|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128116": {
+      "content": "<|reserved_special_token_108|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128117": {
+      "content": "<|reserved_special_token_109|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128118": {
+      "content": "<|reserved_special_token_110|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128119": {
+      "content": "<|reserved_special_token_111|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128120": {
+      "content": "<|reserved_special_token_112|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128121": {
+      "content": "<|reserved_special_token_113|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128122": {
+      "content": "<|reserved_special_token_114|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128123": {
+      "content": "<|reserved_special_token_115|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128124": {
+      "content": "<|reserved_special_token_116|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128125": {
+      "content": "<|reserved_special_token_117|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128126": {
+      "content": "<|reserved_special_token_118|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128127": {
+      "content": "<|reserved_special_token_119|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128128": {
+      "content": "<|reserved_special_token_120|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128129": {
+      "content": "<|reserved_special_token_121|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128130": {
+      "content": "<|reserved_special_token_122|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128131": {
+      "content": "<|reserved_special_token_123|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128132": {
+      "content": "<|reserved_special_token_124|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128133": {
+      "content": "<|reserved_special_token_125|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128134": {
+      "content": "<|reserved_special_token_126|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128135": {
+      "content": "<|reserved_special_token_127|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128136": {
+      "content": "<|reserved_special_token_128|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128137": {
+      "content": "<|reserved_special_token_129|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128138": {
+      "content": "<|reserved_special_token_130|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128139": {
+      "content": "<|reserved_special_token_131|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128140": {
+      "content": "<|reserved_special_token_132|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128141": {
+      "content": "<|reserved_special_token_133|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128142": {
+      "content": "<|reserved_special_token_134|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128143": {
+      "content": "<|reserved_special_token_135|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128144": {
+      "content": "<|reserved_special_token_136|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128145": {
+      "content": "<|reserved_special_token_137|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128146": {
+      "content": "<|reserved_special_token_138|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128147": {
+      "content": "<|reserved_special_token_139|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128148": {
+      "content": "<|reserved_special_token_140|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128149": {
+      "content": "<|reserved_special_token_141|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128150": {
+      "content": "<|reserved_special_token_142|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128151": {
+      "content": "<|reserved_special_token_143|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128152": {
+      "content": "<|reserved_special_token_144|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128153": {
+      "content": "<|reserved_special_token_145|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128154": {
+      "content": "<|reserved_special_token_146|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128155": {
+      "content": "<|reserved_special_token_147|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128156": {
+      "content": "<|reserved_special_token_148|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128157": {
+      "content": "<|reserved_special_token_149|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128158": {
+      "content": "<|reserved_special_token_150|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128159": {
+      "content": "<|reserved_special_token_151|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128160": {
+      "content": "<|reserved_special_token_152|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128161": {
+      "content": "<|reserved_special_token_153|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128162": {
+      "content": "<|reserved_special_token_154|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128163": {
+      "content": "<|reserved_special_token_155|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128164": {
+      "content": "<|reserved_special_token_156|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128165": {
+      "content": "<|reserved_special_token_157|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128166": {
+      "content": "<|reserved_special_token_158|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128167": {
+      "content": "<|reserved_special_token_159|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128168": {
+      "content": "<|reserved_special_token_160|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128169": {
+      "content": "<|reserved_special_token_161|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128170": {
+      "content": "<|reserved_special_token_162|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128171": {
+      "content": "<|reserved_special_token_163|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128172": {
+      "content": "<|reserved_special_token_164|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128173": {
+      "content": "<|reserved_special_token_165|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128174": {
+      "content": "<|reserved_special_token_166|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128175": {
+      "content": "<|reserved_special_token_167|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128176": {
+      "content": "<|reserved_special_token_168|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128177": {
+      "content": "<|reserved_special_token_169|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128178": {
+      "content": "<|reserved_special_token_170|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128179": {
+      "content": "<|reserved_special_token_171|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128180": {
+      "content": "<|reserved_special_token_172|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128181": {
+      "content": "<|reserved_special_token_173|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128182": {
+      "content": "<|reserved_special_token_174|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128183": {
+      "content": "<|reserved_special_token_175|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128184": {
+      "content": "<|reserved_special_token_176|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128185": {
+      "content": "<|reserved_special_token_177|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128186": {
+      "content": "<|reserved_special_token_178|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128187": {
+      "content": "<|reserved_special_token_179|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128188": {
+      "content": "<|reserved_special_token_180|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128189": {
+      "content": "<|reserved_special_token_181|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128190": {
+      "content": "<|reserved_special_token_182|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128191": {
+      "content": "<|reserved_special_token_183|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128192": {
+      "content": "<|reserved_special_token_184|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128193": {
+      "content": "<|reserved_special_token_185|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128194": {
+      "content": "<|reserved_special_token_186|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128195": {
+      "content": "<|reserved_special_token_187|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128196": {
+      "content": "<|reserved_special_token_188|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128197": {
+      "content": "<|reserved_special_token_189|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128198": {
+      "content": "<|reserved_special_token_190|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128199": {
+      "content": "<|reserved_special_token_191|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128200": {
+      "content": "<|reserved_special_token_192|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128201": {
+      "content": "<|reserved_special_token_193|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128202": {
+      "content": "<|reserved_special_token_194|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128203": {
+      "content": "<|reserved_special_token_195|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128204": {
+      "content": "<|reserved_special_token_196|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128205": {
+      "content": "<|reserved_special_token_197|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128206": {
+      "content": "<|reserved_special_token_198|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128207": {
+      "content": "<|reserved_special_token_199|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128208": {
+      "content": "<|reserved_special_token_200|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128209": {
+      "content": "<|reserved_special_token_201|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128210": {
+      "content": "<|reserved_special_token_202|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128211": {
+      "content": "<|reserved_special_token_203|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128212": {
+      "content": "<|reserved_special_token_204|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128213": {
+      "content": "<|reserved_special_token_205|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128214": {
+      "content": "<|reserved_special_token_206|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128215": {
+      "content": "<|reserved_special_token_207|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128216": {
+      "content": "<|reserved_special_token_208|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128217": {
+      "content": "<|reserved_special_token_209|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128218": {
+      "content": "<|reserved_special_token_210|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128219": {
+      "content": "<|reserved_special_token_211|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128220": {
+      "content": "<|reserved_special_token_212|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128221": {
+      "content": "<|reserved_special_token_213|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128222": {
+      "content": "<|reserved_special_token_214|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128223": {
+      "content": "<|reserved_special_token_215|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128224": {
+      "content": "<|reserved_special_token_216|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128225": {
+      "content": "<|reserved_special_token_217|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128226": {
+      "content": "<|reserved_special_token_218|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128227": {
+      "content": "<|reserved_special_token_219|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128228": {
+      "content": "<|reserved_special_token_220|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128229": {
+      "content": "<|reserved_special_token_221|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128230": {
+      "content": "<|reserved_special_token_222|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128231": {
+      "content": "<|reserved_special_token_223|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128232": {
+      "content": "<|reserved_special_token_224|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128233": {
+      "content": "<|reserved_special_token_225|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128234": {
+      "content": "<|reserved_special_token_226|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128235": {
+      "content": "<|reserved_special_token_227|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128236": {
+      "content": "<|reserved_special_token_228|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128237": {
+      "content": "<|reserved_special_token_229|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128238": {
+      "content": "<|reserved_special_token_230|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128239": {
+      "content": "<|reserved_special_token_231|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128240": {
+      "content": "<|reserved_special_token_232|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128241": {
+      "content": "<|reserved_special_token_233|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128242": {
+      "content": "<|reserved_special_token_234|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128243": {
+      "content": "<|reserved_special_token_235|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128244": {
+      "content": "<|reserved_special_token_236|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128245": {
+      "content": "<|reserved_special_token_237|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128246": {
+      "content": "<|reserved_special_token_238|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128247": {
+      "content": "<|reserved_special_token_239|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128248": {
+      "content": "<|reserved_special_token_240|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128249": {
+      "content": "<|reserved_special_token_241|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128250": {
+      "content": "<|reserved_special_token_242|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128251": {
+      "content": "<|reserved_special_token_243|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128252": {
+      "content": "<|reserved_special_token_244|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128253": {
+      "content": "<|reserved_special_token_245|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128254": {
+      "content": "<|reserved_special_token_246|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128255": {
+      "content": "<|reserved_special_token_247|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|begin_of_text|>",
+  "chat_template": "{{- bos_token }}\n{%- if custom_tools is defined %}\n    {%- set tools = custom_tools %}\n{%- endif %}\n{%- if not tools_in_user_message is defined %}\n    {%- set tools_in_user_message = true %}\n{%- endif %}\n{%- if not date_string is defined %}\n    {%- set date_string = \"26 Jul 2024\" %}\n{%- endif %}\n{%- if not tools is defined %}\n    {%- set tools = none %}\n{%- endif %}\n\n{#- This block extracts the system message, so we can slot it into the right place. #}\n{%- if messages[0]['role'] == 'system' %}\n    {%- set system_message = messages[0]['content']|trim %}\n    {%- set messages = messages[1:] %}\n{%- else %}\n    {%- set system_message = \"\" %}\n{%- endif %}\n\n{#- System message + builtin tools #}\n{{- \"<|start_header_id|>system<|end_header_id|>\\n\\n\" }}\n{%- if builtin_tools is defined or tools is not none %}\n    {{- \"Environment: ipython\\n\" }}\n{%- endif %}\n{%- if builtin_tools is defined %}\n    {{- \"Tools: \" + builtin_tools | reject('equalto', 'code_interpreter') | join(\", \") + \"\\n\\n\"}}\n{%- endif %}\n{{- \"Cutting Knowledge Date: December 2023\\n\" }}\n{{- \"Today Date: \" + date_string + \"\\n\\n\" }}\n{%- if tools is not none and not tools_in_user_message %}\n    {{- \"You have access to the following functions. To call a function, please respond with JSON for a function call.\" }}\n    {{- 'Respond in the format {\"name\": function name, \"parameters\": dictionary of argument name and its value}.' }}\n    {{- \"Do not use variables.\\n\\n\" }}\n    {%- for t in tools %}\n        {{- t | tojson(indent=4) }}\n        {{- \"\\n\\n\" }}\n    {%- endfor %}\n{%- endif %}\n{{- system_message }}\n{{- \"<|eot_id|>\" }}\n\n{#- Custom tools are passed in a user message with some extra guidance #}\n{%- if tools_in_user_message and not tools is none %}\n    {#- Extract the first user message so we can plug it in here #}\n    {%- if messages | length != 0 %}\n        {%- set first_user_message = messages[0]['content']|trim %}\n        {%- set messages = messages[1:] %}\n    {%- else %}\n        {{- raise_exception(\"Cannot put tools in the first user message when there's no first user message!\") }}\n{%- endif %}\n    {{- '<|start_header_id|>user<|end_header_id|>\\n\\n' -}}\n    {{- \"Given the following functions, please respond with a JSON for a function call \" }}\n    {{- \"with its proper arguments that best answers the given prompt.\\n\\n\" }}\n    {{- 'Respond in the format {\"name\": function name, \"parameters\": dictionary of argument name and its value}.' }}\n    {{- \"Do not use variables.\\n\\n\" }}\n    {%- for t in tools %}\n        {{- t | tojson(indent=4) }}\n        {{- \"\\n\\n\" }}\n    {%- endfor %}\n    {{- first_user_message + \"<|eot_id|>\"}}\n{%- endif %}\n\n{%- for message in messages %}\n    {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}\n        {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\\n\\n'+ message['content'] | trim + '<|eot_id|>' }}\n    {%- elif 'tool_calls' in message %}\n        {%- if not message.tool_calls|length == 1 %}\n            {{- raise_exception(\"This model only supports single tool-calls at once!\") }}\n        {%- endif %}\n        {%- set tool_call = message.tool_calls[0].function %}\n        {%- if builtin_tools is defined and tool_call.name in builtin_tools %}\n            {{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' -}}\n            {{- \"<|python_tag|>\" + tool_call.name + \".call(\" }}\n            {%- for arg_name, arg_val in tool_call.arguments | items %}\n                {{- arg_name + '=\"' + arg_val + '\"' }}\n                {%- if not loop.last %}\n                    {{- \", \" }}\n                {%- endif %}\n                {%- endfor %}\n            {{- \")\" }}\n        {%- else  %}\n            {{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' -}}\n            {{- '{\"name\": \"' + tool_call.name + '\", ' }}\n            {{- '\"parameters\": ' }}\n            {{- tool_call.arguments | tojson }}\n            {{- \"}\" }}\n        {%- endif %}\n        {%- if builtin_tools is defined %}\n            {#- This means we're in ipython mode #}\n            {{- \"<|eom_id|>\" }}\n        {%- else %}\n            {{- \"<|eot_id|>\" }}\n        {%- endif %}\n    {%- elif message.role == \"tool\" or message.role == \"ipython\" %}\n        {{- \"<|start_header_id|>ipython<|end_header_id|>\\n\\n\" }}\n        {%- if message.content is mapping or message.content is iterable %}\n            {{- message.content | tojson }}\n        {%- else %}\n            {{- message.content }}\n        {%- endif %}\n        {{- \"<|eot_id|>\" }}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' }}\n{%- endif %}\n",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|eot_id|>",
+  "extra_special_tokens": {},
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 131072,
+  "tokenizer_class": "PreTrainedTokenizerFast"
+}

tokenizer_config.json

@@ -0,0 +1 @@
+{"tokenizer_class": "CoEncoderDualTokenizer"}