Upload folder using huggingface_hub

config.json

@@ -0,0 +1,50 @@
+{
+  "_name_or_path": "microsoft/Phi-3-small-8k-instruct",
+  "architectures": [
+    "Phi3SmallForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout_prob": 0.0,
+  "auto_map": {
+    "AutoConfig": "microsoft/Phi-3-small-8k-instruct--configuration_phi3_small.Phi3SmallConfig",
+    "AutoModelForCausalLM": "microsoft/Phi-3-small-8k-instruct--modeling_phi3_small.Phi3SmallForCausalLM",
+    "AutoModelForSequenceClassification": "microsoft/Phi-3-small-8k-instruct--modeling_phi3_small.Phi3SmallForSequenceClassification",
+    "AutoTokenizer": "microsoft/Phi-3-small-8k-instruct--tokenization_phi3_small.Phi3SmallTokenizer"
+  },
+  "blocksparse_block_size": 64,
+  "blocksparse_homo_head_pattern": false,
+  "blocksparse_num_local_blocks": 16,
+  "blocksparse_triton_kernel_block_size": 64,
+  "blocksparse_vert_stride": 8,
+  "bos_token_id": 100257,
+  "dense_attention_every_n_layers": 2,
+  "embedding_dropout_prob": 0.1,
+  "eos_token_id": 100257,
+  "ff_dim_multiplier": null,
+  "ff_intermediate_size": 14336,
+  "ffn_dropout_prob": 0.1,
+  "gegelu_limit": 20.0,
+  "gegelu_pad_to_256": true,
+  "hidden_act": "gegelu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "layer_norm_epsilon": 1e-05,
+  "max_position_embeddings": 8192,
+  "model_type": "phi3small",
+  "mup_attn_multiplier": 1.0,
+  "mup_embedding_multiplier": 10.0,
+  "mup_use_scaling": true,
+  "mup_width_multiplier": 8.0,
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "pad_sequence_to_multiple_of_64": true,
+  "reorder_and_upcast_attn": false,
+  "rope_embedding_base": 1000000,
+  "rope_position_scale": 1.0,
+  "rope_scaling": null,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.44.2",
+  "use_cache": true,
+  "vocab_size": 100352
+}

mergekit_config.yml

@@ -0,0 +1,17 @@
+
+slices:
+  - sources:
+      - model: microsoft/Phi-3-small-128k-instruct
+        layer_range: [0, 32]
+      - model: microsoft/Phi-3-small-8k-instruct
+        layer_range: [0, 32]
+merge_method: slerp
+base_model: microsoft/Phi-3-small-8k-instruct
+parameters:
+  t:
+    - filter: self_attn
+      value: [0, 0.5, 0.3, 0.7, 1]
+    - filter: mlp
+      value: [1, 0.5, 0.7, 0.3, 0]
+    - value: 0.5
+dtype: bfloat16

model-00001-of-00018.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e1a23c9ab1479aa4ac2209ed8adad413250eeba52cf71e83c0d285b36a68ddb0
+size 822083712

model-00002-of-00018.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0316ae332d0c74392d6c8778d992dc3be5894691d6ebd359656a751735f87db1
+size 939623272

model-00003-of-00018.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:250009aa3cbca5fb4b4252d2bbb96c7eb042e1c393ee6cbba4cc0c970ccfc19a
+size 872655528

model-00004-of-00018.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a95de6e95e049b18aa68c0f6a8ffcd5c57cb0720be28d05fad2fce143cfb1367
+size 872655552

model-00005-of-00018.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6e3f3d21d7fe20cb093f1555f42f5f19e7f57ae5e9aaf25b611ab00dae7a6866
+size 872655552

model-00006-of-00018.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d5fcbe4fa88a4808800453c0e6a9c86e362b44ebbd5a39b58664a61a2b8d0de0
+size 872655552

model-00007-of-00018.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8447f677ff87985493161182632c08320805926ff443bc57e6ef16631a1041a8
+size 872655552

model-00008-of-00018.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:38e083ff5b2a0f5466acf16f6fb7f780e3bba083b41429d7fe438807a876e2c9
+size 872655544

model-00009-of-00018.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:34c1996df50d560a5da66e6dc6feb4e4b35dd104a8d2ba046544773a246431ae
+size 872655544

model-00010-of-00018.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5c414d2b47f21d59e3fd9ba7b2659e96d4be78cb5775d31bc125aa43059d13a8
+size 872655552

model-00011-of-00018.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:71822fb744c92fea2b9320528fce99fb79656c8902dbb4a88b2725e8f307e8f0
+size 872655552

model-00012-of-00018.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dcbe64e88193e628a22ea9e6d260cebbd1f1e249526b0418122872e34659d207
+size 872655552

model-00013-of-00018.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:481a436447f7e3b4377b2a67ee3a6616585e897be216fd55ed1ab6b00616cfd4
+size 872655552

model-00014-of-00018.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:66589d4ec8918eb3c23d1d52955851e51ae48276b8d2f1aea37d59dfb0b34810
+size 872655536

model-00015-of-00018.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c2c698dd98fef7b628339fc02964ae766c689e7841da69c6e03e6c35cadf88c2
+size 872655544

model-00016-of-00018.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c07a0132ffad6f513b74f0bf95d0457646495326dcb4c94bbffea79b187117e4
+size 872655528

model-00017-of-00018.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ecbf8e9ef6bff52fb4a352c29df35f6a57b833802978aa791b754379c77df454
+size 872655528

model-00018-of-00018.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4f1f8f6efe2337ebbb99f8664160552c675d65edd4f891b3ebdc28788f989003
+size 755132552

model.safetensors.index.json

@@ -0,0 +1 @@
+{"metadata": {"mergekit_version": "0.0.4.4", "total_size": 15606628352}, "weight_map": {"lm_head.weight": "model-00001-of-00018.safetensors", "model.embed_tokens.weight": "model-00002-of-00018.safetensors", "model.final_layernorm.bias": "model-00002-of-00018.safetensors", "model.final_layernorm.weight": "model-00002-of-00018.safetensors", "model.layers.0.input_layernorm.bias": "model-00002-of-00018.safetensors", "model.layers.0.input_layernorm.weight": "model-00002-of-00018.safetensors", "model.layers.0.mlp.down_proj.bias": "model-00002-of-00018.safetensors", "model.layers.0.mlp.down_proj.weight": "model-00002-of-00018.safetensors", "model.layers.0.mlp.up_proj.bias": "model-00002-of-00018.safetensors", "model.layers.0.mlp.up_proj.weight": "model-00003-of-00018.safetensors", "model.layers.0.post_attention_layernorm.bias": "model-00003-of-00018.safetensors", "model.layers.0.post_attention_layernorm.weight": "model-00003-of-00018.safetensors", "model.layers.0.self_attn.dense.bias": "model-00003-of-00018.safetensors", "model.layers.0.self_attn.dense.weight": "model-00003-of-00018.safetensors", "model.layers.0.self_attn.query_key_value.bias": "model-00003-of-00018.safetensors", "model.layers.0.self_attn.query_key_value.weight": "model-00003-of-00018.safetensors", "model.layers.1.input_layernorm.bias": "model-00003-of-00018.safetensors", "model.layers.1.input_layernorm.weight": "model-00003-of-00018.safetensors", "model.layers.1.mlp.down_proj.bias": "model-00003-of-00018.safetensors", "model.layers.1.mlp.down_proj.weight": "model-00003-of-00018.safetensors", "model.layers.1.mlp.up_proj.bias": "model-00003-of-00018.safetensors", "model.layers.1.mlp.up_proj.weight": "model-00003-of-00018.safetensors", "model.layers.1.post_attention_layernorm.bias": "model-00003-of-00018.safetensors", "model.layers.1.post_attention_layernorm.weight": "model-00003-of-00018.safetensors", "model.layers.1.self_attn.dense.bias": "model-00003-of-00018.safetensors", "model.layers.1.self_attn.dense.weight": "model-00003-of-00018.safetensors", "model.layers.1.self_attn.query_key_value.bias": "model-00003-of-00018.safetensors", "model.layers.1.self_attn.query_key_value.weight": "model-00003-of-00018.safetensors", "model.layers.10.input_layernorm.bias": "model-00003-of-00018.safetensors", "model.layers.10.input_layernorm.weight": "model-00003-of-00018.safetensors", "model.layers.10.mlp.down_proj.bias": "model-00003-of-00018.safetensors", "model.layers.10.mlp.down_proj.weight": "model-00003-of-00018.safetensors", "model.layers.10.mlp.up_proj.bias": "model-00003-of-00018.safetensors", "model.layers.10.mlp.up_proj.weight": "model-00004-of-00018.safetensors", "model.layers.10.post_attention_layernorm.bias": "model-00004-of-00018.safetensors", "model.layers.10.post_attention_layernorm.weight": "model-00004-of-00018.safetensors", "model.layers.10.self_attn.dense.bias": "model-00004-of-00018.safetensors", "model.layers.10.self_attn.dense.weight": "model-00004-of-00018.safetensors", "model.layers.10.self_attn.query_key_value.bias": "model-00004-of-00018.safetensors", "model.layers.10.self_attn.query_key_value.weight": "model-00004-of-00018.safetensors", "model.layers.11.input_layernorm.bias": "model-00004-of-00018.safetensors", "model.layers.11.input_layernorm.weight": "model-00004-of-00018.safetensors", "model.layers.11.mlp.down_proj.bias": "model-00004-of-00018.safetensors", "model.layers.11.mlp.down_proj.weight": "model-00004-of-00018.safetensors", "model.layers.11.mlp.up_proj.bias": "model-00004-of-00018.safetensors", "model.layers.11.mlp.up_proj.weight": "model-00004-of-00018.safetensors", "model.layers.11.post_attention_layernorm.bias": "model-00004-of-00018.safetensors", "model.layers.11.post_attention_layernorm.weight": "model-00004-of-00018.safetensors", "model.layers.11.self_attn.dense.bias": "model-00004-of-00018.safetensors", "model.layers.11.self_attn.dense.weight": "model-00004-of-00018.safetensors", "model.layers.11.self_attn.query_key_value.bias": "model-00004-of-00018.safetensors", "model.layers.11.self_attn.query_key_value.weight": "model-00004-of-00018.safetensors", "model.layers.12.input_layernorm.bias": "model-00004-of-00018.safetensors", "model.layers.12.input_layernorm.weight": "model-00004-of-00018.safetensors", "model.layers.12.mlp.down_proj.bias": "model-00004-of-00018.safetensors", "model.layers.12.mlp.down_proj.weight": "model-00004-of-00018.safetensors", "model.layers.12.mlp.up_proj.bias": "model-00004-of-00018.safetensors", "model.layers.12.mlp.up_proj.weight": "model-00005-of-00018.safetensors", "model.layers.12.post_attention_layernorm.bias": "model-00005-of-00018.safetensors", "model.layers.12.post_attention_layernorm.weight": "model-00005-of-00018.safetensors", "model.layers.12.self_attn.dense.bias": "model-00005-of-00018.safetensors", "model.layers.12.self_attn.dense.weight": "model-00005-of-00018.safetensors", "model.layers.12.self_attn.query_key_value.bias": "model-00005-of-00018.safetensors", "model.layers.12.self_attn.query_key_value.weight": "model-00005-of-00018.safetensors", "model.layers.13.input_layernorm.bias": "model-00005-of-00018.safetensors", "model.layers.13.input_layernorm.weight": "model-00005-of-00018.safetensors", "model.layers.13.mlp.down_proj.bias": "model-00005-of-00018.safetensors", "model.layers.13.mlp.down_proj.weight": "model-00005-of-00018.safetensors", "model.layers.13.mlp.up_proj.bias": "model-00005-of-00018.safetensors", "model.layers.13.mlp.up_proj.weight": "model-00005-of-00018.safetensors", "model.layers.13.post_attention_layernorm.bias": "model-00005-of-00018.safetensors", "model.layers.13.post_attention_layernorm.weight": "model-00005-of-00018.safetensors", "model.layers.13.self_attn.dense.bias": "model-00005-of-00018.safetensors", "model.layers.13.self_attn.dense.weight": "model-00005-of-00018.safetensors", "model.layers.13.self_attn.query_key_value.bias": "model-00005-of-00018.safetensors", "model.layers.13.self_attn.query_key_value.weight": "model-00005-of-00018.safetensors", "model.layers.14.input_layernorm.bias": "model-00005-of-00018.safetensors", "model.layers.14.input_layernorm.weight": "model-00005-of-00018.safetensors", "model.layers.14.mlp.down_proj.bias": "model-00005-of-00018.safetensors", "model.layers.14.mlp.down_proj.weight": "model-00005-of-00018.safetensors", "model.layers.14.mlp.up_proj.bias": "model-00005-of-00018.safetensors", "model.layers.14.mlp.up_proj.weight": "model-00006-of-00018.safetensors", "model.layers.14.post_attention_layernorm.bias": "model-00006-of-00018.safetensors", "model.layers.14.post_attention_layernorm.weight": "model-00006-of-00018.safetensors", "model.layers.14.self_attn.dense.bias": "model-00006-of-00018.safetensors", "model.layers.14.self_attn.dense.weight": "model-00006-of-00018.safetensors", "model.layers.14.self_attn.query_key_value.bias": "model-00006-of-00018.safetensors", "model.layers.14.self_attn.query_key_value.weight": "model-00006-of-00018.safetensors", "model.layers.15.input_layernorm.bias": "model-00006-of-00018.safetensors", "model.layers.15.input_layernorm.weight": "model-00006-of-00018.safetensors", "model.layers.15.mlp.down_proj.bias": "model-00006-of-00018.safetensors", "model.layers.15.mlp.down_proj.weight": "model-00006-of-00018.safetensors", "model.layers.15.mlp.up_proj.bias": "model-00006-of-00018.safetensors", "model.layers.15.mlp.up_proj.weight": "model-00006-of-00018.safetensors", "model.layers.15.post_attention_layernorm.bias": "model-00006-of-00018.safetensors", "model.layers.15.post_attention_layernorm.weight": "model-00006-of-00018.safetensors", "model.layers.15.self_attn.dense.bias": "model-00006-of-00018.safetensors", "model.layers.15.self_attn.dense.weight": "model-00006-of-00018.safetensors", "model.layers.15.self_attn.query_key_value.bias": "model-00006-of-00018.safetensors", "model.layers.15.self_attn.query_key_value.weight": "model-00006-of-00018.safetensors", "model.layers.16.input_layernorm.bias": "model-00006-of-00018.safetensors", "model.layers.16.input_layernorm.weight": "model-00006-of-00018.safetensors", "model.layers.16.mlp.down_proj.bias": "model-00006-of-00018.safetensors", "model.layers.16.mlp.down_proj.weight": "model-00006-of-00018.safetensors", "model.layers.16.mlp.up_proj.bias": "model-00006-of-00018.safetensors", "model.layers.16.mlp.up_proj.weight": "model-00007-of-00018.safetensors", "model.layers.16.post_attention_layernorm.bias": "model-00007-of-00018.safetensors", "model.layers.16.post_attention_layernorm.weight": "model-00007-of-00018.safetensors", "model.layers.16.self_attn.dense.bias": "model-00007-of-00018.safetensors", "model.layers.16.self_attn.dense.weight": "model-00007-of-00018.safetensors", "model.layers.16.self_attn.query_key_value.bias": "model-00007-of-00018.safetensors", "model.layers.16.self_attn.query_key_value.weight": "model-00007-of-00018.safetensors", "model.layers.17.input_layernorm.bias": "model-00007-of-00018.safetensors", "model.layers.17.input_layernorm.weight": "model-00007-of-00018.safetensors", "model.layers.17.mlp.down_proj.bias": "model-00007-of-00018.safetensors", "model.layers.17.mlp.down_proj.weight": "model-00007-of-00018.safetensors", "model.layers.17.mlp.up_proj.bias": "model-00007-of-00018.safetensors", "model.layers.17.mlp.up_proj.weight": "model-00007-of-00018.safetensors", "model.layers.17.post_attention_layernorm.bias": "model-00007-of-00018.safetensors", "model.layers.17.post_attention_layernorm.weight": "model-00007-of-00018.safetensors", "model.layers.17.self_attn.dense.bias": "model-00007-of-00018.safetensors", "model.layers.17.self_attn.dense.weight": "model-00007-of-00018.safetensors", "model.layers.17.self_attn.query_key_value.bias": "model-00007-of-00018.safetensors", "model.layers.17.self_attn.query_key_value.weight": "model-00007-of-00018.safetensors", "model.layers.18.input_layernorm.bias": "model-00007-of-00018.safetensors", "model.layers.18.input_layernorm.weight": "model-00007-of-00018.safetensors", "model.layers.18.mlp.down_proj.bias": "model-00007-of-00018.safetensors", "model.layers.18.mlp.down_proj.weight": "model-00007-of-00018.safetensors", "model.layers.18.mlp.up_proj.bias": "model-00007-of-00018.safetensors", "model.layers.18.mlp.up_proj.weight": "model-00008-of-00018.safetensors", "model.layers.18.post_attention_layernorm.bias": "model-00008-of-00018.safetensors", "model.layers.18.post_attention_layernorm.weight": "model-00008-of-00018.safetensors", "model.layers.18.self_attn.dense.bias": "model-00008-of-00018.safetensors", "model.layers.18.self_attn.dense.weight": "model-00008-of-00018.safetensors", "model.layers.18.self_attn.query_key_value.bias": "model-00008-of-00018.safetensors", "model.layers.18.self_attn.query_key_value.weight": "model-00008-of-00018.safetensors", "model.layers.19.input_layernorm.bias": "model-00008-of-00018.safetensors", "model.layers.19.input_layernorm.weight": "model-00008-of-00018.safetensors", "model.layers.19.mlp.down_proj.bias": "model-00008-of-00018.safetensors", "model.layers.19.mlp.down_proj.weight": "model-00008-of-00018.safetensors", "model.layers.19.mlp.up_proj.bias": "model-00008-of-00018.safetensors", "model.layers.19.mlp.up_proj.weight": "model-00008-of-00018.safetensors", "model.layers.19.post_attention_layernorm.bias": "model-00008-of-00018.safetensors", "model.layers.19.post_attention_layernorm.weight": "model-00008-of-00018.safetensors", "model.layers.19.self_attn.dense.bias": "model-00008-of-00018.safetensors", "model.layers.19.self_attn.dense.weight": "model-00008-of-00018.safetensors", "model.layers.19.self_attn.query_key_value.bias": "model-00008-of-00018.safetensors", "model.layers.19.self_attn.query_key_value.weight": "model-00008-of-00018.safetensors", "model.layers.2.input_layernorm.bias": "model-00008-of-00018.safetensors", "model.layers.2.input_layernorm.weight": "model-00008-of-00018.safetensors", "model.layers.2.mlp.down_proj.bias": "model-00008-of-00018.safetensors", "model.layers.2.mlp.down_proj.weight": "model-00008-of-00018.safetensors", "model.layers.2.mlp.up_proj.bias": "model-00008-of-00018.safetensors", "model.layers.2.mlp.up_proj.weight": "model-00009-of-00018.safetensors", "model.layers.2.post_attention_layernorm.bias": "model-00009-of-00018.safetensors", "model.layers.2.post_attention_layernorm.weight": "model-00009-of-00018.safetensors", "model.layers.2.self_attn.dense.bias": "model-00009-of-00018.safetensors", "model.layers.2.self_attn.dense.weight": "model-00009-of-00018.safetensors", "model.layers.2.self_attn.query_key_value.bias": "model-00009-of-00018.safetensors", "model.layers.2.self_attn.query_key_value.weight": "model-00009-of-00018.safetensors", "model.layers.20.input_layernorm.bias": "model-00009-of-00018.safetensors", "model.layers.20.input_layernorm.weight": "model-00009-of-00018.safetensors", "model.layers.20.mlp.down_proj.bias": "model-00009-of-00018.safetensors", "model.layers.20.mlp.down_proj.weight": "model-00009-of-00018.safetensors", "model.layers.20.mlp.up_proj.bias": "model-00009-of-00018.safetensors", "model.layers.20.mlp.up_proj.weight": "model-00009-of-00018.safetensors", "model.layers.20.post_attention_layernorm.bias": "model-00009-of-00018.safetensors", "model.layers.20.post_attention_layernorm.weight": "model-00009-of-00018.safetensors", "model.layers.20.self_attn.dense.bias": "model-00009-of-00018.safetensors", "model.layers.20.self_attn.dense.weight": "model-00009-of-00018.safetensors", "model.layers.20.self_attn.query_key_value.bias": "model-00009-of-00018.safetensors", "model.layers.20.self_attn.query_key_value.weight": "model-00009-of-00018.safetensors", "model.layers.21.input_layernorm.bias": "model-00009-of-00018.safetensors", "model.layers.21.input_layernorm.weight": "model-00009-of-00018.safetensors", "model.layers.21.mlp.down_proj.bias": "model-00009-of-00018.safetensors", "model.layers.21.mlp.down_proj.weight": "model-00009-of-00018.safetensors", "model.layers.21.mlp.up_proj.bias": "model-00009-of-00018.safetensors", "model.layers.21.mlp.up_proj.weight": "model-00010-of-00018.safetensors", "model.layers.21.post_attention_layernorm.bias": "model-00010-of-00018.safetensors", "model.layers.21.post_attention_layernorm.weight": "model-00010-of-00018.safetensors", "model.layers.21.self_attn.dense.bias": "model-00010-of-00018.safetensors", "model.layers.21.self_attn.dense.weight": "model-00010-of-00018.safetensors", "model.layers.21.self_attn.query_key_value.bias": "model-00010-of-00018.safetensors", "model.layers.21.self_attn.query_key_value.weight": "model-00010-of-00018.safetensors", "model.layers.22.input_layernorm.bias": "model-00010-of-00018.safetensors", "model.layers.22.input_layernorm.weight": "model-00010-of-00018.safetensors", "model.layers.22.mlp.down_proj.bias": "model-00010-of-00018.safetensors", "model.layers.22.mlp.down_proj.weight": "model-00010-of-00018.safetensors", "model.layers.22.mlp.up_proj.bias": "model-00010-of-00018.safetensors", "model.layers.22.mlp.up_proj.weight": "model-00010-of-00018.safetensors", "model.layers.22.post_attention_layernorm.bias": "model-00010-of-00018.safetensors", "model.layers.22.post_attention_layernorm.weight": "model-00010-of-00018.safetensors", "model.layers.22.self_attn.dense.bias": "model-00010-of-00018.safetensors", "model.layers.22.self_attn.dense.weight": "model-00010-of-00018.safetensors", "model.layers.22.self_attn.query_key_value.bias": "model-00010-of-00018.safetensors", "model.layers.22.self_attn.query_key_value.weight": "model-00010-of-00018.safetensors", "model.layers.23.input_layernorm.bias": "model-00010-of-00018.safetensors", "model.layers.23.input_layernorm.weight": "model-00010-of-00018.safetensors", "model.layers.23.mlp.down_proj.bias": "model-00010-of-00018.safetensors", "model.layers.23.mlp.down_proj.weight": "model-00010-of-00018.safetensors", "model.layers.23.mlp.up_proj.bias": "model-00010-of-00018.safetensors", "model.layers.23.mlp.up_proj.weight": "model-00011-of-00018.safetensors", "model.layers.23.post_attention_layernorm.bias": "model-00011-of-00018.safetensors", "model.layers.23.post_attention_layernorm.weight": "model-00011-of-00018.safetensors", "model.layers.23.self_attn.dense.bias": "model-00011-of-00018.safetensors", "model.layers.23.self_attn.dense.weight": "model-00011-of-00018.safetensors", "model.layers.23.self_attn.query_key_value.bias": "model-00011-of-00018.safetensors", "model.layers.23.self_attn.query_key_value.weight": "model-00011-of-00018.safetensors", "model.layers.24.input_layernorm.bias": "model-00011-of-00018.safetensors", "model.layers.24.input_layernorm.weight": "model-00011-of-00018.safetensors", "model.layers.24.mlp.down_proj.bias": "model-00011-of-00018.safetensors", "model.layers.24.mlp.down_proj.weight": "model-00011-of-00018.safetensors", "model.layers.24.mlp.up_proj.bias": "model-00011-of-00018.safetensors", "model.layers.24.mlp.up_proj.weight": "model-00011-of-00018.safetensors", "model.layers.24.post_attention_layernorm.bias": "model-00011-of-00018.safetensors", "model.layers.24.post_attention_layernorm.weight": "model-00011-of-00018.safetensors", "model.layers.24.self_attn.dense.bias": "model-00011-of-00018.safetensors", "model.layers.24.self_attn.dense.weight": "model-00011-of-00018.safetensors", "model.layers.24.self_attn.query_key_value.bias": "model-00011-of-00018.safetensors", "model.layers.24.self_attn.query_key_value.weight": "model-00011-of-00018.safetensors", "model.layers.25.input_layernorm.bias": "model-00011-of-00018.safetensors", "model.layers.25.input_layernorm.weight": "model-00011-of-00018.safetensors", "model.layers.25.mlp.down_proj.bias": "model-00011-of-00018.safetensors", "model.layers.25.mlp.down_proj.weight": "model-00011-of-00018.safetensors", "model.layers.25.mlp.up_proj.bias": "model-00011-of-00018.safetensors", "model.layers.25.mlp.up_proj.weight": "model-00012-of-00018.safetensors", "model.layers.25.post_attention_layernorm.bias": "model-00012-of-00018.safetensors", "model.layers.25.post_attention_layernorm.weight": "model-00012-of-00018.safetensors", "model.layers.25.self_attn.dense.bias": "model-00012-of-00018.safetensors", "model.layers.25.self_attn.dense.weight": "model-00012-of-00018.safetensors", "model.layers.25.self_attn.query_key_value.bias": "model-00012-of-00018.safetensors", "model.layers.25.self_attn.query_key_value.weight": "model-00012-of-00018.safetensors", "model.layers.26.input_layernorm.bias": "model-00012-of-00018.safetensors", "model.layers.26.input_layernorm.weight": "model-00012-of-00018.safetensors", "model.layers.26.mlp.down_proj.bias": "model-00012-of-00018.safetensors", "model.layers.26.mlp.down_proj.weight": "model-00012-of-00018.safetensors", "model.layers.26.mlp.up_proj.bias": "model-00012-of-00018.safetensors", "model.layers.26.mlp.up_proj.weight": "model-00012-of-00018.safetensors", "model.layers.26.post_attention_layernorm.bias": "model-00012-of-00018.safetensors", "model.layers.26.post_attention_layernorm.weight": "model-00012-of-00018.safetensors", "model.layers.26.self_attn.dense.bias": "model-00012-of-00018.safetensors", "model.layers.26.self_attn.dense.weight": "model-00012-of-00018.safetensors", "model.layers.26.self_attn.query_key_value.bias": "model-00012-of-00018.safetensors", "model.layers.26.self_attn.query_key_value.weight": "model-00012-of-00018.safetensors", "model.layers.27.input_layernorm.bias": "model-00012-of-00018.safetensors", "model.layers.27.input_layernorm.weight": "model-00012-of-00018.safetensors", "model.layers.27.mlp.down_proj.bias": "model-00012-of-00018.safetensors", "model.layers.27.mlp.down_proj.weight": "model-00012-of-00018.safetensors", "model.layers.27.mlp.up_proj.bias": "model-00012-of-00018.safetensors", "model.layers.27.mlp.up_proj.weight": "model-00013-of-00018.safetensors", "model.layers.27.post_attention_layernorm.bias": "model-00013-of-00018.safetensors", "model.layers.27.post_attention_layernorm.weight": "model-00013-of-00018.safetensors", "model.layers.27.self_attn.dense.bias": "model-00013-of-00018.safetensors", "model.layers.27.self_attn.dense.weight": "model-00013-of-00018.safetensors", "model.layers.27.self_attn.query_key_value.bias": "model-00013-of-00018.safetensors", "model.layers.27.self_attn.query_key_value.weight": "model-00013-of-00018.safetensors", "model.layers.28.input_layernorm.bias": "model-00013-of-00018.safetensors", "model.layers.28.input_layernorm.weight": "model-00013-of-00018.safetensors", "model.layers.28.mlp.down_proj.bias": "model-00013-of-00018.safetensors", "model.layers.28.mlp.down_proj.weight": "model-00013-of-00018.safetensors", "model.layers.28.mlp.up_proj.bias": "model-00013-of-00018.safetensors", "model.layers.28.mlp.up_proj.weight": "model-00013-of-00018.safetensors", "model.layers.28.post_attention_layernorm.bias": "model-00013-of-00018.safetensors", "model.layers.28.post_attention_layernorm.weight": "model-00013-of-00018.safetensors", "model.layers.28.self_attn.dense.bias": "model-00013-of-00018.safetensors", "model.layers.28.self_attn.dense.weight": "model-00013-of-00018.safetensors", "model.layers.28.self_attn.query_key_value.bias": "model-00013-of-00018.safetensors", "model.layers.28.self_attn.query_key_value.weight": "model-00013-of-00018.safetensors", "model.layers.29.input_layernorm.bias": "model-00013-of-00018.safetensors", "model.layers.29.input_layernorm.weight": "model-00013-of-00018.safetensors", "model.layers.29.mlp.down_proj.bias": "model-00013-of-00018.safetensors", "model.layers.29.mlp.down_proj.weight": "model-00013-of-00018.safetensors", "model.layers.29.mlp.up_proj.bias": "model-00013-of-00018.safetensors", "model.layers.29.mlp.up_proj.weight": "model-00014-of-00018.safetensors", "model.layers.29.post_attention_layernorm.bias": "model-00014-of-00018.safetensors", "model.layers.29.post_attention_layernorm.weight": "model-00014-of-00018.safetensors", "model.layers.29.self_attn.dense.bias": "model-00014-of-00018.safetensors", "model.layers.29.self_attn.dense.weight": "model-00014-of-00018.safetensors", "model.layers.29.self_attn.query_key_value.bias": "model-00014-of-00018.safetensors", "model.layers.29.self_attn.query_key_value.weight": "model-00014-of-00018.safetensors", "model.layers.3.input_layernorm.bias": "model-00014-of-00018.safetensors", "model.layers.3.input_layernorm.weight": "model-00014-of-00018.safetensors", "model.layers.3.mlp.down_proj.bias": "model-00014-of-00018.safetensors", "model.layers.3.mlp.down_proj.weight": "model-00014-of-00018.safetensors", "model.layers.3.mlp.up_proj.bias": "model-00014-of-00018.safetensors", "model.layers.3.mlp.up_proj.weight": "model-00014-of-00018.safetensors", "model.layers.3.post_attention_layernorm.bias": "model-00014-of-00018.safetensors", "model.layers.3.post_attention_layernorm.weight": "model-00014-of-00018.safetensors", "model.layers.3.self_attn.dense.bias": "model-00014-of-00018.safetensors", "model.layers.3.self_attn.dense.weight": "model-00014-of-00018.safetensors", "model.layers.3.self_attn.query_key_value.bias": "model-00014-of-00018.safetensors", "model.layers.3.self_attn.query_key_value.weight": "model-00014-of-00018.safetensors", "model.layers.30.input_layernorm.bias": "model-00014-of-00018.safetensors", "model.layers.30.input_layernorm.weight": "model-00014-of-00018.safetensors", "model.layers.30.mlp.down_proj.bias": "model-00014-of-00018.safetensors", "model.layers.30.mlp.down_proj.weight": "model-00014-of-00018.safetensors", "model.layers.30.mlp.up_proj.bias": "model-00014-of-00018.safetensors", "model.layers.30.mlp.up_proj.weight": "model-00015-of-00018.safetensors", "model.layers.30.post_attention_layernorm.bias": "model-00015-of-00018.safetensors", "model.layers.30.post_attention_layernorm.weight": "model-00015-of-00018.safetensors", "model.layers.30.self_attn.dense.bias": "model-00015-of-00018.safetensors", "model.layers.30.self_attn.dense.weight": "model-00015-of-00018.safetensors", "model.layers.30.self_attn.query_key_value.bias": "model-00015-of-00018.safetensors", "model.layers.30.self_attn.query_key_value.weight": "model-00015-of-00018.safetensors", "model.layers.31.input_layernorm.bias": "model-00015-of-00018.safetensors", "model.layers.31.input_layernorm.weight": "model-00015-of-00018.safetensors", "model.layers.31.mlp.down_proj.bias": "model-00015-of-00018.safetensors", "model.layers.31.mlp.down_proj.weight": "model-00015-of-00018.safetensors", "model.layers.31.mlp.up_proj.bias": "model-00015-of-00018.safetensors", "model.layers.31.mlp.up_proj.weight": "model-00015-of-00018.safetensors", "model.layers.31.post_attention_layernorm.bias": "model-00015-of-00018.safetensors", "model.layers.31.post_attention_layernorm.weight": "model-00015-of-00018.safetensors", "model.layers.31.self_attn.dense.bias": "model-00015-of-00018.safetensors", "model.layers.31.self_attn.dense.weight": "model-00015-of-00018.safetensors", "model.layers.31.self_attn.query_key_value.bias": "model-00015-of-00018.safetensors", "model.layers.31.self_attn.query_key_value.weight": "model-00015-of-00018.safetensors", "model.layers.4.input_layernorm.bias": "model-00015-of-00018.safetensors", "model.layers.4.input_layernorm.weight": "model-00015-of-00018.safetensors", "model.layers.4.mlp.down_proj.bias": "model-00015-of-00018.safetensors", "model.layers.4.mlp.down_proj.weight": "model-00015-of-00018.safetensors", "model.layers.4.mlp.up_proj.bias": "model-00015-of-00018.safetensors", "model.layers.4.mlp.up_proj.weight": "model-00016-of-00018.safetensors", "model.layers.4.post_attention_layernorm.bias": "model-00016-of-00018.safetensors", "model.layers.4.post_attention_layernorm.weight": "model-00016-of-00018.safetensors", "model.layers.4.self_attn.dense.bias": "model-00016-of-00018.safetensors", "model.layers.4.self_attn.dense.weight": "model-00016-of-00018.safetensors", "model.layers.4.self_attn.query_key_value.bias": "model-00016-of-00018.safetensors", "model.layers.4.self_attn.query_key_value.weight": "model-00016-of-00018.safetensors", "model.layers.5.input_layernorm.bias": "model-00016-of-00018.safetensors", "model.layers.5.input_layernorm.weight": "model-00016-of-00018.safetensors", "model.layers.5.mlp.down_proj.bias": "model-00016-of-00018.safetensors", "model.layers.5.mlp.down_proj.weight": "model-00016-of-00018.safetensors", "model.layers.5.mlp.up_proj.bias": "model-00016-of-00018.safetensors", "model.layers.5.mlp.up_proj.weight": "model-00016-of-00018.safetensors", "model.layers.5.post_attention_layernorm.bias": "model-00016-of-00018.safetensors", "model.layers.5.post_attention_layernorm.weight": "model-00016-of-00018.safetensors", "model.layers.5.self_attn.dense.bias": "model-00016-of-00018.safetensors", "model.layers.5.self_attn.dense.weight": "model-00016-of-00018.safetensors", "model.layers.5.self_attn.query_key_value.bias": "model-00016-of-00018.safetensors", "model.layers.5.self_attn.query_key_value.weight": "model-00016-of-00018.safetensors", "model.layers.6.input_layernorm.bias": "model-00016-of-00018.safetensors", "model.layers.6.input_layernorm.weight": "model-00016-of-00018.safetensors", "model.layers.6.mlp.down_proj.bias": "model-00016-of-00018.safetensors", "model.layers.6.mlp.down_proj.weight": "model-00016-of-00018.safetensors", "model.layers.6.mlp.up_proj.bias": "model-00016-of-00018.safetensors", "model.layers.6.mlp.up_proj.weight": "model-00017-of-00018.safetensors", "model.layers.6.post_attention_layernorm.bias": "model-00017-of-00018.safetensors", "model.layers.6.post_attention_layernorm.weight": "model-00017-of-00018.safetensors", "model.layers.6.self_attn.dense.bias": "model-00017-of-00018.safetensors", "model.layers.6.self_attn.dense.weight": "model-00017-of-00018.safetensors", "model.layers.6.self_attn.query_key_value.bias": "model-00017-of-00018.safetensors", "model.layers.6.self_attn.query_key_value.weight": "model-00017-of-00018.safetensors", "model.layers.7.input_layernorm.bias": "model-00017-of-00018.safetensors", "model.layers.7.input_layernorm.weight": "model-00017-of-00018.safetensors", "model.layers.7.mlp.down_proj.bias": "model-00017-of-00018.safetensors", "model.layers.7.mlp.down_proj.weight": "model-00017-of-00018.safetensors", "model.layers.7.mlp.up_proj.bias": "model-00017-of-00018.safetensors", "model.layers.7.mlp.up_proj.weight": "model-00017-of-00018.safetensors", "model.layers.7.post_attention_layernorm.bias": "model-00017-of-00018.safetensors", "model.layers.7.post_attention_layernorm.weight": "model-00017-of-00018.safetensors", "model.layers.7.self_attn.dense.bias": "model-00017-of-00018.safetensors", "model.layers.7.self_attn.dense.weight": "model-00017-of-00018.safetensors", "model.layers.7.self_attn.query_key_value.bias": "model-00017-of-00018.safetensors", "model.layers.7.self_attn.query_key_value.weight": "model-00017-of-00018.safetensors", "model.layers.8.input_layernorm.bias": "model-00017-of-00018.safetensors", "model.layers.8.input_layernorm.weight": "model-00017-of-00018.safetensors", "model.layers.8.mlp.down_proj.bias": "model-00017-of-00018.safetensors", "model.layers.8.mlp.down_proj.weight": "model-00017-of-00018.safetensors", "model.layers.8.mlp.up_proj.bias": "model-00017-of-00018.safetensors", "model.layers.8.mlp.up_proj.weight": "model-00018-of-00018.safetensors", "model.layers.8.post_attention_layernorm.bias": "model-00018-of-00018.safetensors", "model.layers.8.post_attention_layernorm.weight": "model-00018-of-00018.safetensors", "model.layers.8.self_attn.dense.bias": "model-00018-of-00018.safetensors", "model.layers.8.self_attn.dense.weight": "model-00018-of-00018.safetensors", "model.layers.8.self_attn.query_key_value.bias": "model-00018-of-00018.safetensors", "model.layers.8.self_attn.query_key_value.weight": "model-00018-of-00018.safetensors", "model.layers.9.input_layernorm.bias": "model-00018-of-00018.safetensors", "model.layers.9.input_layernorm.weight": "model-00018-of-00018.safetensors", "model.layers.9.mlp.down_proj.bias": "model-00018-of-00018.safetensors", "model.layers.9.mlp.down_proj.weight": "model-00018-of-00018.safetensors", "model.layers.9.mlp.up_proj.bias": "model-00018-of-00018.safetensors", "model.layers.9.mlp.up_proj.weight": "model-00018-of-00018.safetensors", "model.layers.9.post_attention_layernorm.bias": "model-00018-of-00018.safetensors", "model.layers.9.post_attention_layernorm.weight": "model-00018-of-00018.safetensors", "model.layers.9.self_attn.dense.bias": "model-00018-of-00018.safetensors", "model.layers.9.self_attn.dense.weight": "model-00018-of-00018.safetensors", "model.layers.9.self_attn.query_key_value.bias": "model-00018-of-00018.safetensors", "model.layers.9.self_attn.query_key_value.weight": "model-00018-of-00018.safetensors"}}

special_tokens_map.json

@@ -0,0 +1,5 @@
+{
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "pad_token": "<|endoftext|>"
+}

tokenizer_config.json

@@ -0,0 +1,20 @@
+{
+  "_commit_hash": "1535ae26fb4faada95c6950e8bc6e867cdad6b00",
+  "_from_auto": true,
+  "added_tokens_decoder": {},
+  "auto_map": {
+    "AutoTokenizer": [
+      "tokenization_phi3_small.Phi3SmallTokenizer",
+      "tokenization_phi3_small.Phi3SmallTokenizer"
+    ]
+  },
+  "bos_token": "<|endoftext|>",
+  "chat_template": "{{ bos_token }}{% for message in messages %}{{'<|' + message['role'] + '|>' + '\n' + message['content'] + '<|end|>\n' }}{% endfor %}{% if add_generation_prompt %}{{ '<|assistant|>\n' }}{% else %}{{ eos_token }}{% endif %}",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|endoftext|>",
+  "model_max_length": 8192,
+  "pad_token": "<|endoftext|>",
+  "revision": null,
+  "tokenizer_class": "Phi3SmallTokenizer",
+  "trust_remote_code": true
+}