Hugging Face's logo

yahavb
/
Mixtral-8x7B-Instruct-v0.1-vllm

Model card Files Community
yahavb commited on
Commit
8d2bdce
·
verified ·
1 Parent(s): b9c21e4

Add deepseek Mixtral-8x7B-Instruct-v0.1 nxd compiled model

Browse files
Files changed (34) hide show
  1. model.pt +3 -0
  2. neuron_config.json +172 -0
  3. weights/tp0_sharded_checkpoint.safetensors +3 -0
  4. weights/tp10_sharded_checkpoint.safetensors +3 -0
  5. weights/tp11_sharded_checkpoint.safetensors +3 -0
  6. weights/tp12_sharded_checkpoint.safetensors +3 -0
  7. weights/tp13_sharded_checkpoint.safetensors +3 -0
  8. weights/tp14_sharded_checkpoint.safetensors +3 -0
  9. weights/tp15_sharded_checkpoint.safetensors +3 -0
  10. weights/tp16_sharded_checkpoint.safetensors +3 -0
  11. weights/tp17_sharded_checkpoint.safetensors +3 -0
  12. weights/tp18_sharded_checkpoint.safetensors +3 -0
  13. weights/tp19_sharded_checkpoint.safetensors +3 -0
  14. weights/tp1_sharded_checkpoint.safetensors +3 -0
  15. weights/tp20_sharded_checkpoint.safetensors +3 -0
  16. weights/tp21_sharded_checkpoint.safetensors +3 -0
  17. weights/tp22_sharded_checkpoint.safetensors +3 -0
  18. weights/tp23_sharded_checkpoint.safetensors +3 -0
  19. weights/tp24_sharded_checkpoint.safetensors +3 -0
  20. weights/tp25_sharded_checkpoint.safetensors +3 -0
  21. weights/tp26_sharded_checkpoint.safetensors +3 -0
  22. weights/tp27_sharded_checkpoint.safetensors +3 -0
  23. weights/tp28_sharded_checkpoint.safetensors +3 -0
  24. weights/tp29_sharded_checkpoint.safetensors +3 -0
  25. weights/tp2_sharded_checkpoint.safetensors +3 -0
  26. weights/tp30_sharded_checkpoint.safetensors +3 -0
  27. weights/tp31_sharded_checkpoint.safetensors +3 -0
  28. weights/tp3_sharded_checkpoint.safetensors +3 -0
  29. weights/tp4_sharded_checkpoint.safetensors +3 -0
  30. weights/tp5_sharded_checkpoint.safetensors +3 -0
  31. weights/tp6_sharded_checkpoint.safetensors +3 -0
  32. weights/tp7_sharded_checkpoint.safetensors +3 -0
  33. weights/tp8_sharded_checkpoint.safetensors +3 -0
  34. weights/tp9_sharded_checkpoint.safetensors +3 -0
model.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9fbe1c5cdc6a02c03f98e5daf389ac7f47d2f341622a09efb039bd14c7aecf91
3
+ size 2702139
neuron_config.json ADDED
@@ -0,0 +1,172 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_name_or_path": "mistralai/Mixtral-8x7B-Instruct-v0.1",
3
+ "add_cross_attention": false,
4
+ "architectures": [
5
+ "MixtralForCausalLM"
6
+ ],
7
+ "attention_dropout": 0.0,
8
+ "attribute_map": {},
9
+ "bad_words_ids": null,
10
+ "begin_suppress_tokens": null,
11
+ "bos_token_id": 1,
12
+ "chunk_size_feed_forward": 0,
13
+ "cross_attention_hidden_size": null,
14
+ "decoder_start_token_id": null,
15
+ "diversity_penalty": 0.0,
16
+ "do_sample": false,
17
+ "early_stopping": false,
18
+ "encoder_no_repeat_ngram_size": 0,
19
+ "eos_token_id": 2,
20
+ "exponential_decay_length_penalty": null,
21
+ "finetuning_task": null,
22
+ "forced_bos_token_id": null,
23
+ "forced_eos_token_id": null,
24
+ "fused_spec_config": null,
25
+ "hidden_act": "silu",
26
+ "hidden_size": 4096,
27
+ "id2label": {
28
+ "0": "LABEL_0",
29
+ "1": "LABEL_1"
30
+ },
31
+ "initializer_range": 0.02,
32
+ "intermediate_size": 14336,
33
+ "is_decoder": false,
34
+ "is_encoder_decoder": false,
35
+ "label2id": {
36
+ "LABEL_0": 0,
37
+ "LABEL_1": 1
38
+ },
39
+ "length_penalty": 1.0,
40
+ "max_length": 20,
41
+ "max_position_embeddings": 32768,
42
+ "min_length": 0,
43
+ "model_type": "mixtral",
44
+ "neuron_config": {
45
+ "async_mode": false,
46
+ "attn_cls": "NeuronLlamaAttention",
47
+ "attn_kernel_enabled": false,
48
+ "batch_size": 1,
49
+ "bucket_n_active_tokens": false,
50
+ "buckets": [
51
+ 128
52
+ ],
53
+ "capacity_factor": null,
54
+ "cc_pipeline_tiling_factor": 2,
55
+ "context_encoding_buckets": null,
56
+ "ctx_batch_size": 1,
57
+ "enable_bucketing": true,
58
+ "enable_eagle_draft_input_norm": false,
59
+ "enable_eagle_speculation": false,
60
+ "enable_fused_speculation": false,
61
+ "ep_degree": 1,
62
+ "flash_decoding_enabled": false,
63
+ "fused_qkv": false,
64
+ "glu_mlp": true,
65
+ "is_chunked_prefill": false,
66
+ "is_continuous_batching": false,
67
+ "is_eagle_draft": false,
68
+ "is_medusa": false,
69
+ "is_paged_attention": false,
70
+ "kv_cache_quant": false,
71
+ "kv_cache_tiling": false,
72
+ "local_ranks_size": 32,
73
+ "logical_neuron_cores": 1,
74
+ "lora_config": null,
75
+ "max_batch_size": 1,
76
+ "max_context_length": 128,
77
+ "max_length": 128,
78
+ "max_new_tokens": null,
79
+ "medusa_speculation_length": 0,
80
+ "medusa_tree": null,
81
+ "mlp_kernel_enabled": false,
82
+ "mlp_kernel_fuse_residual_add": false,
83
+ "n_active_tokens": 128,
84
+ "n_positions": 128,
85
+ "num_medusa_heads": 0,
86
+ "on_cpu": false,
87
+ "on_device_sampling_config": {
88
+ "deterministic": false,
89
+ "do_sample": true,
90
+ "dynamic": true,
91
+ "global_topk": 64,
92
+ "on_device_sampling_config": true,
93
+ "temperature": 1.0,
94
+ "top_k": 1,
95
+ "top_p": 1.0
96
+ },
97
+ "output_logits": false,
98
+ "overrides_torch_dtype": true,
99
+ "pa_block_size": 128,
100
+ "pa_num_blocks": 1,
101
+ "padding_side": "right",
102
+ "pp_degree": 1,
103
+ "qk_layernorm": false,
104
+ "qkv_kernel_enabled": false,
105
+ "quantization_dtype": "int8",
106
+ "quantization_type": "per_tensor_symmetric",
107
+ "quantized": false,
108
+ "quantized_checkpoints_path": null,
109
+ "quantized_kernel_lower_bound": 1200.0,
110
+ "quantized_mlp_kernel_enabled": false,
111
+ "rmsnorm_quantize_kernel_enabled": false,
112
+ "rpl_reduce_dtype": "bfloat16",
113
+ "save_sharded_checkpoint": true,
114
+ "seq_len": 128,
115
+ "sequence_parallel_enabled": true,
116
+ "spec_batch_size": 1,
117
+ "speculation_length": 0,
118
+ "start_rank_id": 0,
119
+ "target": null,
120
+ "tkg_batch_size": 1,
121
+ "token_generation_buckets": null,
122
+ "torch_dtype": "bfloat16",
123
+ "tp_degree": 32,
124
+ "trace_tokengen_model": true,
125
+ "vocab_parallel": false,
126
+ "weights_to_skip_layout_optimization": [],
127
+ "world_size": 32
128
+ },
129
+ "no_repeat_ngram_size": 0,
130
+ "num_attention_heads": 32,
131
+ "num_beam_groups": 1,
132
+ "num_beams": 1,
133
+ "num_cores_per_group": 1,
134
+ "num_experts_per_tok": 2,
135
+ "num_hidden_layers": 32,
136
+ "num_key_value_heads": 8,
137
+ "num_local_experts": 8,
138
+ "num_return_sequences": 1,
139
+ "output_attentions": false,
140
+ "output_hidden_states": false,
141
+ "output_router_logits": false,
142
+ "output_scores": false,
143
+ "pad_token_id": null,
144
+ "prefix": null,
145
+ "problem_type": null,
146
+ "pruned_heads": {},
147
+ "remove_invalid_values": false,
148
+ "repetition_penalty": 1.0,
149
+ "return_dict": true,
150
+ "return_dict_in_generate": false,
151
+ "rms_norm_eps": 1e-05,
152
+ "rope_theta": 1000000.0,
153
+ "router_aux_loss_coef": 0.02,
154
+ "router_jitter_noise": 0.0,
155
+ "sep_token_id": null,
156
+ "sliding_window": null,
157
+ "suppress_tokens": null,
158
+ "task_specific_params": null,
159
+ "temperature": 1.0,
160
+ "tf_legacy_loss": false,
161
+ "tie_encoder_decoder": false,
162
+ "tie_word_embeddings": false,
163
+ "tokenizer_class": null,
164
+ "top_k": 50,
165
+ "top_p": 1.0,
166
+ "torchscript": false,
167
+ "transformers_version": "4.45.2",
168
+ "typical_p": 1.0,
169
+ "use_bfloat16": false,
170
+ "use_cache": true,
171
+ "vocab_size": 32000
172
+ }
weights/tp0_sharded_checkpoint.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f2cf265cdf4253ee3fc6916220aa7ed8b5f96c362e769c437b05255b2ea1dc4e
3
+ size 2971838096
weights/tp10_sharded_checkpoint.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:55afa9bd733e6d69aeabd4b32846c17b8dcd43a47ad9e93507b48fd6af0c5b0d
3
+ size 2971838096
weights/tp11_sharded_checkpoint.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8f987d98f99cd9d0edde0214b73bf4c656f8ffce83e2f158e7e10aecea4a2b77
3
+ size 2971838096
weights/tp12_sharded_checkpoint.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a74ef7f60564af5d5c105660f7963ff6340dec8b856f098446d627a1290c4242
3
+ size 2971838096
weights/tp13_sharded_checkpoint.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5b4c64912d9281a408a34cf36387a084a2ac54a3ebfd2385957c376bbd46ca78
3
+ size 2971838096
weights/tp14_sharded_checkpoint.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e6d86f1bcdfe5e504db87ed47b177be1e454ee0b9c17a95e3a0ab9967dcd7d88
3
+ size 2971838096
weights/tp15_sharded_checkpoint.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8c538abec3cf7d54baea8eb695261ae8056a47284a3d03e35938b823d704427a
3
+ size 2971838096
weights/tp16_sharded_checkpoint.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:23de7c3117c135222417d052122a80ac2b30c2a69c1b9fb89217708747e8c8a3
3
+ size 2971838096
weights/tp17_sharded_checkpoint.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:da22b30c4c0a6601ef857bc0f9498128f47e821cc94084951a828407891e6b10
3
+ size 2971838096
weights/tp18_sharded_checkpoint.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b75a62706233c5f45bf22dbf222bbb75fce835a4e8f3214307dfbbe4d2d696d8
3
+ size 2971838096
weights/tp19_sharded_checkpoint.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e7a8d77ff435def2fb3e661370eff9988e0caf752b5e93e450dcd3c879b11d6f
3
+ size 2971838096
weights/tp1_sharded_checkpoint.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6710749f189000e9c6ce91c76d3db1942a71f7e43b28cb8fafae0ba1ec4a7baa
3
+ size 2971838096
weights/tp20_sharded_checkpoint.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0301b6c3be78d96c5cccb8af34706f3e86d17baae04469ec488ce679c44432ee
3
+ size 2971838096
weights/tp21_sharded_checkpoint.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:46f050da97397e6113554e1a08cc88130d5a5cb3e39a0703aa22c4254381f4c5
3
+ size 2971838096
weights/tp22_sharded_checkpoint.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:50e9acb37baea058605770fcbfa3b41617debc941810d2bb18d8451a92de55db
3
+ size 2971838096
weights/tp23_sharded_checkpoint.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:996243d38c53779fb4a1e681fd5a7f561ded43b634a6b4ff488de3ec1af8eae5
3
+ size 2971838096
weights/tp24_sharded_checkpoint.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7db4c0b89eba6dd1288d2c1bdb69f4788ed3007053fff68fe272436d74af8052
3
+ size 2971838096
weights/tp25_sharded_checkpoint.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2c2687762476e01d6a85cb1bf170fc6da77a1935866611ce9677efcf7149b23c
3
+ size 2971838096
weights/tp26_sharded_checkpoint.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:38d0a1671ecf6dc4b228c54593f117df4649a1d41f63bbf359d0fae793214960
3
+ size 2971838096
weights/tp27_sharded_checkpoint.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9c6fc961285a0f27c2f2d3a867cb22116684d3e5171cfb7023eb01fc6fd68bbd
3
+ size 2971838096
weights/tp28_sharded_checkpoint.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:54f1060a621de9ecd3499129a974ec3c1774b06d8e729636c840096acf24125d
3
+ size 2971838096
weights/tp29_sharded_checkpoint.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:05dcce702c245aad0d58aecc04db20394a46de565c29b407eb0a18e1e261ea18
3
+ size 2971838096
weights/tp2_sharded_checkpoint.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7835b75441bcd69da8571c967968b2b074a533a4b25b27ab3876201147baf666
3
+ size 2971838096
weights/tp30_sharded_checkpoint.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3fc12818dd8c972c53b0c9b0da1d45f1f18dffcd591e0f8b33172daf3319dc01
3
+ size 2971838096
weights/tp31_sharded_checkpoint.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:42f404a33e9e8b14c395e539edc9ff245a8e5fd7a4e4ca73dfdec9247c9268c1
3
+ size 2971838096
weights/tp3_sharded_checkpoint.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1ddd5cb644b1ca6b566e64f2f99f66912cf5c5720be7ee0915e2c08a3715c316
3
+ size 2971838096
weights/tp4_sharded_checkpoint.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:84e2831df9f9457d3c9a9b63ad8e14c951473cf877ada24d09e406b3f9e4ba35
3
+ size 2971838096
weights/tp5_sharded_checkpoint.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:44bed47612bb8dde0c05e1d758d0f0c1e7a0e867ab0201b45301dbe2bfaf5bc3
3
+ size 2971838096
weights/tp6_sharded_checkpoint.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e878130354cf82625863bbd7e74e6004703f85435951d05d8a93526f9c4bc7fa
3
+ size 2971838096
weights/tp7_sharded_checkpoint.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f21f613c2648788275d091ada9fe92309e143f300c52655bbb535a3e0a47dba1
3
+ size 2971838096
weights/tp8_sharded_checkpoint.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8d88ef71b2e46675bc393be38fa20c47f1fb27da18b8efe30845f1164cb59148
3
+ size 2971838096
weights/tp9_sharded_checkpoint.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b848b06d3fd9ffe955e61b162f6f663ba89fbe790524b5eeb5e451801528e85e
3
+ size 2971838096