Hugging Face's logo

tencent
/
Hunyuan-A13B-Instruct-FP8

Text Generation
Transformers
Safetensors
hunyuan
conversational
custom_code
fp8
Model card Files Community
5
manaestras commited on
Commit
e215d8a
·
verified ·
1 Parent(s): 3e627cc

Upload config.json with huggingface_hub

Browse files
Files changed (1) hide show
  1. config.json +211 -0
config.json ADDED
@@ -0,0 +1,211 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_name_or_path": "/apdcephfs_gy2/share_302508627/darrenhu/llm/model/open_A13B/7b_moe_t1_v53_long_context_stage_daxianwang_fix_renshe_hf",
3
+ "add_classification_head": false,
4
+ "anyres_pooling_size": 2,
5
+ "anyres_vit_max_image_size": null,
6
+ "anyres_vit_two_views": false,
7
+ "architectures": [
8
+ "HunYuanForCausalLM"
9
+ ],
10
+ "attention_bias": false,
11
+ "attention_dropout": 0.1,
12
+ "attention_head_dim": 128,
13
+ "auto_map": {
14
+ "AutoConfig": "configuration_hunyuan.HunYuanConfig",
15
+ "AutoModel": "hunyuan.HunyuanModel",
16
+ "AutoModelForCausalLM": "hunyuan.HunYuanForCausalLM"
17
+ },
18
+ "bos_token_id": 1,
19
+ "cla_share_factor": 2,
20
+ "class_num": 0,
21
+ "dense_list": [
22
+ 4096,
23
+ 0
24
+ ],
25
+ "eod_token_id": 127967,
26
+ "eos_token_id": 127960,
27
+ "group_limited_greedy": false,
28
+ "hidden_act": "silu",
29
+ "hidden_size": 4096,
30
+ "im_end_id": 6,
31
+ "im_newline_id": 12,
32
+ "im_start_id": 5,
33
+ "image_token_id": 9,
34
+ "initializer_range": 0.02,
35
+ "intermediate_size": 3072,
36
+ "kv_lora_rank": null,
37
+ "mask_init_id": 13,
38
+ "max_position_embeddings": 32768,
39
+ "mlp_bias": false,
40
+ "model_type": "hunyuan",
41
+ "moe_drop_tokens": false,
42
+ "moe_intermediate_size": [
43
+ 3072,
44
+ 3072,
45
+ 3072,
46
+ 3072,
47
+ 3072,
48
+ 3072,
49
+ 3072,
50
+ 3072,
51
+ 3072,
52
+ 3072,
53
+ 3072,
54
+ 3072,
55
+ 3072,
56
+ 3072,
57
+ 3072,
58
+ 3072,
59
+ 3072,
60
+ 3072,
61
+ 3072,
62
+ 3072,
63
+ 3072,
64
+ 3072,
65
+ 3072,
66
+ 3072,
67
+ 3072,
68
+ 3072,
69
+ 3072,
70
+ 3072,
71
+ 3072,
72
+ 3072,
73
+ 3072,
74
+ 3072
75
+ ],
76
+ "moe_layer_num_skipped": 0,
77
+ "moe_random_routing_dropped_token": false,
78
+ "moe_topk": [
79
+ 8,
80
+ 8,
81
+ 8,
82
+ 8,
83
+ 8,
84
+ 8,
85
+ 8,
86
+ 8,
87
+ 8,
88
+ 8,
89
+ 8,
90
+ 8,
91
+ 8,
92
+ 8,
93
+ 8,
94
+ 8,
95
+ 8,
96
+ 8,
97
+ 8,
98
+ 8,
99
+ 8,
100
+ 8,
101
+ 8,
102
+ 8,
103
+ 8,
104
+ 8,
105
+ 8,
106
+ 8,
107
+ 8,
108
+ 8,
109
+ 8,
110
+ 8
111
+ ],
112
+ "n_group": null,
113
+ "norm_topk_prob": true,
114
+ "norm_type": "rms",
115
+ "num_attention_heads": 32,
116
+ "num_experts": 64,
117
+ "num_hidden_layers": 32,
118
+ "num_key_value_heads": 8,
119
+ "num_media_embeds": 257,
120
+ "num_shared_expert": [
121
+ 1,
122
+ 1,
123
+ 1,
124
+ 1,
125
+ 1,
126
+ 1,
127
+ 1,
128
+ 1,
129
+ 1,
130
+ 1,
131
+ 1,
132
+ 1,
133
+ 1,
134
+ 1,
135
+ 1,
136
+ 1,
137
+ 1,
138
+ 1,
139
+ 1,
140
+ 1,
141
+ 1,
142
+ 1,
143
+ 1,
144
+ 1,
145
+ 1,
146
+ 1,
147
+ 1,
148
+ 1,
149
+ 1,
150
+ 1,
151
+ 1,
152
+ 1
153
+ ],
154
+ "org_vocab_size": 128167,
155
+ "pad_id": 127961,
156
+ "pad_token_id": 127961,
157
+ "pool_type": "last",
158
+ "position_embedding_xdrope": false,
159
+ "pretraining_tp": 1,
160
+ "q_lora_rank": null,
161
+ "qk_nope_head_dim": null,
162
+ "qk_rope_head_dim": null,
163
+ "quantization_config": {
164
+ "activation_scheme": "static",
165
+ "ignored_layers": [
166
+ "lm_head"
167
+ ],
168
+ "quant_method": "fp8"
169
+ },
170
+ "rms_norm_eps": 1e-05,
171
+ "rope_scaling": {
172
+ "alpha": 1000.0,
173
+ "beta_fast": 32,
174
+ "beta_slow": 1,
175
+ "factor": 1.0,
176
+ "mscale": 1.0,
177
+ "mscale_all_dim": 1.0,
178
+ "type": "dynamic"
179
+ },
180
+ "rope_theta": 10000.0,
181
+ "routed_scaling_factor": 1.0,
182
+ "sep_token_id": 127962,
183
+ "skip_cls_token": false,
184
+ "text_end_id": 8,
185
+ "text_start_id": 7,
186
+ "tie_word_embeddings": true,
187
+ "topk_group": null,
188
+ "torch_dtype": "bfloat16",
189
+ "transformers_version": "4.41.2",
190
+ "use_cache": false,
191
+ "use_cla": false,
192
+ "use_mixed_mlp_moe": true,
193
+ "use_mla": false,
194
+ "use_qk_norm": true,
195
+ "use_rotary_pos_emb": true,
196
+ "v_head_dim": null,
197
+ "video_end_id": 11,
198
+ "video_start_id": 10,
199
+ "vit_add_patchemb_bias": false,
200
+ "vit_input_resolution": 224,
201
+ "vit_mapping_type": "resampler",
202
+ "vit_norm_type": "fused",
203
+ "vit_patch": 1,
204
+ "vit_path": null,
205
+ "vit_remove_prenorm": false,
206
+ "vit_token": 64,
207
+ "vit_type": null,
208
+ "vit_used_rms_norm": false,
209
+ "vocab_size": 128167,
210
+ "xdrope_section": null
211
+ }