{ "attention_bias": false, "attention_dropout": 0.0, "architectures": [ "HunYuanForCausalLM" ], "auto_map": { "AutoConfig": "configuration_hunyuan.HunYuanConfig", "AutoModel": "modeling_hunyuan.HunyuanModel", "AutoModelForCausalLM": "modeling_hunyuan.HunYuanForCausalLM" }, "bos_token_id": 1, "capacity_factor": 1.0, "cla_share_factor": 2, "eos_token_id": 2, "hidden_act": "silu", "hidden_size": 6400, "initializer_range": 0.02, "intermediate_size": 18304, "max_position_embeddings": 262144, "model_type": "hunyuan", "moe_drop_tokens": false, "moe_random_routing_dropped_token": false, "moe_topk": 1, "num_attention_heads": 80, "num_experts": 16, "num_hidden_layers": 64, "num_key_value_heads": 8, "num_shared_expert": 1, "pad_token_id": 0, "pretraining_tp": 1, "rms_norm_eps": 1e-05, "rope_scaling": { "alpha": 100000.0, "factor": 1.0, "type": "dynamic" }, "rope_theta": 10000.0, "tie_word_embeddings": true, "transformers_version": "4.41.2", "use_cache": true, "use_cla": true, "use_mixed_mlp_moe": true, "use_qk_norm": true, "vocab_size": 128512 }