{ | |
"_name_or_path": "MBZUAI-LLM/SlimPajama-DC-6", | |
"architectures": [ | |
"BTLMLMHeadModel" | |
], | |
"auto_map": { | |
"AutoConfig": "MBZUAI-LLM/SlimPajama-DC--configuration_btlm.BTLMConfig", | |
"AutoModel": "MBZUAI-LLM/SlimPajama-DC--modeling_btlm.BTLMModel", | |
"AutoModelForCausalLM": "MBZUAI-LLM/SlimPajama-DC--modeling_btlm.BTLMLMHeadModel" | |
}, | |
"model_type": "btlm", | |
"eos_token_id": 0, | |
"pad_token_id": 0, | |
"attn_pdrop": 0.0, | |
"scale_attn_weights": true, | |
"resid_pdrop": 0.0, | |
"embeddings_scale": 14.6, | |
"n_inner": 5461, | |
"n_embd": 2048, | |
"layer_norm_epsilon": 1e-05, | |
"n_positions": 2048, | |
"activation_function": "swiglu", | |
"n_head": 16, | |
"n_layer": 24, | |
"width_scale": 0.2775, | |
"position_embedding_type": "alibi", | |
"scale_qk_dot_by_d": true, | |
"tie_word_embeddings": true, | |
"vocab_size": 50277, | |
"embd_pdrop": 0.0 | |
} |