|
{ |
|
"_name_or_path": "MBZUAI-LLM/SlimPajama-DC", |
|
"architectures": [ |
|
"BTLMLMHeadModel" |
|
], |
|
"auto_map": { |
|
"AutoConfig": "MBZUAI-LLM/SlimPajama-DC--configuration_btlm.BTLMConfig", |
|
"AutoModel": "MBZUAI-LLM/SlimPajama-DC--modeling_btlm.BTLMModel", |
|
"AutoModelForCausalLM": "MBZUAI-LLM/SlimPajama-DC--modeling_btlm.BTLMLMHeadModel" |
|
}, |
|
"model_type": "btlm", |
|
"eos_token_id": 0, |
|
"pad_token_id": 0, |
|
"attn_pdrop": 0.0, |
|
"scale_attn_weights": true, |
|
"resid_pdrop": 0.0, |
|
"embeddings_scale": 14.6, |
|
"n_inner": 5461, |
|
"n_embd": 2048, |
|
"layer_norm_epsilon": 1e-05, |
|
"n_positions": 2048, |
|
"activation_function": "swiglu", |
|
"n_head": 16, |
|
"n_layer": 24, |
|
"width_scale": 0.2775, |
|
"position_embedding_type": "alibi", |
|
"scale_qk_dot_by_d": true, |
|
"tie_word_embeddings": true, |
|
"vocab_size": 50277, |
|
"embd_pdrop": 0.0 |
|
} |