SlimPajama-DC / params_train_to_hf.json
Jason0214's picture
Upload folder using huggingface_hub
b02c73e verified
raw
history blame
463 Bytes
{
"attn_pdrop": 0.0,
"scale_attn_weights": true,
"resid_pdrop": 0.0,
"embeddings_scale": 14.6,
"n_inner": 5461,
"n_embd": 2048,
"layer_norm_epsilon": 1e-05,
"n_positions": 2048,
"activation_function": "swiglu",
"n_head": 16,
"n_layer": 24,
"width_scale": 0.2775,
"position_embedding_type": "alibi",
"scale_qk_dot_by_d": true,
"tie_word_embeddings": true,
"vocab_size": 50277,
"embd_pdrop": 0.0
}