File size: 2,983 Bytes
dde6157 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 |
from transformers import PretrainedConfig
from typing import List
#STARCODER2_PRETRAINED_CONFIG_ARCHIVE_MAP = {}
class ModularStarEncoderConfig(PretrainedConfig):
model_type = "ModularStarEncoder"
keys_to_ignore_at_inference = ["past_key_values"]
def __init__(
self,
attention_dropout= 0.1,
residual_dropout= 0.1,
embedding_dropout= 0.1,
bos_token_id= 0,
eos_token_id= 0,
hidden_act= "gelu_pytorch_tanh",
_attn_implementation="flash_attention_2",
hidden_size= 1024,
conditional_size= 4,
initializer_range= 0.018042,
intermediate_size= 12288,
max_position_embeddings= 2048,
mlp_type= "default",
model_type= "starcoder2",
torch_dtype= "bfloat16",
layer_matryoshka_loss= True,
matryoshka_layers= [4,9,18,27,36],
norm_epsilon= 1e-05,
layer_norm_eps=1e-05,
norm_type= "layer_norm",
num_attention_heads= 16,
num_hidden_layers= 36,
num_key_value_heads= 4,
rope_theta= 999999.4420358813,
sliding_window= None,
transformers_version= "4.39.3",
use_bias= True,
use_cache= False,
vocab_size= 49156,
pad_token_id=0,
**kwargs,
):
if _attn_implementation not in ["flash_attention_2", "sdpa"]:
raise ValueError(f"`_attn_implementation` must be 'flash_attention_2', 'sdpa', got {_attn_implementation}.")
self.attention_dropout=attention_dropout ,
self.residual_dropout= residual_dropout,
self.embedding_dropout= embedding_dropout,
self.bos_token_id= bos_token_id,
self.eos_token_id= eos_token_id,
self.hidden_act= hidden_act,
self._attn_implementation=_attn_implementation,
self.hidden_size= hidden_size,
self.conditional_size= conditional_size,
self.initializer_range= initializer_range,
self.intermediate_size= intermediate_size,
self.max_position_embeddings= max_position_embeddings,
self.mlp_type= mlp_type,
self.model_type= model_type,
self.torch_dtype= torch_dtype,
self.layer_matryoshka_loss= layer_matryoshka_loss,
self.matryoshka_layers= matryoshka_layers,
self.norm_epsilon= norm_epsilon,
self.layer_norm_eps=layer_norm_eps,
self.norm_type= norm_type,
self.num_attention_heads= num_attention_heads,
self.num_hidden_layers= num_hidden_layers,
self.num_key_value_heads= num_key_value_heads,
self.rope_theta= rope_theta,
self.sliding_window= sliding_window,
self.transformers_version= transformers_version,
self.use_bias= use_bias,
self.use_cache= use_cache,
self.vocab_size= vocab_size,
self.pad_token_id=pad_token_id,
super().__init__(
bos_token_id=bos_token_id,
eos_token_id=eos_token_id,
**kwargs)
|