Upload HGRN2ForCausalLM

README.md

@@ -8,4 +8,4 @@ license: mit
 datasets:
 - cerebras/SlimPajama-627B
 library_name: fla
----
+---

config.json

@@ -3,13 +3,16 @@
   "architectures": [
     "HGRN2ForCausalLM"
   ],
+  "attn": null,
   "attn_mode": "fused_chunk",
   "bos_token_id": 1,
   "clamp_min": null,
   "conv_size": 4,
+  "elementwise_affine": true,
   "eos_token_id": 2,
   "expand_ratio": 128,
   "fuse_cross_entropy": true,
+  "fuse_norm": true,
   "hidden_act": "swish",
   "hidden_ratio": 4,
   "hidden_size": 2048,
@@ -17,13 +20,14 @@
   "intermediate_size": null,
   "max_position_embeddings": 2048,
   "model_type": "hgrn2",
+  "norm_eps": 1e-06,
   "num_heads": null,
   "num_hidden_layers": 24,
   "rms_norm_eps": 1e-06,
   "share_conv_kernel": true,
   "tie_word_embeddings": false,
   "torch_dtype": "bfloat16",
-  "transformers_version": "4.40.0.dev0",
+  "transformers_version": "4.48.2",
   "use_cache": true,
   "use_lower_bound": true,
   "use_short_conv": false,

generation_config.json

@@ -2,5 +2,5 @@
   "_from_model_config": true,
   "bos_token_id": 1,
   "eos_token_id": 2,
-  "transformers_version": "4.40.0.dev0"
+  "transformers_version": "4.48.2"
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f395b111991637a1e613dc3486067d562eb96a70a0a524b30310c97ee776ec46
-size 2728816304
+oid sha256:92ea939b56018e51e696d1cb38970ed2797fad97bfe1eacff144bdc0397cdc7c
+size 2728818968