Upload folder using huggingface_hub

config.json

@@ -0,0 +1,36 @@
+{
+  "_name_or_path": "/workspace/philhoon/models/kv-fusion/tqa-kv-llama3.1-base",
+  "architectures": [
+    "LlamaForEncoderDecoder2"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 128000,
+  "eos_token_id": 128001,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 131072,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": {
+    "factor": 8.0,
+    "high_freq_factor": 4.0,
+    "low_freq_factor": 1.0,
+    "original_max_position_embeddings": 8192,
+    "rope_type": "llama3"
+  },
+  "rope_theta": 500000.0,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.45.0.dev0",
+  "use_cache": true,
+  "vocab_size": 128257
+}

model-00001-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f7285f423d51c18a4421ee370bbae108323c05b94ad51a6ace18633b9b1bdbed
+size 4976707520

model-00002-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:75ca2c523e6061e2df25b30aa7135256f632f216f71430172690f757513a7bbd
+size 4999803552

model-00003-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bfcf774a75f3296b74f9bc535baf303394f20b6f1e013e7f5cad25c29ef342de
+size 4915916976

model-00004-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:72e4259fbe4a9f14aa1e493c671a609215282892314eaed01841e84a62c2dad5
+size 4920065288

model-00005-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:329669675b5f66a5c0225b2fd9f139ace96d9352820eaa6e37672bad5dc65814
+size 4915916144

model-00006-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f7c779dcc2e3eefb7e902859603bda94e830be01458d3367122a8dd476d462d1
+size 4915916176

model-00007-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d3f119a497de9119a0d24f321aebdcca69d5adbb572ca0799fcb847d0b93df81
+size 2476822152

model.safetensors.index.json

@@ -0,0 +1,589 @@
+{
+  "metadata": {
+    "total_size": 32121077760
+  },
+  "weight_map": {
+    "encoder.lm_head.weight": "model-00004-of-00007.safetensors",
+    "encoder.model.embed_tokens.weight": "model-00001-of-00007.safetensors",
+    "encoder.model.layers.0.input_layernorm.weight": "model-00001-of-00007.safetensors",
+    "encoder.model.layers.0.mlp.down_proj.weight": "model-00001-of-00007.safetensors",
+    "encoder.model.layers.0.mlp.gate_proj.weight": "model-00001-of-00007.safetensors",
+    "encoder.model.layers.0.mlp.up_proj.weight": "model-00001-of-00007.safetensors",
+    "encoder.model.layers.0.post_attention_layernorm.weight": "model-00001-of-00007.safetensors",
+    "encoder.model.layers.0.self_attn.k_proj.weight": "model-00001-of-00007.safetensors",
+    "encoder.model.layers.0.self_attn.o_proj.weight": "model-00001-of-00007.safetensors",
+    "encoder.model.layers.0.self_attn.q_proj.weight": "model-00001-of-00007.safetensors",
+    "encoder.model.layers.0.self_attn.v_proj.weight": "model-00001-of-00007.safetensors",
+    "encoder.model.layers.1.input_layernorm.weight": "model-00001-of-00007.safetensors",
+    "encoder.model.layers.1.mlp.down_proj.weight": "model-00001-of-00007.safetensors",
+    "encoder.model.layers.1.mlp.gate_proj.weight": "model-00001-of-00007.safetensors",
+    "encoder.model.layers.1.mlp.up_proj.weight": "model-00001-of-00007.safetensors",
+    "encoder.model.layers.1.post_attention_layernorm.weight": "model-00001-of-00007.safetensors",
+    "encoder.model.layers.1.self_attn.k_proj.weight": "model-00001-of-00007.safetensors",
+    "encoder.model.layers.1.self_attn.o_proj.weight": "model-00001-of-00007.safetensors",
+    "encoder.model.layers.1.self_attn.q_proj.weight": "model-00001-of-00007.safetensors",
+    "encoder.model.layers.1.self_attn.v_proj.weight": "model-00001-of-00007.safetensors",
+    "encoder.model.layers.10.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "encoder.model.layers.10.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "encoder.model.layers.10.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "encoder.model.layers.10.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "encoder.model.layers.10.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "encoder.model.layers.10.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "encoder.model.layers.10.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "encoder.model.layers.10.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "encoder.model.layers.10.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "encoder.model.layers.11.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "encoder.model.layers.11.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "encoder.model.layers.11.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "encoder.model.layers.11.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "encoder.model.layers.11.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "encoder.model.layers.11.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "encoder.model.layers.11.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "encoder.model.layers.11.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "encoder.model.layers.11.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "encoder.model.layers.12.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "encoder.model.layers.12.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "encoder.model.layers.12.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "encoder.model.layers.12.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "encoder.model.layers.12.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "encoder.model.layers.12.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "encoder.model.layers.12.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "encoder.model.layers.12.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "encoder.model.layers.12.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "encoder.model.layers.13.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "encoder.model.layers.13.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "encoder.model.layers.13.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "encoder.model.layers.13.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "encoder.model.layers.13.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "encoder.model.layers.13.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "encoder.model.layers.13.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "encoder.model.layers.13.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "encoder.model.layers.13.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "encoder.model.layers.14.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "encoder.model.layers.14.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "encoder.model.layers.14.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "encoder.model.layers.14.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "encoder.model.layers.14.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "encoder.model.layers.14.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "encoder.model.layers.14.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "encoder.model.layers.14.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "encoder.model.layers.14.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "encoder.model.layers.15.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "encoder.model.layers.15.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "encoder.model.layers.15.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "encoder.model.layers.15.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "encoder.model.layers.15.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "encoder.model.layers.15.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "encoder.model.layers.15.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "encoder.model.layers.15.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "encoder.model.layers.15.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "encoder.model.layers.16.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "encoder.model.layers.16.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "encoder.model.layers.16.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "encoder.model.layers.16.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "encoder.model.layers.16.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "encoder.model.layers.16.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "encoder.model.layers.16.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "encoder.model.layers.16.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "encoder.model.layers.16.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "encoder.model.layers.17.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "encoder.model.layers.17.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "encoder.model.layers.17.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "encoder.model.layers.17.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "encoder.model.layers.17.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "encoder.model.layers.17.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "encoder.model.layers.17.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "encoder.model.layers.17.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "encoder.model.layers.17.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "encoder.model.layers.18.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "encoder.model.layers.18.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "encoder.model.layers.18.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "encoder.model.layers.18.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "encoder.model.layers.18.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "encoder.model.layers.18.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "encoder.model.layers.18.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "encoder.model.layers.18.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "encoder.model.layers.18.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "encoder.model.layers.19.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "encoder.model.layers.19.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "encoder.model.layers.19.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "encoder.model.layers.19.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "encoder.model.layers.19.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "encoder.model.layers.19.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "encoder.model.layers.19.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "encoder.model.layers.19.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "encoder.model.layers.19.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "encoder.model.layers.2.input_layernorm.weight": "model-00001-of-00007.safetensors",
+    "encoder.model.layers.2.mlp.down_proj.weight": "model-00001-of-00007.safetensors",
+    "encoder.model.layers.2.mlp.gate_proj.weight": "model-00001-of-00007.safetensors",
+    "encoder.model.layers.2.mlp.up_proj.weight": "model-00001-of-00007.safetensors",
+    "encoder.model.layers.2.post_attention_layernorm.weight": "model-00001-of-00007.safetensors",
+    "encoder.model.layers.2.self_attn.k_proj.weight": "model-00001-of-00007.safetensors",
+    "encoder.model.layers.2.self_attn.o_proj.weight": "model-00001-of-00007.safetensors",
+    "encoder.model.layers.2.self_attn.q_proj.weight": "model-00001-of-00007.safetensors",
+    "encoder.model.layers.2.self_attn.v_proj.weight": "model-00001-of-00007.safetensors",
+    "encoder.model.layers.20.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "encoder.model.layers.20.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "encoder.model.layers.20.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "encoder.model.layers.20.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
+    "encoder.model.layers.20.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "encoder.model.layers.20.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "encoder.model.layers.20.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "encoder.model.layers.20.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "encoder.model.layers.20.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "encoder.model.layers.21.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "encoder.model.layers.21.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "encoder.model.layers.21.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "encoder.model.layers.21.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
+    "encoder.model.layers.21.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "encoder.model.layers.21.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "encoder.model.layers.21.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "encoder.model.layers.21.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "encoder.model.layers.21.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "encoder.model.layers.22.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "encoder.model.layers.22.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "encoder.model.layers.22.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "encoder.model.layers.22.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
+    "encoder.model.layers.22.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "encoder.model.layers.22.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "encoder.model.layers.22.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "encoder.model.layers.22.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "encoder.model.layers.22.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "encoder.model.layers.23.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "encoder.model.layers.23.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "encoder.model.layers.23.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "encoder.model.layers.23.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
+    "encoder.model.layers.23.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "encoder.model.layers.23.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "encoder.model.layers.23.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "encoder.model.layers.23.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "encoder.model.layers.23.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "encoder.model.layers.24.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "encoder.model.layers.24.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "encoder.model.layers.24.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "encoder.model.layers.24.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
+    "encoder.model.layers.24.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "encoder.model.layers.24.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "encoder.model.layers.24.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "encoder.model.layers.24.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "encoder.model.layers.24.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "encoder.model.layers.25.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "encoder.model.layers.25.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "encoder.model.layers.25.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "encoder.model.layers.25.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
+    "encoder.model.layers.25.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "encoder.model.layers.25.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "encoder.model.layers.25.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "encoder.model.layers.25.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "encoder.model.layers.25.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "encoder.model.layers.26.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "encoder.model.layers.26.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "encoder.model.layers.26.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "encoder.model.layers.26.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
+    "encoder.model.layers.26.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "encoder.model.layers.26.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "encoder.model.layers.26.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "encoder.model.layers.26.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "encoder.model.layers.26.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "encoder.model.layers.27.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "encoder.model.layers.27.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "encoder.model.layers.27.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "encoder.model.layers.27.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
+    "encoder.model.layers.27.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "encoder.model.layers.27.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "encoder.model.layers.27.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "encoder.model.layers.27.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "encoder.model.layers.27.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "encoder.model.layers.28.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "encoder.model.layers.28.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "encoder.model.layers.28.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "encoder.model.layers.28.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
+    "encoder.model.layers.28.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "encoder.model.layers.28.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "encoder.model.layers.28.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "encoder.model.layers.28.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "encoder.model.layers.28.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "encoder.model.layers.29.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "encoder.model.layers.29.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "encoder.model.layers.29.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "encoder.model.layers.29.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
+    "encoder.model.layers.29.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "encoder.model.layers.29.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "encoder.model.layers.29.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "encoder.model.layers.29.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "encoder.model.layers.29.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "encoder.model.layers.3.input_layernorm.weight": "model-00001-of-00007.safetensors",
+    "encoder.model.layers.3.mlp.down_proj.weight": "model-00001-of-00007.safetensors",
+    "encoder.model.layers.3.mlp.gate_proj.weight": "model-00001-of-00007.safetensors",
+    "encoder.model.layers.3.mlp.up_proj.weight": "model-00001-of-00007.safetensors",
+    "encoder.model.layers.3.post_attention_layernorm.weight": "model-00001-of-00007.safetensors",
+    "encoder.model.layers.3.self_attn.k_proj.weight": "model-00001-of-00007.safetensors",
+    "encoder.model.layers.3.self_attn.o_proj.weight": "model-00001-of-00007.safetensors",
+    "encoder.model.layers.3.self_attn.q_proj.weight": "model-00001-of-00007.safetensors",
+    "encoder.model.layers.3.self_attn.v_proj.weight": "model-00001-of-00007.safetensors",
+    "encoder.model.layers.30.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "encoder.model.layers.30.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "encoder.model.layers.30.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "encoder.model.layers.30.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
+    "encoder.model.layers.30.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "encoder.model.layers.30.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "encoder.model.layers.30.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "encoder.model.layers.30.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "encoder.model.layers.30.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "encoder.model.layers.31.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "encoder.model.layers.31.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "encoder.model.layers.31.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "encoder.model.layers.31.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
+    "encoder.model.layers.31.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "encoder.model.layers.31.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "encoder.model.layers.31.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "encoder.model.layers.31.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "encoder.model.layers.31.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "encoder.model.layers.4.input_layernorm.weight": "model-00001-of-00007.safetensors",
+    "encoder.model.layers.4.mlp.down_proj.weight": "model-00001-of-00007.safetensors",
+    "encoder.model.layers.4.mlp.gate_proj.weight": "model-00001-of-00007.safetensors",
+    "encoder.model.layers.4.mlp.up_proj.weight": "model-00001-of-00007.safetensors",
+    "encoder.model.layers.4.post_attention_layernorm.weight": "model-00001-of-00007.safetensors",
+    "encoder.model.layers.4.self_attn.k_proj.weight": "model-00001-of-00007.safetensors",
+    "encoder.model.layers.4.self_attn.o_proj.weight": "model-00001-of-00007.safetensors",
+    "encoder.model.layers.4.self_attn.q_proj.weight": "model-00001-of-00007.safetensors",
+    "encoder.model.layers.4.self_attn.v_proj.weight": "model-00001-of-00007.safetensors",
+    "encoder.model.layers.5.input_layernorm.weight": "model-00001-of-00007.safetensors",
+    "encoder.model.layers.5.mlp.down_proj.weight": "model-00001-of-00007.safetensors",
+    "encoder.model.layers.5.mlp.gate_proj.weight": "model-00001-of-00007.safetensors",
+    "encoder.model.layers.5.mlp.up_proj.weight": "model-00001-of-00007.safetensors",
+    "encoder.model.layers.5.post_attention_layernorm.weight": "model-00001-of-00007.safetensors",
+    "encoder.model.layers.5.self_attn.k_proj.weight": "model-00001-of-00007.safetensors",
+    "encoder.model.layers.5.self_attn.o_proj.weight": "model-00001-of-00007.safetensors",
+    "encoder.model.layers.5.self_attn.q_proj.weight": "model-00001-of-00007.safetensors",
+    "encoder.model.layers.5.self_attn.v_proj.weight": "model-00001-of-00007.safetensors",
+    "encoder.model.layers.6.input_layernorm.weight": "model-00001-of-00007.safetensors",
+    "encoder.model.layers.6.mlp.down_proj.weight": "model-00001-of-00007.safetensors",
+    "encoder.model.layers.6.mlp.gate_proj.weight": "model-00001-of-00007.safetensors",
+    "encoder.model.layers.6.mlp.up_proj.weight": "model-00001-of-00007.safetensors",
+    "encoder.model.layers.6.post_attention_layernorm.weight": "model-00001-of-00007.safetensors",
+    "encoder.model.layers.6.self_attn.k_proj.weight": "model-00001-of-00007.safetensors",
+    "encoder.model.layers.6.self_attn.o_proj.weight": "model-00001-of-00007.safetensors",
+    "encoder.model.layers.6.self_attn.q_proj.weight": "model-00001-of-00007.safetensors",
+    "encoder.model.layers.6.self_attn.v_proj.weight": "model-00001-of-00007.safetensors",
+    "encoder.model.layers.7.input_layernorm.weight": "model-00001-of-00007.safetensors",
+    "encoder.model.layers.7.mlp.down_proj.weight": "model-00001-of-00007.safetensors",
+    "encoder.model.layers.7.mlp.gate_proj.weight": "model-00001-of-00007.safetensors",
+    "encoder.model.layers.7.mlp.up_proj.weight": "model-00001-of-00007.safetensors",
+    "encoder.model.layers.7.post_attention_layernorm.weight": "model-00001-of-00007.safetensors",
+    "encoder.model.layers.7.self_attn.k_proj.weight": "model-00001-of-00007.safetensors",
+    "encoder.model.layers.7.self_attn.o_proj.weight": "model-00001-of-00007.safetensors",
+    "encoder.model.layers.7.self_attn.q_proj.weight": "model-00001-of-00007.safetensors",
+    "encoder.model.layers.7.self_attn.v_proj.weight": "model-00001-of-00007.safetensors",
+    "encoder.model.layers.8.input_layernorm.weight": "model-00001-of-00007.safetensors",
+    "encoder.model.layers.8.mlp.down_proj.weight": "model-00001-of-00007.safetensors",
+    "encoder.model.layers.8.mlp.gate_proj.weight": "model-00001-of-00007.safetensors",
+    "encoder.model.layers.8.mlp.up_proj.weight": "model-00001-of-00007.safetensors",
+    "encoder.model.layers.8.post_attention_layernorm.weight": "model-00001-of-00007.safetensors",
+    "encoder.model.layers.8.self_attn.k_proj.weight": "model-00001-of-00007.safetensors",
+    "encoder.model.layers.8.self_attn.o_proj.weight": "model-00001-of-00007.safetensors",
+    "encoder.model.layers.8.self_attn.q_proj.weight": "model-00001-of-00007.safetensors",
+    "encoder.model.layers.8.self_attn.v_proj.weight": "model-00001-of-00007.safetensors",
+    "encoder.model.layers.9.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "encoder.model.layers.9.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "encoder.model.layers.9.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "encoder.model.layers.9.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "encoder.model.layers.9.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "encoder.model.layers.9.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "encoder.model.layers.9.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "encoder.model.layers.9.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "encoder.model.layers.9.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "encoder.model.norm.weight": "model-00004-of-00007.safetensors",
+    "lm_head.weight": "model-00007-of-00007.safetensors",
+    "model.embed_tokens.weight": "model-00004-of-00007.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00007-of-00007.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00007-of-00007.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00007-of-00007.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00007-of-00007.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00007-of-00007.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00007-of-00007.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00007-of-00007.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00007-of-00007.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00007-of-00007.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00007-of-00007.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00007-of-00007.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00007-of-00007.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00007-of-00007.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00007-of-00007.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00007-of-00007.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00007-of-00007.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00007-of-00007.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00007-of-00007.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00007-of-00007.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00007-of-00007.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00007-of-00007.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00007-of-00007.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00007-of-00007.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00007-of-00007.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00007-of-00007.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00007-of-00007.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00007-of-00007.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00007-of-00007.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00007-of-00007.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00007-of-00007.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "model.norm.weight": "model-00007-of-00007.safetensors"
+  }
+}

rng_state_0.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:92cc13315f24c28015d695b6cde08bb1cd6fea4cbc435998485ed6fbe4c91285
+size 15024

rng_state_1.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f4c154b6a63e0b1f98f7d2847944398f99f1657d35e8eddf7fdf0ae2c24b0552
+size 15024

rng_state_2.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f784c6a9507b51189f2caffbd178ea9882103b75852e31c15f47fdae6a43af1d
+size 15024

rng_state_3.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:34b023e05bc2d12b91dc436d4922b990d50ec8dc56d40dc3e36b3bb34fc81341
+size 15024

scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:754972bd89a0ad01d3e86f4bc7920b352b519a155dc9755a815c98aa76c268ae
+size 1064

trainer_state.json

@@ -0,0 +1,1671 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.9968,
+  "eval_steps": 500,
+  "global_step": 936,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.008533333333333334,
+      "grad_norm": 2.515625,
+      "learning_rate": 1.6000000000000001e-06,
+      "loss": 0.2639,
+      "step": 4
+    },
+    {
+      "epoch": 0.017066666666666667,
+      "grad_norm": 1.96875,
+      "learning_rate": 3.2000000000000003e-06,
+      "loss": 0.2509,
+      "step": 8
+    },
+    {
+      "epoch": 0.0256,
+      "grad_norm": 1.9296875,
+      "learning_rate": 4.800000000000001e-06,
+      "loss": 0.2595,
+      "step": 12
+    },
+    {
+      "epoch": 0.034133333333333335,
+      "grad_norm": 2.046875,
+      "learning_rate": 6.4000000000000006e-06,
+      "loss": 0.2416,
+      "step": 16
+    },
+    {
+      "epoch": 0.042666666666666665,
+      "grad_norm": 2.546875,
+      "learning_rate": 8.000000000000001e-06,
+      "loss": 0.2478,
+      "step": 20
+    },
+    {
+      "epoch": 0.0512,
+      "grad_norm": 2.296875,
+      "learning_rate": 9.600000000000001e-06,
+      "loss": 0.2367,
+      "step": 24
+    },
+    {
+      "epoch": 0.05973333333333333,
+      "grad_norm": 2.03125,
+      "learning_rate": 1.1200000000000001e-05,
+      "loss": 0.2397,
+      "step": 28
+    },
+    {
+      "epoch": 0.06826666666666667,
+      "grad_norm": 1.8359375,
+      "learning_rate": 1.2800000000000001e-05,
+      "loss": 0.2416,
+      "step": 32
+    },
+    {
+      "epoch": 0.0768,
+      "grad_norm": 1.9609375,
+      "learning_rate": 1.4400000000000001e-05,
+      "loss": 0.2421,
+      "step": 36
+    },
+    {
+      "epoch": 0.08533333333333333,
+      "grad_norm": 2.0,
+      "learning_rate": 1.6000000000000003e-05,
+      "loss": 0.2392,
+      "step": 40
+    },
+    {
+      "epoch": 0.09386666666666667,
+      "grad_norm": 2.046875,
+      "learning_rate": 1.76e-05,
+      "loss": 0.2356,
+      "step": 44
+    },
+    {
+      "epoch": 0.1024,
+      "grad_norm": 2.203125,
+      "learning_rate": 1.9200000000000003e-05,
+      "loss": 0.2367,
+      "step": 48
+    },
+    {
+      "epoch": 0.11093333333333333,
+      "grad_norm": 2.09375,
+      "learning_rate": 1.999974854488333e-05,
+      "loss": 0.2418,
+      "step": 52
+    },
+    {
+      "epoch": 0.11946666666666667,
+      "grad_norm": 1.921875,
+      "learning_rate": 1.9997736979824944e-05,
+      "loss": 0.2379,
+      "step": 56
+    },
+    {
+      "epoch": 0.128,
+      "grad_norm": 1.9140625,
+      "learning_rate": 1.999371425435775e-05,
+      "loss": 0.241,
+      "step": 60
+    },
+    {
+      "epoch": 0.13653333333333334,
+      "grad_norm": 1.9453125,
+      "learning_rate": 1.9987681177699486e-05,
+      "loss": 0.2509,
+      "step": 64
+    },
+    {
+      "epoch": 0.14506666666666668,
+      "grad_norm": 1.8828125,
+      "learning_rate": 1.9979638963473294e-05,
+      "loss": 0.2412,
+      "step": 68
+    },
+    {
+      "epoch": 0.1536,
+      "grad_norm": 1.8125,
+      "learning_rate": 1.996958922946357e-05,
+      "loss": 0.2501,
+      "step": 72
+    },
+    {
+      "epoch": 0.16213333333333332,
+      "grad_norm": 2.6875,
+      "learning_rate": 1.9957533997290524e-05,
+      "loss": 0.2482,
+      "step": 76
+    },
+    {
+      "epoch": 0.17066666666666666,
+      "grad_norm": 1.8359375,
+      "learning_rate": 1.9943475692003514e-05,
+      "loss": 0.2371,
+      "step": 80
+    },
+    {
+      "epoch": 0.1792,
+      "grad_norm": 1.9765625,
+      "learning_rate": 1.992741714159322e-05,
+      "loss": 0.2459,
+      "step": 84
+    },
+    {
+      "epoch": 0.18773333333333334,
+      "grad_norm": 2.671875,
+      "learning_rate": 1.990936157642277e-05,
+      "loss": 0.2448,
+      "step": 88
+    },
+    {
+      "epoch": 0.19626666666666667,
+      "grad_norm": 1.8359375,
+      "learning_rate": 1.9889312628577887e-05,
+      "loss": 0.2515,
+      "step": 92
+    },
+    {
+      "epoch": 0.2048,
+      "grad_norm": 2.15625,
+      "learning_rate": 1.9867274331136276e-05,
+      "loss": 0.2417,
+      "step": 96
+    },
+    {
+      "epoch": 0.21333333333333335,
+      "grad_norm": 1.921875,
+      "learning_rate": 1.984325111735633e-05,
+      "loss": 0.2506,
+      "step": 100
+    },
+    {
+      "epoch": 0.22186666666666666,
+      "grad_norm": 2.734375,
+      "learning_rate": 1.9817247819785303e-05,
+      "loss": 0.2513,
+      "step": 104
+    },
+    {
+      "epoch": 0.2304,
+      "grad_norm": 1.8671875,
+      "learning_rate": 1.9789269669287212e-05,
+      "loss": 0.2456,
+      "step": 108
+    },
+    {
+      "epoch": 0.23893333333333333,
+      "grad_norm": 2.15625,
+      "learning_rate": 1.975932229399057e-05,
+      "loss": 0.2572,
+      "step": 112
+    },
+    {
+      "epoch": 0.24746666666666667,
+      "grad_norm": 1.9609375,
+      "learning_rate": 1.972741171815623e-05,
+      "loss": 0.2476,
+      "step": 116
+    },
+    {
+      "epoch": 0.256,
+      "grad_norm": 1.9609375,
+      "learning_rate": 1.9693544360965548e-05,
+      "loss": 0.2442,
+      "step": 120
+    },
+    {
+      "epoch": 0.26453333333333334,
+      "grad_norm": 1.734375,
+      "learning_rate": 1.9657727035229066e-05,
+      "loss": 0.2655,
+      "step": 124
+    },
+    {
+      "epoch": 0.2730666666666667,
+      "grad_norm": 1.9140625,
+      "learning_rate": 1.9619966946016054e-05,
+      "loss": 0.248,
+      "step": 128
+    },
+    {
+      "epoch": 0.2816,
+      "grad_norm": 2.203125,
+      "learning_rate": 1.958027168920512e-05,
+      "loss": 0.2478,
+      "step": 132
+    },
+    {
+      "epoch": 0.29013333333333335,
+      "grad_norm": 1.65625,
+      "learning_rate": 1.953864924995621e-05,
+      "loss": 0.2534,
+      "step": 136
+    },
+    {
+      "epoch": 0.2986666666666667,
+      "grad_norm": 2.015625,
+      "learning_rate": 1.9495108001104312e-05,
+      "loss": 0.2453,
+      "step": 140
+    },
+    {
+      "epoch": 0.3072,
+      "grad_norm": 1.9453125,
+      "learning_rate": 1.9449656701475147e-05,
+      "loss": 0.2313,
+      "step": 144
+    },
+    {
+      "epoch": 0.3157333333333333,
+      "grad_norm": 1.8828125,
+      "learning_rate": 1.940230449412324e-05,
+      "loss": 0.2412,
+      "step": 148
+    },
+    {
+      "epoch": 0.32426666666666665,
+      "grad_norm": 1.9453125,
+      "learning_rate": 1.9353060904492694e-05,
+      "loss": 0.2615,
+      "step": 152
+    },
+    {
+      "epoch": 0.3328,
+      "grad_norm": 2.234375,
+      "learning_rate": 1.930193583850102e-05,
+      "loss": 0.2571,
+      "step": 156
+    },
+    {
+      "epoch": 0.3413333333333333,
+      "grad_norm": 2.078125,
+      "learning_rate": 1.9248939580546453e-05,
+      "loss": 0.2753,
+      "step": 160
+    },
+    {
+      "epoch": 0.34986666666666666,
+      "grad_norm": 1.6953125,
+      "learning_rate": 1.9194082791439146e-05,
+      "loss": 0.2462,
+      "step": 164
+    },
+    {
+      "epoch": 0.3584,
+      "grad_norm": 1.8828125,
+      "learning_rate": 1.91373765062566e-05,
+      "loss": 0.2574,
+      "step": 168
+    },
+    {
+      "epoch": 0.36693333333333333,
+      "grad_norm": 1.7734375,
+      "learning_rate": 1.9078832132123833e-05,
+      "loss": 0.2394,
+      "step": 172
+    },
+    {
+      "epoch": 0.37546666666666667,
+      "grad_norm": 1.8671875,
+      "learning_rate": 1.9018461445918727e-05,
+      "loss": 0.252,
+      "step": 176
+    },
+    {
+      "epoch": 0.384,
+      "grad_norm": 1.8046875,
+      "learning_rate": 1.895627659190294e-05,
+      "loss": 0.2529,
+      "step": 180
+    },
+    {
+      "epoch": 0.39253333333333335,
+      "grad_norm": 1.84375,
+      "learning_rate": 1.889229007927897e-05,
+      "loss": 0.2576,
+      "step": 184
+    },
+    {
+      "epoch": 0.4010666666666667,
+      "grad_norm": 2.015625,
+      "learning_rate": 1.8826514779673792e-05,
+      "loss": 0.2541,
+      "step": 188
+    },
+    {
+      "epoch": 0.4096,
+      "grad_norm": 1.8046875,
+      "learning_rate": 1.875896392454955e-05,
+      "loss": 0.2659,
+      "step": 192
+    },
+    {
+      "epoch": 0.41813333333333336,
+      "grad_norm": 1.6875,
+      "learning_rate": 1.8689651102541915e-05,
+      "loss": 0.2551,
+      "step": 196
+    },
+    {
+      "epoch": 0.4266666666666667,
+      "grad_norm": 1.9296875,
+      "learning_rate": 1.8618590256726587e-05,
+      "loss": 0.2597,
+      "step": 200
+    },
+    {
+      "epoch": 0.4352,
+      "grad_norm": 1.9140625,
+      "learning_rate": 1.854579568181446e-05,
+      "loss": 0.2627,
+      "step": 204
+    },
+    {
+      "epoch": 0.4437333333333333,
+      "grad_norm": 1.9296875,
+      "learning_rate": 1.8471282021276073e-05,
+      "loss": 0.2557,
+      "step": 208
+    },
+    {
+      "epoch": 0.45226666666666665,
+      "grad_norm": 1.9609375,
+      "learning_rate": 1.8395064264395945e-05,
+      "loss": 0.2573,
+      "step": 212
+    },
+    {
+      "epoch": 0.4608,
+      "grad_norm": 1.75,
+      "learning_rate": 1.831715774325726e-05,
+      "loss": 0.2515,
+      "step": 216
+    },
+    {
+      "epoch": 0.4693333333333333,
+      "grad_norm": 1.75,
+      "learning_rate": 1.8237578129657664e-05,
+      "loss": 0.2501,
+      "step": 220
+    },
+    {
+      "epoch": 0.47786666666666666,
+      "grad_norm": 1.6640625,
+      "learning_rate": 1.8156341431956706e-05,
+      "loss": 0.2521,
+      "step": 224
+    },
+    {
+      "epoch": 0.4864,
+      "grad_norm": 1.9296875,
+      "learning_rate": 1.8073463991855562e-05,
+      "loss": 0.2427,
+      "step": 228
+    },
+    {
+      "epoch": 0.49493333333333334,
+      "grad_norm": 1.8671875,
+      "learning_rate": 1.7988962481109716e-05,
+      "loss": 0.2639,
+      "step": 232
+    },
+    {
+      "epoch": 0.5034666666666666,
+      "grad_norm": 1.8203125,
+      "learning_rate": 1.7902853898175244e-05,
+      "loss": 0.2537,
+      "step": 236
+    },
+    {
+      "epoch": 0.512,
+      "grad_norm": 1.9375,
+      "learning_rate": 1.7815155564789374e-05,
+      "loss": 0.2611,
+      "step": 240
+    },
+    {
+      "epoch": 0.5205333333333333,
+      "grad_norm": 1.921875,
+      "learning_rate": 1.772588512248602e-05,
+      "loss": 0.2648,
+      "step": 244
+    },
+    {
+      "epoch": 0.5290666666666667,
+      "grad_norm": 2.03125,
+      "learning_rate": 1.7635060529046994e-05,
+      "loss": 0.2848,
+      "step": 248
+    },
+    {
+      "epoch": 0.5376,
+      "grad_norm": 1.75,
+      "learning_rate": 1.7542700054889572e-05,
+      "loss": 0.2695,
+      "step": 252
+    },
+    {
+      "epoch": 0.5461333333333334,
+      "grad_norm": 2.078125,
+      "learning_rate": 1.7448822279391204e-05,
+      "loss": 0.2653,
+      "step": 256
+    },
+    {
+      "epoch": 0.5546666666666666,
+      "grad_norm": 1.765625,
+      "learning_rate": 1.7353446087152038e-05,
+      "loss": 0.249,
+      "step": 260
+    },
+    {
+      "epoch": 0.5632,
+      "grad_norm": 1.75,
+      "learning_rate": 1.72565906641961e-05,
+      "loss": 0.2462,
+      "step": 264
+    },
+    {
+      "epoch": 0.5717333333333333,
+      "grad_norm": 1.6328125,
+      "learning_rate": 1.7158275494111763e-05,
+      "loss": 0.2373,
+      "step": 268
+    },
+    {
+      "epoch": 0.5802666666666667,
+      "grad_norm": 2.0625,
+      "learning_rate": 1.705852035413242e-05,
+      "loss": 0.2728,
+      "step": 272
+    },
+    {
+      "epoch": 0.5888,
+      "grad_norm": 1.671875,
+      "learning_rate": 1.6957345311158066e-05,
+      "loss": 0.2409,
+      "step": 276
+    },
+    {
+      "epoch": 0.5973333333333334,
+      "grad_norm": 2.015625,
+      "learning_rate": 1.6854770717718587e-05,
+      "loss": 0.254,
+      "step": 280
+    },
+    {
+      "epoch": 0.6058666666666667,
+      "grad_norm": 1.75,
+      "learning_rate": 1.6750817207879655e-05,
+      "loss": 0.2462,
+      "step": 284
+    },
+    {
+      "epoch": 0.6144,
+      "grad_norm": 1.6640625,
+      "learning_rate": 1.6645505693091897e-05,
+      "loss": 0.2561,
+      "step": 288
+    },
+    {
+      "epoch": 0.6229333333333333,
+      "grad_norm": 2.25,
+      "learning_rate": 1.6538857357984358e-05,
+      "loss": 0.2647,
+      "step": 292
+    },
+    {
+      "epoch": 0.6314666666666666,
+      "grad_norm": 1.9140625,
+      "learning_rate": 1.6430893656102942e-05,
+      "loss": 0.2398,
+      "step": 296
+    },
+    {
+      "epoch": 0.64,
+      "grad_norm": 2.078125,
+      "learning_rate": 1.6321636305594784e-05,
+      "loss": 0.2575,
+      "step": 300
+    },
+    {
+      "epoch": 0.6485333333333333,
+      "grad_norm": 1.8125,
+      "learning_rate": 1.6211107284839417e-05,
+      "loss": 0.2436,
+      "step": 304
+    },
+    {
+      "epoch": 0.6570666666666667,
+      "grad_norm": 1.984375,
+      "learning_rate": 1.609932882802753e-05,
+      "loss": 0.2427,
+      "step": 308
+    },
+    {
+      "epoch": 0.6656,
+      "grad_norm": 1.8671875,
+      "learning_rate": 1.5986323420688335e-05,
+      "loss": 0.2391,
+      "step": 312
+    },
+    {
+      "epoch": 0.6741333333333334,
+      "grad_norm": 2.0,
+      "learning_rate": 1.5872113795166337e-05,
+      "loss": 0.2543,
+      "step": 316
+    },
+    {
+      "epoch": 0.6826666666666666,
+      "grad_norm": 1.875,
+      "learning_rate": 1.575672292604844e-05,
+      "loss": 0.2541,
+      "step": 320
+    },
+    {
+      "epoch": 0.6912,
+      "grad_norm": 1.671875,
+      "learning_rate": 1.564017402554237e-05,
+      "loss": 0.253,
+      "step": 324
+    },
+    {
+      "epoch": 0.6997333333333333,
+      "grad_norm": 2.078125,
+      "learning_rate": 1.5522490538807248e-05,
+      "loss": 0.2565,
+      "step": 328
+    },
+    {
+      "epoch": 0.7082666666666667,
+      "grad_norm": 1.7265625,
+      "learning_rate": 1.5403696139237338e-05,
+      "loss": 0.2587,
+      "step": 332
+    },
+    {
+      "epoch": 0.7168,
+      "grad_norm": 1.90625,
+      "learning_rate": 1.5283814723699877e-05,
+      "loss": 0.2609,
+      "step": 336
+    },
+    {
+      "epoch": 0.7253333333333334,
+      "grad_norm": 1.8359375,
+      "learning_rate": 1.5162870407727922e-05,
+      "loss": 0.2522,
+      "step": 340
+    },
+    {
+      "epoch": 0.7338666666666667,
+      "grad_norm": 1.7109375,
+      "learning_rate": 1.5040887520669245e-05,
+      "loss": 0.2532,
+      "step": 344
+    },
+    {
+      "epoch": 0.7424,
+      "grad_norm": 1.859375,
+      "learning_rate": 1.4917890600792215e-05,
+      "loss": 0.2434,
+      "step": 348
+    },
+    {
+      "epoch": 0.7509333333333333,
+      "grad_norm": 1.890625,
+      "learning_rate": 1.4793904390349618e-05,
+      "loss": 0.2491,
+      "step": 352
+    },
+    {
+      "epoch": 0.7594666666666666,
+      "grad_norm": 1.7890625,
+      "learning_rate": 1.4668953830601473e-05,
+      "loss": 0.2336,
+      "step": 356
+    },
+    {
+      "epoch": 0.768,
+      "grad_norm": 1.765625,
+      "learning_rate": 1.4543064056797826e-05,
+      "loss": 0.2665,
+      "step": 360
+    },
+    {
+      "epoch": 0.7765333333333333,
+      "grad_norm": 1.78125,
+      "learning_rate": 1.4416260393122487e-05,
+      "loss": 0.2538,
+      "step": 364
+    },
+    {
+      "epoch": 0.7850666666666667,
+      "grad_norm": 2.0625,
+      "learning_rate": 1.4288568347598777e-05,
+      "loss": 0.2551,
+      "step": 368
+    },
+    {
+      "epoch": 0.7936,
+      "grad_norm": 2.28125,
+      "learning_rate": 1.4160013606958303e-05,
+      "loss": 0.2349,
+      "step": 372
+    },
+    {
+      "epoch": 0.8021333333333334,
+      "grad_norm": 1.7734375,
+      "learning_rate": 1.403062203147377e-05,
+      "loss": 0.2347,
+      "step": 376
+    },
+    {
+      "epoch": 0.8106666666666666,
+      "grad_norm": 1.8359375,
+      "learning_rate": 1.3900419649756895e-05,
+      "loss": 0.2491,
+      "step": 380
+    },
+    {
+      "epoch": 0.8192,
+      "grad_norm": 1.8125,
+      "learning_rate": 1.3769432653522436e-05,
+      "loss": 0.2424,
+      "step": 384
+    },
+    {
+      "epoch": 0.8277333333333333,
+      "grad_norm": 1.828125,
+      "learning_rate": 1.3637687392319443e-05,
+      "loss": 0.2477,
+      "step": 388
+    },
+    {
+      "epoch": 0.8362666666666667,
+      "grad_norm": 1.7265625,
+      "learning_rate": 1.3505210368230723e-05,
+      "loss": 0.249,
+      "step": 392
+    },
+    {
+      "epoch": 0.8448,
+      "grad_norm": 1.7578125,
+      "learning_rate": 1.3372028230541658e-05,
+      "loss": 0.2325,
+      "step": 396
+    },
+    {
+      "epoch": 0.8533333333333334,
+      "grad_norm": 1.640625,
+      "learning_rate": 1.3238167770379384e-05,
+      "loss": 0.2358,
+      "step": 400
+    },
+    {
+      "epoch": 0.8618666666666667,
+      "grad_norm": 1.921875,
+      "learning_rate": 1.3103655915323444e-05,
+      "loss": 0.2371,
+      "step": 404
+    },
+    {
+      "epoch": 0.8704,
+      "grad_norm": 1.8359375,
+      "learning_rate": 1.2968519723988994e-05,
+      "loss": 0.234,
+      "step": 408
+    },
+    {
+      "epoch": 0.8789333333333333,
+      "grad_norm": 1.7734375,
+      "learning_rate": 1.2832786380583664e-05,
+      "loss": 0.256,
+      "step": 412
+    },
+    {
+      "epoch": 0.8874666666666666,
+      "grad_norm": 1.8515625,
+      "learning_rate": 1.2696483189439113e-05,
+      "loss": 0.2462,
+      "step": 416
+    },
+    {
+      "epoch": 0.896,
+      "grad_norm": 1.84375,
+      "learning_rate": 1.2559637569518472e-05,
+      "loss": 0.2554,
+      "step": 420
+    },
+    {
+      "epoch": 0.9045333333333333,
+      "grad_norm": 1.9296875,
+      "learning_rate": 1.2422277048900694e-05,
+      "loss": 0.2495,
+      "step": 424
+    },
+    {
+      "epoch": 0.9130666666666667,
+      "grad_norm": 1.765625,
+      "learning_rate": 1.2284429259242958e-05,
+      "loss": 0.2407,
+      "step": 428
+    },
+    {
+      "epoch": 0.9216,
+      "grad_norm": 1.8671875,
+      "learning_rate": 1.2146121930222241e-05,
+      "loss": 0.2532,
+      "step": 432
+    },
+    {
+      "epoch": 0.9301333333333334,
+      "grad_norm": 1.8046875,
+      "learning_rate": 1.2007382883957186e-05,
+      "loss": 0.2342,
+      "step": 436
+    },
+    {
+      "epoch": 0.9386666666666666,
+      "grad_norm": 1.984375,
+      "learning_rate": 1.1868240029411351e-05,
+      "loss": 0.243,
+      "step": 440
+    },
+    {
+      "epoch": 0.9472,
+      "grad_norm": 1.8671875,
+      "learning_rate": 1.1728721356778994e-05,
+      "loss": 0.2406,
+      "step": 444
+    },
+    {
+      "epoch": 0.9557333333333333,
+      "grad_norm": 1.8359375,
+      "learning_rate": 1.158885493185453e-05,
+      "loss": 0.2393,
+      "step": 448
+    },
+    {
+      "epoch": 0.9642666666666667,
+      "grad_norm": 1.984375,
+      "learning_rate": 1.1448668890386765e-05,
+      "loss": 0.2434,
+      "step": 452
+    },
+    {
+      "epoch": 0.9728,
+      "grad_norm": 1.96875,
+      "learning_rate": 1.1308191432419078e-05,
+      "loss": 0.2312,
+      "step": 456
+    },
+    {
+      "epoch": 0.9813333333333333,
+      "grad_norm": 1.6484375,
+      "learning_rate": 1.1167450816616639e-05,
+      "loss": 0.2402,
+      "step": 460
+    },
+    {
+      "epoch": 0.9898666666666667,
+      "grad_norm": 2.09375,
+      "learning_rate": 1.102647535458186e-05,
+      "loss": 0.2479,
+      "step": 464
+    },
+    {
+      "epoch": 0.9984,
+      "grad_norm": 1.8515625,
+      "learning_rate": 1.0885293405159196e-05,
+      "loss": 0.2402,
+      "step": 468
+    },
+    {
+      "epoch": 1.0069333333333332,
+      "grad_norm": 1.28125,
+      "learning_rate": 1.0743933368730417e-05,
+      "loss": 0.1636,
+      "step": 472
+    },
+    {
+      "epoch": 1.0154666666666667,
+      "grad_norm": 1.8125,
+      "learning_rate": 1.0602423681501564e-05,
+      "loss": 0.1315,
+      "step": 476
+    },
+    {
+      "epoch": 1.024,
+      "grad_norm": 2.265625,
+      "learning_rate": 1.0460792809782659e-05,
+      "loss": 0.1396,
+      "step": 480
+    },
+    {
+      "epoch": 1.0325333333333333,
+      "grad_norm": 1.453125,
+      "learning_rate": 1.031906924426139e-05,
+      "loss": 0.1295,
+      "step": 484
+    },
+    {
+      "epoch": 1.0410666666666666,
+      "grad_norm": 1.484375,
+      "learning_rate": 1.0177281494271873e-05,
+      "loss": 0.1258,
+      "step": 488
+    },
+    {
+      "epoch": 1.0496,
+      "grad_norm": 1.9140625,
+      "learning_rate": 1.0035458082059672e-05,
+      "loss": 0.1304,
+      "step": 492
+    },
+    {
+      "epoch": 1.0581333333333334,
+      "grad_norm": 2.0,
+      "learning_rate": 9.893627537044223e-06,
+      "loss": 0.1347,
+      "step": 496
+    },
+    {
+      "epoch": 1.0666666666666667,
+      "grad_norm": 1.75,
+      "learning_rate": 9.751818390079805e-06,
+      "loss": 0.1332,
+      "step": 500
+    },
+    {
+      "epoch": 1.0752,
+      "grad_norm": 1.6953125,
+      "learning_rate": 9.61005916771623e-06,
+      "loss": 0.1265,
+      "step": 504
+    },
+    {
+      "epoch": 1.0837333333333334,
+      "grad_norm": 1.7578125,
+      "learning_rate": 9.468378386460406e-06,
+      "loss": 0.1304,
+      "step": 508
+    },
+    {
+      "epoch": 1.0922666666666667,
+      "grad_norm": 1.8515625,
+      "learning_rate": 9.326804547039894e-06,
+      "loss": 0.1346,
+      "step": 512
+    },
+    {
+      "epoch": 1.1008,
+      "grad_norm": 1.7421875,
+      "learning_rate": 9.185366128669682e-06,
+      "loss": 0.1264,
+      "step": 516
+    },
+    {
+      "epoch": 1.1093333333333333,
+      "grad_norm": 2.078125,
+      "learning_rate": 9.044091583323231e-06,
+      "loss": 0.1206,
+      "step": 520
+    },
+    {
+      "epoch": 1.1178666666666666,
+      "grad_norm": 1.6328125,
+      "learning_rate": 8.903009330009063e-06,
+      "loss": 0.1307,
+      "step": 524
+    },
+    {
+      "epoch": 1.1264,
+      "grad_norm": 2.15625,
+      "learning_rate": 8.762147749053928e-06,
+      "loss": 0.1338,
+      "step": 528
+    },
+    {
+      "epoch": 1.1349333333333333,
+      "grad_norm": 1.5625,
+      "learning_rate": 8.621535176393776e-06,
+      "loss": 0.1212,
+      "step": 532
+    },
+    {
+      "epoch": 1.1434666666666666,
+      "grad_norm": 1.5078125,
+      "learning_rate": 8.481199897873667e-06,
+      "loss": 0.1301,
+      "step": 536
+    },
+    {
+      "epoch": 1.152,
+      "grad_norm": 1.8203125,
+      "learning_rate": 8.341170143557733e-06,
+      "loss": 0.1252,
+      "step": 540
+    },
+    {
+      "epoch": 1.1605333333333334,
+      "grad_norm": 1.6953125,
+      "learning_rate": 8.201474082050376e-06,
+      "loss": 0.1176,
+      "step": 544
+    },
+    {
+      "epoch": 1.1690666666666667,
+      "grad_norm": 1.6875,
+      "learning_rate": 8.062139814829839e-06,
+      "loss": 0.1164,
+      "step": 548
+    },
+    {
+      "epoch": 1.1776,
+      "grad_norm": 1.703125,
+      "learning_rate": 7.92319537059525e-06,
+      "loss": 0.1326,
+      "step": 552
+    },
+    {
+      "epoch": 1.1861333333333333,
+      "grad_norm": 1.9296875,
+      "learning_rate": 7.784668699628345e-06,
+      "loss": 0.1334,
+      "step": 556
+    },
+    {
+      "epoch": 1.1946666666666665,
+      "grad_norm": 1.6796875,
+      "learning_rate": 7.64658766817093e-06,
+      "loss": 0.1253,
+      "step": 560
+    },
+    {
+      "epoch": 1.2032,
+      "grad_norm": 1.75,
+      "learning_rate": 7.508980052819274e-06,
+      "loss": 0.1327,
+      "step": 564
+    },
+    {
+      "epoch": 1.2117333333333333,
+      "grad_norm": 1.5390625,
+      "learning_rate": 7.371873534936522e-06,
+      "loss": 0.1318,
+      "step": 568
+    },
+    {
+      "epoch": 1.2202666666666666,
+      "grad_norm": 1.6640625,
+      "learning_rate": 7.235295695084259e-06,
+      "loss": 0.1266,
+      "step": 572
+    },
+    {
+      "epoch": 1.2288000000000001,
+      "grad_norm": 1.7890625,
+      "learning_rate": 7.0992740074743835e-06,
+      "loss": 0.1275,
+      "step": 576
+    },
+    {
+      "epoch": 1.2373333333333334,
+      "grad_norm": 1.8671875,
+      "learning_rate": 6.963835834442336e-06,
+      "loss": 0.1215,
+      "step": 580
+    },
+    {
+      "epoch": 1.2458666666666667,
+      "grad_norm": 1.8046875,
+      "learning_rate": 6.829008420942842e-06,
+      "loss": 0.1287,
+      "step": 584
+    },
+    {
+      "epoch": 1.2544,
+      "grad_norm": 2.03125,
+      "learning_rate": 6.694818889069294e-06,
+      "loss": 0.1206,
+      "step": 588
+    },
+    {
+      "epoch": 1.2629333333333332,
+      "grad_norm": 1.6015625,
+      "learning_rate": 6.561294232597817e-06,
+      "loss": 0.1237,
+      "step": 592
+    },
+    {
+      "epoch": 1.2714666666666667,
+      "grad_norm": 1.6875,
+      "learning_rate": 6.428461311557159e-06,
+      "loss": 0.1244,
+      "step": 596
+    },
+    {
+      "epoch": 1.28,
+      "grad_norm": 1.828125,
+      "learning_rate": 6.29634684682549e-06,
+      "loss": 0.1266,
+      "step": 600
+    },
+    {
+      "epoch": 1.2885333333333333,
+      "grad_norm": 1.5703125,
+      "learning_rate": 6.1649774147551755e-06,
+      "loss": 0.1249,
+      "step": 604
+    },
+    {
+      "epoch": 1.2970666666666666,
+      "grad_norm": 1.703125,
+      "learning_rate": 6.034379441826659e-06,
+      "loss": 0.1211,
+      "step": 608
+    },
+    {
+      "epoch": 1.3056,
+      "grad_norm": 1.6796875,
+      "learning_rate": 5.904579199332443e-06,
+      "loss": 0.122,
+      "step": 612
+    },
+    {
+      "epoch": 1.3141333333333334,
+      "grad_norm": 2.28125,
+      "learning_rate": 5.775602798092335e-06,
+      "loss": 0.1299,
+      "step": 616
+    },
+    {
+      "epoch": 1.3226666666666667,
+      "grad_norm": 1.6484375,
+      "learning_rate": 5.6474761832009554e-06,
+      "loss": 0.1238,
+      "step": 620
+    },
+    {
+      "epoch": 1.3312,
+      "grad_norm": 1.6484375,
+      "learning_rate": 5.520225128808555e-06,
+      "loss": 0.1185,
+      "step": 624
+    },
+    {
+      "epoch": 1.3397333333333332,
+      "grad_norm": 1.703125,
+      "learning_rate": 5.393875232936283e-06,
+      "loss": 0.1267,
+      "step": 628
+    },
+    {
+      "epoch": 1.3482666666666667,
+      "grad_norm": 1.734375,
+      "learning_rate": 5.2684519123268155e-06,
+      "loss": 0.1272,
+      "step": 632
+    },
+    {
+      "epoch": 1.3568,
+      "grad_norm": 4.125,
+      "learning_rate": 5.143980397331512e-06,
+      "loss": 0.1348,
+      "step": 636
+    },
+    {
+      "epoch": 1.3653333333333333,
+      "grad_norm": 1.828125,
+      "learning_rate": 5.02048572683502e-06,
+      "loss": 0.1269,
+      "step": 640
+    },
+    {
+      "epoch": 1.3738666666666668,
+      "grad_norm": 1.5625,
+      "learning_rate": 4.897992743218419e-06,
+      "loss": 0.1178,
+      "step": 644
+    },
+    {
+      "epoch": 1.3824,
+      "grad_norm": 1.9375,
+      "learning_rate": 4.776526087361896e-06,
+      "loss": 0.1215,
+      "step": 648
+    },
+    {
+      "epoch": 1.3909333333333334,
+      "grad_norm": 1.671875,
+      "learning_rate": 4.656110193687925e-06,
+      "loss": 0.1246,
+      "step": 652
+    },
+    {
+      "epoch": 1.3994666666666666,
+      "grad_norm": 1.8359375,
+      "learning_rate": 4.536769285246033e-06,
+      "loss": 0.129,
+      "step": 656
+    },
+    {
+      "epoch": 1.408,
+      "grad_norm": 1.6015625,
+      "learning_rate": 4.4185273688400274e-06,
+      "loss": 0.126,
+      "step": 660
+    },
+    {
+      "epoch": 1.4165333333333332,
+      "grad_norm": 1.71875,
+      "learning_rate": 4.301408230198763e-06,
+      "loss": 0.1203,
+      "step": 664
+    },
+    {
+      "epoch": 1.4250666666666667,
+      "grad_norm": 1.890625,
+      "learning_rate": 4.1854354291913594e-06,
+      "loss": 0.126,
+      "step": 668
+    },
+    {
+      "epoch": 1.4336,
+      "grad_norm": 1.7421875,
+      "learning_rate": 4.070632295087863e-06,
+      "loss": 0.1243,
+      "step": 672
+    },
+    {
+      "epoch": 1.4421333333333333,
+      "grad_norm": 2.015625,
+      "learning_rate": 3.957021921866301e-06,
+      "loss": 0.1246,
+      "step": 676
+    },
+    {
+      "epoch": 1.4506666666666668,
+      "grad_norm": 1.8984375,
+      "learning_rate": 3.844627163567059e-06,
+      "loss": 0.128,
+      "step": 680
+    },
+    {
+      "epoch": 1.4592,
+      "grad_norm": 1.6640625,
+      "learning_rate": 3.7334706296955093e-06,
+      "loss": 0.115,
+      "step": 684
+    },
+    {
+      "epoch": 1.4677333333333333,
+      "grad_norm": 1.6328125,
+      "learning_rate": 3.623574680673879e-06,
+      "loss": 0.1265,
+      "step": 688
+    },
+    {
+      "epoch": 1.4762666666666666,
+      "grad_norm": 1.8046875,
+      "learning_rate": 3.5149614233431616e-06,
+      "loss": 0.1273,
+      "step": 692
+    },
+    {
+      "epoch": 1.4848,
+      "grad_norm": 1.921875,
+      "learning_rate": 3.4076527065160914e-06,
+      "loss": 0.1287,
+      "step": 696
+    },
+    {
+      "epoch": 1.4933333333333334,
+      "grad_norm": 1.625,
+      "learning_rate": 3.3016701165819943e-06,
+      "loss": 0.1268,
+      "step": 700
+    },
+    {
+      "epoch": 1.5018666666666667,
+      "grad_norm": 2.015625,
+      "learning_rate": 3.197034973164429e-06,
+      "loss": 0.1173,
+      "step": 704
+    },
+    {
+      "epoch": 1.5104,
+      "grad_norm": 1.7578125,
+      "learning_rate": 3.0937683248325133e-06,
+      "loss": 0.1213,
+      "step": 708
+    },
+    {
+      "epoch": 1.5189333333333335,
+      "grad_norm": 1.8828125,
+      "learning_rate": 2.991890944866752e-06,
+      "loss": 0.1234,
+      "step": 712
+    },
+    {
+      "epoch": 1.5274666666666668,
+      "grad_norm": 1.6875,
+      "learning_rate": 2.891423327080246e-06,
+      "loss": 0.1347,
+      "step": 716
+    },
+    {
+      "epoch": 1.536,
+      "grad_norm": 1.7578125,
+      "learning_rate": 2.792385681696138e-06,
+      "loss": 0.127,
+      "step": 720
+    },
+    {
+      "epoch": 1.5445333333333333,
+      "grad_norm": 1.7265625,
+      "learning_rate": 2.6947979312820825e-06,
+      "loss": 0.1193,
+      "step": 724
+    },
+    {
+      "epoch": 1.5530666666666666,
+      "grad_norm": 1.71875,
+      "learning_rate": 2.5986797067425972e-06,
+      "loss": 0.1336,
+      "step": 728
+    },
+    {
+      "epoch": 1.5615999999999999,
+      "grad_norm": 1.46875,
+      "learning_rate": 2.5040503433700702e-06,
+      "loss": 0.1153,
+      "step": 732
+    },
+    {
+      "epoch": 1.5701333333333334,
+      "grad_norm": 1.484375,
+      "learning_rate": 2.4109288769552518e-06,
+      "loss": 0.1195,
+      "step": 736
+    },
+    {
+      "epoch": 1.5786666666666667,
+      "grad_norm": 1.8359375,
+      "learning_rate": 2.3193340399579865e-06,
+      "loss": 0.1188,
+      "step": 740
+    },
+    {
+      "epoch": 1.5872000000000002,
+      "grad_norm": 1.6484375,
+      "learning_rate": 2.229284257738946e-06,
+      "loss": 0.1203,
+      "step": 744
+    },
+    {
+      "epoch": 1.5957333333333334,
+      "grad_norm": 1.6953125,
+      "learning_rate": 2.1407976448531776e-06,
+      "loss": 0.1249,
+      "step": 748
+    },
+    {
+      "epoch": 1.6042666666666667,
+      "grad_norm": 1.7734375,
+      "learning_rate": 2.053892001406136e-06,
+      "loss": 0.1181,
+      "step": 752
+    },
+    {
+      "epoch": 1.6128,
+      "grad_norm": 1.7265625,
+      "learning_rate": 1.9685848094729853e-06,
+      "loss": 0.129,
+      "step": 756
+    },
+    {
+      "epoch": 1.6213333333333333,
+      "grad_norm": 1.75,
+      "learning_rate": 1.8848932295818945e-06,
+      "loss": 0.1213,
+      "step": 760
+    },
+    {
+      "epoch": 1.6298666666666666,
+      "grad_norm": 1.5859375,
+      "learning_rate": 1.802834097261975e-06,
+      "loss": 0.1248,
+      "step": 764
+    },
+    {
+      "epoch": 1.6383999999999999,
+      "grad_norm": 1.8125,
+      "learning_rate": 1.7224239196566395e-06,
+      "loss": 0.1313,
+      "step": 768
+    },
+    {
+      "epoch": 1.6469333333333334,
+      "grad_norm": 2.0,
+      "learning_rate": 1.6436788722029906e-06,
+      "loss": 0.1283,
+      "step": 772
+    },
+    {
+      "epoch": 1.6554666666666666,
+      "grad_norm": 1.578125,
+      "learning_rate": 1.5666147953779376e-06,
+      "loss": 0.1176,
+      "step": 776
+    },
+    {
+      "epoch": 1.6640000000000001,
+      "grad_norm": 1.6875,
+      "learning_rate": 1.4912471915117189e-06,
+      "loss": 0.1203,
+      "step": 780
+    },
+    {
+      "epoch": 1.6725333333333334,
+      "grad_norm": 1.765625,
+      "learning_rate": 1.417591221669412e-06,
+      "loss": 0.1218,
+      "step": 784
+    },
+    {
+      "epoch": 1.6810666666666667,
+      "grad_norm": 1.578125,
+      "learning_rate": 1.3456617026011233e-06,
+      "loss": 0.1226,
+      "step": 788
+    },
+    {
+      "epoch": 1.6896,
+      "grad_norm": 1.734375,
+      "learning_rate": 1.2754731037614122e-06,
+      "loss": 0.1265,
+      "step": 792
+    },
+    {
+      "epoch": 1.6981333333333333,
+      "grad_norm": 1.8046875,
+      "learning_rate": 1.207039544398607e-06,
+      "loss": 0.12,
+      "step": 796
+    },
+    {
+      "epoch": 1.7066666666666666,
+      "grad_norm": 1.8828125,
+      "learning_rate": 1.1403747907145546e-06,
+      "loss": 0.1325,
+      "step": 800
+    },
+    {
+      "epoch": 1.7151999999999998,
+      "grad_norm": 1.6796875,
+      "learning_rate": 1.0754922530953737e-06,
+      "loss": 0.125,
+      "step": 804
+    },
+    {
+      "epoch": 1.7237333333333333,
+      "grad_norm": 1.625,
+      "learning_rate": 1.0124049834138205e-06,
+      "loss": 0.1161,
+      "step": 808
+    },
+    {
+      "epoch": 1.7322666666666666,
+      "grad_norm": 1.5,
+      "learning_rate": 9.511256724037443e-07,
+      "loss": 0.1244,
+      "step": 812
+    },
+    {
+      "epoch": 1.7408000000000001,
+      "grad_norm": 1.765625,
+      "learning_rate": 8.916666471071922e-07,
+      "loss": 0.1267,
+      "step": 816
+    },
+    {
+      "epoch": 1.7493333333333334,
+      "grad_norm": 1.6640625,
+      "learning_rate": 8.340398683947004e-07,
+      "loss": 0.1303,
+      "step": 820
+    },
+    {
+      "epoch": 1.7578666666666667,
+      "grad_norm": 1.875,
+      "learning_rate": 7.78256928559209e-07,
+      "loss": 0.133,
+      "step": 824
+    },
+    {
+      "epoch": 1.7664,
+      "grad_norm": 1.5859375,
+      "learning_rate": 7.243290489841493e-07,
+      "loss": 0.1188,
+      "step": 828
+    },
+    {
+      "epoch": 1.7749333333333333,
+      "grad_norm": 1.7578125,
+      "learning_rate": 6.722670778861284e-07,
+      "loss": 0.1247,
+      "step": 832
+    },
+    {
+      "epoch": 1.7834666666666665,
+      "grad_norm": 1.671875,
+      "learning_rate": 6.22081488132682e-07,
+      "loss": 0.1215,
+      "step": 836
+    },
+    {
+      "epoch": 1.792,
+      "grad_norm": 1.53125,
+      "learning_rate": 5.737823751355465e-07,
+      "loss": 0.1203,
+      "step": 840
+    },
+    {
+      "epoch": 1.8005333333333333,
+      "grad_norm": 1.6953125,
+      "learning_rate": 5.273794548198374e-07,
+      "loss": 0.1283,
+      "step": 844
+    },
+    {
+      "epoch": 1.8090666666666668,
+      "grad_norm": 1.671875,
+      "learning_rate": 4.828820616695873e-07,
+      "loss": 0.1262,
+      "step": 848
+    },
+    {
+      "epoch": 1.8176,
+      "grad_norm": 1.7265625,
+      "learning_rate": 4.4029914685000176e-07,
+      "loss": 0.1111,
+      "step": 852
+    },
+    {
+      "epoch": 1.8261333333333334,
+      "grad_norm": 1.875,
+      "learning_rate": 3.9963927640683243e-07,
+      "loss": 0.1228,
+      "step": 856
+    },
+    {
+      "epoch": 1.8346666666666667,
+      "grad_norm": 1.59375,
+      "learning_rate": 3.6091062954321634e-07,
+      "loss": 0.1293,
+      "step": 860
+    },
+    {
+      "epoch": 1.8432,
+      "grad_norm": 1.7421875,
+      "learning_rate": 3.241209969743353e-07,
+      "loss": 0.1201,
+      "step": 864
+    },
+    {
+      "epoch": 1.8517333333333332,
+      "grad_norm": 2.390625,
+      "learning_rate": 2.892777793602175e-07,
+      "loss": 0.1187,
+      "step": 868
+    },
+    {
+      "epoch": 1.8602666666666665,
+      "grad_norm": 1.7265625,
+      "learning_rate": 2.563879858170215e-07,
+      "loss": 0.1276,
+      "step": 872
+    },
+    {
+      "epoch": 1.8688,
+      "grad_norm": 1.765625,
+      "learning_rate": 2.2545823250705867e-07,
+      "loss": 0.1334,
+      "step": 876
+    },
+    {
+      "epoch": 1.8773333333333333,
+      "grad_norm": 1.96875,
+      "learning_rate": 1.9649474130788438e-07,
+      "loss": 0.114,
+      "step": 880
+    },
+    {
+      "epoch": 1.8858666666666668,
+      "grad_norm": 1.765625,
+      "learning_rate": 1.6950333856069369e-07,
+      "loss": 0.1221,
+      "step": 884
+    },
+    {
+      "epoch": 1.8944,
+      "grad_norm": 1.8046875,
+      "learning_rate": 1.4448945389827772e-07,
+      "loss": 0.1243,
+      "step": 888
+    },
+    {
+      "epoch": 1.9029333333333334,
+      "grad_norm": 1.765625,
+      "learning_rate": 1.2145811915280414e-07,
+      "loss": 0.1174,
+      "step": 892
+    },
+    {
+      "epoch": 1.9114666666666666,
+      "grad_norm": 1.9921875,
+      "learning_rate": 1.004139673435922e-07,
+      "loss": 0.1165,
+      "step": 896
+    },
+    {
+      "epoch": 1.92,
+      "grad_norm": 2.921875,
+      "learning_rate": 8.136123174513843e-08,
+      "loss": 0.1351,
+      "step": 900
+    },
+    {
+      "epoch": 1.9285333333333332,
+      "grad_norm": 1.640625,
+      "learning_rate": 6.430374503553439e-08,
+      "loss": 0.1208,
+      "step": 904
+    },
+    {
+      "epoch": 1.9370666666666667,
+      "grad_norm": 1.6640625,
+      "learning_rate": 4.924493852549006e-08,
+      "loss": 0.1178,
+      "step": 908
+    },
+    {
+      "epoch": 1.9456,
+      "grad_norm": 1.78125,
+      "learning_rate": 3.618784146807497e-08,
+      "loss": 0.1184,
+      "step": 912
+    },
+    {
+      "epoch": 1.9541333333333335,
+      "grad_norm": 1.7578125,
+      "learning_rate": 2.513508044935775e-08,
+      "loss": 0.1235,
+      "step": 916
+    },
+    {
+      "epoch": 1.9626666666666668,
+      "grad_norm": 1.6171875,
+      "learning_rate": 1.6088878860032187e-08,
+      "loss": 0.1231,
+      "step": 920
+    },
+    {
+      "epoch": 1.9712,
+      "grad_norm": 2.03125,
+      "learning_rate": 9.051056448160511e-09,
+      "loss": 0.1263,
+      "step": 924
+    },
+    {
+      "epoch": 1.9797333333333333,
+      "grad_norm": 1.75,
+      "learning_rate": 4.023028953106245e-09,
+      "loss": 0.1271,
+      "step": 928
+    },
+    {
+      "epoch": 1.9882666666666666,
+      "grad_norm": 1.6875,
+      "learning_rate": 1.0058078207453303e-09,
+      "loss": 0.1201,
+      "step": 932
+    },
+    {
+      "epoch": 1.9968,
+      "grad_norm": 1.953125,
+      "learning_rate": 0.0,
+      "loss": 0.1226,
+      "step": 936
+    }
+  ],
+  "logging_steps": 4,
+  "max_steps": 936,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 2,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 6.214928345734316e+19,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a6bcd441fcd08e9f847dfca832eddb3952efa99e41aac13b551227030ee3795d
+size 5432