Initial commit

README.md

@@ -0,0 +1,60 @@
+---
+license: other
+base_model: meta-llama/Meta-Llama-3-8B-Instruct
+tags:
+- llama-factory
+- full
+- generated_from_trainer
+model-index:
+- name: train_2024-07-11-10-49-45_inst_llama3
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# train_2024-07-11-10-49-45_inst_llama3
+
+This model is a fine-tuned version of [meta-llama/Meta-Llama-3-8B-Instruct](https://huggingface.co/meta-llama/Meta-Llama-3-8B-Instruct) on the truth_train dataset.
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 5e-06
+- train_batch_size: 4
+- eval_batch_size: 8
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 8
+- gradient_accumulation_steps: 8
+- total_train_batch_size: 256
+- total_eval_batch_size: 64
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: cosine
+- lr_scheduler_warmup_steps: 600
+- num_epochs: 5.0
+
+### Training results
+
+
+
+### Framework versions
+
+- Transformers 4.42.3
+- Pytorch 2.3.0a0+ebedce2
+- Datasets 2.20.0
+- Tokenizers 0.19.1

all_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 4.951768488745981,
+    "num_input_tokens_seen": 5776160,
+    "total_flos": 2.6009787341943603e+17,
+    "train_loss": 1.2260490821160424,
+    "train_runtime": 5290.2175,
+    "train_samples_per_second": 18.789,
+    "train_steps_per_second": 0.073
+}

checkpoint-385/config.json

@@ -0,0 +1,29 @@
+{
+  "_name_or_path": "meta-llama/Meta-Llama-3-8B-Instruct",
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 128000,
+  "eos_token_id": 128009,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 8192,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": null,
+  "rope_theta": 500000.0,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.42.3",
+  "use_cache": false,
+  "vocab_size": 128256
+}

checkpoint-385/generation_config.json

@@ -0,0 +1,12 @@
+{
+  "bos_token_id": 128000,
+  "do_sample": true,
+  "eos_token_id": [
+    128001,
+    128009
+  ],
+  "max_length": 4096,
+  "temperature": 0.6,
+  "top_p": 0.9,
+  "transformers_version": "4.42.3"
+}

checkpoint-385/global_step385/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ec53a455f325b35628c9e1bbccb82c9df8d758c387f95f2875efe8ebf758507b
+size 12045399116

checkpoint-385/global_step385/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:58f9a2ec888607bb16d67dcab5e4c8a6efe1828523cef761b7c05ba39d28bbf8
+size 12045399948

checkpoint-385/global_step385/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1f0d474e4325d5d7ea131adaece4c14ab0ae47637d28a07eb375a6de871e154d
+size 12045400204

checkpoint-385/global_step385/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:48adc696909841cc2cfd19367bce6cfef86f8199a5eec7ff7da872cfc078d1a0
+size 12045399948

checkpoint-385/global_step385/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6e2b75d4fc44f85d6cadac5cc7e3da5872f5f85b6b8052700c39cb9e39f51fc8
+size 12045400204

checkpoint-385/global_step385/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:085e5bdfa3b7ad0dfba07600c8b4dd79c6df817e30feb10a041624d4542dbf60
+size 12045400268

checkpoint-385/global_step385/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0f3351d702be1df68648462e75a67b9bb12fdee55c0adec4c5aceafb70b7cd3c
+size 12045399948

checkpoint-385/global_step385/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b0ec8531392025a419e2bde35f157496aaee81eec806c20828b1af6ad4d7eeac
+size 12045398796

checkpoint-385/global_step385/mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ceeb3cefcfd3b7f4163afc92cc259163783fc404b7f62a73d7a2a1871f7c4767
+size 16060610616

checkpoint-385/latest

@@ -0,0 +1 @@
+global_step385

checkpoint-385/model-00001-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:efcd3278f2c803edac2237fe1884ec3368c10c25b63ea6360a1d7b3e7370a29d
+size 4976698672

checkpoint-385/model-00002-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c6c06b5f09b61f3fe2fae5caf38101caa2c98923103d35594d18b36e9c6c2efd
+size 4999802720

checkpoint-385/model-00003-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:099b3b6852cee899327282a532875347bee6f1601ee05bc5860c15305483572f
+size 4915916176

checkpoint-385/model-00004-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a4b4e47f230f79b84a08863d70ee9d25bda8c074e8ccbb54e41596fe5e20d6ca
+size 1168138808

checkpoint-385/model.safetensors.index.json

@@ -0,0 +1,298 @@
+{
+  "metadata": {
+    "total_size": 16060522496
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00004-of-00004.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.norm.weight": "model-00004-of-00004.safetensors"
+  }
+}

checkpoint-385/rng_state_0.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5a0ef6f96a48e59aa52c4b471312c2a62378c19acc7ebbae839612b03a7d775a
+size 15984

checkpoint-385/rng_state_1.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ab11d533c0fdad46ea8b8e295ba5fdb705e078eeb88cc28f37d82913508766e9
+size 15984

checkpoint-385/rng_state_2.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:615c168147e3465ce5bfab6da2ff4afc68566ce00ec0f0c6c9fc988038a58d0a
+size 15984

checkpoint-385/rng_state_3.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:79f71e8f8674ecaef9f8cdcbf7ac457a8b8ff15b12694ba2a2fffcb4b43f0f08
+size 15984

checkpoint-385/rng_state_4.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:88cf6d674dab5545c300a55135f08ca935730a3d35e2c419fb0b333f19482c19
+size 15984

checkpoint-385/rng_state_5.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2754f2cd8824702f027870d93748b3c0491b0ecd30f1e3d8e937116b2be6151f
+size 15984

checkpoint-385/rng_state_6.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1385124ac55604598f45ea6e2d141f29456647d3e7c10d12ca64ec93d312be8d
+size 15984

checkpoint-385/rng_state_7.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:416538efaec7391fa8fe782fb15146b83e5612d9e1961292c34c53e964806873
+size 15984

checkpoint-385/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5773c00222bef1db5ca810359f2616a1fbd633a740c194d6a3070d6b031a0320
+size 1064

checkpoint-385/special_tokens_map.json

@@ -0,0 +1,17 @@
+{
+  "bos_token": {
+    "content": "<|begin_of_text|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|eot_id|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "<|eot_id|>"
+}

checkpoint-385/tokenizer_config.json

@@ -0,0 +1,2065 @@
+{
+  "added_tokens_decoder": {
+    "128000": {
+      "content": "<|begin_of_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128001": {
+      "content": "<|end_of_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128002": {
+      "content": "<|reserved_special_token_0|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128003": {
+      "content": "<|reserved_special_token_1|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128004": {
+      "content": "<|reserved_special_token_2|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128005": {
+      "content": "<|reserved_special_token_3|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128006": {
+      "content": "<|start_header_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128007": {
+      "content": "<|end_header_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128008": {
+      "content": "<|reserved_special_token_4|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128009": {
+      "content": "<|eot_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128010": {
+      "content": "<|reserved_special_token_5|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128011": {
+      "content": "<|reserved_special_token_6|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128012": {
+      "content": "<|reserved_special_token_7|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128013": {
+      "content": "<|reserved_special_token_8|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128014": {
+      "content": "<|reserved_special_token_9|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128015": {
+      "content": "<|reserved_special_token_10|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128016": {
+      "content": "<|reserved_special_token_11|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128017": {
+      "content": "<|reserved_special_token_12|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128018": {
+      "content": "<|reserved_special_token_13|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128019": {
+      "content": "<|reserved_special_token_14|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128020": {
+      "content": "<|reserved_special_token_15|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128021": {
+      "content": "<|reserved_special_token_16|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128022": {
+      "content": "<|reserved_special_token_17|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128023": {
+      "content": "<|reserved_special_token_18|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128024": {
+      "content": "<|reserved_special_token_19|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128025": {
+      "content": "<|reserved_special_token_20|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128026": {
+      "content": "<|reserved_special_token_21|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128027": {
+      "content": "<|reserved_special_token_22|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128028": {
+      "content": "<|reserved_special_token_23|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128029": {
+      "content": "<|reserved_special_token_24|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128030": {
+      "content": "<|reserved_special_token_25|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128031": {
+      "content": "<|reserved_special_token_26|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128032": {
+      "content": "<|reserved_special_token_27|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128033": {
+      "content": "<|reserved_special_token_28|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128034": {
+      "content": "<|reserved_special_token_29|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128035": {
+      "content": "<|reserved_special_token_30|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128036": {
+      "content": "<|reserved_special_token_31|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128037": {
+      "content": "<|reserved_special_token_32|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128038": {
+      "content": "<|reserved_special_token_33|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128039": {
+      "content": "<|reserved_special_token_34|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128040": {
+      "content": "<|reserved_special_token_35|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128041": {
+      "content": "<|reserved_special_token_36|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128042": {
+      "content": "<|reserved_special_token_37|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128043": {
+      "content": "<|reserved_special_token_38|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128044": {
+      "content": "<|reserved_special_token_39|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128045": {
+      "content": "<|reserved_special_token_40|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128046": {
+      "content": "<|reserved_special_token_41|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128047": {
+      "content": "<|reserved_special_token_42|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128048": {
+      "content": "<|reserved_special_token_43|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128049": {
+      "content": "<|reserved_special_token_44|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128050": {
+      "content": "<|reserved_special_token_45|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128051": {
+      "content": "<|reserved_special_token_46|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128052": {
+      "content": "<|reserved_special_token_47|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128053": {
+      "content": "<|reserved_special_token_48|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128054": {
+      "content": "<|reserved_special_token_49|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128055": {
+      "content": "<|reserved_special_token_50|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128056": {
+      "content": "<|reserved_special_token_51|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128057": {
+      "content": "<|reserved_special_token_52|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128058": {
+      "content": "<|reserved_special_token_53|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128059": {
+      "content": "<|reserved_special_token_54|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128060": {
+      "content": "<|reserved_special_token_55|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128061": {
+      "content": "<|reserved_special_token_56|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128062": {
+      "content": "<|reserved_special_token_57|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128063": {
+      "content": "<|reserved_special_token_58|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128064": {
+      "content": "<|reserved_special_token_59|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128065": {
+      "content": "<|reserved_special_token_60|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128066": {
+      "content": "<|reserved_special_token_61|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128067": {
+      "content": "<|reserved_special_token_62|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128068": {
+      "content": "<|reserved_special_token_63|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128069": {
+      "content": "<|reserved_special_token_64|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128070": {
+      "content": "<|reserved_special_token_65|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128071": {
+      "content": "<|reserved_special_token_66|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128072": {
+      "content": "<|reserved_special_token_67|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128073": {
+      "content": "<|reserved_special_token_68|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128074": {
+      "content": "<|reserved_special_token_69|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128075": {
+      "content": "<|reserved_special_token_70|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128076": {
+      "content": "<|reserved_special_token_71|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128077": {
+      "content": "<|reserved_special_token_72|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128078": {
+      "content": "<|reserved_special_token_73|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128079": {
+      "content": "<|reserved_special_token_74|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128080": {
+      "content": "<|reserved_special_token_75|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128081": {
+      "content": "<|reserved_special_token_76|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128082": {
+      "content": "<|reserved_special_token_77|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128083": {
+      "content": "<|reserved_special_token_78|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128084": {
+      "content": "<|reserved_special_token_79|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128085": {
+      "content": "<|reserved_special_token_80|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128086": {
+      "content": "<|reserved_special_token_81|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128087": {
+      "content": "<|reserved_special_token_82|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128088": {
+      "content": "<|reserved_special_token_83|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128089": {
+      "content": "<|reserved_special_token_84|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128090": {
+      "content": "<|reserved_special_token_85|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128091": {
+      "content": "<|reserved_special_token_86|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128092": {
+      "content": "<|reserved_special_token_87|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128093": {
+      "content": "<|reserved_special_token_88|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128094": {
+      "content": "<|reserved_special_token_89|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128095": {
+      "content": "<|reserved_special_token_90|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128096": {
+      "content": "<|reserved_special_token_91|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128097": {
+      "content": "<|reserved_special_token_92|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128098": {
+      "content": "<|reserved_special_token_93|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128099": {
+      "content": "<|reserved_special_token_94|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128100": {
+      "content": "<|reserved_special_token_95|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128101": {
+      "content": "<|reserved_special_token_96|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128102": {
+      "content": "<|reserved_special_token_97|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128103": {
+      "content": "<|reserved_special_token_98|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128104": {
+      "content": "<|reserved_special_token_99|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128105": {
+      "content": "<|reserved_special_token_100|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128106": {
+      "content": "<|reserved_special_token_101|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128107": {
+      "content": "<|reserved_special_token_102|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128108": {
+      "content": "<|reserved_special_token_103|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128109": {
+      "content": "<|reserved_special_token_104|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128110": {
+      "content": "<|reserved_special_token_105|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128111": {
+      "content": "<|reserved_special_token_106|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128112": {
+      "content": "<|reserved_special_token_107|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128113": {
+      "content": "<|reserved_special_token_108|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128114": {
+      "content": "<|reserved_special_token_109|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128115": {
+      "content": "<|reserved_special_token_110|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128116": {
+      "content": "<|reserved_special_token_111|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128117": {
+      "content": "<|reserved_special_token_112|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128118": {
+      "content": "<|reserved_special_token_113|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128119": {
+      "content": "<|reserved_special_token_114|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128120": {
+      "content": "<|reserved_special_token_115|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128121": {
+      "content": "<|reserved_special_token_116|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128122": {
+      "content": "<|reserved_special_token_117|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128123": {
+      "content": "<|reserved_special_token_118|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128124": {
+      "content": "<|reserved_special_token_119|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128125": {
+      "content": "<|reserved_special_token_120|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128126": {
+      "content": "<|reserved_special_token_121|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128127": {
+      "content": "<|reserved_special_token_122|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128128": {
+      "content": "<|reserved_special_token_123|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128129": {
+      "content": "<|reserved_special_token_124|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128130": {
+      "content": "<|reserved_special_token_125|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128131": {
+      "content": "<|reserved_special_token_126|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128132": {
+      "content": "<|reserved_special_token_127|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128133": {
+      "content": "<|reserved_special_token_128|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128134": {
+      "content": "<|reserved_special_token_129|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128135": {
+      "content": "<|reserved_special_token_130|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128136": {
+      "content": "<|reserved_special_token_131|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128137": {
+      "content": "<|reserved_special_token_132|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128138": {
+      "content": "<|reserved_special_token_133|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128139": {
+      "content": "<|reserved_special_token_134|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128140": {
+      "content": "<|reserved_special_token_135|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128141": {
+      "content": "<|reserved_special_token_136|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128142": {
+      "content": "<|reserved_special_token_137|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128143": {
+      "content": "<|reserved_special_token_138|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128144": {
+      "content": "<|reserved_special_token_139|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128145": {
+      "content": "<|reserved_special_token_140|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128146": {
+      "content": "<|reserved_special_token_141|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128147": {
+      "content": "<|reserved_special_token_142|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128148": {
+      "content": "<|reserved_special_token_143|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128149": {
+      "content": "<|reserved_special_token_144|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128150": {
+      "content": "<|reserved_special_token_145|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128151": {
+      "content": "<|reserved_special_token_146|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128152": {
+      "content": "<|reserved_special_token_147|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128153": {
+      "content": "<|reserved_special_token_148|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128154": {
+      "content": "<|reserved_special_token_149|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128155": {
+      "content": "<|reserved_special_token_150|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128156": {
+      "content": "<|reserved_special_token_151|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128157": {
+      "content": "<|reserved_special_token_152|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128158": {
+      "content": "<|reserved_special_token_153|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128159": {
+      "content": "<|reserved_special_token_154|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128160": {
+      "content": "<|reserved_special_token_155|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128161": {
+      "content": "<|reserved_special_token_156|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128162": {
+      "content": "<|reserved_special_token_157|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128163": {
+      "content": "<|reserved_special_token_158|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128164": {
+      "content": "<|reserved_special_token_159|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128165": {
+      "content": "<|reserved_special_token_160|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128166": {
+      "content": "<|reserved_special_token_161|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128167": {
+      "content": "<|reserved_special_token_162|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128168": {
+      "content": "<|reserved_special_token_163|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128169": {
+      "content": "<|reserved_special_token_164|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128170": {
+      "content": "<|reserved_special_token_165|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128171": {
+      "content": "<|reserved_special_token_166|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128172": {
+      "content": "<|reserved_special_token_167|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128173": {
+      "content": "<|reserved_special_token_168|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128174": {
+      "content": "<|reserved_special_token_169|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128175": {
+      "content": "<|reserved_special_token_170|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128176": {
+      "content": "<|reserved_special_token_171|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128177": {
+      "content": "<|reserved_special_token_172|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128178": {
+      "content": "<|reserved_special_token_173|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128179": {
+      "content": "<|reserved_special_token_174|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128180": {
+      "content": "<|reserved_special_token_175|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128181": {
+      "content": "<|reserved_special_token_176|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128182": {
+      "content": "<|reserved_special_token_177|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128183": {
+      "content": "<|reserved_special_token_178|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128184": {
+      "content": "<|reserved_special_token_179|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128185": {
+      "content": "<|reserved_special_token_180|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128186": {
+      "content": "<|reserved_special_token_181|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128187": {
+      "content": "<|reserved_special_token_182|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128188": {
+      "content": "<|reserved_special_token_183|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128189": {
+      "content": "<|reserved_special_token_184|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128190": {
+      "content": "<|reserved_special_token_185|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128191": {
+      "content": "<|reserved_special_token_186|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128192": {
+      "content": "<|reserved_special_token_187|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128193": {
+      "content": "<|reserved_special_token_188|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128194": {
+      "content": "<|reserved_special_token_189|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128195": {
+      "content": "<|reserved_special_token_190|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128196": {
+      "content": "<|reserved_special_token_191|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128197": {
+      "content": "<|reserved_special_token_192|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128198": {
+      "content": "<|reserved_special_token_193|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128199": {
+      "content": "<|reserved_special_token_194|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128200": {
+      "content": "<|reserved_special_token_195|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128201": {
+      "content": "<|reserved_special_token_196|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128202": {
+      "content": "<|reserved_special_token_197|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128203": {
+      "content": "<|reserved_special_token_198|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128204": {
+      "content": "<|reserved_special_token_199|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128205": {
+      "content": "<|reserved_special_token_200|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128206": {
+      "content": "<|reserved_special_token_201|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128207": {
+      "content": "<|reserved_special_token_202|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128208": {
+      "content": "<|reserved_special_token_203|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128209": {
+      "content": "<|reserved_special_token_204|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128210": {
+      "content": "<|reserved_special_token_205|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128211": {
+      "content": "<|reserved_special_token_206|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128212": {
+      "content": "<|reserved_special_token_207|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128213": {
+      "content": "<|reserved_special_token_208|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128214": {
+      "content": "<|reserved_special_token_209|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128215": {
+      "content": "<|reserved_special_token_210|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128216": {
+      "content": "<|reserved_special_token_211|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128217": {
+      "content": "<|reserved_special_token_212|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128218": {
+      "content": "<|reserved_special_token_213|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128219": {
+      "content": "<|reserved_special_token_214|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128220": {
+      "content": "<|reserved_special_token_215|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128221": {
+      "content": "<|reserved_special_token_216|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128222": {
+      "content": "<|reserved_special_token_217|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128223": {
+      "content": "<|reserved_special_token_218|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128224": {
+      "content": "<|reserved_special_token_219|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128225": {
+      "content": "<|reserved_special_token_220|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128226": {
+      "content": "<|reserved_special_token_221|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128227": {
+      "content": "<|reserved_special_token_222|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128228": {
+      "content": "<|reserved_special_token_223|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128229": {
+      "content": "<|reserved_special_token_224|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128230": {
+      "content": "<|reserved_special_token_225|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128231": {
+      "content": "<|reserved_special_token_226|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128232": {
+      "content": "<|reserved_special_token_227|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128233": {
+      "content": "<|reserved_special_token_228|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128234": {
+      "content": "<|reserved_special_token_229|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128235": {
+      "content": "<|reserved_special_token_230|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128236": {
+      "content": "<|reserved_special_token_231|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128237": {
+      "content": "<|reserved_special_token_232|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128238": {
+      "content": "<|reserved_special_token_233|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128239": {
+      "content": "<|reserved_special_token_234|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128240": {
+      "content": "<|reserved_special_token_235|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128241": {
+      "content": "<|reserved_special_token_236|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128242": {
+      "content": "<|reserved_special_token_237|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128243": {
+      "content": "<|reserved_special_token_238|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128244": {
+      "content": "<|reserved_special_token_239|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128245": {
+      "content": "<|reserved_special_token_240|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128246": {
+      "content": "<|reserved_special_token_241|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128247": {
+      "content": "<|reserved_special_token_242|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128248": {
+      "content": "<|reserved_special_token_243|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128249": {
+      "content": "<|reserved_special_token_244|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128250": {
+      "content": "<|reserved_special_token_245|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128251": {
+      "content": "<|reserved_special_token_246|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128252": {
+      "content": "<|reserved_special_token_247|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128253": {
+      "content": "<|reserved_special_token_248|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128254": {
+      "content": "<|reserved_special_token_249|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128255": {
+      "content": "<|reserved_special_token_250|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|begin_of_text|>",
+  "chat_template": "{{ '<|begin_of_text|>' }}{% if messages[0]['role'] == 'system' %}{% set system_message = messages[0]['content'] %}{% endif %}{% if system_message is defined %}{{ '<|start_header_id|>system<|end_header_id|>\n\n' + system_message + '<|eot_id|>' }}{% endif %}{% for message in messages %}{% set content = message['content'] %}{% if message['role'] == 'user' %}{{ '<|start_header_id|>user<|end_header_id|>\n\n' + content + '<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n' }}{% elif message['role'] == 'assistant' %}{{ content + '<|eot_id|>' }}{% endif %}{% endfor %}",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|eot_id|>",
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<|eot_id|>",
+  "padding_side": "right",
+  "split_special_tokens": false,
+  "tokenizer_class": "PreTrainedTokenizerFast"
+}

checkpoint-385/trainer_state.json

@@ -0,0 +1,3113 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 4.951768488745981,
+  "eval_steps": 500,
+  "global_step": 385,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.012861736334405145,
+      "grad_norm": 577.1012573242188,
+      "learning_rate": 8.333333333333335e-09,
+      "loss": 13.9619,
+      "num_input_tokens_seen": 15136,
+      "step": 1
+    },
+    {
+      "epoch": 0.02572347266881029,
+      "grad_norm": 552.5532836914062,
+      "learning_rate": 1.666666666666667e-08,
+      "loss": 13.7129,
+      "num_input_tokens_seen": 30208,
+      "step": 2
+    },
+    {
+      "epoch": 0.03858520900321544,
+      "grad_norm": 562.699462890625,
+      "learning_rate": 2.5000000000000002e-08,
+      "loss": 13.8474,
+      "num_input_tokens_seen": 45376,
+      "step": 3
+    },
+    {
+      "epoch": 0.05144694533762058,
+      "grad_norm": 585.796630859375,
+      "learning_rate": 3.333333333333334e-08,
+      "loss": 13.8844,
+      "num_input_tokens_seen": 59968,
+      "step": 4
+    },
+    {
+      "epoch": 0.06430868167202572,
+      "grad_norm": 592.56689453125,
+      "learning_rate": 4.166666666666667e-08,
+      "loss": 14.138,
+      "num_input_tokens_seen": 75136,
+      "step": 5
+    },
+    {
+      "epoch": 0.07717041800643087,
+      "grad_norm": 569.5825805664062,
+      "learning_rate": 5.0000000000000004e-08,
+      "loss": 13.9077,
+      "num_input_tokens_seen": 90816,
+      "step": 6
+    },
+    {
+      "epoch": 0.09003215434083602,
+      "grad_norm": 578.7579956054688,
+      "learning_rate": 5.833333333333334e-08,
+      "loss": 13.8396,
+      "num_input_tokens_seen": 105600,
+      "step": 7
+    },
+    {
+      "epoch": 0.10289389067524116,
+      "grad_norm": 577.330322265625,
+      "learning_rate": 6.666666666666668e-08,
+      "loss": 13.9828,
+      "num_input_tokens_seen": 119872,
+      "step": 8
+    },
+    {
+      "epoch": 0.1157556270096463,
+      "grad_norm": 575.2134399414062,
+      "learning_rate": 7.500000000000001e-08,
+      "loss": 14.0361,
+      "num_input_tokens_seen": 135104,
+      "step": 9
+    },
+    {
+      "epoch": 0.12861736334405144,
+      "grad_norm": 579.0157470703125,
+      "learning_rate": 8.333333333333334e-08,
+      "loss": 13.9392,
+      "num_input_tokens_seen": 150016,
+      "step": 10
+    },
+    {
+      "epoch": 0.1414790996784566,
+      "grad_norm": 585.5687255859375,
+      "learning_rate": 9.166666666666668e-08,
+      "loss": 14.0256,
+      "num_input_tokens_seen": 165216,
+      "step": 11
+    },
+    {
+      "epoch": 0.15434083601286175,
+      "grad_norm": 566.5042114257812,
+      "learning_rate": 1.0000000000000001e-07,
+      "loss": 13.6693,
+      "num_input_tokens_seen": 179296,
+      "step": 12
+    },
+    {
+      "epoch": 0.16720257234726688,
+      "grad_norm": 574.4198608398438,
+      "learning_rate": 1.0833333333333335e-07,
+      "loss": 13.9031,
+      "num_input_tokens_seen": 193312,
+      "step": 13
+    },
+    {
+      "epoch": 0.18006430868167203,
+      "grad_norm": 588.1041259765625,
+      "learning_rate": 1.1666666666666668e-07,
+      "loss": 13.8575,
+      "num_input_tokens_seen": 208768,
+      "step": 14
+    },
+    {
+      "epoch": 0.19292604501607716,
+      "grad_norm": 573.4750366210938,
+      "learning_rate": 1.2500000000000002e-07,
+      "loss": 13.8366,
+      "num_input_tokens_seen": 223456,
+      "step": 15
+    },
+    {
+      "epoch": 0.2057877813504823,
+      "grad_norm": 572.26220703125,
+      "learning_rate": 1.3333333333333336e-07,
+      "loss": 13.8705,
+      "num_input_tokens_seen": 238656,
+      "step": 16
+    },
+    {
+      "epoch": 0.21864951768488747,
+      "grad_norm": 566.3518676757812,
+      "learning_rate": 1.4166666666666668e-07,
+      "loss": 13.1816,
+      "num_input_tokens_seen": 253312,
+      "step": 17
+    },
+    {
+      "epoch": 0.2315112540192926,
+      "grad_norm": 560.5255126953125,
+      "learning_rate": 1.5000000000000002e-07,
+      "loss": 13.2292,
+      "num_input_tokens_seen": 268352,
+      "step": 18
+    },
+    {
+      "epoch": 0.24437299035369775,
+      "grad_norm": 583.0404663085938,
+      "learning_rate": 1.5833333333333336e-07,
+      "loss": 13.4366,
+      "num_input_tokens_seen": 282560,
+      "step": 19
+    },
+    {
+      "epoch": 0.2572347266881029,
+      "grad_norm": 555.8986206054688,
+      "learning_rate": 1.6666666666666668e-07,
+      "loss": 12.9904,
+      "num_input_tokens_seen": 297120,
+      "step": 20
+    },
+    {
+      "epoch": 0.27009646302250806,
+      "grad_norm": 557.931396484375,
+      "learning_rate": 1.7500000000000002e-07,
+      "loss": 12.878,
+      "num_input_tokens_seen": 312416,
+      "step": 21
+    },
+    {
+      "epoch": 0.2829581993569132,
+      "grad_norm": 562.4014282226562,
+      "learning_rate": 1.8333333333333336e-07,
+      "loss": 12.7794,
+      "num_input_tokens_seen": 327968,
+      "step": 22
+    },
+    {
+      "epoch": 0.2958199356913183,
+      "grad_norm": 548.7069702148438,
+      "learning_rate": 1.9166666666666668e-07,
+      "loss": 11.3144,
+      "num_input_tokens_seen": 342880,
+      "step": 23
+    },
+    {
+      "epoch": 0.3086816720257235,
+      "grad_norm": 511.2892761230469,
+      "learning_rate": 2.0000000000000002e-07,
+      "loss": 10.8531,
+      "num_input_tokens_seen": 358848,
+      "step": 24
+    },
+    {
+      "epoch": 0.3215434083601286,
+      "grad_norm": 496.71868896484375,
+      "learning_rate": 2.0833333333333333e-07,
+      "loss": 10.7149,
+      "num_input_tokens_seen": 373792,
+      "step": 25
+    },
+    {
+      "epoch": 0.33440514469453375,
+      "grad_norm": 504.7909851074219,
+      "learning_rate": 2.166666666666667e-07,
+      "loss": 10.5802,
+      "num_input_tokens_seen": 389248,
+      "step": 26
+    },
+    {
+      "epoch": 0.34726688102893893,
+      "grad_norm": 495.0346984863281,
+      "learning_rate": 2.2500000000000002e-07,
+      "loss": 10.3671,
+      "num_input_tokens_seen": 404320,
+      "step": 27
+    },
+    {
+      "epoch": 0.36012861736334406,
+      "grad_norm": 482.26318359375,
+      "learning_rate": 2.3333333333333336e-07,
+      "loss": 10.1751,
+      "num_input_tokens_seen": 419296,
+      "step": 28
+    },
+    {
+      "epoch": 0.3729903536977492,
+      "grad_norm": 480.48193359375,
+      "learning_rate": 2.416666666666667e-07,
+      "loss": 9.7707,
+      "num_input_tokens_seen": 433632,
+      "step": 29
+    },
+    {
+      "epoch": 0.3858520900321543,
+      "grad_norm": 467.7600402832031,
+      "learning_rate": 2.5000000000000004e-07,
+      "loss": 9.6489,
+      "num_input_tokens_seen": 448736,
+      "step": 30
+    },
+    {
+      "epoch": 0.3987138263665595,
+      "grad_norm": 468.9175720214844,
+      "learning_rate": 2.5833333333333333e-07,
+      "loss": 8.6805,
+      "num_input_tokens_seen": 463840,
+      "step": 31
+    },
+    {
+      "epoch": 0.4115755627009646,
+      "grad_norm": 394.77691650390625,
+      "learning_rate": 2.666666666666667e-07,
+      "loss": 5.9207,
+      "num_input_tokens_seen": 478816,
+      "step": 32
+    },
+    {
+      "epoch": 0.42443729903536975,
+      "grad_norm": 440.14697265625,
+      "learning_rate": 2.75e-07,
+      "loss": 5.7661,
+      "num_input_tokens_seen": 494464,
+      "step": 33
+    },
+    {
+      "epoch": 0.43729903536977494,
+      "grad_norm": 436.6060791015625,
+      "learning_rate": 2.8333333333333336e-07,
+      "loss": 5.6168,
+      "num_input_tokens_seen": 510080,
+      "step": 34
+    },
+    {
+      "epoch": 0.45016077170418006,
+      "grad_norm": 388.576416015625,
+      "learning_rate": 2.916666666666667e-07,
+      "loss": 5.3367,
+      "num_input_tokens_seen": 525216,
+      "step": 35
+    },
+    {
+      "epoch": 0.4630225080385852,
+      "grad_norm": 327.9267272949219,
+      "learning_rate": 3.0000000000000004e-07,
+      "loss": 4.9751,
+      "num_input_tokens_seen": 539648,
+      "step": 36
+    },
+    {
+      "epoch": 0.4758842443729904,
+      "grad_norm": 322.6181335449219,
+      "learning_rate": 3.083333333333334e-07,
+      "loss": 4.7041,
+      "num_input_tokens_seen": 554080,
+      "step": 37
+    },
+    {
+      "epoch": 0.4887459807073955,
+      "grad_norm": 313.6160888671875,
+      "learning_rate": 3.166666666666667e-07,
+      "loss": 4.4631,
+      "num_input_tokens_seen": 568960,
+      "step": 38
+    },
+    {
+      "epoch": 0.5016077170418006,
+      "grad_norm": 299.7319641113281,
+      "learning_rate": 3.25e-07,
+      "loss": 4.1912,
+      "num_input_tokens_seen": 583904,
+      "step": 39
+    },
+    {
+      "epoch": 0.5144694533762058,
+      "grad_norm": 308.4530944824219,
+      "learning_rate": 3.3333333333333335e-07,
+      "loss": 3.9146,
+      "num_input_tokens_seen": 598848,
+      "step": 40
+    },
+    {
+      "epoch": 0.5273311897106109,
+      "grad_norm": 315.37396240234375,
+      "learning_rate": 3.416666666666667e-07,
+      "loss": 3.053,
+      "num_input_tokens_seen": 614208,
+      "step": 41
+    },
+    {
+      "epoch": 0.5401929260450161,
+      "grad_norm": 263.33258056640625,
+      "learning_rate": 3.5000000000000004e-07,
+      "loss": 1.5544,
+      "num_input_tokens_seen": 629280,
+      "step": 42
+    },
+    {
+      "epoch": 0.5530546623794212,
+      "grad_norm": 177.95458984375,
+      "learning_rate": 3.583333333333334e-07,
+      "loss": 1.0549,
+      "num_input_tokens_seen": 643968,
+      "step": 43
+    },
+    {
+      "epoch": 0.5659163987138264,
+      "grad_norm": 142.09307861328125,
+      "learning_rate": 3.666666666666667e-07,
+      "loss": 0.711,
+      "num_input_tokens_seen": 659040,
+      "step": 44
+    },
+    {
+      "epoch": 0.5787781350482315,
+      "grad_norm": 93.70928192138672,
+      "learning_rate": 3.75e-07,
+      "loss": 0.5127,
+      "num_input_tokens_seen": 674464,
+      "step": 45
+    },
+    {
+      "epoch": 0.5916398713826366,
+      "grad_norm": 65.4582290649414,
+      "learning_rate": 3.8333333333333335e-07,
+      "loss": 0.4143,
+      "num_input_tokens_seen": 689216,
+      "step": 46
+    },
+    {
+      "epoch": 0.6045016077170418,
+      "grad_norm": 51.321414947509766,
+      "learning_rate": 3.9166666666666675e-07,
+      "loss": 0.4014,
+      "num_input_tokens_seen": 704448,
+      "step": 47
+    },
+    {
+      "epoch": 0.617363344051447,
+      "grad_norm": 38.20650100708008,
+      "learning_rate": 4.0000000000000003e-07,
+      "loss": 0.384,
+      "num_input_tokens_seen": 719520,
+      "step": 48
+    },
+    {
+      "epoch": 0.6302250803858521,
+      "grad_norm": 70.43089294433594,
+      "learning_rate": 4.083333333333334e-07,
+      "loss": 0.3017,
+      "num_input_tokens_seen": 735520,
+      "step": 49
+    },
+    {
+      "epoch": 0.6430868167202572,
+      "grad_norm": 47.982505798339844,
+      "learning_rate": 4.1666666666666667e-07,
+      "loss": 0.3031,
+      "num_input_tokens_seen": 750976,
+      "step": 50
+    },
+    {
+      "epoch": 0.6559485530546624,
+      "grad_norm": 27.063344955444336,
+      "learning_rate": 4.2500000000000006e-07,
+      "loss": 0.2818,
+      "num_input_tokens_seen": 765728,
+      "step": 51
+    },
+    {
+      "epoch": 0.6688102893890675,
+      "grad_norm": 60.75875473022461,
+      "learning_rate": 4.333333333333334e-07,
+      "loss": 0.2881,
+      "num_input_tokens_seen": 780608,
+      "step": 52
+    },
+    {
+      "epoch": 0.6816720257234726,
+      "grad_norm": 56.81061935424805,
+      "learning_rate": 4.416666666666667e-07,
+      "loss": 0.2943,
+      "num_input_tokens_seen": 796192,
+      "step": 53
+    },
+    {
+      "epoch": 0.6945337620578779,
+      "grad_norm": 31.529571533203125,
+      "learning_rate": 4.5000000000000003e-07,
+      "loss": 0.2781,
+      "num_input_tokens_seen": 810496,
+      "step": 54
+    },
+    {
+      "epoch": 0.707395498392283,
+      "grad_norm": 33.71590805053711,
+      "learning_rate": 4.583333333333333e-07,
+      "loss": 0.2724,
+      "num_input_tokens_seen": 826048,
+      "step": 55
+    },
+    {
+      "epoch": 0.7202572347266881,
+      "grad_norm": 26.352554321289062,
+      "learning_rate": 4.666666666666667e-07,
+      "loss": 0.252,
+      "num_input_tokens_seen": 840512,
+      "step": 56
+    },
+    {
+      "epoch": 0.7331189710610932,
+      "grad_norm": 15.606438636779785,
+      "learning_rate": 4.7500000000000006e-07,
+      "loss": 0.2482,
+      "num_input_tokens_seen": 855808,
+      "step": 57
+    },
+    {
+      "epoch": 0.7459807073954984,
+      "grad_norm": 31.445789337158203,
+      "learning_rate": 4.833333333333334e-07,
+      "loss": 0.2212,
+      "num_input_tokens_seen": 871520,
+      "step": 58
+    },
+    {
+      "epoch": 0.7588424437299035,
+      "grad_norm": 9.967268943786621,
+      "learning_rate": 4.916666666666667e-07,
+      "loss": 0.2408,
+      "num_input_tokens_seen": 886144,
+      "step": 59
+    },
+    {
+      "epoch": 0.7717041800643086,
+      "grad_norm": 23.841659545898438,
+      "learning_rate": 5.000000000000001e-07,
+      "loss": 0.2133,
+      "num_input_tokens_seen": 901408,
+      "step": 60
+    },
+    {
+      "epoch": 0.7845659163987139,
+      "grad_norm": 16.51668357849121,
+      "learning_rate": 5.083333333333334e-07,
+      "loss": 0.2319,
+      "num_input_tokens_seen": 916672,
+      "step": 61
+    },
+    {
+      "epoch": 0.797427652733119,
+      "grad_norm": 51.05732345581055,
+      "learning_rate": 5.166666666666667e-07,
+      "loss": 0.2703,
+      "num_input_tokens_seen": 931168,
+      "step": 62
+    },
+    {
+      "epoch": 0.8102893890675241,
+      "grad_norm": 61.92572784423828,
+      "learning_rate": 5.250000000000001e-07,
+      "loss": 0.2647,
+      "num_input_tokens_seen": 946816,
+      "step": 63
+    },
+    {
+      "epoch": 0.8231511254019293,
+      "grad_norm": 10.283011436462402,
+      "learning_rate": 5.333333333333335e-07,
+      "loss": 0.2288,
+      "num_input_tokens_seen": 961248,
+      "step": 64
+    },
+    {
+      "epoch": 0.8360128617363344,
+      "grad_norm": 45.712772369384766,
+      "learning_rate": 5.416666666666667e-07,
+      "loss": 0.2426,
+      "num_input_tokens_seen": 976736,
+      "step": 65
+    },
+    {
+      "epoch": 0.8488745980707395,
+      "grad_norm": 22.493711471557617,
+      "learning_rate": 5.5e-07,
+      "loss": 0.1936,
+      "num_input_tokens_seen": 992192,
+      "step": 66
+    },
+    {
+      "epoch": 0.8617363344051447,
+      "grad_norm": 10.771052360534668,
+      "learning_rate": 5.583333333333333e-07,
+      "loss": 0.192,
+      "num_input_tokens_seen": 1007808,
+      "step": 67
+    },
+    {
+      "epoch": 0.8745980707395499,
+      "grad_norm": 11.903575897216797,
+      "learning_rate": 5.666666666666667e-07,
+      "loss": 0.1956,
+      "num_input_tokens_seen": 1022176,
+      "step": 68
+    },
+    {
+      "epoch": 0.887459807073955,
+      "grad_norm": 23.1299991607666,
+      "learning_rate": 5.750000000000001e-07,
+      "loss": 0.1862,
+      "num_input_tokens_seen": 1037280,
+      "step": 69
+    },
+    {
+      "epoch": 0.9003215434083601,
+      "grad_norm": 11.018325805664062,
+      "learning_rate": 5.833333333333334e-07,
+      "loss": 0.195,
+      "num_input_tokens_seen": 1051392,
+      "step": 70
+    },
+    {
+      "epoch": 0.9131832797427653,
+      "grad_norm": 27.96196174621582,
+      "learning_rate": 5.916666666666667e-07,
+      "loss": 0.1971,
+      "num_input_tokens_seen": 1066432,
+      "step": 71
+    },
+    {
+      "epoch": 0.9260450160771704,
+      "grad_norm": 14.441658020019531,
+      "learning_rate": 6.000000000000001e-07,
+      "loss": 0.1683,
+      "num_input_tokens_seen": 1081696,
+      "step": 72
+    },
+    {
+      "epoch": 0.9389067524115756,
+      "grad_norm": 17.725786209106445,
+      "learning_rate": 6.083333333333334e-07,
+      "loss": 0.2003,
+      "num_input_tokens_seen": 1096672,
+      "step": 73
+    },
+    {
+      "epoch": 0.9517684887459807,
+      "grad_norm": 26.286319732666016,
+      "learning_rate": 6.166666666666668e-07,
+      "loss": 0.1543,
+      "num_input_tokens_seen": 1111488,
+      "step": 74
+    },
+    {
+      "epoch": 0.9646302250803859,
+      "grad_norm": 38.698097229003906,
+      "learning_rate": 6.25e-07,
+      "loss": 0.1982,
+      "num_input_tokens_seen": 1126368,
+      "step": 75
+    },
+    {
+      "epoch": 0.977491961414791,
+      "grad_norm": 8.287293434143066,
+      "learning_rate": 6.333333333333334e-07,
+      "loss": 0.1545,
+      "num_input_tokens_seen": 1141536,
+      "step": 76
+    },
+    {
+      "epoch": 0.9903536977491961,
+      "grad_norm": 44.253807067871094,
+      "learning_rate": 6.416666666666667e-07,
+      "loss": 0.1573,
+      "num_input_tokens_seen": 1156320,
+      "step": 77
+    },
+    {
+      "epoch": 1.0032154340836013,
+      "grad_norm": 57.11432647705078,
+      "learning_rate": 6.5e-07,
+      "loss": 0.1788,
+      "num_input_tokens_seen": 1171904,
+      "step": 78
+    },
+    {
+      "epoch": 1.0160771704180065,
+      "grad_norm": 21.049449920654297,
+      "learning_rate": 6.583333333333333e-07,
+      "loss": 0.187,
+      "num_input_tokens_seen": 1187232,
+      "step": 79
+    },
+    {
+      "epoch": 1.0289389067524115,
+      "grad_norm": 67.61766052246094,
+      "learning_rate": 6.666666666666667e-07,
+      "loss": 0.2009,
+      "num_input_tokens_seen": 1201632,
+      "step": 80
+    },
+    {
+      "epoch": 1.0418006430868167,
+      "grad_norm": 91.35673522949219,
+      "learning_rate": 6.750000000000001e-07,
+      "loss": 0.2655,
+      "num_input_tokens_seen": 1216832,
+      "step": 81
+    },
+    {
+      "epoch": 1.0546623794212218,
+      "grad_norm": 54.41016387939453,
+      "learning_rate": 6.833333333333334e-07,
+      "loss": 0.2041,
+      "num_input_tokens_seen": 1232128,
+      "step": 82
+    },
+    {
+      "epoch": 1.067524115755627,
+      "grad_norm": 10.082817077636719,
+      "learning_rate": 6.916666666666668e-07,
+      "loss": 0.1675,
+      "num_input_tokens_seen": 1246944,
+      "step": 83
+    },
+    {
+      "epoch": 1.0803858520900322,
+      "grad_norm": 43.82609558105469,
+      "learning_rate": 7.000000000000001e-07,
+      "loss": 0.2097,
+      "num_input_tokens_seen": 1262624,
+      "step": 84
+    },
+    {
+      "epoch": 1.0932475884244373,
+      "grad_norm": 51.51130676269531,
+      "learning_rate": 7.083333333333334e-07,
+      "loss": 0.2171,
+      "num_input_tokens_seen": 1278080,
+      "step": 85
+    },
+    {
+      "epoch": 1.1061093247588425,
+      "grad_norm": 35.92145538330078,
+      "learning_rate": 7.166666666666668e-07,
+      "loss": 0.1702,
+      "num_input_tokens_seen": 1293280,
+      "step": 86
+    },
+    {
+      "epoch": 1.1189710610932475,
+      "grad_norm": 8.067242622375488,
+      "learning_rate": 7.25e-07,
+      "loss": 0.1255,
+      "num_input_tokens_seen": 1307456,
+      "step": 87
+    },
+    {
+      "epoch": 1.1318327974276527,
+      "grad_norm": 28.8727970123291,
+      "learning_rate": 7.333333333333334e-07,
+      "loss": 0.1826,
+      "num_input_tokens_seen": 1323168,
+      "step": 88
+    },
+    {
+      "epoch": 1.144694533762058,
+      "grad_norm": 45.813865661621094,
+      "learning_rate": 7.416666666666668e-07,
+      "loss": 0.2039,
+      "num_input_tokens_seen": 1337984,
+      "step": 89
+    },
+    {
+      "epoch": 1.157556270096463,
+      "grad_norm": 53.024105072021484,
+      "learning_rate": 7.5e-07,
+      "loss": 0.2337,
+      "num_input_tokens_seen": 1353568,
+      "step": 90
+    },
+    {
+      "epoch": 1.1704180064308682,
+      "grad_norm": 17.642776489257812,
+      "learning_rate": 7.583333333333334e-07,
+      "loss": 0.1466,
+      "num_input_tokens_seen": 1368800,
+      "step": 91
+    },
+    {
+      "epoch": 1.1832797427652733,
+      "grad_norm": 17.741811752319336,
+      "learning_rate": 7.666666666666667e-07,
+      "loss": 0.1295,
+      "num_input_tokens_seen": 1384192,
+      "step": 92
+    },
+    {
+      "epoch": 1.1961414790996785,
+      "grad_norm": 24.114030838012695,
+      "learning_rate": 7.750000000000001e-07,
+      "loss": 0.1525,
+      "num_input_tokens_seen": 1398784,
+      "step": 93
+    },
+    {
+      "epoch": 1.2090032154340835,
+      "grad_norm": 27.668102264404297,
+      "learning_rate": 7.833333333333335e-07,
+      "loss": 0.1735,
+      "num_input_tokens_seen": 1414304,
+      "step": 94
+    },
+    {
+      "epoch": 1.2218649517684887,
+      "grad_norm": 7.892695426940918,
+      "learning_rate": 7.916666666666667e-07,
+      "loss": 0.1484,
+      "num_input_tokens_seen": 1430880,
+      "step": 95
+    },
+    {
+      "epoch": 1.234726688102894,
+      "grad_norm": 12.985011100769043,
+      "learning_rate": 8.000000000000001e-07,
+      "loss": 0.1517,
+      "num_input_tokens_seen": 1445760,
+      "step": 96
+    },
+    {
+      "epoch": 1.247588424437299,
+      "grad_norm": 20.373502731323242,
+      "learning_rate": 8.083333333333334e-07,
+      "loss": 0.1429,
+      "num_input_tokens_seen": 1460576,
+      "step": 97
+    },
+    {
+      "epoch": 1.2604501607717042,
+      "grad_norm": 8.579370498657227,
+      "learning_rate": 8.166666666666668e-07,
+      "loss": 0.151,
+      "num_input_tokens_seen": 1476032,
+      "step": 98
+    },
+    {
+      "epoch": 1.2733118971061093,
+      "grad_norm": 6.366987705230713,
+      "learning_rate": 8.250000000000001e-07,
+      "loss": 0.1217,
+      "num_input_tokens_seen": 1491008,
+      "step": 99
+    },
+    {
+      "epoch": 1.2861736334405145,
+      "grad_norm": 19.89562225341797,
+      "learning_rate": 8.333333333333333e-07,
+      "loss": 0.1366,
+      "num_input_tokens_seen": 1505120,
+      "step": 100
+    },
+    {
+      "epoch": 1.2990353697749195,
+      "grad_norm": 10.997241020202637,
+      "learning_rate": 8.416666666666667e-07,
+      "loss": 0.1534,
+      "num_input_tokens_seen": 1520736,
+      "step": 101
+    },
+    {
+      "epoch": 1.3118971061093248,
+      "grad_norm": 17.20572280883789,
+      "learning_rate": 8.500000000000001e-07,
+      "loss": 0.141,
+      "num_input_tokens_seen": 1535424,
+      "step": 102
+    },
+    {
+      "epoch": 1.32475884244373,
+      "grad_norm": 10.88858413696289,
+      "learning_rate": 8.583333333333334e-07,
+      "loss": 0.1238,
+      "num_input_tokens_seen": 1549856,
+      "step": 103
+    },
+    {
+      "epoch": 1.337620578778135,
+      "grad_norm": 6.872950077056885,
+      "learning_rate": 8.666666666666668e-07,
+      "loss": 0.1241,
+      "num_input_tokens_seen": 1564512,
+      "step": 104
+    },
+    {
+      "epoch": 1.3504823151125402,
+      "grad_norm": 6.931344509124756,
+      "learning_rate": 8.75e-07,
+      "loss": 0.1414,
+      "num_input_tokens_seen": 1578976,
+      "step": 105
+    },
+    {
+      "epoch": 1.3633440514469453,
+      "grad_norm": 12.237205505371094,
+      "learning_rate": 8.833333333333334e-07,
+      "loss": 0.1296,
+      "num_input_tokens_seen": 1593952,
+      "step": 106
+    },
+    {
+      "epoch": 1.3762057877813505,
+      "grad_norm": 13.147500991821289,
+      "learning_rate": 8.916666666666668e-07,
+      "loss": 0.1232,
+      "num_input_tokens_seen": 1608704,
+      "step": 107
+    },
+    {
+      "epoch": 1.3890675241157555,
+      "grad_norm": 23.525880813598633,
+      "learning_rate": 9.000000000000001e-07,
+      "loss": 0.1625,
+      "num_input_tokens_seen": 1623936,
+      "step": 108
+    },
+    {
+      "epoch": 1.4019292604501608,
+      "grad_norm": 25.576210021972656,
+      "learning_rate": 9.083333333333335e-07,
+      "loss": 0.1509,
+      "num_input_tokens_seen": 1639264,
+      "step": 109
+    },
+    {
+      "epoch": 1.414790996784566,
+      "grad_norm": 18.12908935546875,
+      "learning_rate": 9.166666666666666e-07,
+      "loss": 0.1416,
+      "num_input_tokens_seen": 1654528,
+      "step": 110
+    },
+    {
+      "epoch": 1.427652733118971,
+      "grad_norm": 14.664992332458496,
+      "learning_rate": 9.25e-07,
+      "loss": 0.1481,
+      "num_input_tokens_seen": 1669824,
+      "step": 111
+    },
+    {
+      "epoch": 1.4405144694533762,
+      "grad_norm": 17.804119110107422,
+      "learning_rate": 9.333333333333334e-07,
+      "loss": 0.1303,
+      "num_input_tokens_seen": 1684800,
+      "step": 112
+    },
+    {
+      "epoch": 1.4533762057877815,
+      "grad_norm": 10.886981964111328,
+      "learning_rate": 9.416666666666667e-07,
+      "loss": 0.116,
+      "num_input_tokens_seen": 1699712,
+      "step": 113
+    },
+    {
+      "epoch": 1.4662379421221865,
+      "grad_norm": 14.883463859558105,
+      "learning_rate": 9.500000000000001e-07,
+      "loss": 0.0981,
+      "num_input_tokens_seen": 1714208,
+      "step": 114
+    },
+    {
+      "epoch": 1.4790996784565915,
+      "grad_norm": 9.800952911376953,
+      "learning_rate": 9.583333333333334e-07,
+      "loss": 0.1174,
+      "num_input_tokens_seen": 1728672,
+      "step": 115
+    },
+    {
+      "epoch": 1.4919614147909968,
+      "grad_norm": 8.804801940917969,
+      "learning_rate": 9.666666666666668e-07,
+      "loss": 0.1458,
+      "num_input_tokens_seen": 1743008,
+      "step": 116
+    },
+    {
+      "epoch": 1.504823151125402,
+      "grad_norm": 9.836427688598633,
+      "learning_rate": 9.750000000000002e-07,
+      "loss": 0.0952,
+      "num_input_tokens_seen": 1758016,
+      "step": 117
+    },
+    {
+      "epoch": 1.517684887459807,
+      "grad_norm": 7.373986721038818,
+      "learning_rate": 9.833333333333334e-07,
+      "loss": 0.1233,
+      "num_input_tokens_seen": 1772512,
+      "step": 118
+    },
+    {
+      "epoch": 1.5305466237942122,
+      "grad_norm": 16.394031524658203,
+      "learning_rate": 9.916666666666668e-07,
+      "loss": 0.127,
+      "num_input_tokens_seen": 1787808,
+      "step": 119
+    },
+    {
+      "epoch": 1.5434083601286175,
+      "grad_norm": 8.336946487426758,
+      "learning_rate": 1.0000000000000002e-06,
+      "loss": 0.1121,
+      "num_input_tokens_seen": 1803104,
+      "step": 120
+    },
+    {
+      "epoch": 1.5562700964630225,
+      "grad_norm": 28.298938751220703,
+      "learning_rate": 1.0083333333333333e-06,
+      "loss": 0.1432,
+      "num_input_tokens_seen": 1817632,
+      "step": 121
+    },
+    {
+      "epoch": 1.5691318327974275,
+      "grad_norm": 25.474973678588867,
+      "learning_rate": 1.0166666666666667e-06,
+      "loss": 0.1446,
+      "num_input_tokens_seen": 1832576,
+      "step": 122
+    },
+    {
+      "epoch": 1.5819935691318328,
+      "grad_norm": 12.33470630645752,
+      "learning_rate": 1.025e-06,
+      "loss": 0.1056,
+      "num_input_tokens_seen": 1847872,
+      "step": 123
+    },
+    {
+      "epoch": 1.594855305466238,
+      "grad_norm": 16.17420768737793,
+      "learning_rate": 1.0333333333333333e-06,
+      "loss": 0.1193,
+      "num_input_tokens_seen": 1863008,
+      "step": 124
+    },
+    {
+      "epoch": 1.607717041800643,
+      "grad_norm": 23.109798431396484,
+      "learning_rate": 1.0416666666666667e-06,
+      "loss": 0.1409,
+      "num_input_tokens_seen": 1878688,
+      "step": 125
+    },
+    {
+      "epoch": 1.6205787781350482,
+      "grad_norm": 15.091007232666016,
+      "learning_rate": 1.0500000000000001e-06,
+      "loss": 0.1154,
+      "num_input_tokens_seen": 1894144,
+      "step": 126
+    },
+    {
+      "epoch": 1.6334405144694535,
+      "grad_norm": 7.409396171569824,
+      "learning_rate": 1.0583333333333335e-06,
+      "loss": 0.1046,
+      "num_input_tokens_seen": 1909504,
+      "step": 127
+    },
+    {
+      "epoch": 1.6463022508038585,
+      "grad_norm": 12.553366661071777,
+      "learning_rate": 1.066666666666667e-06,
+      "loss": 0.09,
+      "num_input_tokens_seen": 1924896,
+      "step": 128
+    },
+    {
+      "epoch": 1.6591639871382635,
+      "grad_norm": 10.23271369934082,
+      "learning_rate": 1.075e-06,
+      "loss": 0.0858,
+      "num_input_tokens_seen": 1939232,
+      "step": 129
+    },
+    {
+      "epoch": 1.6720257234726688,
+      "grad_norm": 6.091500282287598,
+      "learning_rate": 1.0833333333333335e-06,
+      "loss": 0.0782,
+      "num_input_tokens_seen": 1954720,
+      "step": 130
+    },
+    {
+      "epoch": 1.684887459807074,
+      "grad_norm": 12.670690536499023,
+      "learning_rate": 1.0916666666666667e-06,
+      "loss": 0.1429,
+      "num_input_tokens_seen": 1969632,
+      "step": 131
+    },
+    {
+      "epoch": 1.697749196141479,
+      "grad_norm": 12.42375659942627,
+      "learning_rate": 1.1e-06,
+      "loss": 0.1121,
+      "num_input_tokens_seen": 1984544,
+      "step": 132
+    },
+    {
+      "epoch": 1.7106109324758842,
+      "grad_norm": 4.910019874572754,
+      "learning_rate": 1.1083333333333335e-06,
+      "loss": 0.0458,
+      "num_input_tokens_seen": 1999584,
+      "step": 133
+    },
+    {
+      "epoch": 1.7234726688102895,
+      "grad_norm": 8.262114524841309,
+      "learning_rate": 1.1166666666666666e-06,
+      "loss": 0.1217,
+      "num_input_tokens_seen": 2014048,
+      "step": 134
+    },
+    {
+      "epoch": 1.7363344051446945,
+      "grad_norm": 21.927522659301758,
+      "learning_rate": 1.125e-06,
+      "loss": 0.1253,
+      "num_input_tokens_seen": 2029312,
+      "step": 135
+    },
+    {
+      "epoch": 1.7491961414790995,
+      "grad_norm": 8.816388130187988,
+      "learning_rate": 1.1333333333333334e-06,
+      "loss": 0.077,
+      "num_input_tokens_seen": 2044192,
+      "step": 136
+    },
+    {
+      "epoch": 1.762057877813505,
+      "grad_norm": 6.877133846282959,
+      "learning_rate": 1.1416666666666668e-06,
+      "loss": 0.0719,
+      "num_input_tokens_seen": 2059552,
+      "step": 137
+    },
+    {
+      "epoch": 1.77491961414791,
+      "grad_norm": 5.565485000610352,
+      "learning_rate": 1.1500000000000002e-06,
+      "loss": 0.0916,
+      "num_input_tokens_seen": 2074336,
+      "step": 138
+    },
+    {
+      "epoch": 1.787781350482315,
+      "grad_norm": 11.287223815917969,
+      "learning_rate": 1.1583333333333334e-06,
+      "loss": 0.0812,
+      "num_input_tokens_seen": 2089280,
+      "step": 139
+    },
+    {
+      "epoch": 1.8006430868167203,
+      "grad_norm": 7.971100807189941,
+      "learning_rate": 1.1666666666666668e-06,
+      "loss": 0.1176,
+      "num_input_tokens_seen": 2104256,
+      "step": 140
+    },
+    {
+      "epoch": 1.8135048231511255,
+      "grad_norm": 7.78786563873291,
+      "learning_rate": 1.175e-06,
+      "loss": 0.0631,
+      "num_input_tokens_seen": 2119232,
+      "step": 141
+    },
+    {
+      "epoch": 1.8263665594855305,
+      "grad_norm": 8.365852355957031,
+      "learning_rate": 1.1833333333333334e-06,
+      "loss": 0.1137,
+      "num_input_tokens_seen": 2133824,
+      "step": 142
+    },
+    {
+      "epoch": 1.8392282958199357,
+      "grad_norm": 16.2335262298584,
+      "learning_rate": 1.1916666666666668e-06,
+      "loss": 0.0958,
+      "num_input_tokens_seen": 2148480,
+      "step": 143
+    },
+    {
+      "epoch": 1.852090032154341,
+      "grad_norm": 8.455872535705566,
+      "learning_rate": 1.2000000000000002e-06,
+      "loss": 0.1343,
+      "num_input_tokens_seen": 2163328,
+      "step": 144
+    },
+    {
+      "epoch": 1.864951768488746,
+      "grad_norm": 13.241278648376465,
+      "learning_rate": 1.2083333333333333e-06,
+      "loss": 0.1101,
+      "num_input_tokens_seen": 2177568,
+      "step": 145
+    },
+    {
+      "epoch": 1.877813504823151,
+      "grad_norm": 13.46755313873291,
+      "learning_rate": 1.2166666666666667e-06,
+      "loss": 0.0914,
+      "num_input_tokens_seen": 2192384,
+      "step": 146
+    },
+    {
+      "epoch": 1.8906752411575563,
+      "grad_norm": 7.21525764465332,
+      "learning_rate": 1.2250000000000001e-06,
+      "loss": 0.1114,
+      "num_input_tokens_seen": 2207584,
+      "step": 147
+    },
+    {
+      "epoch": 1.9035369774919615,
+      "grad_norm": 5.000830173492432,
+      "learning_rate": 1.2333333333333335e-06,
+      "loss": 0.083,
+      "num_input_tokens_seen": 2222208,
+      "step": 148
+    },
+    {
+      "epoch": 1.9163987138263665,
+      "grad_norm": 8.995044708251953,
+      "learning_rate": 1.2416666666666667e-06,
+      "loss": 0.1095,
+      "num_input_tokens_seen": 2237760,
+      "step": 149
+    },
+    {
+      "epoch": 1.9292604501607717,
+      "grad_norm": 7.872910976409912,
+      "learning_rate": 1.25e-06,
+      "loss": 0.0662,
+      "num_input_tokens_seen": 2252768,
+      "step": 150
+    },
+    {
+      "epoch": 1.942122186495177,
+      "grad_norm": 11.594476699829102,
+      "learning_rate": 1.2583333333333333e-06,
+      "loss": 0.0979,
+      "num_input_tokens_seen": 2268736,
+      "step": 151
+    },
+    {
+      "epoch": 1.954983922829582,
+      "grad_norm": 14.245850563049316,
+      "learning_rate": 1.2666666666666669e-06,
+      "loss": 0.0847,
+      "num_input_tokens_seen": 2283840,
+      "step": 152
+    },
+    {
+      "epoch": 1.967845659163987,
+      "grad_norm": 12.000141143798828,
+      "learning_rate": 1.275e-06,
+      "loss": 0.0949,
+      "num_input_tokens_seen": 2297696,
+      "step": 153
+    },
+    {
+      "epoch": 1.9807073954983923,
+      "grad_norm": 16.763721466064453,
+      "learning_rate": 1.2833333333333335e-06,
+      "loss": 0.1206,
+      "num_input_tokens_seen": 2311936,
+      "step": 154
+    },
+    {
+      "epoch": 1.9935691318327975,
+      "grad_norm": 22.926712036132812,
+      "learning_rate": 1.2916666666666669e-06,
+      "loss": 0.139,
+      "num_input_tokens_seen": 2327392,
+      "step": 155
+    },
+    {
+      "epoch": 2.0064308681672025,
+      "grad_norm": 7.715733528137207,
+      "learning_rate": 1.3e-06,
+      "loss": 0.07,
+      "num_input_tokens_seen": 2341952,
+      "step": 156
+    },
+    {
+      "epoch": 2.0192926045016075,
+      "grad_norm": 5.047181129455566,
+      "learning_rate": 1.3083333333333334e-06,
+      "loss": 0.0562,
+      "num_input_tokens_seen": 2356768,
+      "step": 157
+    },
+    {
+      "epoch": 2.032154340836013,
+      "grad_norm": 10.79956340789795,
+      "learning_rate": 1.3166666666666666e-06,
+      "loss": 0.0456,
+      "num_input_tokens_seen": 2371584,
+      "step": 158
+    },
+    {
+      "epoch": 2.045016077170418,
+      "grad_norm": 13.864229202270508,
+      "learning_rate": 1.3250000000000002e-06,
+      "loss": 0.0582,
+      "num_input_tokens_seen": 2386880,
+      "step": 159
+    },
+    {
+      "epoch": 2.057877813504823,
+      "grad_norm": 7.294124603271484,
+      "learning_rate": 1.3333333333333334e-06,
+      "loss": 0.0452,
+      "num_input_tokens_seen": 2401248,
+      "step": 160
+    },
+    {
+      "epoch": 2.0707395498392285,
+      "grad_norm": 5.838871479034424,
+      "learning_rate": 1.3416666666666666e-06,
+      "loss": 0.0553,
+      "num_input_tokens_seen": 2415968,
+      "step": 161
+    },
+    {
+      "epoch": 2.0836012861736335,
+      "grad_norm": 20.286243438720703,
+      "learning_rate": 1.3500000000000002e-06,
+      "loss": 0.1108,
+      "num_input_tokens_seen": 2430368,
+      "step": 162
+    },
+    {
+      "epoch": 2.0964630225080385,
+      "grad_norm": 12.720942497253418,
+      "learning_rate": 1.3583333333333334e-06,
+      "loss": 0.0791,
+      "num_input_tokens_seen": 2445056,
+      "step": 163
+    },
+    {
+      "epoch": 2.1093247588424435,
+      "grad_norm": 9.107832908630371,
+      "learning_rate": 1.3666666666666668e-06,
+      "loss": 0.0637,
+      "num_input_tokens_seen": 2459744,
+      "step": 164
+    },
+    {
+      "epoch": 2.122186495176849,
+      "grad_norm": 6.473385334014893,
+      "learning_rate": 1.3750000000000002e-06,
+      "loss": 0.0404,
+      "num_input_tokens_seen": 2474400,
+      "step": 165
+    },
+    {
+      "epoch": 2.135048231511254,
+      "grad_norm": 6.510437488555908,
+      "learning_rate": 1.3833333333333336e-06,
+      "loss": 0.0372,
+      "num_input_tokens_seen": 2489280,
+      "step": 166
+    },
+    {
+      "epoch": 2.147909967845659,
+      "grad_norm": 8.90833854675293,
+      "learning_rate": 1.3916666666666668e-06,
+      "loss": 0.045,
+      "num_input_tokens_seen": 2503776,
+      "step": 167
+    },
+    {
+      "epoch": 2.1607717041800645,
+      "grad_norm": 11.84021282196045,
+      "learning_rate": 1.4000000000000001e-06,
+      "loss": 0.0964,
+      "num_input_tokens_seen": 2518752,
+      "step": 168
+    },
+    {
+      "epoch": 2.1736334405144695,
+      "grad_norm": 5.904385566711426,
+      "learning_rate": 1.4083333333333335e-06,
+      "loss": 0.0543,
+      "num_input_tokens_seen": 2533568,
+      "step": 169
+    },
+    {
+      "epoch": 2.1864951768488745,
+      "grad_norm": 6.935682773590088,
+      "learning_rate": 1.4166666666666667e-06,
+      "loss": 0.071,
+      "num_input_tokens_seen": 2548640,
+      "step": 170
+    },
+    {
+      "epoch": 2.19935691318328,
+      "grad_norm": 3.85262131690979,
+      "learning_rate": 1.425e-06,
+      "loss": 0.0285,
+      "num_input_tokens_seen": 2563232,
+      "step": 171
+    },
+    {
+      "epoch": 2.212218649517685,
+      "grad_norm": 5.231224060058594,
+      "learning_rate": 1.4333333333333335e-06,
+      "loss": 0.0399,
+      "num_input_tokens_seen": 2578336,
+      "step": 172
+    },
+    {
+      "epoch": 2.22508038585209,
+      "grad_norm": 6.751733779907227,
+      "learning_rate": 1.4416666666666667e-06,
+      "loss": 0.067,
+      "num_input_tokens_seen": 2594336,
+      "step": 173
+    },
+    {
+      "epoch": 2.237942122186495,
+      "grad_norm": 4.619236469268799,
+      "learning_rate": 1.45e-06,
+      "loss": 0.0436,
+      "num_input_tokens_seen": 2609408,
+      "step": 174
+    },
+    {
+      "epoch": 2.2508038585209005,
+      "grad_norm": 4.799732685089111,
+      "learning_rate": 1.4583333333333335e-06,
+      "loss": 0.0522,
+      "num_input_tokens_seen": 2624288,
+      "step": 175
+    },
+    {
+      "epoch": 2.2636655948553055,
+      "grad_norm": 7.050292015075684,
+      "learning_rate": 1.4666666666666669e-06,
+      "loss": 0.0521,
+      "num_input_tokens_seen": 2639296,
+      "step": 176
+    },
+    {
+      "epoch": 2.2765273311897105,
+      "grad_norm": 6.788993835449219,
+      "learning_rate": 1.475e-06,
+      "loss": 0.0446,
+      "num_input_tokens_seen": 2653856,
+      "step": 177
+    },
+    {
+      "epoch": 2.289389067524116,
+      "grad_norm": 7.368555545806885,
+      "learning_rate": 1.4833333333333337e-06,
+      "loss": 0.0378,
+      "num_input_tokens_seen": 2669216,
+      "step": 178
+    },
+    {
+      "epoch": 2.302250803858521,
+      "grad_norm": 6.674912452697754,
+      "learning_rate": 1.4916666666666669e-06,
+      "loss": 0.0387,
+      "num_input_tokens_seen": 2683264,
+      "step": 179
+    },
+    {
+      "epoch": 2.315112540192926,
+      "grad_norm": 4.754708766937256,
+      "learning_rate": 1.5e-06,
+      "loss": 0.036,
+      "num_input_tokens_seen": 2698112,
+      "step": 180
+    },
+    {
+      "epoch": 2.327974276527331,
+      "grad_norm": 11.09841251373291,
+      "learning_rate": 1.5083333333333336e-06,
+      "loss": 0.0765,
+      "num_input_tokens_seen": 2712800,
+      "step": 181
+    },
+    {
+      "epoch": 2.3408360128617365,
+      "grad_norm": 13.097764015197754,
+      "learning_rate": 1.5166666666666668e-06,
+      "loss": 0.0884,
+      "num_input_tokens_seen": 2728032,
+      "step": 182
+    },
+    {
+      "epoch": 2.3536977491961415,
+      "grad_norm": 14.366617202758789,
+      "learning_rate": 1.525e-06,
+      "loss": 0.0801,
+      "num_input_tokens_seen": 2742432,
+      "step": 183
+    },
+    {
+      "epoch": 2.3665594855305465,
+      "grad_norm": 6.947451591491699,
+      "learning_rate": 1.5333333333333334e-06,
+      "loss": 0.0276,
+      "num_input_tokens_seen": 2758176,
+      "step": 184
+    },
+    {
+      "epoch": 2.379421221864952,
+      "grad_norm": 11.02580738067627,
+      "learning_rate": 1.5416666666666668e-06,
+      "loss": 0.0778,
+      "num_input_tokens_seen": 2772768,
+      "step": 185
+    },
+    {
+      "epoch": 2.392282958199357,
+      "grad_norm": 15.404431343078613,
+      "learning_rate": 1.5500000000000002e-06,
+      "loss": 0.0726,
+      "num_input_tokens_seen": 2787968,
+      "step": 186
+    },
+    {
+      "epoch": 2.405144694533762,
+      "grad_norm": 19.688905715942383,
+      "learning_rate": 1.5583333333333334e-06,
+      "loss": 0.1381,
+      "num_input_tokens_seen": 2802560,
+      "step": 187
+    },
+    {
+      "epoch": 2.418006430868167,
+      "grad_norm": 5.732620716094971,
+      "learning_rate": 1.566666666666667e-06,
+      "loss": 0.0408,
+      "num_input_tokens_seen": 2817856,
+      "step": 188
+    },
+    {
+      "epoch": 2.4308681672025725,
+      "grad_norm": 9.736536026000977,
+      "learning_rate": 1.5750000000000002e-06,
+      "loss": 0.1066,
+      "num_input_tokens_seen": 2833408,
+      "step": 189
+    },
+    {
+      "epoch": 2.4437299035369775,
+      "grad_norm": 9.312838554382324,
+      "learning_rate": 1.5833333333333333e-06,
+      "loss": 0.0686,
+      "num_input_tokens_seen": 2848832,
+      "step": 190
+    },
+    {
+      "epoch": 2.4565916398713825,
+      "grad_norm": 7.711926460266113,
+      "learning_rate": 1.591666666666667e-06,
+      "loss": 0.0428,
+      "num_input_tokens_seen": 2863968,
+      "step": 191
+    },
+    {
+      "epoch": 2.469453376205788,
+      "grad_norm": 6.855398654937744,
+      "learning_rate": 1.6000000000000001e-06,
+      "loss": 0.0387,
+      "num_input_tokens_seen": 2878944,
+      "step": 192
+    },
+    {
+      "epoch": 2.482315112540193,
+      "grad_norm": 3.374630928039551,
+      "learning_rate": 1.6083333333333333e-06,
+      "loss": 0.0489,
+      "num_input_tokens_seen": 2894112,
+      "step": 193
+    },
+    {
+      "epoch": 2.495176848874598,
+      "grad_norm": 7.452945709228516,
+      "learning_rate": 1.6166666666666667e-06,
+      "loss": 0.0621,
+      "num_input_tokens_seen": 2908928,
+      "step": 194
+    },
+    {
+      "epoch": 2.508038585209003,
+      "grad_norm": 12.478065490722656,
+      "learning_rate": 1.6250000000000001e-06,
+      "loss": 0.0651,
+      "num_input_tokens_seen": 2923680,
+      "step": 195
+    },
+    {
+      "epoch": 2.5209003215434085,
+      "grad_norm": 4.181528091430664,
+      "learning_rate": 1.6333333333333335e-06,
+      "loss": 0.0398,
+      "num_input_tokens_seen": 2939136,
+      "step": 196
+    },
+    {
+      "epoch": 2.5337620578778135,
+      "grad_norm": 5.443283557891846,
+      "learning_rate": 1.6416666666666667e-06,
+      "loss": 0.0369,
+      "num_input_tokens_seen": 2953760,
+      "step": 197
+    },
+    {
+      "epoch": 2.5466237942122185,
+      "grad_norm": 7.586729526519775,
+      "learning_rate": 1.6500000000000003e-06,
+      "loss": 0.0582,
+      "num_input_tokens_seen": 2968640,
+      "step": 198
+    },
+    {
+      "epoch": 2.559485530546624,
+      "grad_norm": 6.357609272003174,
+      "learning_rate": 1.6583333333333335e-06,
+      "loss": 0.0479,
+      "num_input_tokens_seen": 2983456,
+      "step": 199
+    },
+    {
+      "epoch": 2.572347266881029,
+      "grad_norm": 8.761418342590332,
+      "learning_rate": 1.6666666666666667e-06,
+      "loss": 0.0561,
+      "num_input_tokens_seen": 2998144,
+      "step": 200
+    },
+    {
+      "epoch": 2.585209003215434,
+      "grad_norm": 7.058523654937744,
+      "learning_rate": 1.6750000000000003e-06,
+      "loss": 0.0497,
+      "num_input_tokens_seen": 3013248,
+      "step": 201
+    },
+    {
+      "epoch": 2.598070739549839,
+      "grad_norm": 8.091582298278809,
+      "learning_rate": 1.6833333333333335e-06,
+      "loss": 0.063,
+      "num_input_tokens_seen": 3028320,
+      "step": 202
+    },
+    {
+      "epoch": 2.6109324758842445,
+      "grad_norm": 9.102255821228027,
+      "learning_rate": 1.6916666666666666e-06,
+      "loss": 0.054,
+      "num_input_tokens_seen": 3043232,
+      "step": 203
+    },
+    {
+      "epoch": 2.6237942122186495,
+      "grad_norm": 9.521942138671875,
+      "learning_rate": 1.7000000000000002e-06,
+      "loss": 0.0729,
+      "num_input_tokens_seen": 3058848,
+      "step": 204
+    },
+    {
+      "epoch": 2.6366559485530545,
+      "grad_norm": 7.026569843292236,
+      "learning_rate": 1.7083333333333334e-06,
+      "loss": 0.0685,
+      "num_input_tokens_seen": 3074080,
+      "step": 205
+    },
+    {
+      "epoch": 2.64951768488746,
+      "grad_norm": 13.492867469787598,
+      "learning_rate": 1.7166666666666668e-06,
+      "loss": 0.0823,
+      "num_input_tokens_seen": 3089152,
+      "step": 206
+    },
+    {
+      "epoch": 2.662379421221865,
+      "grad_norm": 6.880157947540283,
+      "learning_rate": 1.725e-06,
+      "loss": 0.0322,
+      "num_input_tokens_seen": 3103616,
+      "step": 207
+    },
+    {
+      "epoch": 2.67524115755627,
+      "grad_norm": 8.232446670532227,
+      "learning_rate": 1.7333333333333336e-06,
+      "loss": 0.0821,
+      "num_input_tokens_seen": 3119200,
+      "step": 208
+    },
+    {
+      "epoch": 2.688102893890675,
+      "grad_norm": 7.365771293640137,
+      "learning_rate": 1.7416666666666668e-06,
+      "loss": 0.0561,
+      "num_input_tokens_seen": 3134400,
+      "step": 209
+    },
+    {
+      "epoch": 2.7009646302250805,
+      "grad_norm": 9.879140853881836,
+      "learning_rate": 1.75e-06,
+      "loss": 0.0468,
+      "num_input_tokens_seen": 3149248,
+      "step": 210
+    },
+    {
+      "epoch": 2.7138263665594855,
+      "grad_norm": 8.636083602905273,
+      "learning_rate": 1.7583333333333336e-06,
+      "loss": 0.0593,
+      "num_input_tokens_seen": 3164800,
+      "step": 211
+    },
+    {
+      "epoch": 2.7266881028938905,
+      "grad_norm": 5.015254497528076,
+      "learning_rate": 1.7666666666666668e-06,
+      "loss": 0.0403,
+      "num_input_tokens_seen": 3179520,
+      "step": 212
+    },
+    {
+      "epoch": 2.739549839228296,
+      "grad_norm": 7.730808258056641,
+      "learning_rate": 1.7750000000000002e-06,
+      "loss": 0.0459,
+      "num_input_tokens_seen": 3194016,
+      "step": 213
+    },
+    {
+      "epoch": 2.752411575562701,
+      "grad_norm": 6.433826923370361,
+      "learning_rate": 1.7833333333333336e-06,
+      "loss": 0.0509,
+      "num_input_tokens_seen": 3208608,
+      "step": 214
+    },
+    {
+      "epoch": 2.765273311897106,
+      "grad_norm": 7.32877779006958,
+      "learning_rate": 1.7916666666666667e-06,
+      "loss": 0.0873,
+      "num_input_tokens_seen": 3224096,
+      "step": 215
+    },
+    {
+      "epoch": 2.778135048231511,
+      "grad_norm": 7.283654689788818,
+      "learning_rate": 1.8000000000000001e-06,
+      "loss": 0.0618,
+      "num_input_tokens_seen": 3239136,
+      "step": 216
+    },
+    {
+      "epoch": 2.7909967845659165,
+      "grad_norm": 11.044650077819824,
+      "learning_rate": 1.8083333333333335e-06,
+      "loss": 0.1025,
+      "num_input_tokens_seen": 3253184,
+      "step": 217
+    },
+    {
+      "epoch": 2.8038585209003215,
+      "grad_norm": 7.287780284881592,
+      "learning_rate": 1.816666666666667e-06,
+      "loss": 0.0669,
+      "num_input_tokens_seen": 3268000,
+      "step": 218
+    },
+    {
+      "epoch": 2.816720257234727,
+      "grad_norm": 7.542795658111572,
+      "learning_rate": 1.825e-06,
+      "loss": 0.0447,
+      "num_input_tokens_seen": 3282528,
+      "step": 219
+    },
+    {
+      "epoch": 2.829581993569132,
+      "grad_norm": 9.812615394592285,
+      "learning_rate": 1.8333333333333333e-06,
+      "loss": 0.0635,
+      "num_input_tokens_seen": 3297824,
+      "step": 220
+    },
+    {
+      "epoch": 2.842443729903537,
+      "grad_norm": 5.7050580978393555,
+      "learning_rate": 1.8416666666666669e-06,
+      "loss": 0.0614,
+      "num_input_tokens_seen": 3312480,
+      "step": 221
+    },
+    {
+      "epoch": 2.855305466237942,
+      "grad_norm": 13.298068046569824,
+      "learning_rate": 1.85e-06,
+      "loss": 0.0646,
+      "num_input_tokens_seen": 3327392,
+      "step": 222
+    },
+    {
+      "epoch": 2.868167202572347,
+      "grad_norm": 13.815607070922852,
+      "learning_rate": 1.8583333333333335e-06,
+      "loss": 0.0893,
+      "num_input_tokens_seen": 3342944,
+      "step": 223
+    },
+    {
+      "epoch": 2.8810289389067525,
+      "grad_norm": 6.954331398010254,
+      "learning_rate": 1.8666666666666669e-06,
+      "loss": 0.0653,
+      "num_input_tokens_seen": 3358112,
+      "step": 224
+    },
+    {
+      "epoch": 2.8938906752411575,
+      "grad_norm": 5.133541584014893,
+      "learning_rate": 1.8750000000000003e-06,
+      "loss": 0.0402,
+      "num_input_tokens_seen": 3373472,
+      "step": 225
+    },
+    {
+      "epoch": 2.906752411575563,
+      "grad_norm": 7.754403114318848,
+      "learning_rate": 1.8833333333333334e-06,
+      "loss": 0.0407,
+      "num_input_tokens_seen": 3389088,
+      "step": 226
+    },
+    {
+      "epoch": 2.919614147909968,
+      "grad_norm": 9.570775985717773,
+      "learning_rate": 1.8916666666666668e-06,
+      "loss": 0.0949,
+      "num_input_tokens_seen": 3403616,
+      "step": 227
+    },
+    {
+      "epoch": 2.932475884244373,
+      "grad_norm": 9.21279525756836,
+      "learning_rate": 1.9000000000000002e-06,
+      "loss": 0.0789,
+      "num_input_tokens_seen": 3418944,
+      "step": 228
+    },
+    {
+      "epoch": 2.945337620578778,
+      "grad_norm": 4.255781650543213,
+      "learning_rate": 1.9083333333333334e-06,
+      "loss": 0.0438,
+      "num_input_tokens_seen": 3434048,
+      "step": 229
+    },
+    {
+      "epoch": 2.958199356913183,
+      "grad_norm": 10.596036911010742,
+      "learning_rate": 1.916666666666667e-06,
+      "loss": 0.0905,
+      "num_input_tokens_seen": 3449952,
+      "step": 230
+    },
+    {
+      "epoch": 2.9710610932475885,
+      "grad_norm": 9.656909942626953,
+      "learning_rate": 1.925e-06,
+      "loss": 0.0495,
+      "num_input_tokens_seen": 3464736,
+      "step": 231
+    },
+    {
+      "epoch": 2.9839228295819935,
+      "grad_norm": 9.526041984558105,
+      "learning_rate": 1.9333333333333336e-06,
+      "loss": 0.0642,
+      "num_input_tokens_seen": 3480416,
+      "step": 232
+    },
+    {
+      "epoch": 2.996784565916399,
+      "grad_norm": 5.788015365600586,
+      "learning_rate": 1.9416666666666666e-06,
+      "loss": 0.0607,
+      "num_input_tokens_seen": 3495264,
+      "step": 233
+    },
+    {
+      "epoch": 3.009646302250804,
+      "grad_norm": 6.213415145874023,
+      "learning_rate": 1.9500000000000004e-06,
+      "loss": 0.0363,
+      "num_input_tokens_seen": 3510080,
+      "step": 234
+    },
+    {
+      "epoch": 3.022508038585209,
+      "grad_norm": 7.382534980773926,
+      "learning_rate": 1.9583333333333334e-06,
+      "loss": 0.0492,
+      "num_input_tokens_seen": 3524736,
+      "step": 235
+    },
+    {
+      "epoch": 3.035369774919614,
+      "grad_norm": 9.54261302947998,
+      "learning_rate": 1.9666666666666668e-06,
+      "loss": 0.0373,
+      "num_input_tokens_seen": 3539968,
+      "step": 236
+    },
+    {
+      "epoch": 3.0482315112540195,
+      "grad_norm": 3.8869571685791016,
+      "learning_rate": 1.975e-06,
+      "loss": 0.0203,
+      "num_input_tokens_seen": 3554464,
+      "step": 237
+    },
+    {
+      "epoch": 3.0610932475884245,
+      "grad_norm": 2.183758497238159,
+      "learning_rate": 1.9833333333333335e-06,
+      "loss": 0.0175,
+      "num_input_tokens_seen": 3569664,
+      "step": 238
+    },
+    {
+      "epoch": 3.0739549839228295,
+      "grad_norm": 3.851379871368408,
+      "learning_rate": 1.991666666666667e-06,
+      "loss": 0.0497,
+      "num_input_tokens_seen": 3585248,
+      "step": 239
+    },
+    {
+      "epoch": 3.0868167202572345,
+      "grad_norm": 6.622572898864746,
+      "learning_rate": 2.0000000000000003e-06,
+      "loss": 0.0361,
+      "num_input_tokens_seen": 3600160,
+      "step": 240
+    },
+    {
+      "epoch": 3.09967845659164,
+      "grad_norm": 3.4328978061676025,
+      "learning_rate": 2.0083333333333337e-06,
+      "loss": 0.0193,
+      "num_input_tokens_seen": 3615232,
+      "step": 241
+    },
+    {
+      "epoch": 3.112540192926045,
+      "grad_norm": 8.150415420532227,
+      "learning_rate": 2.0166666666666667e-06,
+      "loss": 0.0142,
+      "num_input_tokens_seen": 3629312,
+      "step": 242
+    },
+    {
+      "epoch": 3.12540192926045,
+      "grad_norm": 5.3073835372924805,
+      "learning_rate": 2.025e-06,
+      "loss": 0.0415,
+      "num_input_tokens_seen": 3644064,
+      "step": 243
+    },
+    {
+      "epoch": 3.1382636655948555,
+      "grad_norm": 4.33547830581665,
+      "learning_rate": 2.0333333333333335e-06,
+      "loss": 0.0178,
+      "num_input_tokens_seen": 3659328,
+      "step": 244
+    },
+    {
+      "epoch": 3.1511254019292605,
+      "grad_norm": 2.770535469055176,
+      "learning_rate": 2.041666666666667e-06,
+      "loss": 0.0166,
+      "num_input_tokens_seen": 3674720,
+      "step": 245
+    },
+    {
+      "epoch": 3.1639871382636655,
+      "grad_norm": 8.939215660095215,
+      "learning_rate": 2.05e-06,
+      "loss": 0.0424,
+      "num_input_tokens_seen": 3689984,
+      "step": 246
+    },
+    {
+      "epoch": 3.176848874598071,
+      "grad_norm": 4.35013484954834,
+      "learning_rate": 2.0583333333333337e-06,
+      "loss": 0.0464,
+      "num_input_tokens_seen": 3705216,
+      "step": 247
+    },
+    {
+      "epoch": 3.189710610932476,
+      "grad_norm": 4.8758368492126465,
+      "learning_rate": 2.0666666666666666e-06,
+      "loss": 0.0235,
+      "num_input_tokens_seen": 3719712,
+      "step": 248
+    },
+    {
+      "epoch": 3.202572347266881,
+      "grad_norm": 2.769258499145508,
+      "learning_rate": 2.075e-06,
+      "loss": 0.0128,
+      "num_input_tokens_seen": 3734272,
+      "step": 249
+    },
+    {
+      "epoch": 3.215434083601286,
+      "grad_norm": 4.699342727661133,
+      "learning_rate": 2.0833333333333334e-06,
+      "loss": 0.0319,
+      "num_input_tokens_seen": 3750240,
+      "step": 250
+    },
+    {
+      "epoch": 3.2282958199356915,
+      "grad_norm": 3.377748966217041,
+      "learning_rate": 2.091666666666667e-06,
+      "loss": 0.0196,
+      "num_input_tokens_seen": 3764736,
+      "step": 251
+    },
+    {
+      "epoch": 3.2411575562700965,
+      "grad_norm": 2.9469077587127686,
+      "learning_rate": 2.1000000000000002e-06,
+      "loss": 0.0326,
+      "num_input_tokens_seen": 3779744,
+      "step": 252
+    },
+    {
+      "epoch": 3.2540192926045015,
+      "grad_norm": 3.4729647636413574,
+      "learning_rate": 2.1083333333333336e-06,
+      "loss": 0.017,
+      "num_input_tokens_seen": 3794208,
+      "step": 253
+    },
+    {
+      "epoch": 3.266881028938907,
+      "grad_norm": 3.1130521297454834,
+      "learning_rate": 2.116666666666667e-06,
+      "loss": 0.0372,
+      "num_input_tokens_seen": 3809376,
+      "step": 254
+    },
+    {
+      "epoch": 3.279742765273312,
+      "grad_norm": 3.0094053745269775,
+      "learning_rate": 2.125e-06,
+      "loss": 0.0165,
+      "num_input_tokens_seen": 3824384,
+      "step": 255
+    },
+    {
+      "epoch": 3.292604501607717,
+      "grad_norm": 2.0758721828460693,
+      "learning_rate": 2.133333333333334e-06,
+      "loss": 0.0142,
+      "num_input_tokens_seen": 3840000,
+      "step": 256
+    },
+    {
+      "epoch": 3.305466237942122,
+      "grad_norm": 2.0254931449890137,
+      "learning_rate": 2.1416666666666668e-06,
+      "loss": 0.0117,
+      "num_input_tokens_seen": 3854240,
+      "step": 257
+    },
+    {
+      "epoch": 3.3183279742765275,
+      "grad_norm": 5.01600980758667,
+      "learning_rate": 2.15e-06,
+      "loss": 0.0264,
+      "num_input_tokens_seen": 3869792,
+      "step": 258
+    },
+    {
+      "epoch": 3.3311897106109325,
+      "grad_norm": 7.374160289764404,
+      "learning_rate": 2.1583333333333336e-06,
+      "loss": 0.034,
+      "num_input_tokens_seen": 3885152,
+      "step": 259
+    },
+    {
+      "epoch": 3.3440514469453375,
+      "grad_norm": 7.036656856536865,
+      "learning_rate": 2.166666666666667e-06,
+      "loss": 0.031,
+      "num_input_tokens_seen": 3900032,
+      "step": 260
+    },
+    {
+      "epoch": 3.356913183279743,
+      "grad_norm": 4.439852237701416,
+      "learning_rate": 2.1750000000000004e-06,
+      "loss": 0.018,
+      "num_input_tokens_seen": 3914432,
+      "step": 261
+    },
+    {
+      "epoch": 3.369774919614148,
+      "grad_norm": 8.757259368896484,
+      "learning_rate": 2.1833333333333333e-06,
+      "loss": 0.0461,
+      "num_input_tokens_seen": 3929600,
+      "step": 262
+    },
+    {
+      "epoch": 3.382636655948553,
+      "grad_norm": 4.993197917938232,
+      "learning_rate": 2.191666666666667e-06,
+      "loss": 0.0372,
+      "num_input_tokens_seen": 3944032,
+      "step": 263
+    },
+    {
+      "epoch": 3.395498392282958,
+      "grad_norm": 5.627339839935303,
+      "learning_rate": 2.2e-06,
+      "loss": 0.0471,
+      "num_input_tokens_seen": 3958464,
+      "step": 264
+    },
+    {
+      "epoch": 3.4083601286173635,
+      "grad_norm": 6.644600868225098,
+      "learning_rate": 2.2083333333333335e-06,
+      "loss": 0.037,
+      "num_input_tokens_seen": 3973024,
+      "step": 265
+    },
+    {
+      "epoch": 3.4212218649517685,
+      "grad_norm": 4.708102703094482,
+      "learning_rate": 2.216666666666667e-06,
+      "loss": 0.0245,
+      "num_input_tokens_seen": 3987520,
+      "step": 266
+    },
+    {
+      "epoch": 3.4340836012861735,
+      "grad_norm": 3.4480528831481934,
+      "learning_rate": 2.2250000000000003e-06,
+      "loss": 0.0233,
+      "num_input_tokens_seen": 4002656,
+      "step": 267
+    },
+    {
+      "epoch": 3.446945337620579,
+      "grad_norm": 5.0224175453186035,
+      "learning_rate": 2.2333333333333333e-06,
+      "loss": 0.0256,
+      "num_input_tokens_seen": 4017824,
+      "step": 268
+    },
+    {
+      "epoch": 3.459807073954984,
+      "grad_norm": 9.12070369720459,
+      "learning_rate": 2.2416666666666667e-06,
+      "loss": 0.0345,
+      "num_input_tokens_seen": 4032480,
+      "step": 269
+    },
+    {
+      "epoch": 3.472668810289389,
+      "grad_norm": 5.336618900299072,
+      "learning_rate": 2.25e-06,
+      "loss": 0.0656,
+      "num_input_tokens_seen": 4047648,
+      "step": 270
+    },
+    {
+      "epoch": 3.485530546623794,
+      "grad_norm": 4.003684997558594,
+      "learning_rate": 2.2583333333333335e-06,
+      "loss": 0.0247,
+      "num_input_tokens_seen": 4062688,
+      "step": 271
+    },
+    {
+      "epoch": 3.4983922829581995,
+      "grad_norm": 5.631625175476074,
+      "learning_rate": 2.266666666666667e-06,
+      "loss": 0.051,
+      "num_input_tokens_seen": 4077856,
+      "step": 272
+    },
+    {
+      "epoch": 3.5112540192926045,
+      "grad_norm": 4.748563766479492,
+      "learning_rate": 2.2750000000000002e-06,
+      "loss": 0.0417,
+      "num_input_tokens_seen": 4092864,
+      "step": 273
+    },
+    {
+      "epoch": 3.5241157556270095,
+      "grad_norm": 3.368995189666748,
+      "learning_rate": 2.2833333333333336e-06,
+      "loss": 0.0176,
+      "num_input_tokens_seen": 4107584,
+      "step": 274
+    },
+    {
+      "epoch": 3.536977491961415,
+      "grad_norm": 3.463752508163452,
+      "learning_rate": 2.2916666666666666e-06,
+      "loss": 0.0155,
+      "num_input_tokens_seen": 4122080,
+      "step": 275
+    },
+    {
+      "epoch": 3.54983922829582,
+      "grad_norm": 4.7195234298706055,
+      "learning_rate": 2.3000000000000004e-06,
+      "loss": 0.0135,
+      "num_input_tokens_seen": 4137600,
+      "step": 276
+    },
+    {
+      "epoch": 3.562700964630225,
+      "grad_norm": 6.3303680419921875,
+      "learning_rate": 2.3083333333333334e-06,
+      "loss": 0.0222,
+      "num_input_tokens_seen": 4152960,
+      "step": 277
+    },
+    {
+      "epoch": 3.57556270096463,
+      "grad_norm": 3.4051268100738525,
+      "learning_rate": 2.316666666666667e-06,
+      "loss": 0.0289,
+      "num_input_tokens_seen": 4168640,
+      "step": 278
+    },
+    {
+      "epoch": 3.5884244372990355,
+      "grad_norm": 3.971222162246704,
+      "learning_rate": 2.325e-06,
+      "loss": 0.0314,
+      "num_input_tokens_seen": 4183616,
+      "step": 279
+    },
+    {
+      "epoch": 3.6012861736334405,
+      "grad_norm": 5.509104251861572,
+      "learning_rate": 2.3333333333333336e-06,
+      "loss": 0.0582,
+      "num_input_tokens_seen": 4197536,
+      "step": 280
+    },
+    {
+      "epoch": 3.6141479099678455,
+      "grad_norm": 4.121878623962402,
+      "learning_rate": 2.341666666666667e-06,
+      "loss": 0.0339,
+      "num_input_tokens_seen": 4213152,
+      "step": 281
+    },
+    {
+      "epoch": 3.627009646302251,
+      "grad_norm": 6.977634429931641,
+      "learning_rate": 2.35e-06,
+      "loss": 0.0656,
+      "num_input_tokens_seen": 4228320,
+      "step": 282
+    },
+    {
+      "epoch": 3.639871382636656,
+      "grad_norm": 3.6625123023986816,
+      "learning_rate": 2.3583333333333338e-06,
+      "loss": 0.0355,
+      "num_input_tokens_seen": 4243680,
+      "step": 283
+    },
+    {
+      "epoch": 3.652733118971061,
+      "grad_norm": 4.691339492797852,
+      "learning_rate": 2.3666666666666667e-06,
+      "loss": 0.037,
+      "num_input_tokens_seen": 4258464,
+      "step": 284
+    },
+    {
+      "epoch": 3.665594855305466,
+      "grad_norm": 3.1770055294036865,
+      "learning_rate": 2.375e-06,
+      "loss": 0.0206,
+      "num_input_tokens_seen": 4273728,
+      "step": 285
+    },
+    {
+      "epoch": 3.6784565916398715,
+      "grad_norm": 2.424788475036621,
+      "learning_rate": 2.3833333333333335e-06,
+      "loss": 0.0286,
+      "num_input_tokens_seen": 4288416,
+      "step": 286
+    },
+    {
+      "epoch": 3.6913183279742765,
+      "grad_norm": 5.731730937957764,
+      "learning_rate": 2.391666666666667e-06,
+      "loss": 0.0316,
+      "num_input_tokens_seen": 4303712,
+      "step": 287
+    },
+    {
+      "epoch": 3.7041800643086815,
+      "grad_norm": 4.939082622528076,
+      "learning_rate": 2.4000000000000003e-06,
+      "loss": 0.0263,
+      "num_input_tokens_seen": 4319104,
+      "step": 288
+    },
+    {
+      "epoch": 3.717041800643087,
+      "grad_norm": 2.817589521408081,
+      "learning_rate": 2.4083333333333337e-06,
+      "loss": 0.0265,
+      "num_input_tokens_seen": 4334912,
+      "step": 289
+    },
+    {
+      "epoch": 3.729903536977492,
+      "grad_norm": 2.6983306407928467,
+      "learning_rate": 2.4166666666666667e-06,
+      "loss": 0.0251,
+      "num_input_tokens_seen": 4349280,
+      "step": 290
+    },
+    {
+      "epoch": 3.742765273311897,
+      "grad_norm": 5.260397434234619,
+      "learning_rate": 2.425e-06,
+      "loss": 0.0293,
+      "num_input_tokens_seen": 4363872,
+      "step": 291
+    },
+    {
+      "epoch": 3.755627009646302,
+      "grad_norm": 4.8456034660339355,
+      "learning_rate": 2.4333333333333335e-06,
+      "loss": 0.0278,
+      "num_input_tokens_seen": 4378752,
+      "step": 292
+    },
+    {
+      "epoch": 3.7684887459807075,
+      "grad_norm": 4.560141563415527,
+      "learning_rate": 2.441666666666667e-06,
+      "loss": 0.035,
+      "num_input_tokens_seen": 4393792,
+      "step": 293
+    },
+    {
+      "epoch": 3.7813504823151125,
+      "grad_norm": 6.035860538482666,
+      "learning_rate": 2.4500000000000003e-06,
+      "loss": 0.0455,
+      "num_input_tokens_seen": 4408192,
+      "step": 294
+    },
+    {
+      "epoch": 3.7942122186495175,
+      "grad_norm": 6.634167194366455,
+      "learning_rate": 2.4583333333333332e-06,
+      "loss": 0.0429,
+      "num_input_tokens_seen": 4422432,
+      "step": 295
+    },
+    {
+      "epoch": 3.807073954983923,
+      "grad_norm": 9.78260326385498,
+      "learning_rate": 2.466666666666667e-06,
+      "loss": 0.056,
+      "num_input_tokens_seen": 4437728,
+      "step": 296
+    },
+    {
+      "epoch": 3.819935691318328,
+      "grad_norm": 6.473577499389648,
+      "learning_rate": 2.475e-06,
+      "loss": 0.0325,
+      "num_input_tokens_seen": 4452928,
+      "step": 297
+    },
+    {
+      "epoch": 3.832797427652733,
+      "grad_norm": 8.688652992248535,
+      "learning_rate": 2.4833333333333334e-06,
+      "loss": 0.0401,
+      "num_input_tokens_seen": 4468096,
+      "step": 298
+    },
+    {
+      "epoch": 3.845659163987138,
+      "grad_norm": 4.41790771484375,
+      "learning_rate": 2.491666666666667e-06,
+      "loss": 0.0182,
+      "num_input_tokens_seen": 4482976,
+      "step": 299
+    },
+    {
+      "epoch": 3.8585209003215435,
+      "grad_norm": 3.7182106971740723,
+      "learning_rate": 2.5e-06,
+      "loss": 0.0344,
+      "num_input_tokens_seen": 4497952,
+      "step": 300
+    },
+    {
+      "epoch": 3.8713826366559485,
+      "grad_norm": 4.0538225173950195,
+      "learning_rate": 2.5083333333333336e-06,
+      "loss": 0.0111,
+      "num_input_tokens_seen": 4512896,
+      "step": 301
+    },
+    {
+      "epoch": 3.884244372990354,
+      "grad_norm": 3.3492507934570312,
+      "learning_rate": 2.5166666666666666e-06,
+      "loss": 0.0088,
+      "num_input_tokens_seen": 4528448,
+      "step": 302
+    },
+    {
+      "epoch": 3.897106109324759,
+      "grad_norm": 3.518641710281372,
+      "learning_rate": 2.5250000000000004e-06,
+      "loss": 0.013,
+      "num_input_tokens_seen": 4543712,
+      "step": 303
+    },
+    {
+      "epoch": 3.909967845659164,
+      "grad_norm": 5.015272617340088,
+      "learning_rate": 2.5333333333333338e-06,
+      "loss": 0.0433,
+      "num_input_tokens_seen": 4559008,
+      "step": 304
+    },
+    {
+      "epoch": 3.922829581993569,
+      "grad_norm": 3.2924585342407227,
+      "learning_rate": 2.5416666666666668e-06,
+      "loss": 0.0214,
+      "num_input_tokens_seen": 4573696,
+      "step": 305
+    },
+    {
+      "epoch": 3.935691318327974,
+      "grad_norm": 8.486410140991211,
+      "learning_rate": 2.55e-06,
+      "loss": 0.0416,
+      "num_input_tokens_seen": 4588384,
+      "step": 306
+    },
+    {
+      "epoch": 3.9485530546623795,
+      "grad_norm": 6.4795002937316895,
+      "learning_rate": 2.558333333333334e-06,
+      "loss": 0.0321,
+      "num_input_tokens_seen": 4602944,
+      "step": 307
+    },
+    {
+      "epoch": 3.9614147909967845,
+      "grad_norm": 6.211554050445557,
+      "learning_rate": 2.566666666666667e-06,
+      "loss": 0.0526,
+      "num_input_tokens_seen": 4618336,
+      "step": 308
+    },
+    {
+      "epoch": 3.97427652733119,
+      "grad_norm": 5.602302074432373,
+      "learning_rate": 2.5750000000000003e-06,
+      "loss": 0.025,
+      "num_input_tokens_seen": 4633472,
+      "step": 309
+    },
+    {
+      "epoch": 3.987138263665595,
+      "grad_norm": 9.136455535888672,
+      "learning_rate": 2.5833333333333337e-06,
+      "loss": 0.0787,
+      "num_input_tokens_seen": 4649472,
+      "step": 310
+    },
+    {
+      "epoch": 4.0,
+      "grad_norm": 8.398818969726562,
+      "learning_rate": 2.5916666666666667e-06,
+      "loss": 0.0559,
+      "num_input_tokens_seen": 4664384,
+      "step": 311
+    },
+    {
+      "epoch": 4.012861736334405,
+      "grad_norm": 2.286210536956787,
+      "learning_rate": 2.6e-06,
+      "loss": 0.0259,
+      "num_input_tokens_seen": 4679200,
+      "step": 312
+    },
+    {
+      "epoch": 4.02572347266881,
+      "grad_norm": 6.477627754211426,
+      "learning_rate": 2.608333333333333e-06,
+      "loss": 0.037,
+      "num_input_tokens_seen": 4693696,
+      "step": 313
+    },
+    {
+      "epoch": 4.038585209003215,
+      "grad_norm": 4.953974723815918,
+      "learning_rate": 2.616666666666667e-06,
+      "loss": 0.0293,
+      "num_input_tokens_seen": 4708608,
+      "step": 314
+    },
+    {
+      "epoch": 4.051446945337621,
+      "grad_norm": 2.7035064697265625,
+      "learning_rate": 2.6250000000000003e-06,
+      "loss": 0.0153,
+      "num_input_tokens_seen": 4722944,
+      "step": 315
+    },
+    {
+      "epoch": 4.064308681672026,
+      "grad_norm": 3.8599040508270264,
+      "learning_rate": 2.6333333333333332e-06,
+      "loss": 0.0234,
+      "num_input_tokens_seen": 4739296,
+      "step": 316
+    },
+    {
+      "epoch": 4.077170418006431,
+      "grad_norm": 5.4720072746276855,
+      "learning_rate": 2.6416666666666666e-06,
+      "loss": 0.0212,
+      "num_input_tokens_seen": 4754048,
+      "step": 317
+    },
+    {
+      "epoch": 4.090032154340836,
+      "grad_norm": 3.8060128688812256,
+      "learning_rate": 2.6500000000000005e-06,
+      "loss": 0.0187,
+      "num_input_tokens_seen": 4769376,
+      "step": 318
+    },
+    {
+      "epoch": 4.102893890675241,
+      "grad_norm": 5.436901092529297,
+      "learning_rate": 2.6583333333333334e-06,
+      "loss": 0.0245,
+      "num_input_tokens_seen": 4784640,
+      "step": 319
+    },
+    {
+      "epoch": 4.115755627009646,
+      "grad_norm": 1.5260390043258667,
+      "learning_rate": 2.666666666666667e-06,
+      "loss": 0.0032,
+      "num_input_tokens_seen": 4799392,
+      "step": 320
+    },
+    {
+      "epoch": 4.128617363344051,
+      "grad_norm": 11.145827293395996,
+      "learning_rate": 2.6750000000000002e-06,
+      "loss": 0.0367,
+      "num_input_tokens_seen": 4813696,
+      "step": 321
+    },
+    {
+      "epoch": 4.141479099678457,
+      "grad_norm": 5.737987518310547,
+      "learning_rate": 2.683333333333333e-06,
+      "loss": 0.038,
+      "num_input_tokens_seen": 4828928,
+      "step": 322
+    },
+    {
+      "epoch": 4.154340836012862,
+      "grad_norm": 12.955948829650879,
+      "learning_rate": 2.691666666666667e-06,
+      "loss": 0.0194,
+      "num_input_tokens_seen": 4843232,
+      "step": 323
+    },
+    {
+      "epoch": 4.167202572347267,
+      "grad_norm": 2.663821220397949,
+      "learning_rate": 2.7000000000000004e-06,
+      "loss": 0.0262,
+      "num_input_tokens_seen": 4858592,
+      "step": 324
+    },
+    {
+      "epoch": 4.180064308681672,
+      "grad_norm": 7.183210372924805,
+      "learning_rate": 2.7083333333333334e-06,
+      "loss": 0.0151,
+      "num_input_tokens_seen": 4872928,
+      "step": 325
+    },
+    {
+      "epoch": 4.192926045016077,
+      "grad_norm": 5.024378776550293,
+      "learning_rate": 2.7166666666666668e-06,
+      "loss": 0.0267,
+      "num_input_tokens_seen": 4887968,
+      "step": 326
+    },
+    {
+      "epoch": 4.205787781350482,
+      "grad_norm": 1.0667839050292969,
+      "learning_rate": 2.7250000000000006e-06,
+      "loss": 0.0041,
+      "num_input_tokens_seen": 4902752,
+      "step": 327
+    },
+    {
+      "epoch": 4.218649517684887,
+      "grad_norm": 6.243584632873535,
+      "learning_rate": 2.7333333333333336e-06,
+      "loss": 0.0365,
+      "num_input_tokens_seen": 4917280,
+      "step": 328
+    },
+    {
+      "epoch": 4.231511254019293,
+      "grad_norm": 3.255237579345703,
+      "learning_rate": 2.741666666666667e-06,
+      "loss": 0.0151,
+      "num_input_tokens_seen": 4933088,
+      "step": 329
+    },
+    {
+      "epoch": 4.244372990353698,
+      "grad_norm": 1.9820326566696167,
+      "learning_rate": 2.7500000000000004e-06,
+      "loss": 0.006,
+      "num_input_tokens_seen": 4948128,
+      "step": 330
+    },
+    {
+      "epoch": 4.257234726688103,
+      "grad_norm": 4.553264141082764,
+      "learning_rate": 2.7583333333333333e-06,
+      "loss": 0.0247,
+      "num_input_tokens_seen": 4962752,
+      "step": 331
+    },
+    {
+      "epoch": 4.270096463022508,
+      "grad_norm": 3.139723062515259,
+      "learning_rate": 2.766666666666667e-06,
+      "loss": 0.0147,
+      "num_input_tokens_seen": 4978272,
+      "step": 332
+    },
+    {
+      "epoch": 4.282958199356913,
+      "grad_norm": 9.011144638061523,
+      "learning_rate": 2.7750000000000005e-06,
+      "loss": 0.0413,
+      "num_input_tokens_seen": 4992384,
+      "step": 333
+    },
+    {
+      "epoch": 4.295819935691318,
+      "grad_norm": 11.36248779296875,
+      "learning_rate": 2.7833333333333335e-06,
+      "loss": 0.0318,
+      "num_input_tokens_seen": 5007136,
+      "step": 334
+    },
+    {
+      "epoch": 4.308681672025724,
+      "grad_norm": 7.9140305519104,
+      "learning_rate": 2.791666666666667e-06,
+      "loss": 0.0513,
+      "num_input_tokens_seen": 5021600,
+      "step": 335
+    },
+    {
+      "epoch": 4.321543408360129,
+      "grad_norm": 4.12211799621582,
+      "learning_rate": 2.8000000000000003e-06,
+      "loss": 0.0165,
+      "num_input_tokens_seen": 5037216,
+      "step": 336
+    },
+    {
+      "epoch": 4.334405144694534,
+      "grad_norm": 4.724823951721191,
+      "learning_rate": 2.8083333333333333e-06,
+      "loss": 0.0131,
+      "num_input_tokens_seen": 5052320,
+      "step": 337
+    },
+    {
+      "epoch": 4.347266881028939,
+      "grad_norm": 6.9196085929870605,
+      "learning_rate": 2.816666666666667e-06,
+      "loss": 0.0246,
+      "num_input_tokens_seen": 5067552,
+      "step": 338
+    },
+    {
+      "epoch": 4.360128617363344,
+      "grad_norm": 5.785276412963867,
+      "learning_rate": 2.825e-06,
+      "loss": 0.0465,
+      "num_input_tokens_seen": 5082240,
+      "step": 339
+    },
+    {
+      "epoch": 4.372990353697749,
+      "grad_norm": 1.6201478242874146,
+      "learning_rate": 2.8333333333333335e-06,
+      "loss": 0.0084,
+      "num_input_tokens_seen": 5097408,
+      "step": 340
+    },
+    {
+      "epoch": 4.385852090032154,
+      "grad_norm": 5.511898994445801,
+      "learning_rate": 2.841666666666667e-06,
+      "loss": 0.0105,
+      "num_input_tokens_seen": 5111680,
+      "step": 341
+    },
+    {
+      "epoch": 4.39871382636656,
+      "grad_norm": 8.025418281555176,
+      "learning_rate": 2.85e-06,
+      "loss": 0.0388,
+      "num_input_tokens_seen": 5127264,
+      "step": 342
+    },
+    {
+      "epoch": 4.411575562700965,
+      "grad_norm": 12.538291931152344,
+      "learning_rate": 2.8583333333333336e-06,
+      "loss": 0.0486,
+      "num_input_tokens_seen": 5142464,
+      "step": 343
+    },
+    {
+      "epoch": 4.42443729903537,
+      "grad_norm": 10.100396156311035,
+      "learning_rate": 2.866666666666667e-06,
+      "loss": 0.0354,
+      "num_input_tokens_seen": 5157376,
+      "step": 344
+    },
+    {
+      "epoch": 4.437299035369775,
+      "grad_norm": 4.218758583068848,
+      "learning_rate": 2.875e-06,
+      "loss": 0.0464,
+      "num_input_tokens_seen": 5171648,
+      "step": 345
+    },
+    {
+      "epoch": 4.45016077170418,
+      "grad_norm": 5.751765251159668,
+      "learning_rate": 2.8833333333333334e-06,
+      "loss": 0.0312,
+      "num_input_tokens_seen": 5186752,
+      "step": 346
+    },
+    {
+      "epoch": 4.463022508038585,
+      "grad_norm": 6.100085258483887,
+      "learning_rate": 2.8916666666666672e-06,
+      "loss": 0.0352,
+      "num_input_tokens_seen": 5202176,
+      "step": 347
+    },
+    {
+      "epoch": 4.47588424437299,
+      "grad_norm": 6.4870991706848145,
+      "learning_rate": 2.9e-06,
+      "loss": 0.0116,
+      "num_input_tokens_seen": 5216896,
+      "step": 348
+    },
+    {
+      "epoch": 4.488745980707396,
+      "grad_norm": 1.8258070945739746,
+      "learning_rate": 2.9083333333333336e-06,
+      "loss": 0.0118,
+      "num_input_tokens_seen": 5232448,
+      "step": 349
+    },
+    {
+      "epoch": 4.501607717041801,
+      "grad_norm": 2.6930766105651855,
+      "learning_rate": 2.916666666666667e-06,
+      "loss": 0.0138,
+      "num_input_tokens_seen": 5247296,
+      "step": 350
+    },
+    {
+      "epoch": 4.514469453376206,
+      "grad_norm": 5.420289993286133,
+      "learning_rate": 2.925e-06,
+      "loss": 0.0344,
+      "num_input_tokens_seen": 5262304,
+      "step": 351
+    },
+    {
+      "epoch": 4.527331189710611,
+      "grad_norm": 4.682693958282471,
+      "learning_rate": 2.9333333333333338e-06,
+      "loss": 0.0231,
+      "num_input_tokens_seen": 5277664,
+      "step": 352
+    },
+    {
+      "epoch": 4.540192926045016,
+      "grad_norm": 4.194357872009277,
+      "learning_rate": 2.941666666666667e-06,
+      "loss": 0.0267,
+      "num_input_tokens_seen": 5292576,
+      "step": 353
+    },
+    {
+      "epoch": 4.553054662379421,
+      "grad_norm": 1.0653674602508545,
+      "learning_rate": 2.95e-06,
+      "loss": 0.0109,
+      "num_input_tokens_seen": 5306304,
+      "step": 354
+    },
+    {
+      "epoch": 4.565916398713826,
+      "grad_norm": 1.8720684051513672,
+      "learning_rate": 2.9583333333333335e-06,
+      "loss": 0.0444,
+      "num_input_tokens_seen": 5320768,
+      "step": 355
+    },
+    {
+      "epoch": 4.578778135048232,
+      "grad_norm": 4.005693435668945,
+      "learning_rate": 2.9666666666666673e-06,
+      "loss": 0.0142,
+      "num_input_tokens_seen": 5336128,
+      "step": 356
+    },
+    {
+      "epoch": 4.591639871382637,
+      "grad_norm": 5.997867107391357,
+      "learning_rate": 2.9750000000000003e-06,
+      "loss": 0.0361,
+      "num_input_tokens_seen": 5351168,
+      "step": 357
+    },
+    {
+      "epoch": 4.604501607717042,
+      "grad_norm": 3.8871259689331055,
+      "learning_rate": 2.9833333333333337e-06,
+      "loss": 0.0197,
+      "num_input_tokens_seen": 5366112,
+      "step": 358
+    },
+    {
+      "epoch": 4.617363344051447,
+      "grad_norm": 4.269320011138916,
+      "learning_rate": 2.991666666666667e-06,
+      "loss": 0.0373,
+      "num_input_tokens_seen": 5381440,
+      "step": 359
+    },
+    {
+      "epoch": 4.630225080385852,
+      "grad_norm": 3.6487338542938232,
+      "learning_rate": 3e-06,
+      "loss": 0.0469,
+      "num_input_tokens_seen": 5397152,
+      "step": 360
+    },
+    {
+      "epoch": 4.643086816720257,
+      "grad_norm": 3.9794583320617676,
+      "learning_rate": 3.0083333333333335e-06,
+      "loss": 0.0283,
+      "num_input_tokens_seen": 5412128,
+      "step": 361
+    },
+    {
+      "epoch": 4.655948553054662,
+      "grad_norm": 5.071774482727051,
+      "learning_rate": 3.0166666666666673e-06,
+      "loss": 0.0271,
+      "num_input_tokens_seen": 5426816,
+      "step": 362
+    },
+    {
+      "epoch": 4.668810289389068,
+      "grad_norm": 3.651871681213379,
+      "learning_rate": 3.0250000000000003e-06,
+      "loss": 0.0303,
+      "num_input_tokens_seen": 5441760,
+      "step": 363
+    },
+    {
+      "epoch": 4.681672025723473,
+      "grad_norm": 3.1399481296539307,
+      "learning_rate": 3.0333333333333337e-06,
+      "loss": 0.0207,
+      "num_input_tokens_seen": 5457504,
+      "step": 364
+    },
+    {
+      "epoch": 4.694533762057878,
+      "grad_norm": 3.089629888534546,
+      "learning_rate": 3.0416666666666666e-06,
+      "loss": 0.0157,
+      "num_input_tokens_seen": 5473088,
+      "step": 365
+    },
+    {
+      "epoch": 4.707395498392283,
+      "grad_norm": 1.4184813499450684,
+      "learning_rate": 3.05e-06,
+      "loss": 0.0128,
+      "num_input_tokens_seen": 5487712,
+      "step": 366
+    },
+    {
+      "epoch": 4.720257234726688,
+      "grad_norm": 5.21925687789917,
+      "learning_rate": 3.058333333333334e-06,
+      "loss": 0.0293,
+      "num_input_tokens_seen": 5503680,
+      "step": 367
+    },
+    {
+      "epoch": 4.733118971061093,
+      "grad_norm": 3.7801499366760254,
+      "learning_rate": 3.066666666666667e-06,
+      "loss": 0.0211,
+      "num_input_tokens_seen": 5518464,
+      "step": 368
+    },
+    {
+      "epoch": 4.745980707395498,
+      "grad_norm": 9.715234756469727,
+      "learning_rate": 3.075e-06,
+      "loss": 0.0929,
+      "num_input_tokens_seen": 5533504,
+      "step": 369
+    },
+    {
+      "epoch": 4.758842443729904,
+      "grad_norm": 6.241518974304199,
+      "learning_rate": 3.0833333333333336e-06,
+      "loss": 0.0195,
+      "num_input_tokens_seen": 5548832,
+      "step": 370
+    },
+    {
+      "epoch": 4.771704180064309,
+      "grad_norm": 2.6658706665039062,
+      "learning_rate": 3.0916666666666666e-06,
+      "loss": 0.0114,
+      "num_input_tokens_seen": 5564256,
+      "step": 371
+    },
+    {
+      "epoch": 4.784565916398714,
+      "grad_norm": 7.38095235824585,
+      "learning_rate": 3.1000000000000004e-06,
+      "loss": 0.0207,
+      "num_input_tokens_seen": 5580064,
+      "step": 372
+    },
+    {
+      "epoch": 4.797427652733119,
+      "grad_norm": 5.605347156524658,
+      "learning_rate": 3.1083333333333338e-06,
+      "loss": 0.0144,
+      "num_input_tokens_seen": 5595008,
+      "step": 373
+    },
+    {
+      "epoch": 4.810289389067524,
+      "grad_norm": 4.410486221313477,
+      "learning_rate": 3.1166666666666668e-06,
+      "loss": 0.0197,
+      "num_input_tokens_seen": 5610304,
+      "step": 374
+    },
+    {
+      "epoch": 4.823151125401929,
+      "grad_norm": 5.259794235229492,
+      "learning_rate": 3.125e-06,
+      "loss": 0.0669,
+      "num_input_tokens_seen": 5626464,
+      "step": 375
+    },
+    {
+      "epoch": 4.836012861736334,
+      "grad_norm": 5.21687650680542,
+      "learning_rate": 3.133333333333334e-06,
+      "loss": 0.0156,
+      "num_input_tokens_seen": 5641216,
+      "step": 376
+    },
+    {
+      "epoch": 4.84887459807074,
+      "grad_norm": 3.3515796661376953,
+      "learning_rate": 3.141666666666667e-06,
+      "loss": 0.018,
+      "num_input_tokens_seen": 5656832,
+      "step": 377
+    },
+    {
+      "epoch": 4.861736334405145,
+      "grad_norm": 7.488304138183594,
+      "learning_rate": 3.1500000000000003e-06,
+      "loss": 0.064,
+      "num_input_tokens_seen": 5671744,
+      "step": 378
+    },
+    {
+      "epoch": 4.87459807073955,
+      "grad_norm": 2.625629186630249,
+      "learning_rate": 3.1583333333333337e-06,
+      "loss": 0.0263,
+      "num_input_tokens_seen": 5686496,
+      "step": 379
+    },
+    {
+      "epoch": 4.887459807073955,
+      "grad_norm": 3.379544496536255,
+      "learning_rate": 3.1666666666666667e-06,
+      "loss": 0.0256,
+      "num_input_tokens_seen": 5701216,
+      "step": 380
+    },
+    {
+      "epoch": 4.90032154340836,
+      "grad_norm": 2.0447638034820557,
+      "learning_rate": 3.175e-06,
+      "loss": 0.0269,
+      "num_input_tokens_seen": 5715904,
+      "step": 381
+    },
+    {
+      "epoch": 4.913183279742765,
+      "grad_norm": 3.5290844440460205,
+      "learning_rate": 3.183333333333334e-06,
+      "loss": 0.0281,
+      "num_input_tokens_seen": 5730976,
+      "step": 382
+    },
+    {
+      "epoch": 4.92604501607717,
+      "grad_norm": 2.183229923248291,
+      "learning_rate": 3.191666666666667e-06,
+      "loss": 0.0199,
+      "num_input_tokens_seen": 5745824,
+      "step": 383
+    },
+    {
+      "epoch": 4.938906752411576,
+      "grad_norm": 1.9661577939987183,
+      "learning_rate": 3.2000000000000003e-06,
+      "loss": 0.0277,
+      "num_input_tokens_seen": 5760160,
+      "step": 384
+    },
+    {
+      "epoch": 4.951768488745981,
+      "grad_norm": 5.09304666519165,
+      "learning_rate": 3.2083333333333337e-06,
+      "loss": 0.0158,
+      "num_input_tokens_seen": 5776160,
+      "step": 385
+    }
+  ],
+  "logging_steps": 1,
+  "max_steps": 385,
+  "num_input_tokens_seen": 5776160,
+  "num_train_epochs": 5,
+  "save_steps": 1000,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 2.6009787341943603e+17,
+  "train_batch_size": 4,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-385/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f37864624d2abe9284ede821087a8ef2503abe21bd8ee765f3fbb1875521922c
+size 6584

checkpoint-385/zero_to_fp32.py

@@ -0,0 +1,604 @@
+#!/usr/bin/env python
+
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: Apache-2.0
+
+# DeepSpeed Team
+
+# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets
+# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in
+# the future. Once extracted, the weights don't require DeepSpeed and can be used in any
+# application.
+#
+# example: python zero_to_fp32.py . pytorch_model.bin
+
+import argparse
+import torch
+import glob
+import math
+import os
+import re
+from collections import OrderedDict
+from dataclasses import dataclass
+
+# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with
+# DeepSpeed data structures it has to be available in the current python environment.
+from deepspeed.utils import logger
+from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS,
+                                            FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES,
+                                            FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS)
+
+
+@dataclass
+class zero_model_state:
+    buffers: dict()
+    param_shapes: dict()
+    shared_params: list
+    ds_version: int
+    frozen_param_shapes: dict()
+    frozen_param_fragments: dict()
+
+
+debug = 0
+
+# load to cpu
+device = torch.device('cpu')
+
+
+def atoi(text):
+    return int(text) if text.isdigit() else text
+
+
+def natural_keys(text):
+    '''
+    alist.sort(key=natural_keys) sorts in human order
+    http://nedbatchelder.com/blog/200712/human_sorting.html
+    (See Toothy's implementation in the comments)
+    '''
+    return [atoi(c) for c in re.split(r'(\d+)', text)]
+
+
+def get_model_state_file(checkpoint_dir, zero_stage):
+    if not os.path.isdir(checkpoint_dir):
+        raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist")
+
+    # there should be only one file
+    if zero_stage <= 2:
+        file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt")
+    elif zero_stage == 3:
+        file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt")
+
+    if not os.path.exists(file):
+        raise FileNotFoundError(f"can't find model states file at '{file}'")
+
+    return file
+
+
+def get_checkpoint_files(checkpoint_dir, glob_pattern):
+    # XXX: need to test that this simple glob rule works for multi-node setup too
+    ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys)
+
+    if len(ckpt_files) == 0:
+        raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'")
+
+    return ckpt_files
+
+
+def get_optim_files(checkpoint_dir):
+    return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt")
+
+
+def get_model_state_files(checkpoint_dir):
+    return get_checkpoint_files(checkpoint_dir, "*_model_states.pt")
+
+
+def parse_model_states(files):
+    zero_model_states = []
+    for file in files:
+        state_dict = torch.load(file, map_location=device)
+
+        if BUFFER_NAMES not in state_dict:
+            raise ValueError(f"{file} is not a model state checkpoint")
+        buffer_names = state_dict[BUFFER_NAMES]
+        if debug:
+            print("Found buffers:", buffer_names)
+
+        # recover just the buffers while restoring them to fp32 if they were saved in fp16
+        buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names}
+        param_shapes = state_dict[PARAM_SHAPES]
+
+        # collect parameters that are included in param_shapes
+        param_names = []
+        for s in param_shapes:
+            for name in s.keys():
+                param_names.append(name)
+
+        # update with frozen parameters
+        frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None)
+        if frozen_param_shapes is not None:
+            if debug:
+                print(f"Found frozen_param_shapes: {frozen_param_shapes}")
+            param_names += list(frozen_param_shapes.keys())
+
+        # handle shared params
+        shared_params = [[k, v] for k, v in state_dict["shared_params"].items()]
+
+        ds_version = state_dict.get(DS_VERSION, None)
+
+        frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None)
+
+        z_model_state = zero_model_state(buffers=buffers,
+                                         param_shapes=param_shapes,
+                                         shared_params=shared_params,
+                                         ds_version=ds_version,
+                                         frozen_param_shapes=frozen_param_shapes,
+                                         frozen_param_fragments=frozen_param_fragments)
+        zero_model_states.append(z_model_state)
+
+    return zero_model_states
+
+
+def parse_optim_states(files, ds_checkpoint_dir):
+
+    total_files = len(files)
+    state_dicts = []
+    for f in files:
+        state_dict = torch.load(f, map_location=device)
+        # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights
+        # and also handle the case where it was already removed by another helper script
+        state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None)
+        state_dicts.append(state_dict)
+
+    if not ZERO_STAGE in state_dicts[0][OPTIMIZER_STATE_DICT]:
+        raise ValueError(f"{files[0]} is not a zero checkpoint")
+    zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE]
+    world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT]
+
+    # For ZeRO-2 each param group can have different partition_count as data parallelism for expert
+    # parameters can be different from data parallelism for non-expert parameters. So we can just
+    # use the max of the partition_count to get the dp world_size.
+
+    if type(world_size) is list:
+        world_size = max(world_size)
+
+    if world_size != total_files:
+        raise ValueError(
+            f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. "
+            "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes."
+        )
+
+    # the groups are named differently in each stage
+    if zero_stage <= 2:
+        fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS
+    elif zero_stage == 3:
+        fp32_groups_key = FP32_FLAT_GROUPS
+    else:
+        raise ValueError(f"unknown zero stage {zero_stage}")
+
+    if zero_stage <= 2:
+        fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))]
+    elif zero_stage == 3:
+        # if there is more than one param group, there will be multiple flattened tensors - one
+        # flattened tensor per group - for simplicity merge them into a single tensor
+        #
+        # XXX: could make the script more memory efficient for when there are multiple groups - it
+        # will require matching the sub-lists of param_shapes for each param group flattened tensor
+
+        fp32_flat_groups = [
+            torch.cat(state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key], 0) for i in range(len(state_dicts))
+        ]
+
+    return zero_stage, world_size, fp32_flat_groups
+
+
+def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters):
+    """
+    Returns fp32 state_dict reconstructed from ds checkpoint
+
+    Args:
+        - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are)
+
+    """
+    print(f"Processing zero checkpoint '{ds_checkpoint_dir}'")
+
+    optim_files = get_optim_files(ds_checkpoint_dir)
+    zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir)
+    print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}")
+
+    model_files = get_model_state_files(ds_checkpoint_dir)
+
+    zero_model_states = parse_model_states(model_files)
+    print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}')
+
+    if zero_stage <= 2:
+        return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                                          exclude_frozen_parameters)
+    elif zero_stage == 3:
+        return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                                          exclude_frozen_parameters)
+
+
+def _zero2_merge_frozen_params(state_dict, zero_model_states):
+    if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+        return
+
+    frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+    frozen_param_fragments = zero_model_states[0].frozen_param_fragments
+
+    if debug:
+        num_elem = sum(s.numel() for s in frozen_param_shapes.values())
+        print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+        wanted_params = len(frozen_param_shapes)
+        wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+        avail_numel = sum([p.numel() for p in frozen_param_fragments.values()])
+        print(f'Frozen params: Have {avail_numel} numels to process.')
+        print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+    total_params = 0
+    total_numel = 0
+    for name, shape in frozen_param_shapes.items():
+        total_params += 1
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+
+        state_dict[name] = frozen_param_fragments[name]
+
+        if debug:
+            print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+
+    print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _has_callable(obj, fn):
+    attr = getattr(obj, fn, None)
+    return callable(attr)
+
+
+def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+    param_shapes = zero_model_states[0].param_shapes
+
+    # Reconstruction protocol:
+    #
+    # XXX: document this
+
+    if debug:
+        for i in range(world_size):
+            for j in range(len(fp32_flat_groups[0])):
+                print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}")
+
+    # XXX: memory usage doubles here (zero2)
+    num_param_groups = len(fp32_flat_groups[0])
+    merged_single_partition_of_fp32_groups = []
+    for i in range(num_param_groups):
+        merged_partitions = [sd[i] for sd in fp32_flat_groups]
+        full_single_fp32_vector = torch.cat(merged_partitions, 0)
+        merged_single_partition_of_fp32_groups.append(full_single_fp32_vector)
+    avail_numel = sum(
+        [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups])
+
+    if debug:
+        wanted_params = sum([len(shapes) for shapes in param_shapes])
+        wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes])
+        # not asserting if there is a mismatch due to possible padding
+        print(f"Have {avail_numel} numels to process.")
+        print(f"Need {wanted_numel} numels in {wanted_params} params.")
+
+    # params
+    # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+    # out-of-core computing solution
+    total_numel = 0
+    total_params = 0
+    for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups):
+        offset = 0
+        avail_numel = full_single_fp32_vector.numel()
+        for name, shape in shapes.items():
+
+            unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape)
+            total_numel += unpartitioned_numel
+            total_params += 1
+
+            if debug:
+                print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+            state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape)
+            offset += unpartitioned_numel
+
+        # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and
+        # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex
+        # paddings performed in the code it's almost impossible to predict the exact numbers w/o the
+        # live optimizer object, so we are checking that the numbers are within the right range
+        align_to = 2 * world_size
+
+        def zero2_align(x):
+            return align_to * math.ceil(x / align_to)
+
+        if debug:
+            print(f"original offset={offset}, avail_numel={avail_numel}")
+
+        offset = zero2_align(offset)
+        avail_numel = zero2_align(avail_numel)
+
+        if debug:
+            print(f"aligned  offset={offset}, avail_numel={avail_numel}")
+
+        # Sanity check
+        if offset != avail_numel:
+            raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+    print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                               exclude_frozen_parameters):
+    state_dict = OrderedDict()
+
+    # buffers
+    buffers = zero_model_states[0].buffers
+    state_dict.update(buffers)
+    if debug:
+        print(f"added {len(buffers)} buffers")
+
+    if not exclude_frozen_parameters:
+        _zero2_merge_frozen_params(state_dict, zero_model_states)
+
+    _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+    # recover shared parameters
+    for pair in zero_model_states[0].shared_params:
+        if pair[1] in state_dict:
+            state_dict[pair[0]] = state_dict[pair[1]]
+
+    return state_dict
+
+
+def zero3_partitioned_param_info(unpartitioned_numel, world_size):
+    remainder = unpartitioned_numel % world_size
+    padding_numel = (world_size - remainder) if remainder else 0
+    partitioned_numel = math.ceil(unpartitioned_numel / world_size)
+    return partitioned_numel, padding_numel
+
+
+def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states):
+    if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+        return
+
+    if debug:
+        for i in range(world_size):
+            num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values())
+            print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+        frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+        wanted_params = len(frozen_param_shapes)
+        wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+        avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size
+        print(f'Frozen params: Have {avail_numel} numels to process.')
+        print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+    total_params = 0
+    total_numel = 0
+    for name, shape in zero_model_states[0].frozen_param_shapes.items():
+        total_params += 1
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+
+        param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states)
+        state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape)
+
+        partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+        if debug:
+            print(
+                f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+            )
+
+    print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+    param_shapes = zero_model_states[0].param_shapes
+    avail_numel = fp32_flat_groups[0].numel() * world_size
+    # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each
+    # param, re-consolidating each param, while dealing with padding if any
+
+    # merge list of dicts, preserving order
+    param_shapes = {k: v for d in param_shapes for k, v in d.items()}
+
+    if debug:
+        for i in range(world_size):
+            print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}")
+
+        wanted_params = len(param_shapes)
+        wanted_numel = sum(shape.numel() for shape in param_shapes.values())
+        # not asserting if there is a mismatch due to possible padding
+        avail_numel = fp32_flat_groups[0].numel() * world_size
+        print(f"Trainable params: Have {avail_numel} numels to process.")
+        print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.")
+
+    # params
+    # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+    # out-of-core computing solution
+    offset = 0
+    total_numel = 0
+    total_params = 0
+    for name, shape in param_shapes.items():
+
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+        total_params += 1
+
+        partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+        if debug:
+            print(
+                f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+            )
+
+        # XXX: memory usage doubles here
+        state_dict[name] = torch.cat(
+            tuple(fp32_flat_groups[i].narrow(0, offset, partitioned_numel) for i in range(world_size)),
+            0).narrow(0, 0, unpartitioned_numel).view(shape)
+        offset += partitioned_numel
+
+    offset *= world_size
+
+    # Sanity check
+    if offset != avail_numel:
+        raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+    print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                               exclude_frozen_parameters):
+    state_dict = OrderedDict()
+
+    # buffers
+    buffers = zero_model_states[0].buffers
+    state_dict.update(buffers)
+    if debug:
+        print(f"added {len(buffers)} buffers")
+
+    if not exclude_frozen_parameters:
+        _zero3_merge_frozen_params(state_dict, world_size, zero_model_states)
+
+    _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+    # recover shared parameters
+    for pair in zero_model_states[0].shared_params:
+        if pair[1] in state_dict:
+            state_dict[pair[0]] = state_dict[pair[1]]
+
+    return state_dict
+
+
+def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag=None, exclude_frozen_parameters=False):
+    """
+    Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with
+    ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example
+    via a model hub.
+
+    Args:
+        - ``checkpoint_dir``: path to the desired checkpoint folder
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14``
+        - ``exclude_frozen_parameters``: exclude frozen parameters
+
+    Returns:
+        - pytorch ``state_dict``
+
+    Note: this approach may not work if your application doesn't have sufficient free CPU memory and
+    you may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with
+    the checkpoint.
+
+    A typical usage might be ::
+
+        from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+        # do the training and checkpoint saving
+        state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu
+        model = model.cpu() # move to cpu
+        model.load_state_dict(state_dict)
+        # submit to model hub or save the model to share with others
+
+    In this example the ``model`` will no longer be usable in the deepspeed context of the same
+    application. i.e. you will need to re-initialize the deepspeed engine, since
+    ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+    If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead.
+
+    """
+    if tag is None:
+        latest_path = os.path.join(checkpoint_dir, 'latest')
+        if os.path.isfile(latest_path):
+            with open(latest_path, 'r') as fd:
+                tag = fd.read().strip()
+        else:
+            raise ValueError(f"Unable to find 'latest' file at {latest_path}")
+
+    ds_checkpoint_dir = os.path.join(checkpoint_dir, tag)
+
+    if not os.path.isdir(ds_checkpoint_dir):
+        raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist")
+
+    return _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters)
+
+
+def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, output_file, tag=None, exclude_frozen_parameters=False):
+    """
+    Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be
+    loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed.
+
+    Args:
+        - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+        - ``output_file``: path to the pytorch fp32 state_dict output file (e.g. path/pytorch_model.bin)
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+        - ``exclude_frozen_parameters``: exclude frozen parameters
+    """
+
+    state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag, exclude_frozen_parameters)
+    print(f"Saving fp32 state dict to {output_file}")
+    torch.save(state_dict, output_file)
+
+
+def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None):
+    """
+    1. Put the provided model to cpu
+    2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict``
+    3. Load it into the provided model
+
+    Args:
+        - ``model``: the model object to update
+        - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+
+    Returns:
+        - ``model`: modified model
+
+    Make sure you have plenty of CPU memory available before you call this function. If you don't
+    have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it
+    conveniently placed for you in the checkpoint folder.
+
+    A typical usage might be ::
+
+        from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint
+        model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir)
+        # submit to model hub or save the model to share with others
+
+    Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context
+    of the same application. i.e. you will need to re-initialize the deepspeed engine, since
+    ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+    """
+    logger.info(f"Extracting fp32 weights")
+    state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag)
+
+    logger.info(f"Overwriting model with fp32 weights")
+    model = model.cpu()
+    model.load_state_dict(state_dict, strict=False)
+
+    return model
+
+
+if __name__ == "__main__":
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("checkpoint_dir",
+                        type=str,
+                        help="path to the desired checkpoint folder, e.g., path/checkpoint-12")
+    parser.add_argument(
+        "output_file",
+        type=str,
+        help="path to the pytorch fp32 state_dict output file (e.g. path/checkpoint-12/pytorch_model.bin)")
+    parser.add_argument("-t",
+                        "--tag",
+                        type=str,
+                        default=None,
+                        help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1")
+    parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters")
+    parser.add_argument("-d", "--debug", action='store_true', help="enable debug")
+    args = parser.parse_args()
+
+    debug = args.debug
+
+    convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir,
+                                               args.output_file,
+                                               tag=args.tag,
+                                               exclude_frozen_parameters=args.exclude_frozen_parameters)

config.json

@@ -0,0 +1,29 @@
+{
+  "_name_or_path": "meta-llama/Meta-Llama-3-8B-Instruct",
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 128000,
+  "eos_token_id": 128009,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 8192,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": null,
+  "rope_theta": 500000.0,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.42.3",
+  "use_cache": false,
+  "vocab_size": 128256
+}

generation_config.json

@@ -0,0 +1,12 @@
+{
+  "bos_token_id": 128000,
+  "do_sample": true,
+  "eos_token_id": [
+    128001,
+    128009
+  ],
+  "max_length": 4096,
+  "temperature": 0.6,
+  "top_p": 0.9,
+  "transformers_version": "4.42.3"
+}

llamaboard_config.yaml

@@ -0,0 +1,65 @@
+top.booster: auto
+top.checkpoint_path: null
+top.finetuning_type: full
+top.model_name: LLaMA3-8B-Chat
+top.quantization_bit: none
+top.quantization_method: bitsandbytes
+top.rope_scaling: none
+top.template: llama3
+top.visual_inputs: false
+train.additional_target: ''
+train.badam_mode: layer
+train.badam_switch_interval: 50
+train.badam_switch_mode: ascending
+train.badam_update_ratio: 0.05
+train.batch_size: 4
+train.compute_type: bf16
+train.create_new_adapter: false
+train.cutoff_len: 1024
+train.dataset:
+- truth_train
+train.dataset_dir: data
+train.ds_offload: false
+train.ds_stage: '2'
+train.freeze_extra_modules: ''
+train.freeze_trainable_layers: 2
+train.freeze_trainable_modules: all
+train.galore_rank: 16
+train.galore_scale: 0.25
+train.galore_target: all
+train.galore_update_interval: 200
+train.gradient_accumulation_steps: 8
+train.learning_rate: 5e-6
+train.logging_steps: 1
+train.lora_alpha: 16
+train.lora_dropout: 0
+train.lora_rank: 8
+train.lora_target: ''
+train.loraplus_lr_ratio: 0
+train.lr_scheduler_type: cosine
+train.max_grad_norm: '1.0'
+train.max_samples: '100000'
+train.neat_packing: false
+train.neftune_alpha: 0
+train.num_train_epochs: '5.0'
+train.optim: adamw_torch
+train.packing: false
+train.ppo_score_norm: false
+train.ppo_whiten_rewards: false
+train.pref_beta: 0.1
+train.pref_ftx: 0
+train.pref_loss: sigmoid
+train.report_to: false
+train.resize_vocab: false
+train.reward_model: null
+train.save_steps: 1000
+train.shift_attn: false
+train.training_stage: Supervised Fine-Tuning
+train.use_badam: false
+train.use_dora: false
+train.use_galore: false
+train.use_llama_pro: false
+train.use_pissa: false
+train.use_rslora: false
+train.val_size: 0
+train.warmup_steps: 600

model-00001-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:efcd3278f2c803edac2237fe1884ec3368c10c25b63ea6360a1d7b3e7370a29d
+size 4976698672

model-00002-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c6c06b5f09b61f3fe2fae5caf38101caa2c98923103d35594d18b36e9c6c2efd
+size 4999802720

model-00003-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:099b3b6852cee899327282a532875347bee6f1601ee05bc5860c15305483572f
+size 4915916176

model-00004-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a4b4e47f230f79b84a08863d70ee9d25bda8c074e8ccbb54e41596fe5e20d6ca
+size 1168138808

model.safetensors.index.json

@@ -0,0 +1,298 @@
+{
+  "metadata": {
+    "total_size": 16060522496
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00004-of-00004.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.norm.weight": "model-00004-of-00004.safetensors"
+  }
+}

running_log.txt

@@ -0,0 +1,1057 @@
+[INFO|parser.py:325] 2024-07-11 11:02:30,806 >> Process rank: 0, device: cuda:0, n_gpu: 1, distributed training: True, compute dtype: torch.bfloat16
+
+07/11/2024 11:02:31 - INFO - llamafactory.hparams.parser - Process rank: 6, device: cuda:6, n_gpu: 1, distributed training: True, compute dtype: torch.bfloat16
+
+07/11/2024 11:02:31 - INFO - llamafactory.hparams.parser - Process rank: 2, device: cuda:2, n_gpu: 1, distributed training: True, compute dtype: torch.bfloat16
+
+07/11/2024 11:02:31 - INFO - llamafactory.hparams.parser - Process rank: 7, device: cuda:7, n_gpu: 1, distributed training: True, compute dtype: torch.bfloat16
+
+07/11/2024 11:02:31 - INFO - llamafactory.hparams.parser - Process rank: 4, device: cuda:4, n_gpu: 1, distributed training: True, compute dtype: torch.bfloat16
+
+07/11/2024 11:02:31 - INFO - llamafactory.hparams.parser - Process rank: 5, device: cuda:5, n_gpu: 1, distributed training: True, compute dtype: torch.bfloat16
+
+07/11/2024 11:02:31 - INFO - llamafactory.hparams.parser - Process rank: 3, device: cuda:3, n_gpu: 1, distributed training: True, compute dtype: torch.bfloat16
+
+07/11/2024 11:02:31 - INFO - llamafactory.hparams.parser - Process rank: 1, device: cuda:1, n_gpu: 1, distributed training: True, compute dtype: torch.bfloat16
+
+[INFO|tokenization_utils_base.py:2161] 2024-07-11 11:02:33,758 >> loading file tokenizer.json from cache at /root/.cache/huggingface/hub/models--meta-llama--Meta-Llama-3-8B-Instruct/snapshots/e1945c40cd546c78e41f1151f4db032b271faeaa/tokenizer.json
+
+07/11/2024 11:02:34 - WARNING - transformers.tokenization_utils_base - Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.
+
+07/11/2024 11:02:34 - INFO - llamafactory.data.template - Replace eos token: <|eot_id|>
+
+07/11/2024 11:02:34 - INFO - llamafactory.data.template - Add pad token: <|eot_id|>
+
+07/11/2024 11:02:34 - WARNING - transformers.tokenization_utils_base - Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.
+
+07/11/2024 11:02:34 - INFO - llamafactory.data.template - Replace eos token: <|eot_id|>
+
+07/11/2024 11:02:34 - INFO - llamafactory.data.template - Add pad token: <|eot_id|>
+
+[INFO|tokenization_utils_base.py:2161] 2024-07-11 11:02:33,758 >> loading file added_tokens.json from cache at None
+
+[INFO|tokenization_utils_base.py:2161] 2024-07-11 11:02:33,758 >> loading file special_tokens_map.json from cache at /root/.cache/huggingface/hub/models--meta-llama--Meta-Llama-3-8B-Instruct/snapshots/e1945c40cd546c78e41f1151f4db032b271faeaa/special_tokens_map.json
+
+[INFO|tokenization_utils_base.py:2161] 2024-07-11 11:02:33,758 >> loading file tokenizer_config.json from cache at /root/.cache/huggingface/hub/models--meta-llama--Meta-Llama-3-8B-Instruct/snapshots/e1945c40cd546c78e41f1151f4db032b271faeaa/tokenizer_config.json
+
+[WARNING|logging.py:313] 2024-07-11 11:02:34,060 >> Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.
+
+[INFO|template.py:270] 2024-07-11 11:02:34,061 >> Replace eos token: <|eot_id|>
+
+[INFO|template.py:372] 2024-07-11 11:02:34,061 >> Add pad token: <|eot_id|>
+
+[INFO|loader.py:50] 2024-07-11 11:02:34,061 >> Loading dataset train_output.json...
+
+07/11/2024 11:02:34 - WARNING - transformers.tokenization_utils_base - Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.
+
+07/11/2024 11:02:34 - INFO - llamafactory.data.template - Replace eos token: <|eot_id|>
+
+07/11/2024 11:02:34 - INFO - llamafactory.data.template - Add pad token: <|eot_id|>
+
+07/11/2024 11:02:34 - WARNING - transformers.tokenization_utils_base - Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.
+
+07/11/2024 11:02:34 - INFO - llamafactory.data.template - Replace eos token: <|eot_id|>
+
+07/11/2024 11:02:34 - INFO - llamafactory.data.template - Add pad token: <|eot_id|>
+
+07/11/2024 11:02:34 - WARNING - transformers.tokenization_utils_base - Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.
+
+07/11/2024 11:02:34 - INFO - llamafactory.data.template - Replace eos token: <|eot_id|>
+
+07/11/2024 11:02:34 - INFO - llamafactory.data.template - Add pad token: <|eot_id|>
+
+07/11/2024 11:02:34 - WARNING - transformers.tokenization_utils_base - Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.
+
+07/11/2024 11:02:34 - INFO - llamafactory.data.template - Replace eos token: <|eot_id|>
+
+07/11/2024 11:02:34 - INFO - llamafactory.data.template - Add pad token: <|eot_id|>
+
+07/11/2024 11:02:34 - WARNING - transformers.tokenization_utils_base - Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.
+
+07/11/2024 11:02:34 - INFO - llamafactory.data.template - Replace eos token: <|eot_id|>
+
+07/11/2024 11:02:34 - INFO - llamafactory.data.template - Add pad token: <|eot_id|>
+
+07/11/2024 11:02:35 - INFO - llamafactory.data.loader - Loading dataset train_output.json...
+
+07/11/2024 11:02:35 - INFO - llamafactory.data.loader - Loading dataset train_output.json...
+
+07/11/2024 11:02:35 - INFO - llamafactory.data.loader - Loading dataset train_output.json...
+
+07/11/2024 11:02:35 - INFO - llamafactory.data.loader - Loading dataset train_output.json...
+
+07/11/2024 11:02:35 - INFO - llamafactory.data.loader - Loading dataset train_output.json...
+
+07/11/2024 11:02:35 - INFO - llamafactory.data.loader - Loading dataset train_output.json...
+
+07/11/2024 11:02:35 - INFO - llamafactory.data.loader - Loading dataset train_output.json...
+
+[INFO|configuration_utils.py:733] 2024-07-11 11:02:39,782 >> loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--meta-llama--Meta-Llama-3-8B-Instruct/snapshots/e1945c40cd546c78e41f1151f4db032b271faeaa/config.json
+
+[INFO|configuration_utils.py:800] 2024-07-11 11:02:39,785 >> Model config LlamaConfig {
+  "_name_or_path": "meta-llama/Meta-Llama-3-8B-Instruct",
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 128000,
+  "eos_token_id": 128009,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 8192,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": null,
+  "rope_theta": 500000.0,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.42.3",
+  "use_cache": true,
+  "vocab_size": 128256
+}
+
+
+[INFO|modeling_utils.py:3556] 2024-07-11 11:02:40,864 >> loading weights file model.safetensors from cache at /root/.cache/huggingface/hub/models--meta-llama--Meta-Llama-3-8B-Instruct/snapshots/e1945c40cd546c78e41f1151f4db032b271faeaa/model.safetensors.index.json
+
+[INFO|modeling_utils.py:1531] 2024-07-11 11:05:05,483 >> Instantiating LlamaForCausalLM model under default dtype torch.bfloat16.
+
+[INFO|configuration_utils.py:1000] 2024-07-11 11:05:05,489 >> Generate config GenerationConfig {
+  "bos_token_id": 128000,
+  "eos_token_id": 128009
+}
+
+
+07/11/2024 11:05:10 - INFO - llamafactory.model.model_utils.checkpointing - Gradient checkpointing enabled.
+
+07/11/2024 11:05:10 - INFO - llamafactory.model.model_utils.attention - Using torch SDPA for faster training and inference.
+
+07/11/2024 11:05:10 - INFO - llamafactory.model.adapter - Upcasting trainable params to float32.
+
+07/11/2024 11:05:10 - INFO - llamafactory.model.adapter - Fine-tuning method: Full
+
+07/11/2024 11:05:10 - INFO - llamafactory.model.loader - trainable params: 8,030,261,248 || all params: 8,030,261,248 || trainable%: 100.0000
+
+[INFO|modeling_utils.py:4364] 2024-07-11 11:05:10,887 >> All model checkpoint weights were used when initializing LlamaForCausalLM.
+
+
+[INFO|modeling_utils.py:4372] 2024-07-11 11:05:10,887 >> All the weights of LlamaForCausalLM were initialized from the model checkpoint at meta-llama/Meta-Llama-3-8B-Instruct.
+If your task is similar to the task the model of the checkpoint was trained on, you can already use LlamaForCausalLM for predictions without further training.
+
+[INFO|configuration_utils.py:955] 2024-07-11 11:05:11,059 >> loading configuration file generation_config.json from cache at /root/.cache/huggingface/hub/models--meta-llama--Meta-Llama-3-8B-Instruct/snapshots/e1945c40cd546c78e41f1151f4db032b271faeaa/generation_config.json
+
+[INFO|configuration_utils.py:1000] 2024-07-11 11:05:11,059 >> Generate config GenerationConfig {
+  "bos_token_id": 128000,
+  "do_sample": true,
+  "eos_token_id": [
+    128001,
+    128009
+  ],
+  "max_length": 4096,
+  "temperature": 0.6,
+  "top_p": 0.9
+}
+
+
+[INFO|checkpointing.py:103] 2024-07-11 11:05:11,066 >> Gradient checkpointing enabled.
+
+[INFO|attention.py:80] 2024-07-11 11:05:11,066 >> Using torch SDPA for faster training and inference.
+
+[INFO|adapter.py:302] 2024-07-11 11:05:11,066 >> Upcasting trainable params to float32.
+
+[INFO|adapter.py:48] 2024-07-11 11:05:11,066 >> Fine-tuning method: Full
+
+07/11/2024 11:05:11 - INFO - llamafactory.model.model_utils.checkpointing - Gradient checkpointing enabled.
+
+07/11/2024 11:05:11 - INFO - llamafactory.model.model_utils.attention - Using torch SDPA for faster training and inference.
+
+07/11/2024 11:05:11 - INFO - llamafactory.model.adapter - Upcasting trainable params to float32.
+
+07/11/2024 11:05:11 - INFO - llamafactory.model.adapter - Fine-tuning method: Full
+
+07/11/2024 11:05:11 - INFO - llamafactory.model.model_utils.checkpointing - Gradient checkpointing enabled.
+
+07/11/2024 11:05:11 - INFO - llamafactory.model.model_utils.attention - Using torch SDPA for faster training and inference.
+
+07/11/2024 11:05:11 - INFO - llamafactory.model.adapter - Upcasting trainable params to float32.
+
+07/11/2024 11:05:11 - INFO - llamafactory.model.adapter - Fine-tuning method: Full
+
+07/11/2024 11:05:11 - INFO - llamafactory.model.model_utils.checkpointing - Gradient checkpointing enabled.
+
+07/11/2024 11:05:11 - INFO - llamafactory.model.model_utils.attention - Using torch SDPA for faster training and inference.
+
+07/11/2024 11:05:11 - INFO - llamafactory.model.adapter - Upcasting trainable params to float32.
+
+07/11/2024 11:05:11 - INFO - llamafactory.model.adapter - Fine-tuning method: Full
+
+07/11/2024 11:05:11 - INFO - llamafactory.model.model_utils.checkpointing - Gradient checkpointing enabled.
+
+07/11/2024 11:05:11 - INFO - llamafactory.model.model_utils.attention - Using torch SDPA for faster training and inference.
+
+07/11/2024 11:05:11 - INFO - llamafactory.model.adapter - Upcasting trainable params to float32.
+
+07/11/2024 11:05:11 - INFO - llamafactory.model.adapter - Fine-tuning method: Full
+
+07/11/2024 11:05:11 - INFO - llamafactory.model.model_utils.checkpointing - Gradient checkpointing enabled.
+
+07/11/2024 11:05:11 - INFO - llamafactory.model.model_utils.attention - Using torch SDPA for faster training and inference.
+
+07/11/2024 11:05:11 - INFO - llamafactory.model.adapter - Upcasting trainable params to float32.
+
+07/11/2024 11:05:11 - INFO - llamafactory.model.adapter - Fine-tuning method: Full
+
+07/11/2024 11:05:11 - INFO - llamafactory.model.model_utils.checkpointing - Gradient checkpointing enabled.
+
+07/11/2024 11:05:11 - INFO - llamafactory.model.model_utils.attention - Using torch SDPA for faster training and inference.
+
+07/11/2024 11:05:11 - INFO - llamafactory.model.adapter - Upcasting trainable params to float32.
+
+07/11/2024 11:05:11 - INFO - llamafactory.model.adapter - Fine-tuning method: Full
+
+[INFO|loader.py:196] 2024-07-11 11:05:11,161 >> trainable params: 8,030,261,248 || all params: 8,030,261,248 || trainable%: 100.0000
+
+07/11/2024 11:05:11 - INFO - llamafactory.model.loader - trainable params: 8,030,261,248 || all params: 8,030,261,248 || trainable%: 100.0000
+
+07/11/2024 11:05:11 - INFO - llamafactory.model.loader - trainable params: 8,030,261,248 || all params: 8,030,261,248 || trainable%: 100.0000
+
+07/11/2024 11:05:11 - INFO - llamafactory.model.loader - trainable params: 8,030,261,248 || all params: 8,030,261,248 || trainable%: 100.0000
+
+07/11/2024 11:05:11 - INFO - llamafactory.model.loader - trainable params: 8,030,261,248 || all params: 8,030,261,248 || trainable%: 100.0000
+
+07/11/2024 11:05:11 - INFO - llamafactory.model.loader - trainable params: 8,030,261,248 || all params: 8,030,261,248 || trainable%: 100.0000
+
+07/11/2024 11:05:11 - INFO - llamafactory.model.loader - trainable params: 8,030,261,248 || all params: 8,030,261,248 || trainable%: 100.0000
+
+[INFO|trainer.py:642] 2024-07-11 11:05:11,167 >> Using auto half precision backend
+
+[INFO|trainer.py:2128] 2024-07-11 11:05:33,940 >> ***** Running training *****
+
+[INFO|trainer.py:2129] 2024-07-11 11:05:33,940 >>   Num examples = 19,880
+
+[INFO|trainer.py:2130] 2024-07-11 11:05:33,940 >>   Num Epochs = 5
+
+[INFO|trainer.py:2131] 2024-07-11 11:05:33,940 >>   Instantaneous batch size per device = 4
+
+[INFO|trainer.py:2134] 2024-07-11 11:05:33,940 >>   Total train batch size (w. parallel, distributed & accumulation) = 256
+
+[INFO|trainer.py:2135] 2024-07-11 11:05:33,940 >>   Gradient Accumulation steps = 8
+
+[INFO|trainer.py:2136] 2024-07-11 11:05:33,940 >>   Total optimization steps = 385
+
+[INFO|trainer.py:2137] 2024-07-11 11:05:33,941 >>   Number of trainable parameters = 8,030,261,248
+
+[INFO|callbacks.py:310] 2024-07-11 11:05:52,173 >> {'loss': 13.9619, 'learning_rate': 8.3333e-09, 'epoch': 0.01, 'throughput': 830.24}
+
+[INFO|callbacks.py:310] 2024-07-11 11:06:05,761 >> {'loss': 13.7129, 'learning_rate': 1.6667e-08, 'epoch': 0.03, 'throughput': 949.38}
+
+[INFO|callbacks.py:310] 2024-07-11 11:06:19,304 >> {'loss': 13.8474, 'learning_rate': 2.5000e-08, 'epoch': 0.04, 'throughput': 1000.31}
+
+[INFO|callbacks.py:310] 2024-07-11 11:06:32,865 >> {'loss': 13.8844, 'learning_rate': 3.3333e-08, 'epoch': 0.05, 'throughput': 1017.74}
+
+[INFO|callbacks.py:310] 2024-07-11 11:06:46,377 >> {'loss': 14.1380, 'learning_rate': 4.1667e-08, 'epoch': 0.06, 'throughput': 1037.30}
+
+[INFO|callbacks.py:310] 2024-07-11 11:06:59,966 >> {'loss': 13.9077, 'learning_rate': 5.0000e-08, 'epoch': 0.08, 'throughput': 1055.71}
+
+[INFO|callbacks.py:310] 2024-07-11 11:07:13,513 >> {'loss': 13.8396, 'learning_rate': 5.8333e-08, 'epoch': 0.09, 'throughput': 1060.56}
+
+[INFO|callbacks.py:310] 2024-07-11 11:07:27,086 >> {'loss': 13.9828, 'learning_rate': 6.6667e-08, 'epoch': 0.10, 'throughput': 1059.47}
+
+[INFO|callbacks.py:310] 2024-07-11 11:07:40,694 >> {'loss': 14.0361, 'learning_rate': 7.5000e-08, 'epoch': 0.12, 'throughput': 1065.89}
+
+[INFO|callbacks.py:310] 2024-07-11 11:07:54,240 >> {'loss': 13.9392, 'learning_rate': 8.3333e-08, 'epoch': 0.13, 'throughput': 1069.27}
+
+[INFO|callbacks.py:310] 2024-07-11 11:08:07,884 >> {'loss': 14.0256, 'learning_rate': 9.1667e-08, 'epoch': 0.14, 'throughput': 1073.24}
+
+[INFO|callbacks.py:310] 2024-07-11 11:08:21,425 >> {'loss': 13.6693, 'learning_rate': 1.0000e-07, 'epoch': 0.15, 'throughput': 1070.54}
+
+[INFO|callbacks.py:310] 2024-07-11 11:08:34,965 >> {'loss': 13.9031, 'learning_rate': 1.0833e-07, 'epoch': 0.17, 'throughput': 1067.89}
+
+[INFO|callbacks.py:310] 2024-07-11 11:08:48,495 >> {'loss': 13.8575, 'learning_rate': 1.1667e-07, 'epoch': 0.18, 'throughput': 1073.07}
+
+[INFO|callbacks.py:310] 2024-07-11 11:09:02,114 >> {'loss': 13.8366, 'learning_rate': 1.2500e-07, 'epoch': 0.19, 'throughput': 1073.42}
+
+[INFO|callbacks.py:310] 2024-07-11 11:09:15,701 >> {'loss': 13.8705, 'learning_rate': 1.3333e-07, 'epoch': 0.21, 'throughput': 1076.20}
+
+[INFO|callbacks.py:310] 2024-07-11 11:09:29,305 >> {'loss': 13.1816, 'learning_rate': 1.4167e-07, 'epoch': 0.22, 'throughput': 1076.26}
+
+[INFO|callbacks.py:310] 2024-07-11 11:09:42,873 >> {'loss': 13.2292, 'learning_rate': 1.5000e-07, 'epoch': 0.23, 'throughput': 1078.02}
+
+[INFO|callbacks.py:310] 2024-07-11 11:09:56,380 >> {'loss': 13.4366, 'learning_rate': 1.5833e-07, 'epoch': 0.24, 'throughput': 1076.68}
+
+[INFO|callbacks.py:310] 2024-07-11 11:10:09,892 >> {'loss': 12.9904, 'learning_rate': 1.6667e-07, 'epoch': 0.26, 'throughput': 1076.72}
+
+[INFO|callbacks.py:310] 2024-07-11 11:10:23,491 >> {'loss': 12.8780, 'learning_rate': 1.7500e-07, 'epoch': 0.27, 'throughput': 1078.97}
+
+[INFO|callbacks.py:310] 2024-07-11 11:10:37,113 >> {'loss': 12.7794, 'learning_rate': 1.8333e-07, 'epoch': 0.28, 'throughput': 1081.79}
+
+[INFO|callbacks.py:310] 2024-07-11 11:10:50,680 >> {'loss': 11.3144, 'learning_rate': 1.9167e-07, 'epoch': 0.30, 'throughput': 1082.54}
+
+[INFO|callbacks.py:310] 2024-07-11 11:11:04,260 >> {'loss': 10.8531, 'learning_rate': 2.0000e-07, 'epoch': 0.31, 'throughput': 1086.37}
+
+[INFO|callbacks.py:310] 2024-07-11 11:11:17,809 >> {'loss': 10.7149, 'learning_rate': 2.0833e-07, 'epoch': 0.32, 'throughput': 1087.02}
+
+[INFO|callbacks.py:310] 2024-07-11 11:11:31,356 >> {'loss': 10.5802, 'learning_rate': 2.1667e-07, 'epoch': 0.33, 'throughput': 1089.07}
+
+[INFO|callbacks.py:310] 2024-07-11 11:11:44,856 >> {'loss': 10.3671, 'learning_rate': 2.2500e-07, 'epoch': 0.35, 'throughput': 1090.07}
+
+[INFO|callbacks.py:310] 2024-07-11 11:11:58,442 >> {'loss': 10.1751, 'learning_rate': 2.3333e-07, 'epoch': 0.36, 'throughput': 1090.50}
+
+[INFO|callbacks.py:310] 2024-07-11 11:12:11,969 >> {'loss': 9.7707, 'learning_rate': 2.4167e-07, 'epoch': 0.37, 'throughput': 1089.46}
+
+[INFO|callbacks.py:310] 2024-07-11 11:12:25,588 >> {'loss': 9.6489, 'learning_rate': 2.5000e-07, 'epoch': 0.39, 'throughput': 1090.10}
+
+[INFO|callbacks.py:310] 2024-07-11 11:12:39,159 >> {'loss': 8.6805, 'learning_rate': 2.5833e-07, 'epoch': 0.40, 'throughput': 1090.83}
+
+[INFO|callbacks.py:310] 2024-07-11 11:12:52,757 >> {'loss': 5.9207, 'learning_rate': 2.6667e-07, 'epoch': 0.41, 'throughput': 1091.16}
+
+[INFO|callbacks.py:310] 2024-07-11 11:13:06,329 >> {'loss': 5.7661, 'learning_rate': 2.7500e-07, 'epoch': 0.42, 'throughput': 1093.01}
+
+[INFO|callbacks.py:310] 2024-07-11 11:13:19,934 >> {'loss': 5.6168, 'learning_rate': 2.8333e-07, 'epoch': 0.44, 'throughput': 1094.61}
+
+[INFO|callbacks.py:310] 2024-07-11 11:13:33,491 >> {'loss': 5.3367, 'learning_rate': 2.9167e-07, 'epoch': 0.45, 'throughput': 1095.23}
+
+[INFO|callbacks.py:310] 2024-07-11 11:13:47,054 >> {'loss': 4.9751, 'learning_rate': 3.0000e-07, 'epoch': 0.46, 'throughput': 1094.37}
+
+[INFO|callbacks.py:310] 2024-07-11 11:14:00,636 >> {'loss': 4.7041, 'learning_rate': 3.0833e-07, 'epoch': 0.48, 'throughput': 1093.52}
+
+[INFO|callbacks.py:310] 2024-07-11 11:14:14,270 >> {'loss': 4.4631, 'learning_rate': 3.1667e-07, 'epoch': 0.49, 'throughput': 1093.46}
+
+[INFO|callbacks.py:310] 2024-07-11 11:14:27,823 >> {'loss': 4.1912, 'learning_rate': 3.2500e-07, 'epoch': 0.50, 'throughput': 1093.70}
+
+[INFO|callbacks.py:310] 2024-07-11 11:14:41,381 >> {'loss': 3.9146, 'learning_rate': 3.3333e-07, 'epoch': 0.51, 'throughput': 1093.91}
+
+[INFO|callbacks.py:310] 2024-07-11 11:14:55,032 >> {'loss': 3.0530, 'learning_rate': 3.4167e-07, 'epoch': 0.53, 'throughput': 1094.67}
+
+[INFO|callbacks.py:310] 2024-07-11 11:15:08,599 >> {'loss': 1.5544, 'learning_rate': 3.5000e-07, 'epoch': 0.54, 'throughput': 1095.05}
+
+[INFO|callbacks.py:310] 2024-07-11 11:15:22,200 >> {'loss': 1.0549, 'learning_rate': 3.5833e-07, 'epoch': 0.55, 'throughput': 1094.70}
+
+[INFO|callbacks.py:310] 2024-07-11 11:15:35,757 >> {'loss': 0.7110, 'learning_rate': 3.6667e-07, 'epoch': 0.57, 'throughput': 1095.09}
+
+[INFO|callbacks.py:310] 2024-07-11 11:15:49,369 >> {'loss': 0.5127, 'learning_rate': 3.7500e-07, 'epoch': 0.58, 'throughput': 1095.93}
+
+[INFO|callbacks.py:310] 2024-07-11 11:16:02,947 >> {'loss': 0.4143, 'learning_rate': 3.8333e-07, 'epoch': 0.59, 'throughput': 1095.72}
+
+[INFO|callbacks.py:310] 2024-07-11 11:16:16,516 >> {'loss': 0.4014, 'learning_rate': 3.9167e-07, 'epoch': 0.60, 'throughput': 1096.29}
+
+[INFO|callbacks.py:310] 2024-07-11 11:16:30,096 >> {'loss': 0.3840, 'learning_rate': 4.0000e-07, 'epoch': 0.62, 'throughput': 1096.57}
+
+[INFO|callbacks.py:310] 2024-07-11 11:16:43,682 >> {'loss': 0.3017, 'learning_rate': 4.0833e-07, 'epoch': 0.63, 'throughput': 1098.22}
+
+[INFO|callbacks.py:310] 2024-07-11 11:16:57,245 >> {'loss': 0.3031, 'learning_rate': 4.1667e-07, 'epoch': 0.64, 'throughput': 1099.04}
+
+[INFO|callbacks.py:310] 2024-07-11 11:17:10,757 >> {'loss': 0.2818, 'learning_rate': 4.2500e-07, 'epoch': 0.66, 'throughput': 1098.90}
+
+[INFO|callbacks.py:310] 2024-07-11 11:17:24,345 >> {'loss': 0.2881, 'learning_rate': 4.3333e-07, 'epoch': 0.67, 'throughput': 1098.82}
+
+[INFO|callbacks.py:310] 2024-07-11 11:17:37,932 >> {'loss': 0.2943, 'learning_rate': 4.4167e-07, 'epoch': 0.68, 'throughput': 1099.73}
+
+[INFO|callbacks.py:310] 2024-07-11 11:17:51,440 >> {'loss': 0.2781, 'learning_rate': 4.5000e-07, 'epoch': 0.69, 'throughput': 1098.98}
+
+[INFO|callbacks.py:310] 2024-07-11 11:18:04,974 >> {'loss': 0.2724, 'learning_rate': 4.5833e-07, 'epoch': 0.71, 'throughput': 1099.88}
+
+[INFO|callbacks.py:310] 2024-07-11 11:18:18,537 >> {'loss': 0.2520, 'learning_rate': 4.6667e-07, 'epoch': 0.72, 'throughput': 1099.29}
+
+[INFO|callbacks.py:310] 2024-07-11 11:18:32,042 >> {'loss': 0.2482, 'learning_rate': 4.7500e-07, 'epoch': 0.73, 'throughput': 1099.87}
+
+[INFO|callbacks.py:310] 2024-07-11 11:18:45,666 >> {'loss': 0.2212, 'learning_rate': 4.8333e-07, 'epoch': 0.75, 'throughput': 1100.79}
+
+[INFO|callbacks.py:310] 2024-07-11 11:18:59,238 >> {'loss': 0.2408, 'learning_rate': 4.9167e-07, 'epoch': 0.76, 'throughput': 1100.40}
+
+[INFO|callbacks.py:310] 2024-07-11 11:19:12,837 >> {'loss': 0.2133, 'learning_rate': 5.0000e-07, 'epoch': 0.77, 'throughput': 1100.76}
+
+[INFO|callbacks.py:310] 2024-07-11 11:19:26,415 >> {'loss': 0.2319, 'learning_rate': 5.0833e-07, 'epoch': 0.78, 'throughput': 1101.14}
+
+[INFO|callbacks.py:310] 2024-07-11 11:19:39,970 >> {'loss': 0.2703, 'learning_rate': 5.1667e-07, 'epoch': 0.80, 'throughput': 1100.64}
+
+[INFO|callbacks.py:310] 2024-07-11 11:19:53,514 >> {'loss': 0.2647, 'learning_rate': 5.2500e-07, 'epoch': 0.81, 'throughput': 1101.50}
+
+[INFO|callbacks.py:310] 2024-07-11 11:20:07,052 >> {'loss': 0.2288, 'learning_rate': 5.3333e-07, 'epoch': 0.82, 'throughput': 1100.95}
+
+[INFO|callbacks.py:310] 2024-07-11 11:20:20,590 >> {'loss': 0.2426, 'learning_rate': 5.4167e-07, 'epoch': 0.84, 'throughput': 1101.61}
+
+[INFO|callbacks.py:310] 2024-07-11 11:20:34,211 >> {'loss': 0.1936, 'learning_rate': 5.5000e-07, 'epoch': 0.85, 'throughput': 1102.11}
+
+[INFO|callbacks.py:310] 2024-07-11 11:20:47,778 >> {'loss': 0.1920, 'learning_rate': 5.5833e-07, 'epoch': 0.86, 'throughput': 1102.83}
+
+[INFO|callbacks.py:310] 2024-07-11 11:21:01,424 >> {'loss': 0.1956, 'learning_rate': 5.6667e-07, 'epoch': 0.87, 'throughput': 1102.10}
+
+[INFO|callbacks.py:310] 2024-07-11 11:21:15,063 >> {'loss': 0.1862, 'learning_rate': 5.7500e-07, 'epoch': 0.89, 'throughput': 1102.18}
+
+[INFO|callbacks.py:310] 2024-07-11 11:21:28,594 >> {'loss': 0.1950, 'learning_rate': 5.8333e-07, 'epoch': 0.90, 'throughput': 1101.34}
+
+[INFO|callbacks.py:310] 2024-07-11 11:21:42,130 >> {'loss': 0.1971, 'learning_rate': 5.9167e-07, 'epoch': 0.91, 'throughput': 1101.47}
+
+[INFO|callbacks.py:310] 2024-07-11 11:21:55,703 >> {'loss': 0.1683, 'learning_rate': 6.0000e-07, 'epoch': 0.93, 'throughput': 1101.79}
+
+[INFO|callbacks.py:310] 2024-07-11 11:22:09,285 >> {'loss': 0.2003, 'learning_rate': 6.0833e-07, 'epoch': 0.94, 'throughput': 1101.80}
+
+[INFO|callbacks.py:310] 2024-07-11 11:22:22,833 >> {'loss': 0.1543, 'learning_rate': 6.1667e-07, 'epoch': 0.95, 'throughput': 1101.69}
+
+[INFO|callbacks.py:310] 2024-07-11 11:22:36,430 >> {'loss': 0.1982, 'learning_rate': 6.2500e-07, 'epoch': 0.96, 'throughput': 1101.60}
+
+[INFO|callbacks.py:310] 2024-07-11 11:22:50,024 >> {'loss': 0.1545, 'learning_rate': 6.3333e-07, 'epoch': 0.98, 'throughput': 1101.78}
+
+[INFO|callbacks.py:310] 2024-07-11 11:23:03,557 >> {'loss': 0.1573, 'learning_rate': 6.4167e-07, 'epoch': 0.99, 'throughput': 1101.66}
+
+[INFO|callbacks.py:310] 2024-07-11 11:23:17,169 >> {'loss': 0.1788, 'learning_rate': 6.5000e-07, 'epoch': 1.00, 'throughput': 1102.22}
+
+[INFO|callbacks.py:310] 2024-07-11 11:23:30,798 >> {'loss': 0.1870, 'learning_rate': 6.5833e-07, 'epoch': 1.02, 'throughput': 1102.50}
+
+[INFO|callbacks.py:310] 2024-07-11 11:23:44,378 >> {'loss': 0.2009, 'learning_rate': 6.6667e-07, 'epoch': 1.03, 'throughput': 1101.97}
+
+[INFO|callbacks.py:310] 2024-07-11 11:23:57,975 >> {'loss': 0.2655, 'learning_rate': 6.7500e-07, 'epoch': 1.04, 'throughput': 1102.17}
+
+[INFO|callbacks.py:310] 2024-07-11 11:24:11,555 >> {'loss': 0.2041, 'learning_rate': 6.8333e-07, 'epoch': 1.05, 'throughput': 1102.46}
+
+[INFO|callbacks.py:310] 2024-07-11 11:24:25,136 >> {'loss': 0.1675, 'learning_rate': 6.9167e-07, 'epoch': 1.07, 'throughput': 1102.33}
+
+[INFO|callbacks.py:310] 2024-07-11 11:24:38,737 >> {'loss': 0.2097, 'learning_rate': 7.0000e-07, 'epoch': 1.08, 'throughput': 1102.93}
+
+[INFO|callbacks.py:310] 2024-07-11 11:24:52,408 >> {'loss': 0.2171, 'learning_rate': 7.0833e-07, 'epoch': 1.09, 'throughput': 1103.25}
+
+[INFO|callbacks.py:310] 2024-07-11 11:25:05,972 >> {'loss': 0.1702, 'learning_rate': 7.1667e-07, 'epoch': 1.11, 'throughput': 1103.45}
+
+[INFO|callbacks.py:310] 2024-07-11 11:25:19,464 >> {'loss': 0.1255, 'learning_rate': 7.2500e-07, 'epoch': 1.12, 'throughput': 1102.85}
+
+[INFO|callbacks.py:310] 2024-07-11 11:25:33,033 >> {'loss': 0.1826, 'learning_rate': 7.3333e-07, 'epoch': 1.13, 'throughput': 1103.48}
+
+[INFO|callbacks.py:310] 2024-07-11 11:25:46,607 >> {'loss': 0.2039, 'learning_rate': 7.4167e-07, 'epoch': 1.14, 'throughput': 1103.34}
+
+[INFO|callbacks.py:310] 2024-07-11 11:26:00,228 >> {'loss': 0.2337, 'learning_rate': 7.5000e-07, 'epoch': 1.16, 'throughput': 1103.79}
+
+[INFO|callbacks.py:310] 2024-07-11 11:26:13,832 >> {'loss': 0.1466, 'learning_rate': 7.5833e-07, 'epoch': 1.17, 'throughput': 1103.97}
+
+[INFO|callbacks.py:310] 2024-07-11 11:26:27,410 >> {'loss': 0.1295, 'learning_rate': 7.6667e-07, 'epoch': 1.18, 'throughput': 1104.29}
+
+[INFO|callbacks.py:310] 2024-07-11 11:26:40,924 >> {'loss': 0.1525, 'learning_rate': 7.7500e-07, 'epoch': 1.20, 'throughput': 1104.03}
+
+[INFO|callbacks.py:310] 2024-07-11 11:26:54,523 >> {'loss': 0.1735, 'learning_rate': 7.8333e-07, 'epoch': 1.21, 'throughput': 1104.42}
+
+[INFO|callbacks.py:310] 2024-07-11 11:27:08,160 >> {'loss': 0.1484, 'learning_rate': 7.9167e-07, 'epoch': 1.22, 'throughput': 1105.59}
+
+[INFO|callbacks.py:310] 2024-07-11 11:27:21,753 >> {'loss': 0.1517, 'learning_rate': 8.0000e-07, 'epoch': 1.23, 'throughput': 1105.48}
+
+[INFO|callbacks.py:310] 2024-07-11 11:27:35,370 >> {'loss': 0.1429, 'learning_rate': 8.0833e-07, 'epoch': 1.25, 'throughput': 1105.30}
+
+[INFO|callbacks.py:310] 2024-07-11 11:27:49,027 >> {'loss': 0.1510, 'learning_rate': 8.1667e-07, 'epoch': 1.26, 'throughput': 1105.57}
+
+[INFO|callbacks.py:310] 2024-07-11 11:28:02,582 >> {'loss': 0.1217, 'learning_rate': 8.2500e-07, 'epoch': 1.27, 'throughput': 1105.56}
+
+[INFO|callbacks.py:310] 2024-07-11 11:28:16,078 >> {'loss': 0.1366, 'learning_rate': 8.3333e-07, 'epoch': 1.29, 'throughput': 1104.97}
+
+[INFO|callbacks.py:310] 2024-07-11 11:28:29,637 >> {'loss': 0.1534, 'learning_rate': 8.4167e-07, 'epoch': 1.30, 'throughput': 1105.43}
+
+[INFO|callbacks.py:310] 2024-07-11 11:28:43,152 >> {'loss': 0.1410, 'learning_rate': 8.5000e-07, 'epoch': 1.31, 'throughput': 1105.25}
+
+[INFO|callbacks.py:310] 2024-07-11 11:28:56,697 >> {'loss': 0.1238, 'learning_rate': 8.5833e-07, 'epoch': 1.32, 'throughput': 1104.87}
+
+[INFO|callbacks.py:310] 2024-07-11 11:29:10,263 >> {'loss': 0.1241, 'learning_rate': 8.6667e-07, 'epoch': 1.34, 'throughput': 1104.63}
+
+[INFO|callbacks.py:310] 2024-07-11 11:29:23,827 >> {'loss': 0.1414, 'learning_rate': 8.7500e-07, 'epoch': 1.35, 'throughput': 1104.27}
+
+[INFO|callbacks.py:310] 2024-07-11 11:29:37,411 >> {'loss': 0.1296, 'learning_rate': 8.8333e-07, 'epoch': 1.36, 'throughput': 1104.25}
+
+[INFO|callbacks.py:310] 2024-07-11 11:29:50,947 >> {'loss': 0.1232, 'learning_rate': 8.9167e-07, 'epoch': 1.38, 'throughput': 1104.12}
+
+[INFO|callbacks.py:310] 2024-07-11 11:30:04,589 >> {'loss': 0.1625, 'learning_rate': 9.0000e-07, 'epoch': 1.39, 'throughput': 1104.23}
+
+[INFO|callbacks.py:310] 2024-07-11 11:30:18,108 >> {'loss': 0.1509, 'learning_rate': 9.0833e-07, 'epoch': 1.40, 'throughput': 1104.50}
+
+[INFO|callbacks.py:310] 2024-07-11 11:30:31,663 >> {'loss': 0.1416, 'learning_rate': 9.1667e-07, 'epoch': 1.41, 'throughput': 1104.70}
+
+[INFO|callbacks.py:310] 2024-07-11 11:30:45,219 >> {'loss': 0.1481, 'learning_rate': 9.2500e-07, 'epoch': 1.43, 'throughput': 1104.91}
+
+[INFO|callbacks.py:310] 2024-07-11 11:30:58,830 >> {'loss': 0.1303, 'learning_rate': 9.3333e-07, 'epoch': 1.44, 'throughput': 1104.87}
+
+[INFO|callbacks.py:310] 2024-07-11 11:31:12,462 >> {'loss': 0.1160, 'learning_rate': 9.4167e-07, 'epoch': 1.45, 'throughput': 1104.77}
+
+[INFO|callbacks.py:310] 2024-07-11 11:31:25,996 >> {'loss': 0.0981, 'learning_rate': 9.5000e-07, 'epoch': 1.47, 'throughput': 1104.48}
+
+[INFO|callbacks.py:310] 2024-07-11 11:31:39,542 >> {'loss': 0.1174, 'learning_rate': 9.5833e-07, 'epoch': 1.48, 'throughput': 1104.16}
+
+[INFO|callbacks.py:310] 2024-07-11 11:31:53,119 >> {'loss': 0.1458, 'learning_rate': 9.6667e-07, 'epoch': 1.49, 'throughput': 1103.74}
+
+[INFO|callbacks.py:310] 2024-07-11 11:32:06,699 >> {'loss': 0.0952, 'learning_rate': 9.7500e-07, 'epoch': 1.50, 'throughput': 1103.76}
+
+[INFO|callbacks.py:310] 2024-07-11 11:32:20,301 >> {'loss': 0.1233, 'learning_rate': 9.8333e-07, 'epoch': 1.52, 'throughput': 1103.43}
+
+[INFO|callbacks.py:310] 2024-07-11 11:32:33,923 >> {'loss': 0.1270, 'learning_rate': 9.9167e-07, 'epoch': 1.53, 'throughput': 1103.60}
+
+[INFO|callbacks.py:310] 2024-07-11 11:32:47,494 >> {'loss': 0.1121, 'learning_rate': 1.0000e-06, 'epoch': 1.54, 'throughput': 1103.79}
+
+[INFO|callbacks.py:310] 2024-07-11 11:33:01,024 >> {'loss': 0.1432, 'learning_rate': 1.0083e-06, 'epoch': 1.56, 'throughput': 1103.55}
+
+[INFO|callbacks.py:310] 2024-07-11 11:33:14,596 >> {'loss': 0.1446, 'learning_rate': 1.0167e-06, 'epoch': 1.57, 'throughput': 1103.53}
+
+[INFO|callbacks.py:310] 2024-07-11 11:33:28,145 >> {'loss': 0.1056, 'learning_rate': 1.0250e-06, 'epoch': 1.58, 'throughput': 1103.73}
+
+[INFO|callbacks.py:310] 2024-07-11 11:33:41,757 >> {'loss': 0.1193, 'learning_rate': 1.0333e-06, 'epoch': 1.59, 'throughput': 1103.80}
+
+[INFO|callbacks.py:310] 2024-07-11 11:33:55,372 >> {'loss': 0.1409, 'learning_rate': 1.0417e-06, 'epoch': 1.61, 'throughput': 1104.18}
+
+[INFO|callbacks.py:310] 2024-07-11 11:34:08,971 >> {'loss': 0.1154, 'learning_rate': 1.0500e-06, 'epoch': 1.62, 'throughput': 1104.44}
+
+[INFO|callbacks.py:310] 2024-07-11 11:34:22,537 >> {'loss': 0.1046, 'learning_rate': 1.0583e-06, 'epoch': 1.63, 'throughput': 1104.66}
+
+[INFO|callbacks.py:310] 2024-07-11 11:34:36,093 >> {'loss': 0.0900, 'learning_rate': 1.0667e-06, 'epoch': 1.65, 'throughput': 1104.90}
+
+[INFO|callbacks.py:310] 2024-07-11 11:34:49,618 >> {'loss': 0.0858, 'learning_rate': 1.0750e-06, 'epoch': 1.66, 'throughput': 1104.55}
+
+[INFO|callbacks.py:310] 2024-07-11 11:35:03,212 >> {'loss': 0.0782, 'learning_rate': 1.0833e-06, 'epoch': 1.67, 'throughput': 1104.82}
+
+[INFO|callbacks.py:310] 2024-07-11 11:35:16,738 >> {'loss': 0.1429, 'learning_rate': 1.0917e-06, 'epoch': 1.68, 'throughput': 1104.80}
+
+[INFO|callbacks.py:310] 2024-07-11 11:35:30,359 >> {'loss': 0.1121, 'learning_rate': 1.1000e-06, 'epoch': 1.70, 'throughput': 1104.72}
+
+[INFO|callbacks.py:310] 2024-07-11 11:35:43,915 >> {'loss': 0.0458, 'learning_rate': 1.1083e-06, 'epoch': 1.71, 'throughput': 1104.76}
+
+[INFO|callbacks.py:310] 2024-07-11 11:35:57,555 >> {'loss': 0.1217, 'learning_rate': 1.1167e-06, 'epoch': 1.72, 'throughput': 1104.43}
+
+[INFO|callbacks.py:310] 2024-07-11 11:36:11,110 >> {'loss': 0.1253, 'learning_rate': 1.1250e-06, 'epoch': 1.74, 'throughput': 1104.59}
+
+[INFO|callbacks.py:310] 2024-07-11 11:36:24,626 >> {'loss': 0.0770, 'learning_rate': 1.1333e-06, 'epoch': 1.75, 'throughput': 1104.56}
+
+[INFO|callbacks.py:310] 2024-07-11 11:36:38,205 >> {'loss': 0.0719, 'learning_rate': 1.1417e-06, 'epoch': 1.76, 'throughput': 1104.75}
+
+[INFO|callbacks.py:310] 2024-07-11 11:36:51,771 >> {'loss': 0.0916, 'learning_rate': 1.1500e-06, 'epoch': 1.77, 'throughput': 1104.65}
+
+[INFO|callbacks.py:310] 2024-07-11 11:37:05,307 >> {'loss': 0.0812, 'learning_rate': 1.1583e-06, 'epoch': 1.79, 'throughput': 1104.64}
+
+[INFO|callbacks.py:310] 2024-07-11 11:37:18,865 >> {'loss': 0.1176, 'learning_rate': 1.1667e-06, 'epoch': 1.80, 'throughput': 1104.64}
+
+[INFO|callbacks.py:310] 2024-07-11 11:37:32,443 >> {'loss': 0.0631, 'learning_rate': 1.1750e-06, 'epoch': 1.81, 'throughput': 1104.63}
+
+[INFO|callbacks.py:310] 2024-07-11 11:37:45,990 >> {'loss': 0.1137, 'learning_rate': 1.1833e-06, 'epoch': 1.83, 'throughput': 1104.44}
+
+[INFO|callbacks.py:310] 2024-07-11 11:37:59,512 >> {'loss': 0.0958, 'learning_rate': 1.1917e-06, 'epoch': 1.84, 'throughput': 1104.29}
+
+[INFO|callbacks.py:310] 2024-07-11 11:38:13,063 >> {'loss': 0.1343, 'learning_rate': 1.2000e-06, 'epoch': 1.85, 'throughput': 1104.23}
+
+[INFO|callbacks.py:310] 2024-07-11 11:38:26,593 >> {'loss': 0.1101, 'learning_rate': 1.2083e-06, 'epoch': 1.86, 'throughput': 1103.88}
+
+[INFO|callbacks.py:310] 2024-07-11 11:38:40,125 >> {'loss': 0.0914, 'learning_rate': 1.2167e-06, 'epoch': 1.88, 'throughput': 1103.82}
+
+[INFO|callbacks.py:310] 2024-07-11 11:38:53,738 >> {'loss': 0.1114, 'learning_rate': 1.2250e-06, 'epoch': 1.89, 'throughput': 1103.91}
+
+[INFO|callbacks.py:310] 2024-07-11 11:39:07,254 >> {'loss': 0.0830, 'learning_rate': 1.2333e-06, 'epoch': 1.90, 'throughput': 1103.76}
+
+[INFO|callbacks.py:310] 2024-07-11 11:39:20,835 >> {'loss': 0.1095, 'learning_rate': 1.2417e-06, 'epoch': 1.92, 'throughput': 1104.03}
+
+[INFO|callbacks.py:310] 2024-07-11 11:39:34,388 >> {'loss': 0.0662, 'learning_rate': 1.2500e-06, 'epoch': 1.93, 'throughput': 1104.06}
+
+[INFO|callbacks.py:310] 2024-07-11 11:39:47,946 >> {'loss': 0.0979, 'learning_rate': 1.2583e-06, 'epoch': 1.94, 'throughput': 1104.54}
+
+[INFO|callbacks.py:310] 2024-07-11 11:40:01,559 >> {'loss': 0.0847, 'learning_rate': 1.2667e-06, 'epoch': 1.95, 'throughput': 1104.58}
+
+[INFO|callbacks.py:310] 2024-07-11 11:40:15,082 >> {'loss': 0.0949, 'learning_rate': 1.2750e-06, 'epoch': 1.97, 'throughput': 1104.06}
+
+[INFO|callbacks.py:310] 2024-07-11 11:40:28,684 >> {'loss': 0.1206, 'learning_rate': 1.2833e-06, 'epoch': 1.98, 'throughput': 1103.69}
+
+[INFO|callbacks.py:310] 2024-07-11 11:40:42,245 >> {'loss': 0.1390, 'learning_rate': 1.2917e-06, 'epoch': 1.99, 'throughput': 1103.92}
+
+[INFO|callbacks.py:310] 2024-07-11 11:40:55,833 >> {'loss': 0.0700, 'learning_rate': 1.3000e-06, 'epoch': 2.01, 'throughput': 1103.71}
+
+[INFO|callbacks.py:310] 2024-07-11 11:41:09,423 >> {'loss': 0.0562, 'learning_rate': 1.3083e-06, 'epoch': 2.02, 'throughput': 1103.62}
+
+[INFO|callbacks.py:310] 2024-07-11 11:41:23,078 >> {'loss': 0.0456, 'learning_rate': 1.3167e-06, 'epoch': 2.03, 'throughput': 1103.51}
+
+[INFO|callbacks.py:310] 2024-07-11 11:41:36,635 >> {'loss': 0.0582, 'learning_rate': 1.3250e-06, 'epoch': 2.05, 'throughput': 1103.66}
+
+[INFO|callbacks.py:310] 2024-07-11 11:41:50,227 >> {'loss': 0.0452, 'learning_rate': 1.3333e-06, 'epoch': 2.06, 'throughput': 1103.37}
+
+[INFO|callbacks.py:310] 2024-07-11 11:42:03,798 >> {'loss': 0.0553, 'learning_rate': 1.3417e-06, 'epoch': 2.07, 'throughput': 1103.25}
+
+[INFO|callbacks.py:310] 2024-07-11 11:42:17,344 >> {'loss': 0.1108, 'learning_rate': 1.3500e-06, 'epoch': 2.08, 'throughput': 1103.01}
+
+[INFO|callbacks.py:310] 2024-07-11 11:42:30,912 >> {'loss': 0.0791, 'learning_rate': 1.3583e-06, 'epoch': 2.10, 'throughput': 1102.88}
+
+[INFO|callbacks.py:310] 2024-07-11 11:42:44,458 >> {'loss': 0.0637, 'learning_rate': 1.3667e-06, 'epoch': 2.11, 'throughput': 1102.77}
+
+[INFO|callbacks.py:310] 2024-07-11 11:42:57,976 >> {'loss': 0.0404, 'learning_rate': 1.3750e-06, 'epoch': 2.12, 'throughput': 1102.66}
+
+[INFO|callbacks.py:310] 2024-07-11 11:43:11,545 >> {'loss': 0.0372, 'learning_rate': 1.3833e-06, 'epoch': 2.14, 'throughput': 1102.62}
+
+[INFO|callbacks.py:310] 2024-07-11 11:43:25,078 >> {'loss': 0.0450, 'learning_rate': 1.3917e-06, 'epoch': 2.15, 'throughput': 1102.43}
+
+[INFO|callbacks.py:310] 2024-07-11 11:43:38,684 >> {'loss': 0.0964, 'learning_rate': 1.4000e-06, 'epoch': 2.16, 'throughput': 1102.42}
+
+[INFO|callbacks.py:310] 2024-07-11 11:43:52,267 >> {'loss': 0.0543, 'learning_rate': 1.4083e-06, 'epoch': 2.17, 'throughput': 1102.35}
+
+[INFO|callbacks.py:310] 2024-07-11 11:44:05,854 >> {'loss': 0.0710, 'learning_rate': 1.4167e-06, 'epoch': 2.19, 'throughput': 1102.39}
+
+[INFO|callbacks.py:310] 2024-07-11 11:44:19,373 >> {'loss': 0.0285, 'learning_rate': 1.4250e-06, 'epoch': 2.20, 'throughput': 1102.26}
+
+[INFO|callbacks.py:310] 2024-07-11 11:44:32,944 >> {'loss': 0.0399, 'learning_rate': 1.4333e-06, 'epoch': 2.21, 'throughput': 1102.32}
+
+[INFO|callbacks.py:310] 2024-07-11 11:44:46,571 >> {'loss': 0.0670, 'learning_rate': 1.4417e-06, 'epoch': 2.23, 'throughput': 1102.74}
+
+[INFO|callbacks.py:310] 2024-07-11 11:45:00,151 >> {'loss': 0.0436, 'learning_rate': 1.4500e-06, 'epoch': 2.24, 'throughput': 1102.78}
+
+[INFO|callbacks.py:310] 2024-07-11 11:45:13,718 >> {'loss': 0.0522, 'learning_rate': 1.4583e-06, 'epoch': 2.25, 'throughput': 1102.75}
+
+[INFO|callbacks.py:310] 2024-07-11 11:45:27,246 >> {'loss': 0.0521, 'learning_rate': 1.4667e-06, 'epoch': 2.26, 'throughput': 1102.78}
+
+[INFO|callbacks.py:310] 2024-07-11 11:45:40,843 >> {'loss': 0.0446, 'learning_rate': 1.4750e-06, 'epoch': 2.28, 'throughput': 1102.60}
+
+[INFO|callbacks.py:310] 2024-07-11 11:45:54,432 >> {'loss': 0.0378, 'learning_rate': 1.4833e-06, 'epoch': 2.29, 'throughput': 1102.76}
+
+[INFO|callbacks.py:310] 2024-07-11 11:46:08,019 >> {'loss': 0.0387, 'learning_rate': 1.4917e-06, 'epoch': 2.30, 'throughput': 1102.37}
+
+[INFO|callbacks.py:310] 2024-07-11 11:46:21,587 >> {'loss': 0.0360, 'learning_rate': 1.5000e-06, 'epoch': 2.32, 'throughput': 1102.33}
+
+[INFO|callbacks.py:310] 2024-07-11 11:46:35,138 >> {'loss': 0.0765, 'learning_rate': 1.5083e-06, 'epoch': 2.33, 'throughput': 1102.23}
+
+[INFO|callbacks.py:310] 2024-07-11 11:46:48,709 >> {'loss': 0.0884, 'learning_rate': 1.5167e-06, 'epoch': 2.34, 'throughput': 1102.34}
+
+[INFO|callbacks.py:310] 2024-07-11 11:47:02,246 >> {'loss': 0.0801, 'learning_rate': 1.5250e-06, 'epoch': 2.35, 'throughput': 1102.13}
+
+[INFO|callbacks.py:310] 2024-07-11 11:47:15,861 >> {'loss': 0.0276, 'learning_rate': 1.5333e-06, 'epoch': 2.37, 'throughput': 1102.42}
+
+[INFO|callbacks.py:310] 2024-07-11 11:47:29,458 >> {'loss': 0.0778, 'learning_rate': 1.5417e-06, 'epoch': 2.38, 'throughput': 1102.27}
+
+[INFO|callbacks.py:310] 2024-07-11 11:47:42,996 >> {'loss': 0.0726, 'learning_rate': 1.5500e-06, 'epoch': 2.39, 'throughput': 1102.38}
+
+[INFO|callbacks.py:310] 2024-07-11 11:47:56,560 >> {'loss': 0.1381, 'learning_rate': 1.5583e-06, 'epoch': 2.41, 'throughput': 1102.23}
+
+[INFO|callbacks.py:310] 2024-07-11 11:48:10,123 >> {'loss': 0.0408, 'learning_rate': 1.5667e-06, 'epoch': 2.42, 'throughput': 1102.37}
+
+[INFO|callbacks.py:310] 2024-07-11 11:48:23,689 >> {'loss': 0.1066, 'learning_rate': 1.5750e-06, 'epoch': 2.43, 'throughput': 1102.60}
+
+[INFO|callbacks.py:310] 2024-07-11 11:48:37,250 >> {'loss': 0.0686, 'learning_rate': 1.5833e-06, 'epoch': 2.44, 'throughput': 1102.78}
+
+[INFO|callbacks.py:310] 2024-07-11 11:48:50,791 >> {'loss': 0.0428, 'learning_rate': 1.5917e-06, 'epoch': 2.46, 'throughput': 1102.86}
+
+[INFO|callbacks.py:310] 2024-07-11 11:49:04,395 >> {'loss': 0.0387, 'learning_rate': 1.6000e-06, 'epoch': 2.47, 'throughput': 1102.85}
+
+[INFO|callbacks.py:310] 2024-07-11 11:49:17,985 >> {'loss': 0.0489, 'learning_rate': 1.6083e-06, 'epoch': 2.48, 'throughput': 1102.92}
+
+[INFO|callbacks.py:310] 2024-07-11 11:49:31,560 >> {'loss': 0.0621, 'learning_rate': 1.6167e-06, 'epoch': 2.50, 'throughput': 1102.86}
+
+[INFO|callbacks.py:310] 2024-07-11 11:49:45,187 >> {'loss': 0.0651, 'learning_rate': 1.6250e-06, 'epoch': 2.51, 'throughput': 1102.76}
+
+[INFO|callbacks.py:310] 2024-07-11 11:49:58,749 >> {'loss': 0.0398, 'learning_rate': 1.6333e-06, 'epoch': 2.52, 'throughput': 1102.95}
+
+[INFO|callbacks.py:310] 2024-07-11 11:50:12,256 >> {'loss': 0.0369, 'learning_rate': 1.6417e-06, 'epoch': 2.53, 'throughput': 1102.84}
+
+[INFO|callbacks.py:310] 2024-07-11 11:50:25,823 >> {'loss': 0.0582, 'learning_rate': 1.6500e-06, 'epoch': 2.55, 'throughput': 1102.81}
+
+[INFO|callbacks.py:310] 2024-07-11 11:50:39,396 >> {'loss': 0.0479, 'learning_rate': 1.6583e-06, 'epoch': 2.56, 'throughput': 1102.76}
+
+[INFO|callbacks.py:310] 2024-07-11 11:50:52,966 >> {'loss': 0.0561, 'learning_rate': 1.6667e-06, 'epoch': 2.57, 'throughput': 1102.65}
+
+[INFO|callbacks.py:310] 2024-07-11 11:51:06,538 >> {'loss': 0.0497, 'learning_rate': 1.6750e-06, 'epoch': 2.59, 'throughput': 1102.71}
+
+[INFO|callbacks.py:310] 2024-07-11 11:51:20,116 >> {'loss': 0.0630, 'learning_rate': 1.6833e-06, 'epoch': 2.60, 'throughput': 1102.74}
+
+[INFO|callbacks.py:310] 2024-07-11 11:51:33,695 >> {'loss': 0.0540, 'learning_rate': 1.6917e-06, 'epoch': 2.61, 'throughput': 1102.72}
+
+[INFO|callbacks.py:310] 2024-07-11 11:51:47,299 >> {'loss': 0.0729, 'learning_rate': 1.7000e-06, 'epoch': 2.62, 'throughput': 1102.94}
+
+[INFO|callbacks.py:310] 2024-07-11 11:52:00,917 >> {'loss': 0.0685, 'learning_rate': 1.7083e-06, 'epoch': 2.64, 'throughput': 1103.02}
+
+[INFO|callbacks.py:310] 2024-07-11 11:52:14,561 >> {'loss': 0.0823, 'learning_rate': 1.7167e-06, 'epoch': 2.65, 'throughput': 1103.02}
+
+[INFO|callbacks.py:310] 2024-07-11 11:52:28,087 >> {'loss': 0.0322, 'learning_rate': 1.7250e-06, 'epoch': 2.66, 'throughput': 1102.86}
+
+[INFO|callbacks.py:310] 2024-07-11 11:52:41,692 >> {'loss': 0.0821, 'learning_rate': 1.7333e-06, 'epoch': 2.68, 'throughput': 1103.07}
+
+[INFO|callbacks.py:310] 2024-07-11 11:52:55,309 >> {'loss': 0.0561, 'learning_rate': 1.7417e-06, 'epoch': 2.69, 'throughput': 1103.13}
+
+[INFO|callbacks.py:310] 2024-07-11 11:53:08,887 >> {'loss': 0.0468, 'learning_rate': 1.7500e-06, 'epoch': 2.70, 'throughput': 1103.09}
+
+[INFO|callbacks.py:310] 2024-07-11 11:53:22,397 >> {'loss': 0.0593, 'learning_rate': 1.7583e-06, 'epoch': 2.71, 'throughput': 1103.31}
+
+[INFO|callbacks.py:310] 2024-07-11 11:53:35,991 >> {'loss': 0.0403, 'learning_rate': 1.7667e-06, 'epoch': 2.73, 'throughput': 1103.22}
+
+[INFO|callbacks.py:310] 2024-07-11 11:53:49,564 >> {'loss': 0.0459, 'learning_rate': 1.7750e-06, 'epoch': 2.74, 'throughput': 1103.05}
+
+[INFO|callbacks.py:310] 2024-07-11 11:54:03,161 >> {'loss': 0.0509, 'learning_rate': 1.7833e-06, 'epoch': 2.75, 'throughput': 1102.91}
+
+[INFO|callbacks.py:310] 2024-07-11 11:54:16,779 >> {'loss': 0.0873, 'learning_rate': 1.7917e-06, 'epoch': 2.77, 'throughput': 1103.07}
+
+[INFO|callbacks.py:310] 2024-07-11 11:54:30,334 >> {'loss': 0.0618, 'learning_rate': 1.8000e-06, 'epoch': 2.78, 'throughput': 1103.10}
+
+[INFO|callbacks.py:310] 2024-07-11 11:54:43,886 >> {'loss': 0.1025, 'learning_rate': 1.8083e-06, 'epoch': 2.79, 'throughput': 1102.80}
+
+[INFO|callbacks.py:310] 2024-07-11 11:54:57,475 >> {'loss': 0.0669, 'learning_rate': 1.8167e-06, 'epoch': 2.80, 'throughput': 1102.74}
+
+[INFO|callbacks.py:310] 2024-07-11 11:55:11,084 >> {'loss': 0.0447, 'learning_rate': 1.8250e-06, 'epoch': 2.82, 'throughput': 1102.58}
+
+[INFO|callbacks.py:310] 2024-07-11 11:55:24,676 >> {'loss': 0.0635, 'learning_rate': 1.8333e-06, 'epoch': 2.83, 'throughput': 1102.68}
+
+[INFO|callbacks.py:310] 2024-07-11 11:55:38,261 >> {'loss': 0.0614, 'learning_rate': 1.8417e-06, 'epoch': 2.84, 'throughput': 1102.57}
+
+[INFO|callbacks.py:310] 2024-07-11 11:55:51,820 >> {'loss': 0.0646, 'learning_rate': 1.8500e-06, 'epoch': 2.86, 'throughput': 1102.56}
+
+[INFO|callbacks.py:310] 2024-07-11 11:56:05,370 >> {'loss': 0.0893, 'learning_rate': 1.8583e-06, 'epoch': 2.87, 'throughput': 1102.76}
+
+[INFO|callbacks.py:310] 2024-07-11 11:56:18,931 >> {'loss': 0.0653, 'learning_rate': 1.8667e-06, 'epoch': 2.88, 'throughput': 1102.83}
+
+[INFO|callbacks.py:310] 2024-07-11 11:56:32,505 >> {'loss': 0.0402, 'learning_rate': 1.8750e-06, 'epoch': 2.89, 'throughput': 1102.96}
+
+[INFO|callbacks.py:310] 2024-07-11 11:56:46,074 >> {'loss': 0.0407, 'learning_rate': 1.8833e-06, 'epoch': 2.91, 'throughput': 1103.17}
+
+[INFO|callbacks.py:310] 2024-07-11 11:56:59,621 >> {'loss': 0.0949, 'learning_rate': 1.8917e-06, 'epoch': 2.92, 'throughput': 1103.04}
+
+[INFO|callbacks.py:310] 2024-07-11 11:57:13,230 >> {'loss': 0.0789, 'learning_rate': 1.9000e-06, 'epoch': 2.93, 'throughput': 1103.14}
+
+[INFO|callbacks.py:310] 2024-07-11 11:57:26,812 >> {'loss': 0.0438, 'learning_rate': 1.9083e-06, 'epoch': 2.95, 'throughput': 1103.18}
+
+[INFO|callbacks.py:310] 2024-07-11 11:57:40,386 >> {'loss': 0.0905, 'learning_rate': 1.9167e-06, 'epoch': 2.96, 'throughput': 1103.48}
+
+[INFO|callbacks.py:310] 2024-07-11 11:57:53,898 >> {'loss': 0.0495, 'learning_rate': 1.9250e-06, 'epoch': 2.97, 'throughput': 1103.43}
+
+[INFO|callbacks.py:310] 2024-07-11 11:58:07,535 >> {'loss': 0.0642, 'learning_rate': 1.9333e-06, 'epoch': 2.98, 'throughput': 1103.64}
+
+[INFO|callbacks.py:310] 2024-07-11 11:58:21,083 >> {'loss': 0.0607, 'learning_rate': 1.9417e-06, 'epoch': 3.00, 'throughput': 1103.60}
+
+[INFO|callbacks.py:310] 2024-07-11 11:58:34,682 >> {'loss': 0.0363, 'learning_rate': 1.9500e-06, 'epoch': 3.01, 'throughput': 1103.54}
+
+[INFO|callbacks.py:310] 2024-07-11 11:58:48,313 >> {'loss': 0.0492, 'learning_rate': 1.9583e-06, 'epoch': 3.02, 'throughput': 1103.42}
+
+[INFO|callbacks.py:310] 2024-07-11 11:59:01,897 >> {'loss': 0.0373, 'learning_rate': 1.9667e-06, 'epoch': 3.04, 'throughput': 1103.50}
+
+[INFO|callbacks.py:310] 2024-07-11 11:59:15,458 >> {'loss': 0.0203, 'learning_rate': 1.9750e-06, 'epoch': 3.05, 'throughput': 1103.35}
+
+[INFO|callbacks.py:310] 2024-07-11 11:59:28,966 >> {'loss': 0.0175, 'learning_rate': 1.9833e-06, 'epoch': 3.06, 'throughput': 1103.44}
+
+[INFO|callbacks.py:310] 2024-07-11 11:59:42,557 >> {'loss': 0.0497, 'learning_rate': 1.9917e-06, 'epoch': 3.07, 'throughput': 1103.62}
+
+[INFO|callbacks.py:310] 2024-07-11 11:59:56,112 >> {'loss': 0.0361, 'learning_rate': 2.0000e-06, 'epoch': 3.09, 'throughput': 1103.61}
+
+[INFO|callbacks.py:310] 2024-07-11 12:00:09,694 >> {'loss': 0.0193, 'learning_rate': 2.0083e-06, 'epoch': 3.10, 'throughput': 1103.63}
+
+[INFO|callbacks.py:310] 2024-07-11 12:00:23,246 >> {'loss': 0.0142, 'learning_rate': 2.0167e-06, 'epoch': 3.11, 'throughput': 1103.37}
+
+[INFO|callbacks.py:310] 2024-07-11 12:00:36,823 >> {'loss': 0.0415, 'learning_rate': 2.0250e-06, 'epoch': 3.13, 'throughput': 1103.30}
+
+[INFO|callbacks.py:310] 2024-07-11 12:00:50,432 >> {'loss': 0.0178, 'learning_rate': 2.0333e-06, 'epoch': 3.14, 'throughput': 1103.37}
+
+[INFO|callbacks.py:310] 2024-07-11 12:01:03,966 >> {'loss': 0.0166, 'learning_rate': 2.0417e-06, 'epoch': 3.15, 'throughput': 1103.51}
+
+[INFO|callbacks.py:310] 2024-07-11 12:01:17,574 >> {'loss': 0.0424, 'learning_rate': 2.0500e-06, 'epoch': 3.16, 'throughput': 1103.59}
+
+[INFO|callbacks.py:310] 2024-07-11 12:01:31,112 >> {'loss': 0.0464, 'learning_rate': 2.0583e-06, 'epoch': 3.18, 'throughput': 1103.67}
+
+[INFO|callbacks.py:310] 2024-07-11 12:01:44,672 >> {'loss': 0.0235, 'learning_rate': 2.0667e-06, 'epoch': 3.19, 'throughput': 1103.53}
+
+[INFO|callbacks.py:310] 2024-07-11 12:01:58,233 >> {'loss': 0.0128, 'learning_rate': 2.0750e-06, 'epoch': 3.20, 'throughput': 1103.41}
+
+[INFO|callbacks.py:310] 2024-07-11 12:02:11,831 >> {'loss': 0.0319, 'learning_rate': 2.0833e-06, 'epoch': 3.22, 'throughput': 1103.70}
+
+[INFO|callbacks.py:310] 2024-07-11 12:02:25,358 >> {'loss': 0.0196, 'learning_rate': 2.0917e-06, 'epoch': 3.23, 'throughput': 1103.57}
+
+[INFO|callbacks.py:310] 2024-07-11 12:02:38,937 >> {'loss': 0.0326, 'learning_rate': 2.1000e-06, 'epoch': 3.24, 'throughput': 1103.58}
+
+[INFO|callbacks.py:310] 2024-07-11 12:02:52,507 >> {'loss': 0.0170, 'learning_rate': 2.1083e-06, 'epoch': 3.25, 'throughput': 1103.43}
+
+[INFO|callbacks.py:310] 2024-07-11 12:03:06,088 >> {'loss': 0.0372, 'learning_rate': 2.1167e-06, 'epoch': 3.27, 'throughput': 1103.48}
+
+[INFO|callbacks.py:310] 2024-07-11 12:03:19,699 >> {'loss': 0.0165, 'learning_rate': 2.1250e-06, 'epoch': 3.28, 'throughput': 1103.48}
+
+[INFO|callbacks.py:310] 2024-07-11 12:03:33,259 >> {'loss': 0.0142, 'learning_rate': 2.1333e-06, 'epoch': 3.29, 'throughput': 1103.67}
+
+[INFO|callbacks.py:310] 2024-07-11 12:03:46,842 >> {'loss': 0.0117, 'learning_rate': 2.1417e-06, 'epoch': 3.31, 'throughput': 1103.45}
+
+[INFO|callbacks.py:310] 2024-07-11 12:04:00,471 >> {'loss': 0.0264, 'learning_rate': 2.1500e-06, 'epoch': 3.32, 'throughput': 1103.60}
+
+[INFO|callbacks.py:310] 2024-07-11 12:04:14,075 >> {'loss': 0.0340, 'learning_rate': 2.1583e-06, 'epoch': 3.33, 'throughput': 1103.69}
+
+[INFO|callbacks.py:310] 2024-07-11 12:04:27,601 >> {'loss': 0.0310, 'learning_rate': 2.1667e-06, 'epoch': 3.34, 'throughput': 1103.68}
+
+[INFO|callbacks.py:310] 2024-07-11 12:04:41,161 >> {'loss': 0.0180, 'learning_rate': 2.1750e-06, 'epoch': 3.36, 'throughput': 1103.52}
+
+[INFO|callbacks.py:310] 2024-07-11 12:04:54,755 >> {'loss': 0.0461, 'learning_rate': 2.1833e-06, 'epoch': 3.37, 'throughput': 1103.57}
+
+[INFO|callbacks.py:310] 2024-07-11 12:05:08,257 >> {'loss': 0.0372, 'learning_rate': 2.1917e-06, 'epoch': 3.38, 'throughput': 1103.44}
+
+[INFO|callbacks.py:310] 2024-07-11 12:05:21,809 >> {'loss': 0.0471, 'learning_rate': 2.2000e-06, 'epoch': 3.40, 'throughput': 1103.29}
+
+[INFO|callbacks.py:310] 2024-07-11 12:05:35,395 >> {'loss': 0.0370, 'learning_rate': 2.2083e-06, 'epoch': 3.41, 'throughput': 1103.17}
+
+[INFO|callbacks.py:310] 2024-07-11 12:05:48,966 >> {'loss': 0.0245, 'learning_rate': 2.2167e-06, 'epoch': 3.42, 'throughput': 1103.04}
+
+[INFO|callbacks.py:310] 2024-07-11 12:06:02,486 >> {'loss': 0.0233, 'learning_rate': 2.2250e-06, 'epoch': 3.43, 'throughput': 1103.10}
+
+[INFO|callbacks.py:310] 2024-07-11 12:06:16,087 >> {'loss': 0.0256, 'learning_rate': 2.2333e-06, 'epoch': 3.45, 'throughput': 1103.15}
+
+[INFO|callbacks.py:310] 2024-07-11 12:06:29,677 >> {'loss': 0.0345, 'learning_rate': 2.2417e-06, 'epoch': 3.46, 'throughput': 1103.06}
+
+[INFO|callbacks.py:310] 2024-07-11 12:06:43,259 >> {'loss': 0.0656, 'learning_rate': 2.2500e-06, 'epoch': 3.47, 'throughput': 1103.11}
+
+[INFO|callbacks.py:310] 2024-07-11 12:06:56,838 >> {'loss': 0.0247, 'learning_rate': 2.2583e-06, 'epoch': 3.49, 'throughput': 1103.12}
+
+[INFO|callbacks.py:310] 2024-07-11 12:07:10,402 >> {'loss': 0.0510, 'learning_rate': 2.2667e-06, 'epoch': 3.50, 'throughput': 1103.18}
+
+[INFO|callbacks.py:310] 2024-07-11 12:07:23,955 >> {'loss': 0.0417, 'learning_rate': 2.2750e-06, 'epoch': 3.51, 'throughput': 1103.19}
+
+[INFO|callbacks.py:310] 2024-07-11 12:07:37,561 >> {'loss': 0.0176, 'learning_rate': 2.2833e-06, 'epoch': 3.52, 'throughput': 1103.12}
+
+[INFO|callbacks.py:310] 2024-07-11 12:07:51,125 >> {'loss': 0.0155, 'learning_rate': 2.2917e-06, 'epoch': 3.54, 'throughput': 1102.99}
+
+[INFO|callbacks.py:310] 2024-07-11 12:08:04,721 >> {'loss': 0.0135, 'learning_rate': 2.3000e-06, 'epoch': 3.55, 'throughput': 1103.13}
+
+[INFO|callbacks.py:310] 2024-07-11 12:08:18,303 >> {'loss': 0.0222, 'learning_rate': 2.3083e-06, 'epoch': 3.56, 'throughput': 1103.23}
+
+[INFO|callbacks.py:310] 2024-07-11 12:08:31,900 >> {'loss': 0.0289, 'learning_rate': 2.3167e-06, 'epoch': 3.58, 'throughput': 1103.41}
+
+[INFO|callbacks.py:310] 2024-07-11 12:08:45,544 >> {'loss': 0.0314, 'learning_rate': 2.3250e-06, 'epoch': 3.59, 'throughput': 1103.39}
+
+[INFO|callbacks.py:310] 2024-07-11 12:08:59,085 >> {'loss': 0.0582, 'learning_rate': 2.3333e-06, 'epoch': 3.60, 'throughput': 1103.12}
+
+[INFO|callbacks.py:310] 2024-07-11 12:09:12,702 >> {'loss': 0.0339, 'learning_rate': 2.3417e-06, 'epoch': 3.61, 'throughput': 1103.28}
+
+[INFO|callbacks.py:310] 2024-07-11 12:09:26,307 >> {'loss': 0.0656, 'learning_rate': 2.3500e-06, 'epoch': 3.63, 'throughput': 1103.32}
+
+[INFO|callbacks.py:310] 2024-07-11 12:09:39,915 >> {'loss': 0.0355, 'learning_rate': 2.3583e-06, 'epoch': 3.64, 'throughput': 1103.41}
+
+[INFO|callbacks.py:310] 2024-07-11 12:09:53,473 >> {'loss': 0.0370, 'learning_rate': 2.3667e-06, 'epoch': 3.65, 'throughput': 1103.36}
+
+[INFO|callbacks.py:310] 2024-07-11 12:10:07,008 >> {'loss': 0.0206, 'learning_rate': 2.3750e-06, 'epoch': 3.67, 'throughput': 1103.45}
+
+[INFO|callbacks.py:310] 2024-07-11 12:10:20,629 >> {'loss': 0.0286, 'learning_rate': 2.3833e-06, 'epoch': 3.68, 'throughput': 1103.36}
+
+[INFO|callbacks.py:310] 2024-07-11 12:10:34,205 >> {'loss': 0.0316, 'learning_rate': 2.3917e-06, 'epoch': 3.69, 'throughput': 1103.44}
+
+[INFO|callbacks.py:310] 2024-07-11 12:10:47,785 >> {'loss': 0.0263, 'learning_rate': 2.4000e-06, 'epoch': 3.70, 'throughput': 1103.55}
+
+[INFO|callbacks.py:310] 2024-07-11 12:11:01,368 >> {'loss': 0.0265, 'learning_rate': 2.4083e-06, 'epoch': 3.72, 'throughput': 1103.75}
+
+[INFO|callbacks.py:310] 2024-07-11 12:11:14,942 >> {'loss': 0.0251, 'learning_rate': 2.4167e-06, 'epoch': 3.73, 'throughput': 1103.60}
+
+[INFO|callbacks.py:310] 2024-07-11 12:11:28,479 >> {'loss': 0.0293, 'learning_rate': 2.4250e-06, 'epoch': 3.74, 'throughput': 1103.51}
+
+[INFO|callbacks.py:310] 2024-07-11 12:11:42,009 >> {'loss': 0.0278, 'learning_rate': 2.4333e-06, 'epoch': 3.76, 'throughput': 1103.50}
+
+[INFO|callbacks.py:310] 2024-07-11 12:11:55,561 >> {'loss': 0.0350, 'learning_rate': 2.4417e-06, 'epoch': 3.77, 'throughput': 1103.52}
+
+[INFO|callbacks.py:310] 2024-07-11 12:12:09,121 >> {'loss': 0.0455, 'learning_rate': 2.4500e-06, 'epoch': 3.78, 'throughput': 1103.38}
+
+[INFO|callbacks.py:310] 2024-07-11 12:12:22,668 >> {'loss': 0.0429, 'learning_rate': 2.4583e-06, 'epoch': 3.79, 'throughput': 1103.20}
+
+[INFO|callbacks.py:310] 2024-07-11 12:12:36,239 >> {'loss': 0.0560, 'learning_rate': 2.4667e-06, 'epoch': 3.81, 'throughput': 1103.28}
+
+[INFO|callbacks.py:310] 2024-07-11 12:12:49,764 >> {'loss': 0.0325, 'learning_rate': 2.4750e-06, 'epoch': 3.82, 'throughput': 1103.35}
+
+[INFO|callbacks.py:310] 2024-07-11 12:13:03,351 >> {'loss': 0.0401, 'learning_rate': 2.4833e-06, 'epoch': 3.83, 'throughput': 1103.39}
+
+[INFO|callbacks.py:310] 2024-07-11 12:13:16,918 >> {'loss': 0.0182, 'learning_rate': 2.4917e-06, 'epoch': 3.85, 'throughput': 1103.37}
+
+[INFO|callbacks.py:310] 2024-07-11 12:13:30,518 >> {'loss': 0.0344, 'learning_rate': 2.5000e-06, 'epoch': 3.86, 'throughput': 1103.37}
+
+[INFO|callbacks.py:310] 2024-07-11 12:13:44,052 >> {'loss': 0.0111, 'learning_rate': 2.5083e-06, 'epoch': 3.87, 'throughput': 1103.37}
+
+[INFO|callbacks.py:310] 2024-07-11 12:13:57,592 >> {'loss': 0.0088, 'learning_rate': 2.5167e-06, 'epoch': 3.88, 'throughput': 1103.52}
+
+[INFO|callbacks.py:310] 2024-07-11 12:14:11,256 >> {'loss': 0.0130, 'learning_rate': 2.5250e-06, 'epoch': 3.90, 'throughput': 1103.56}
+
+[INFO|callbacks.py:310] 2024-07-11 12:14:24,831 >> {'loss': 0.0433, 'learning_rate': 2.5333e-06, 'epoch': 3.91, 'throughput': 1103.64}
+
+[INFO|callbacks.py:310] 2024-07-11 12:14:38,383 >> {'loss': 0.0214, 'learning_rate': 2.5417e-06, 'epoch': 3.92, 'throughput': 1103.57}
+
+[INFO|callbacks.py:310] 2024-07-11 12:14:51,973 >> {'loss': 0.0416, 'learning_rate': 2.5500e-06, 'epoch': 3.94, 'throughput': 1103.50}
+
+[INFO|callbacks.py:310] 2024-07-11 12:15:05,507 >> {'loss': 0.0321, 'learning_rate': 2.5583e-06, 'epoch': 3.95, 'throughput': 1103.41}
+
+[INFO|callbacks.py:310] 2024-07-11 12:15:19,098 >> {'loss': 0.0526, 'learning_rate': 2.5667e-06, 'epoch': 3.96, 'throughput': 1103.50}
+
+[INFO|callbacks.py:310] 2024-07-11 12:15:32,712 >> {'loss': 0.0250, 'learning_rate': 2.5750e-06, 'epoch': 3.97, 'throughput': 1103.53}
+
+[INFO|callbacks.py:310] 2024-07-11 12:15:46,305 >> {'loss': 0.0787, 'learning_rate': 2.5833e-06, 'epoch': 3.99, 'throughput': 1103.77}
+
+[INFO|callbacks.py:310] 2024-07-11 12:15:59,841 >> {'loss': 0.0559, 'learning_rate': 2.5917e-06, 'epoch': 4.00, 'throughput': 1103.76}
+
+[INFO|callbacks.py:310] 2024-07-11 12:16:13,395 >> {'loss': 0.0259, 'learning_rate': 2.6000e-06, 'epoch': 4.01, 'throughput': 1103.73}
+
+[INFO|callbacks.py:310] 2024-07-11 12:16:27,011 >> {'loss': 0.0370, 'learning_rate': 2.6083e-06, 'epoch': 4.03, 'throughput': 1103.60}
+
+[INFO|callbacks.py:310] 2024-07-11 12:16:40,583 >> {'loss': 0.0293, 'learning_rate': 2.6167e-06, 'epoch': 4.04, 'throughput': 1103.59}
+
+[INFO|callbacks.py:310] 2024-07-11 12:16:54,092 >> {'loss': 0.0153, 'learning_rate': 2.6250e-06, 'epoch': 4.05, 'throughput': 1103.45}
+
+[INFO|callbacks.py:310] 2024-07-11 12:17:07,743 >> {'loss': 0.0234, 'learning_rate': 2.6333e-06, 'epoch': 4.06, 'throughput': 1103.75}
+
+[INFO|callbacks.py:310] 2024-07-11 12:17:21,282 >> {'loss': 0.0212, 'learning_rate': 2.6417e-06, 'epoch': 4.08, 'throughput': 1103.71}
+
+[INFO|callbacks.py:310] 2024-07-11 12:17:34,845 >> {'loss': 0.0187, 'learning_rate': 2.6500e-06, 'epoch': 4.09, 'throughput': 1103.79}
+
+[INFO|callbacks.py:310] 2024-07-11 12:17:48,409 >> {'loss': 0.0245, 'learning_rate': 2.6583e-06, 'epoch': 4.10, 'throughput': 1103.86}
+
+[INFO|callbacks.py:310] 2024-07-11 12:18:01,997 >> {'loss': 0.0032, 'learning_rate': 2.6667e-06, 'epoch': 4.12, 'throughput': 1103.80}
+
+[INFO|callbacks.py:310] 2024-07-11 12:18:15,570 >> {'loss': 0.0367, 'learning_rate': 2.6750e-06, 'epoch': 4.13, 'throughput': 1103.65}
+
+[INFO|callbacks.py:310] 2024-07-11 12:18:29,133 >> {'loss': 0.0380, 'learning_rate': 2.6833e-06, 'epoch': 4.14, 'throughput': 1103.71}
+
+[INFO|callbacks.py:310] 2024-07-11 12:18:42,699 >> {'loss': 0.0194, 'learning_rate': 2.6917e-06, 'epoch': 4.15, 'throughput': 1103.55}
+
+[INFO|callbacks.py:310] 2024-07-11 12:18:56,286 >> {'loss': 0.0262, 'learning_rate': 2.7000e-06, 'epoch': 4.17, 'throughput': 1103.64}
+
+[INFO|callbacks.py:310] 2024-07-11 12:19:09,822 >> {'loss': 0.0151, 'learning_rate': 2.7083e-06, 'epoch': 4.18, 'throughput': 1103.50}
+
+[INFO|callbacks.py:310] 2024-07-11 12:19:23,377 >> {'loss': 0.0267, 'learning_rate': 2.7167e-06, 'epoch': 4.19, 'throughput': 1103.52}
+
+[INFO|callbacks.py:310] 2024-07-11 12:19:36,893 >> {'loss': 0.0041, 'learning_rate': 2.7250e-06, 'epoch': 4.21, 'throughput': 1103.49}
+
+[INFO|callbacks.py:310] 2024-07-11 12:19:50,423 >> {'loss': 0.0365, 'learning_rate': 2.7333e-06, 'epoch': 4.22, 'throughput': 1103.40}
+
+[INFO|callbacks.py:310] 2024-07-11 12:20:04,049 >> {'loss': 0.0151, 'learning_rate': 2.7417e-06, 'epoch': 4.23, 'throughput': 1103.57}
+
+[INFO|callbacks.py:310] 2024-07-11 12:20:17,604 >> {'loss': 0.0060, 'learning_rate': 2.7500e-06, 'epoch': 4.24, 'throughput': 1103.59}
+
+[INFO|callbacks.py:310] 2024-07-11 12:20:31,180 >> {'loss': 0.0247, 'learning_rate': 2.7583e-06, 'epoch': 4.26, 'throughput': 1103.51}
+
+[INFO|callbacks.py:310] 2024-07-11 12:20:44,824 >> {'loss': 0.0147, 'learning_rate': 2.7667e-06, 'epoch': 4.27, 'throughput': 1103.61}
+
+[INFO|callbacks.py:310] 2024-07-11 12:20:58,301 >> {'loss': 0.0413, 'learning_rate': 2.7750e-06, 'epoch': 4.28, 'throughput': 1103.45}
+
+[INFO|callbacks.py:310] 2024-07-11 12:21:11,864 >> {'loss': 0.0318, 'learning_rate': 2.7833e-06, 'epoch': 4.30, 'throughput': 1103.40}
+
+[INFO|callbacks.py:310] 2024-07-11 12:21:25,395 >> {'loss': 0.0513, 'learning_rate': 2.7917e-06, 'epoch': 4.31, 'throughput': 1103.30}
+
+[INFO|callbacks.py:310] 2024-07-11 12:21:38,939 >> {'loss': 0.0165, 'learning_rate': 2.8000e-06, 'epoch': 4.32, 'throughput': 1103.44}
+
+[INFO|callbacks.py:310] 2024-07-11 12:21:52,483 >> {'loss': 0.0131, 'learning_rate': 2.8083e-06, 'epoch': 4.33, 'throughput': 1103.48}
+
+[INFO|callbacks.py:310] 2024-07-11 12:22:06,128 >> {'loss': 0.0246, 'learning_rate': 2.8167e-06, 'epoch': 4.35, 'throughput': 1103.52}
+
+[INFO|callbacks.py:310] 2024-07-11 12:22:19,671 >> {'loss': 0.0465, 'learning_rate': 2.8250e-06, 'epoch': 4.36, 'throughput': 1103.46}
+
+[INFO|callbacks.py:310] 2024-07-11 12:22:33,289 >> {'loss': 0.0084, 'learning_rate': 2.8333e-06, 'epoch': 4.37, 'throughput': 1103.49}
+
+[INFO|callbacks.py:310] 2024-07-11 12:22:46,833 >> {'loss': 0.0105, 'learning_rate': 2.8417e-06, 'epoch': 4.39, 'throughput': 1103.35}
+
+[INFO|callbacks.py:310] 2024-07-11 12:23:00,393 >> {'loss': 0.0388, 'learning_rate': 2.8500e-06, 'epoch': 4.40, 'throughput': 1103.48}
+
+[INFO|callbacks.py:310] 2024-07-11 12:23:13,920 >> {'loss': 0.0486, 'learning_rate': 2.8583e-06, 'epoch': 4.41, 'throughput': 1103.54}
+
+[INFO|callbacks.py:310] 2024-07-11 12:23:27,450 >> {'loss': 0.0354, 'learning_rate': 2.8667e-06, 'epoch': 4.42, 'throughput': 1103.53}
+
+[INFO|callbacks.py:310] 2024-07-11 12:23:41,045 >> {'loss': 0.0464, 'learning_rate': 2.8750e-06, 'epoch': 4.44, 'throughput': 1103.38}
+
+[INFO|callbacks.py:310] 2024-07-11 12:23:54,593 >> {'loss': 0.0312, 'learning_rate': 2.8833e-06, 'epoch': 4.45, 'throughput': 1103.41}
+
+[INFO|callbacks.py:310] 2024-07-11 12:24:08,195 >> {'loss': 0.0352, 'learning_rate': 2.8917e-06, 'epoch': 4.46, 'throughput': 1103.50}
+
+[INFO|callbacks.py:310] 2024-07-11 12:24:21,721 >> {'loss': 0.0116, 'learning_rate': 2.9000e-06, 'epoch': 4.48, 'throughput': 1103.46}
+
+[INFO|callbacks.py:310] 2024-07-11 12:24:35,342 >> {'loss': 0.0118, 'learning_rate': 2.9083e-06, 'epoch': 4.49, 'throughput': 1103.57}
+
+[INFO|callbacks.py:310] 2024-07-11 12:24:48,908 >> {'loss': 0.0138, 'learning_rate': 2.9167e-06, 'epoch': 4.50, 'throughput': 1103.54}
+
+[INFO|callbacks.py:310] 2024-07-11 12:25:02,509 >> {'loss': 0.0344, 'learning_rate': 2.9250e-06, 'epoch': 4.51, 'throughput': 1103.54}
+
+[INFO|callbacks.py:310] 2024-07-11 12:25:16,091 >> {'loss': 0.0231, 'learning_rate': 2.9333e-06, 'epoch': 4.53, 'throughput': 1103.62}
+
+[INFO|callbacks.py:310] 2024-07-11 12:25:29,594 >> {'loss': 0.0267, 'learning_rate': 2.9417e-06, 'epoch': 4.54, 'throughput': 1103.62}
+
+[INFO|callbacks.py:310] 2024-07-11 12:25:43,090 >> {'loss': 0.0109, 'learning_rate': 2.9500e-06, 'epoch': 4.55, 'throughput': 1103.38}
+
+[INFO|callbacks.py:310] 2024-07-11 12:25:56,700 >> {'loss': 0.0444, 'learning_rate': 2.9583e-06, 'epoch': 4.57, 'throughput': 1103.26}
+
+[INFO|callbacks.py:310] 2024-07-11 12:26:10,257 >> {'loss': 0.0142, 'learning_rate': 2.9667e-06, 'epoch': 4.58, 'throughput': 1103.35}
+
+[INFO|callbacks.py:310] 2024-07-11 12:26:23,765 >> {'loss': 0.0361, 'learning_rate': 2.9750e-06, 'epoch': 4.59, 'throughput': 1103.37}
+
+[INFO|callbacks.py:310] 2024-07-11 12:26:37,356 >> {'loss': 0.0197, 'learning_rate': 2.9833e-06, 'epoch': 4.60, 'throughput': 1103.36}
+
+[INFO|callbacks.py:310] 2024-07-11 12:26:50,915 >> {'loss': 0.0373, 'learning_rate': 2.9917e-06, 'epoch': 4.62, 'throughput': 1103.44}
+
+[INFO|callbacks.py:310] 2024-07-11 12:27:04,473 >> {'loss': 0.0469, 'learning_rate': 3.0000e-06, 'epoch': 4.63, 'throughput': 1103.59}
+
+[INFO|callbacks.py:310] 2024-07-11 12:27:17,984 >> {'loss': 0.0283, 'learning_rate': 3.0083e-06, 'epoch': 4.64, 'throughput': 1103.61}
+
+[INFO|callbacks.py:310] 2024-07-11 12:27:31,534 >> {'loss': 0.0271, 'learning_rate': 3.0167e-06, 'epoch': 4.66, 'throughput': 1103.55}
+
+[INFO|callbacks.py:310] 2024-07-11 12:27:45,113 >> {'loss': 0.0303, 'learning_rate': 3.0250e-06, 'epoch': 4.67, 'throughput': 1103.54}
+
+[INFO|callbacks.py:310] 2024-07-11 12:27:58,741 >> {'loss': 0.0207, 'learning_rate': 3.0333e-06, 'epoch': 4.68, 'throughput': 1103.69}
+
+[INFO|callbacks.py:310] 2024-07-11 12:28:12,347 >> {'loss': 0.0157, 'learning_rate': 3.0417e-06, 'epoch': 4.69, 'throughput': 1103.80}
+
+[INFO|callbacks.py:310] 2024-07-11 12:28:25,988 >> {'loss': 0.0128, 'learning_rate': 3.0500e-06, 'epoch': 4.71, 'throughput': 1103.71}
+
+[INFO|callbacks.py:310] 2024-07-11 12:28:39,517 >> {'loss': 0.0293, 'learning_rate': 3.0583e-06, 'epoch': 4.72, 'throughput': 1103.92}
+
+[INFO|callbacks.py:310] 2024-07-11 12:28:53,124 >> {'loss': 0.0211, 'learning_rate': 3.0667e-06, 'epoch': 4.73, 'throughput': 1103.87}
+
+[INFO|callbacks.py:310] 2024-07-11 12:29:06,615 >> {'loss': 0.0929, 'learning_rate': 3.0750e-06, 'epoch': 4.75, 'throughput': 1103.90}
+
+[INFO|callbacks.py:310] 2024-07-11 12:29:20,204 >> {'loss': 0.0195, 'learning_rate': 3.0833e-06, 'epoch': 4.76, 'throughput': 1103.97}
+
+[INFO|callbacks.py:310] 2024-07-11 12:29:33,779 >> {'loss': 0.0114, 'learning_rate': 3.0917e-06, 'epoch': 4.77, 'throughput': 1104.05}
+
+[INFO|callbacks.py:310] 2024-07-11 12:29:47,300 >> {'loss': 0.0207, 'learning_rate': 3.1000e-06, 'epoch': 4.78, 'throughput': 1104.23}
+
+[INFO|callbacks.py:310] 2024-07-11 12:30:00,871 >> {'loss': 0.0144, 'learning_rate': 3.1083e-06, 'epoch': 4.80, 'throughput': 1104.22}
+
+[INFO|callbacks.py:310] 2024-07-11 12:30:14,485 >> {'loss': 0.0197, 'learning_rate': 3.1167e-06, 'epoch': 4.81, 'throughput': 1104.27}
+
+[INFO|callbacks.py:310] 2024-07-11 12:30:28,105 >> {'loss': 0.0669, 'learning_rate': 3.1250e-06, 'epoch': 4.82, 'throughput': 1104.49}
+
+[INFO|callbacks.py:310] 2024-07-11 12:30:41,666 >> {'loss': 0.0156, 'learning_rate': 3.1333e-06, 'epoch': 4.84, 'throughput': 1104.45}
+
+[INFO|callbacks.py:310] 2024-07-11 12:30:55,231 >> {'loss': 0.0180, 'learning_rate': 3.1417e-06, 'epoch': 4.85, 'throughput': 1104.57}
+
+[INFO|callbacks.py:310] 2024-07-11 12:31:08,788 >> {'loss': 0.0640, 'learning_rate': 3.1500e-06, 'epoch': 4.86, 'throughput': 1104.56}
+
+[INFO|callbacks.py:310] 2024-07-11 12:31:22,373 >> {'loss': 0.0263, 'learning_rate': 3.1583e-06, 'epoch': 4.87, 'throughput': 1104.51}
+
+[INFO|callbacks.py:310] 2024-07-11 12:31:35,946 >> {'loss': 0.0256, 'learning_rate': 3.1667e-06, 'epoch': 4.89, 'throughput': 1104.46}
+
+[INFO|callbacks.py:310] 2024-07-11 12:31:49,501 >> {'loss': 0.0269, 'learning_rate': 3.1750e-06, 'epoch': 4.90, 'throughput': 1104.40}
+
+[INFO|callbacks.py:310] 2024-07-11 12:32:03,102 >> {'loss': 0.0281, 'learning_rate': 3.1833e-06, 'epoch': 4.91, 'throughput': 1104.41}
+
+[INFO|callbacks.py:310] 2024-07-11 12:32:16,664 >> {'loss': 0.0199, 'learning_rate': 3.1917e-06, 'epoch': 4.93, 'throughput': 1104.39}
+
+[INFO|callbacks.py:310] 2024-07-11 12:32:30,171 >> {'loss': 0.0277, 'learning_rate': 3.2000e-06, 'epoch': 4.94, 'throughput': 1104.28}
+
+[INFO|callbacks.py:310] 2024-07-11 12:32:43,768 >> {'loss': 0.0158, 'learning_rate': 3.2083e-06, 'epoch': 4.95, 'throughput': 1104.47}
+
+[INFO|trainer.py:3478] 2024-07-11 12:32:51,369 >> Saving model checkpoint to saves/LLaMA3-8B-Chat/full/train_2024-07-11-10-49-45_inst_llama3/checkpoint-385
+
+[INFO|configuration_utils.py:472] 2024-07-11 12:32:51,372 >> Configuration saved in saves/LLaMA3-8B-Chat/full/train_2024-07-11-10-49-45_inst_llama3/checkpoint-385/config.json
+
+[INFO|configuration_utils.py:769] 2024-07-11 12:32:51,373 >> Configuration saved in saves/LLaMA3-8B-Chat/full/train_2024-07-11-10-49-45_inst_llama3/checkpoint-385/generation_config.json
+
+[INFO|modeling_utils.py:2698] 2024-07-11 12:33:07,622 >> The model is bigger than the maximum size per checkpoint (5GB) and is going to be split in 4 checkpoint shards. You can find where each parameters has been saved in the index located at saves/LLaMA3-8B-Chat/full/train_2024-07-11-10-49-45_inst_llama3/checkpoint-385/model.safetensors.index.json.
+
+[INFO|tokenization_utils_base.py:2574] 2024-07-11 12:33:07,625 >> tokenizer config file saved in saves/LLaMA3-8B-Chat/full/train_2024-07-11-10-49-45_inst_llama3/checkpoint-385/tokenizer_config.json
+
+[INFO|tokenization_utils_base.py:2583] 2024-07-11 12:33:07,626 >> Special tokens file saved in saves/LLaMA3-8B-Chat/full/train_2024-07-11-10-49-45_inst_llama3/checkpoint-385/special_tokens_map.json
+
+[INFO|trainer.py:2383] 2024-07-11 12:33:44,158 >> 
+
+Training completed. Do not forget to share your model on huggingface.co/models =)
+
+
+
+[INFO|trainer.py:3478] 2024-07-11 12:33:51,651 >> Saving model checkpoint to saves/LLaMA3-8B-Chat/full/train_2024-07-11-10-49-45_inst_llama3
+
+[INFO|configuration_utils.py:472] 2024-07-11 12:33:51,653 >> Configuration saved in saves/LLaMA3-8B-Chat/full/train_2024-07-11-10-49-45_inst_llama3/config.json
+
+[INFO|configuration_utils.py:769] 2024-07-11 12:33:51,654 >> Configuration saved in saves/LLaMA3-8B-Chat/full/train_2024-07-11-10-49-45_inst_llama3/generation_config.json
+
+[INFO|modeling_utils.py:2698] 2024-07-11 12:34:08,270 >> The model is bigger than the maximum size per checkpoint (5GB) and is going to be split in 4 checkpoint shards. You can find where each parameters has been saved in the index located at saves/LLaMA3-8B-Chat/full/train_2024-07-11-10-49-45_inst_llama3/model.safetensors.index.json.
+
+[INFO|tokenization_utils_base.py:2574] 2024-07-11 12:34:08,273 >> tokenizer config file saved in saves/LLaMA3-8B-Chat/full/train_2024-07-11-10-49-45_inst_llama3/tokenizer_config.json
+
+[INFO|tokenization_utils_base.py:2583] 2024-07-11 12:34:08,273 >> Special tokens file saved in saves/LLaMA3-8B-Chat/full/train_2024-07-11-10-49-45_inst_llama3/special_tokens_map.json
+
+[WARNING|ploting.py:89] 2024-07-11 12:34:09,588 >> No metric eval_loss to plot.
+
+[WARNING|ploting.py:89] 2024-07-11 12:34:09,588 >> No metric eval_accuracy to plot.
+
+[INFO|modelcard.py:449] 2024-07-11 12:34:09,589 >> Dropping the following result as it does not have all the necessary fields:
+{'task': {'name': 'Causal Language Modeling', 'type': 'text-generation'}}
+

special_tokens_map.json

@@ -0,0 +1,17 @@
+{
+  "bos_token": {
+    "content": "<|begin_of_text|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|eot_id|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "<|eot_id|>"
+}

tokenizer_config.json

@@ -0,0 +1,2065 @@
+{
+  "added_tokens_decoder": {
+    "128000": {
+      "content": "<|begin_of_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128001": {
+      "content": "<|end_of_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128002": {
+      "content": "<|reserved_special_token_0|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128003": {
+      "content": "<|reserved_special_token_1|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128004": {
+      "content": "<|reserved_special_token_2|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128005": {
+      "content": "<|reserved_special_token_3|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128006": {
+      "content": "<|start_header_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128007": {
+      "content": "<|end_header_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128008": {
+      "content": "<|reserved_special_token_4|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128009": {
+      "content": "<|eot_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128010": {
+      "content": "<|reserved_special_token_5|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128011": {
+      "content": "<|reserved_special_token_6|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128012": {
+      "content": "<|reserved_special_token_7|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128013": {
+      "content": "<|reserved_special_token_8|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128014": {
+      "content": "<|reserved_special_token_9|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128015": {
+      "content": "<|reserved_special_token_10|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128016": {
+      "content": "<|reserved_special_token_11|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128017": {
+      "content": "<|reserved_special_token_12|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128018": {
+      "content": "<|reserved_special_token_13|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128019": {
+      "content": "<|reserved_special_token_14|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128020": {
+      "content": "<|reserved_special_token_15|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128021": {
+      "content": "<|reserved_special_token_16|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128022": {
+      "content": "<|reserved_special_token_17|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128023": {
+      "content": "<|reserved_special_token_18|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128024": {
+      "content": "<|reserved_special_token_19|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128025": {
+      "content": "<|reserved_special_token_20|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128026": {
+      "content": "<|reserved_special_token_21|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128027": {
+      "content": "<|reserved_special_token_22|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128028": {
+      "content": "<|reserved_special_token_23|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128029": {
+      "content": "<|reserved_special_token_24|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128030": {
+      "content": "<|reserved_special_token_25|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128031": {
+      "content": "<|reserved_special_token_26|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128032": {
+      "content": "<|reserved_special_token_27|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128033": {
+      "content": "<|reserved_special_token_28|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128034": {
+      "content": "<|reserved_special_token_29|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128035": {
+      "content": "<|reserved_special_token_30|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128036": {
+      "content": "<|reserved_special_token_31|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128037": {
+      "content": "<|reserved_special_token_32|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128038": {
+      "content": "<|reserved_special_token_33|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128039": {
+      "content": "<|reserved_special_token_34|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128040": {
+      "content": "<|reserved_special_token_35|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128041": {
+      "content": "<|reserved_special_token_36|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128042": {
+      "content": "<|reserved_special_token_37|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128043": {
+      "content": "<|reserved_special_token_38|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128044": {
+      "content": "<|reserved_special_token_39|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128045": {
+      "content": "<|reserved_special_token_40|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128046": {
+      "content": "<|reserved_special_token_41|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128047": {
+      "content": "<|reserved_special_token_42|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128048": {
+      "content": "<|reserved_special_token_43|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128049": {
+      "content": "<|reserved_special_token_44|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128050": {
+      "content": "<|reserved_special_token_45|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128051": {
+      "content": "<|reserved_special_token_46|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128052": {
+      "content": "<|reserved_special_token_47|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128053": {
+      "content": "<|reserved_special_token_48|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128054": {
+      "content": "<|reserved_special_token_49|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128055": {
+      "content": "<|reserved_special_token_50|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128056": {
+      "content": "<|reserved_special_token_51|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128057": {
+      "content": "<|reserved_special_token_52|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128058": {
+      "content": "<|reserved_special_token_53|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128059": {
+      "content": "<|reserved_special_token_54|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128060": {
+      "content": "<|reserved_special_token_55|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128061": {
+      "content": "<|reserved_special_token_56|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128062": {
+      "content": "<|reserved_special_token_57|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128063": {
+      "content": "<|reserved_special_token_58|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128064": {
+      "content": "<|reserved_special_token_59|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128065": {
+      "content": "<|reserved_special_token_60|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128066": {
+      "content": "<|reserved_special_token_61|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128067": {
+      "content": "<|reserved_special_token_62|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128068": {
+      "content": "<|reserved_special_token_63|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128069": {
+      "content": "<|reserved_special_token_64|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128070": {
+      "content": "<|reserved_special_token_65|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128071": {
+      "content": "<|reserved_special_token_66|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128072": {
+      "content": "<|reserved_special_token_67|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128073": {
+      "content": "<|reserved_special_token_68|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128074": {
+      "content": "<|reserved_special_token_69|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128075": {
+      "content": "<|reserved_special_token_70|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128076": {
+      "content": "<|reserved_special_token_71|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128077": {
+      "content": "<|reserved_special_token_72|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128078": {
+      "content": "<|reserved_special_token_73|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128079": {
+      "content": "<|reserved_special_token_74|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128080": {
+      "content": "<|reserved_special_token_75|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128081": {
+      "content": "<|reserved_special_token_76|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128082": {
+      "content": "<|reserved_special_token_77|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128083": {
+      "content": "<|reserved_special_token_78|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128084": {
+      "content": "<|reserved_special_token_79|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128085": {
+      "content": "<|reserved_special_token_80|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128086": {
+      "content": "<|reserved_special_token_81|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128087": {
+      "content": "<|reserved_special_token_82|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128088": {
+      "content": "<|reserved_special_token_83|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128089": {
+      "content": "<|reserved_special_token_84|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128090": {
+      "content": "<|reserved_special_token_85|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128091": {
+      "content": "<|reserved_special_token_86|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128092": {
+      "content": "<|reserved_special_token_87|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128093": {
+      "content": "<|reserved_special_token_88|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128094": {
+      "content": "<|reserved_special_token_89|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128095": {
+      "content": "<|reserved_special_token_90|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128096": {
+      "content": "<|reserved_special_token_91|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128097": {
+      "content": "<|reserved_special_token_92|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128098": {
+      "content": "<|reserved_special_token_93|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128099": {
+      "content": "<|reserved_special_token_94|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128100": {
+      "content": "<|reserved_special_token_95|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128101": {
+      "content": "<|reserved_special_token_96|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128102": {
+      "content": "<|reserved_special_token_97|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128103": {
+      "content": "<|reserved_special_token_98|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128104": {
+      "content": "<|reserved_special_token_99|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128105": {
+      "content": "<|reserved_special_token_100|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128106": {
+      "content": "<|reserved_special_token_101|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128107": {
+      "content": "<|reserved_special_token_102|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128108": {
+      "content": "<|reserved_special_token_103|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128109": {
+      "content": "<|reserved_special_token_104|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128110": {
+      "content": "<|reserved_special_token_105|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128111": {
+      "content": "<|reserved_special_token_106|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128112": {
+      "content": "<|reserved_special_token_107|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128113": {
+      "content": "<|reserved_special_token_108|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128114": {
+      "content": "<|reserved_special_token_109|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128115": {
+      "content": "<|reserved_special_token_110|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128116": {
+      "content": "<|reserved_special_token_111|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128117": {
+      "content": "<|reserved_special_token_112|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128118": {
+      "content": "<|reserved_special_token_113|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128119": {
+      "content": "<|reserved_special_token_114|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128120": {
+      "content": "<|reserved_special_token_115|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128121": {
+      "content": "<|reserved_special_token_116|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128122": {
+      "content": "<|reserved_special_token_117|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128123": {
+      "content": "<|reserved_special_token_118|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128124": {
+      "content": "<|reserved_special_token_119|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128125": {
+      "content": "<|reserved_special_token_120|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128126": {
+      "content": "<|reserved_special_token_121|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128127": {
+      "content": "<|reserved_special_token_122|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128128": {
+      "content": "<|reserved_special_token_123|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128129": {
+      "content": "<|reserved_special_token_124|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128130": {
+      "content": "<|reserved_special_token_125|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128131": {
+      "content": "<|reserved_special_token_126|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128132": {
+      "content": "<|reserved_special_token_127|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128133": {
+      "content": "<|reserved_special_token_128|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128134": {
+      "content": "<|reserved_special_token_129|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128135": {
+      "content": "<|reserved_special_token_130|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128136": {
+      "content": "<|reserved_special_token_131|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128137": {
+      "content": "<|reserved_special_token_132|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128138": {
+      "content": "<|reserved_special_token_133|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128139": {
+      "content": "<|reserved_special_token_134|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128140": {
+      "content": "<|reserved_special_token_135|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128141": {
+      "content": "<|reserved_special_token_136|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128142": {
+      "content": "<|reserved_special_token_137|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128143": {
+      "content": "<|reserved_special_token_138|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128144": {
+      "content": "<|reserved_special_token_139|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128145": {
+      "content": "<|reserved_special_token_140|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128146": {
+      "content": "<|reserved_special_token_141|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128147": {
+      "content": "<|reserved_special_token_142|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128148": {
+      "content": "<|reserved_special_token_143|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128149": {
+      "content": "<|reserved_special_token_144|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128150": {
+      "content": "<|reserved_special_token_145|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128151": {
+      "content": "<|reserved_special_token_146|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128152": {
+      "content": "<|reserved_special_token_147|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128153": {
+      "content": "<|reserved_special_token_148|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128154": {
+      "content": "<|reserved_special_token_149|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128155": {
+      "content": "<|reserved_special_token_150|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128156": {
+      "content": "<|reserved_special_token_151|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128157": {
+      "content": "<|reserved_special_token_152|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128158": {
+      "content": "<|reserved_special_token_153|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128159": {
+      "content": "<|reserved_special_token_154|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128160": {
+      "content": "<|reserved_special_token_155|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128161": {
+      "content": "<|reserved_special_token_156|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128162": {
+      "content": "<|reserved_special_token_157|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128163": {
+      "content": "<|reserved_special_token_158|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128164": {
+      "content": "<|reserved_special_token_159|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128165": {
+      "content": "<|reserved_special_token_160|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128166": {
+      "content": "<|reserved_special_token_161|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128167": {
+      "content": "<|reserved_special_token_162|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128168": {
+      "content": "<|reserved_special_token_163|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128169": {
+      "content": "<|reserved_special_token_164|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128170": {
+      "content": "<|reserved_special_token_165|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128171": {
+      "content": "<|reserved_special_token_166|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128172": {
+      "content": "<|reserved_special_token_167|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128173": {
+      "content": "<|reserved_special_token_168|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128174": {
+      "content": "<|reserved_special_token_169|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128175": {
+      "content": "<|reserved_special_token_170|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128176": {
+      "content": "<|reserved_special_token_171|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128177": {
+      "content": "<|reserved_special_token_172|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128178": {
+      "content": "<|reserved_special_token_173|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128179": {
+      "content": "<|reserved_special_token_174|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128180": {
+      "content": "<|reserved_special_token_175|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128181": {
+      "content": "<|reserved_special_token_176|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128182": {
+      "content": "<|reserved_special_token_177|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128183": {
+      "content": "<|reserved_special_token_178|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128184": {
+      "content": "<|reserved_special_token_179|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128185": {
+      "content": "<|reserved_special_token_180|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128186": {
+      "content": "<|reserved_special_token_181|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128187": {
+      "content": "<|reserved_special_token_182|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128188": {
+      "content": "<|reserved_special_token_183|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128189": {
+      "content": "<|reserved_special_token_184|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128190": {
+      "content": "<|reserved_special_token_185|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128191": {
+      "content": "<|reserved_special_token_186|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128192": {
+      "content": "<|reserved_special_token_187|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128193": {
+      "content": "<|reserved_special_token_188|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128194": {
+      "content": "<|reserved_special_token_189|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128195": {
+      "content": "<|reserved_special_token_190|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128196": {
+      "content": "<|reserved_special_token_191|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128197": {
+      "content": "<|reserved_special_token_192|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128198": {
+      "content": "<|reserved_special_token_193|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128199": {
+      "content": "<|reserved_special_token_194|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128200": {
+      "content": "<|reserved_special_token_195|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128201": {
+      "content": "<|reserved_special_token_196|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128202": {
+      "content": "<|reserved_special_token_197|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128203": {
+      "content": "<|reserved_special_token_198|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128204": {
+      "content": "<|reserved_special_token_199|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128205": {
+      "content": "<|reserved_special_token_200|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128206": {
+      "content": "<|reserved_special_token_201|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128207": {
+      "content": "<|reserved_special_token_202|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128208": {
+      "content": "<|reserved_special_token_203|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128209": {
+      "content": "<|reserved_special_token_204|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128210": {
+      "content": "<|reserved_special_token_205|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128211": {
+      "content": "<|reserved_special_token_206|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128212": {
+      "content": "<|reserved_special_token_207|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128213": {
+      "content": "<|reserved_special_token_208|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128214": {
+      "content": "<|reserved_special_token_209|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128215": {
+      "content": "<|reserved_special_token_210|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128216": {
+      "content": "<|reserved_special_token_211|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128217": {
+      "content": "<|reserved_special_token_212|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128218": {
+      "content": "<|reserved_special_token_213|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128219": {
+      "content": "<|reserved_special_token_214|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128220": {
+      "content": "<|reserved_special_token_215|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128221": {
+      "content": "<|reserved_special_token_216|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128222": {
+      "content": "<|reserved_special_token_217|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128223": {
+      "content": "<|reserved_special_token_218|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128224": {
+      "content": "<|reserved_special_token_219|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128225": {
+      "content": "<|reserved_special_token_220|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128226": {
+      "content": "<|reserved_special_token_221|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128227": {
+      "content": "<|reserved_special_token_222|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128228": {
+      "content": "<|reserved_special_token_223|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128229": {
+      "content": "<|reserved_special_token_224|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128230": {
+      "content": "<|reserved_special_token_225|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128231": {
+      "content": "<|reserved_special_token_226|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128232": {
+      "content": "<|reserved_special_token_227|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128233": {
+      "content": "<|reserved_special_token_228|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128234": {
+      "content": "<|reserved_special_token_229|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128235": {
+      "content": "<|reserved_special_token_230|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128236": {
+      "content": "<|reserved_special_token_231|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128237": {
+      "content": "<|reserved_special_token_232|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128238": {
+      "content": "<|reserved_special_token_233|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128239": {
+      "content": "<|reserved_special_token_234|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128240": {
+      "content": "<|reserved_special_token_235|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128241": {
+      "content": "<|reserved_special_token_236|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128242": {
+      "content": "<|reserved_special_token_237|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128243": {
+      "content": "<|reserved_special_token_238|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128244": {
+      "content": "<|reserved_special_token_239|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128245": {
+      "content": "<|reserved_special_token_240|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128246": {
+      "content": "<|reserved_special_token_241|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128247": {
+      "content": "<|reserved_special_token_242|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128248": {
+      "content": "<|reserved_special_token_243|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128249": {
+      "content": "<|reserved_special_token_244|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128250": {
+      "content": "<|reserved_special_token_245|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128251": {
+      "content": "<|reserved_special_token_246|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128252": {
+      "content": "<|reserved_special_token_247|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128253": {
+      "content": "<|reserved_special_token_248|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128254": {
+      "content": "<|reserved_special_token_249|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128255": {
+      "content": "<|reserved_special_token_250|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|begin_of_text|>",
+  "chat_template": "{{ '<|begin_of_text|>' }}{% if messages[0]['role'] == 'system' %}{% set system_message = messages[0]['content'] %}{% endif %}{% if system_message is defined %}{{ '<|start_header_id|>system<|end_header_id|>\n\n' + system_message + '<|eot_id|>' }}{% endif %}{% for message in messages %}{% set content = message['content'] %}{% if message['role'] == 'user' %}{{ '<|start_header_id|>user<|end_header_id|>\n\n' + content + '<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n' }}{% elif message['role'] == 'assistant' %}{{ content + '<|eot_id|>' }}{% endif %}{% endfor %}",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|eot_id|>",
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<|eot_id|>",
+  "padding_side": "right",
+  "split_special_tokens": false,
+  "tokenizer_class": "PreTrainedTokenizerFast"
+}

train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 4.951768488745981,
+    "num_input_tokens_seen": 5776160,
+    "total_flos": 2.6009787341943603e+17,
+    "train_loss": 1.2260490821160424,
+    "train_runtime": 5290.2175,
+    "train_samples_per_second": 18.789,
+    "train_steps_per_second": 0.073
+}

trainer_log.jsonl

@@ -0,0 +1,386 @@
+{"current_steps": 1, "total_steps": 385, "loss": 13.9619, "learning_rate": 8.333333333333335e-09, "epoch": 0.012861736334405145, "percentage": 0.26, "elapsed_time": "0:00:18", "remaining_time": "1:56:40", "throughput": "830.24", "total_tokens": 15136}
+{"current_steps": 2, "total_steps": 385, "loss": 13.7129, "learning_rate": 1.666666666666667e-08, "epoch": 0.02572347266881029, "percentage": 0.52, "elapsed_time": "0:00:31", "remaining_time": "1:41:33", "throughput": "949.38", "total_tokens": 30208}
+{"current_steps": 3, "total_steps": 385, "loss": 13.8474, "learning_rate": 2.5000000000000002e-08, "epoch": 0.03858520900321544, "percentage": 0.78, "elapsed_time": "0:00:45", "remaining_time": "1:36:16", "throughput": "1000.31", "total_tokens": 45376}
+{"current_steps": 4, "total_steps": 385, "loss": 13.8844, "learning_rate": 3.333333333333334e-08, "epoch": 0.05144694533762058, "percentage": 1.04, "elapsed_time": "0:00:58", "remaining_time": "1:33:32", "throughput": "1017.74", "total_tokens": 59968}
+{"current_steps": 5, "total_steps": 385, "loss": 14.138, "learning_rate": 4.166666666666667e-08, "epoch": 0.06430868167202572, "percentage": 1.3, "elapsed_time": "0:01:12", "remaining_time": "1:31:45", "throughput": "1037.30", "total_tokens": 75136}
+{"current_steps": 6, "total_steps": 385, "loss": 13.9077, "learning_rate": 5.0000000000000004e-08, "epoch": 0.07717041800643087, "percentage": 1.56, "elapsed_time": "0:01:26", "remaining_time": "1:30:33", "throughput": "1055.71", "total_tokens": 90816}
+{"current_steps": 7, "total_steps": 385, "loss": 13.8396, "learning_rate": 5.833333333333334e-08, "epoch": 0.09003215434083602, "percentage": 1.82, "elapsed_time": "0:01:39", "remaining_time": "1:29:36", "throughput": "1060.56", "total_tokens": 105600}
+{"current_steps": 8, "total_steps": 385, "loss": 13.9828, "learning_rate": 6.666666666666668e-08, "epoch": 0.10289389067524116, "percentage": 2.08, "elapsed_time": "0:01:53", "remaining_time": "1:28:51", "throughput": "1059.47", "total_tokens": 119872}
+{"current_steps": 9, "total_steps": 385, "loss": 14.0361, "learning_rate": 7.500000000000001e-08, "epoch": 0.1157556270096463, "percentage": 2.34, "elapsed_time": "0:02:06", "remaining_time": "1:28:15", "throughput": "1065.89", "total_tokens": 135104}
+{"current_steps": 10, "total_steps": 385, "loss": 13.9392, "learning_rate": 8.333333333333334e-08, "epoch": 0.12861736334405144, "percentage": 2.6, "elapsed_time": "0:02:20", "remaining_time": "1:27:41", "throughput": "1069.27", "total_tokens": 150016}
+{"current_steps": 11, "total_steps": 385, "loss": 14.0256, "learning_rate": 9.166666666666668e-08, "epoch": 0.1414790996784566, "percentage": 2.86, "elapsed_time": "0:02:33", "remaining_time": "1:27:14", "throughput": "1073.24", "total_tokens": 165216}
+{"current_steps": 12, "total_steps": 385, "loss": 13.6693, "learning_rate": 1.0000000000000001e-07, "epoch": 0.15434083601286175, "percentage": 3.12, "elapsed_time": "0:02:47", "remaining_time": "1:26:45", "throughput": "1070.54", "total_tokens": 179296}
+{"current_steps": 13, "total_steps": 385, "loss": 13.9031, "learning_rate": 1.0833333333333335e-07, "epoch": 0.16720257234726688, "percentage": 3.38, "elapsed_time": "0:03:01", "remaining_time": "1:26:20", "throughput": "1067.89", "total_tokens": 193312}
+{"current_steps": 14, "total_steps": 385, "loss": 13.8575, "learning_rate": 1.1666666666666668e-07, "epoch": 0.18006430868167203, "percentage": 3.64, "elapsed_time": "0:03:14", "remaining_time": "1:25:55", "throughput": "1073.07", "total_tokens": 208768}
+{"current_steps": 15, "total_steps": 385, "loss": 13.8366, "learning_rate": 1.2500000000000002e-07, "epoch": 0.19292604501607716, "percentage": 3.9, "elapsed_time": "0:03:28", "remaining_time": "1:25:34", "throughput": "1073.42", "total_tokens": 223456}
+{"current_steps": 16, "total_steps": 385, "loss": 13.8705, "learning_rate": 1.3333333333333336e-07, "epoch": 0.2057877813504823, "percentage": 4.16, "elapsed_time": "0:03:41", "remaining_time": "1:25:14", "throughput": "1076.20", "total_tokens": 238656}
+{"current_steps": 17, "total_steps": 385, "loss": 13.1816, "learning_rate": 1.4166666666666668e-07, "epoch": 0.21864951768488747, "percentage": 4.42, "elapsed_time": "0:03:55", "remaining_time": "1:24:54", "throughput": "1076.26", "total_tokens": 253312}
+{"current_steps": 18, "total_steps": 385, "loss": 13.2292, "learning_rate": 1.5000000000000002e-07, "epoch": 0.2315112540192926, "percentage": 4.68, "elapsed_time": "0:04:08", "remaining_time": "1:24:35", "throughput": "1078.02", "total_tokens": 268352}
+{"current_steps": 19, "total_steps": 385, "loss": 13.4366, "learning_rate": 1.5833333333333336e-07, "epoch": 0.24437299035369775, "percentage": 4.94, "elapsed_time": "0:04:22", "remaining_time": "1:24:15", "throughput": "1076.68", "total_tokens": 282560}
+{"current_steps": 20, "total_steps": 385, "loss": 12.9904, "learning_rate": 1.6666666666666668e-07, "epoch": 0.2572347266881029, "percentage": 5.19, "elapsed_time": "0:04:35", "remaining_time": "1:23:56", "throughput": "1076.72", "total_tokens": 297120}
+{"current_steps": 21, "total_steps": 385, "loss": 12.878, "learning_rate": 1.7500000000000002e-07, "epoch": 0.27009646302250806, "percentage": 5.45, "elapsed_time": "0:04:49", "remaining_time": "1:23:38", "throughput": "1078.97", "total_tokens": 312416}
+{"current_steps": 22, "total_steps": 385, "loss": 12.7794, "learning_rate": 1.8333333333333336e-07, "epoch": 0.2829581993569132, "percentage": 5.71, "elapsed_time": "0:05:03", "remaining_time": "1:23:22", "throughput": "1081.79", "total_tokens": 327968}
+{"current_steps": 23, "total_steps": 385, "loss": 11.3144, "learning_rate": 1.9166666666666668e-07, "epoch": 0.2958199356913183, "percentage": 5.97, "elapsed_time": "0:05:16", "remaining_time": "1:23:05", "throughput": "1082.54", "total_tokens": 342880}
+{"current_steps": 24, "total_steps": 385, "loss": 10.8531, "learning_rate": 2.0000000000000002e-07, "epoch": 0.3086816720257235, "percentage": 6.23, "elapsed_time": "0:05:30", "remaining_time": "1:22:48", "throughput": "1086.37", "total_tokens": 358848}
+{"current_steps": 25, "total_steps": 385, "loss": 10.7149, "learning_rate": 2.0833333333333333e-07, "epoch": 0.3215434083601286, "percentage": 6.49, "elapsed_time": "0:05:43", "remaining_time": "1:22:31", "throughput": "1087.02", "total_tokens": 373792}
+{"current_steps": 26, "total_steps": 385, "loss": 10.5802, "learning_rate": 2.166666666666667e-07, "epoch": 0.33440514469453375, "percentage": 6.75, "elapsed_time": "0:05:57", "remaining_time": "1:22:15", "throughput": "1089.07", "total_tokens": 389248}
+{"current_steps": 27, "total_steps": 385, "loss": 10.3671, "learning_rate": 2.2500000000000002e-07, "epoch": 0.34726688102893893, "percentage": 7.01, "elapsed_time": "0:06:10", "remaining_time": "1:21:58", "throughput": "1090.07", "total_tokens": 404320}
+{"current_steps": 28, "total_steps": 385, "loss": 10.1751, "learning_rate": 2.3333333333333336e-07, "epoch": 0.36012861736334406, "percentage": 7.27, "elapsed_time": "0:06:24", "remaining_time": "1:21:42", "throughput": "1090.50", "total_tokens": 419296}
+{"current_steps": 29, "total_steps": 385, "loss": 9.7707, "learning_rate": 2.416666666666667e-07, "epoch": 0.3729903536977492, "percentage": 7.53, "elapsed_time": "0:06:38", "remaining_time": "1:21:26", "throughput": "1089.46", "total_tokens": 433632}
+{"current_steps": 30, "total_steps": 385, "loss": 9.6489, "learning_rate": 2.5000000000000004e-07, "epoch": 0.3858520900321543, "percentage": 7.79, "elapsed_time": "0:06:51", "remaining_time": "1:21:11", "throughput": "1090.10", "total_tokens": 448736}
+{"current_steps": 31, "total_steps": 385, "loss": 8.6805, "learning_rate": 2.5833333333333333e-07, "epoch": 0.3987138263665595, "percentage": 8.05, "elapsed_time": "0:07:05", "remaining_time": "1:20:55", "throughput": "1090.83", "total_tokens": 463840}
+{"current_steps": 32, "total_steps": 385, "loss": 5.9207, "learning_rate": 2.666666666666667e-07, "epoch": 0.4115755627009646, "percentage": 8.31, "elapsed_time": "0:07:18", "remaining_time": "1:20:40", "throughput": "1091.16", "total_tokens": 478816}
+{"current_steps": 33, "total_steps": 385, "loss": 5.7661, "learning_rate": 2.75e-07, "epoch": 0.42443729903536975, "percentage": 8.57, "elapsed_time": "0:07:32", "remaining_time": "1:20:25", "throughput": "1093.01", "total_tokens": 494464}
+{"current_steps": 34, "total_steps": 385, "loss": 5.6168, "learning_rate": 2.8333333333333336e-07, "epoch": 0.43729903536977494, "percentage": 8.83, "elapsed_time": "0:07:45", "remaining_time": "1:20:10", "throughput": "1094.61", "total_tokens": 510080}
+{"current_steps": 35, "total_steps": 385, "loss": 5.3367, "learning_rate": 2.916666666666667e-07, "epoch": 0.45016077170418006, "percentage": 9.09, "elapsed_time": "0:07:59", "remaining_time": "1:19:55", "throughput": "1095.23", "total_tokens": 525216}
+{"current_steps": 36, "total_steps": 385, "loss": 4.9751, "learning_rate": 3.0000000000000004e-07, "epoch": 0.4630225080385852, "percentage": 9.35, "elapsed_time": "0:08:13", "remaining_time": "1:19:40", "throughput": "1094.37", "total_tokens": 539648}
+{"current_steps": 37, "total_steps": 385, "loss": 4.7041, "learning_rate": 3.083333333333334e-07, "epoch": 0.4758842443729904, "percentage": 9.61, "elapsed_time": "0:08:26", "remaining_time": "1:19:25", "throughput": "1093.52", "total_tokens": 554080}
+{"current_steps": 38, "total_steps": 385, "loss": 4.4631, "learning_rate": 3.166666666666667e-07, "epoch": 0.4887459807073955, "percentage": 9.87, "elapsed_time": "0:08:40", "remaining_time": "1:19:11", "throughput": "1093.46", "total_tokens": 568960}
+{"current_steps": 39, "total_steps": 385, "loss": 4.1912, "learning_rate": 3.25e-07, "epoch": 0.5016077170418006, "percentage": 10.13, "elapsed_time": "0:08:53", "remaining_time": "1:18:56", "throughput": "1093.70", "total_tokens": 583904}
+{"current_steps": 40, "total_steps": 385, "loss": 3.9146, "learning_rate": 3.3333333333333335e-07, "epoch": 0.5144694533762058, "percentage": 10.39, "elapsed_time": "0:09:07", "remaining_time": "1:18:41", "throughput": "1093.91", "total_tokens": 598848}
+{"current_steps": 41, "total_steps": 385, "loss": 3.053, "learning_rate": 3.416666666666667e-07, "epoch": 0.5273311897106109, "percentage": 10.65, "elapsed_time": "0:09:21", "remaining_time": "1:18:27", "throughput": "1094.67", "total_tokens": 614208}
+{"current_steps": 42, "total_steps": 385, "loss": 1.5544, "learning_rate": 3.5000000000000004e-07, "epoch": 0.5401929260450161, "percentage": 10.91, "elapsed_time": "0:09:34", "remaining_time": "1:18:13", "throughput": "1095.05", "total_tokens": 629280}
+{"current_steps": 43, "total_steps": 385, "loss": 1.0549, "learning_rate": 3.583333333333334e-07, "epoch": 0.5530546623794212, "percentage": 11.17, "elapsed_time": "0:09:48", "remaining_time": "1:17:58", "throughput": "1094.70", "total_tokens": 643968}
+{"current_steps": 44, "total_steps": 385, "loss": 0.711, "learning_rate": 3.666666666666667e-07, "epoch": 0.5659163987138264, "percentage": 11.43, "elapsed_time": "0:10:01", "remaining_time": "1:17:44", "throughput": "1095.09", "total_tokens": 659040}
+{"current_steps": 45, "total_steps": 385, "loss": 0.5127, "learning_rate": 3.75e-07, "epoch": 0.5787781350482315, "percentage": 11.69, "elapsed_time": "0:10:15", "remaining_time": "1:17:29", "throughput": "1095.93", "total_tokens": 674464}
+{"current_steps": 46, "total_steps": 385, "loss": 0.4143, "learning_rate": 3.8333333333333335e-07, "epoch": 0.5916398713826366, "percentage": 11.95, "elapsed_time": "0:10:29", "remaining_time": "1:17:15", "throughput": "1095.72", "total_tokens": 689216}
+{"current_steps": 47, "total_steps": 385, "loss": 0.4014, "learning_rate": 3.9166666666666675e-07, "epoch": 0.6045016077170418, "percentage": 12.21, "elapsed_time": "0:10:42", "remaining_time": "1:17:01", "throughput": "1096.29", "total_tokens": 704448}
+{"current_steps": 48, "total_steps": 385, "loss": 0.384, "learning_rate": 4.0000000000000003e-07, "epoch": 0.617363344051447, "percentage": 12.47, "elapsed_time": "0:10:56", "remaining_time": "1:16:46", "throughput": "1096.57", "total_tokens": 719520}
+{"current_steps": 49, "total_steps": 385, "loss": 0.3017, "learning_rate": 4.083333333333334e-07, "epoch": 0.6302250803858521, "percentage": 12.73, "elapsed_time": "0:11:09", "remaining_time": "1:16:32", "throughput": "1098.22", "total_tokens": 735520}
+{"current_steps": 50, "total_steps": 385, "loss": 0.3031, "learning_rate": 4.1666666666666667e-07, "epoch": 0.6430868167202572, "percentage": 12.99, "elapsed_time": "0:11:23", "remaining_time": "1:16:18", "throughput": "1099.04", "total_tokens": 750976}
+{"current_steps": 51, "total_steps": 385, "loss": 0.2818, "learning_rate": 4.2500000000000006e-07, "epoch": 0.6559485530546624, "percentage": 13.25, "elapsed_time": "0:11:36", "remaining_time": "1:16:03", "throughput": "1098.90", "total_tokens": 765728}
+{"current_steps": 52, "total_steps": 385, "loss": 0.2881, "learning_rate": 4.333333333333334e-07, "epoch": 0.6688102893890675, "percentage": 13.51, "elapsed_time": "0:11:50", "remaining_time": "1:15:49", "throughput": "1098.82", "total_tokens": 780608}
+{"current_steps": 53, "total_steps": 385, "loss": 0.2943, "learning_rate": 4.416666666666667e-07, "epoch": 0.6816720257234726, "percentage": 13.77, "elapsed_time": "0:12:03", "remaining_time": "1:15:35", "throughput": "1099.73", "total_tokens": 796192}
+{"current_steps": 54, "total_steps": 385, "loss": 0.2781, "learning_rate": 4.5000000000000003e-07, "epoch": 0.6945337620578779, "percentage": 14.03, "elapsed_time": "0:12:17", "remaining_time": "1:15:20", "throughput": "1098.98", "total_tokens": 810496}
+{"current_steps": 55, "total_steps": 385, "loss": 0.2724, "learning_rate": 4.583333333333333e-07, "epoch": 0.707395498392283, "percentage": 14.29, "elapsed_time": "0:12:31", "remaining_time": "1:15:06", "throughput": "1099.88", "total_tokens": 826048}
+{"current_steps": 56, "total_steps": 385, "loss": 0.252, "learning_rate": 4.666666666666667e-07, "epoch": 0.7202572347266881, "percentage": 14.55, "elapsed_time": "0:12:44", "remaining_time": "1:14:51", "throughput": "1099.29", "total_tokens": 840512}
+{"current_steps": 57, "total_steps": 385, "loss": 0.2482, "learning_rate": 4.7500000000000006e-07, "epoch": 0.7331189710610932, "percentage": 14.81, "elapsed_time": "0:12:58", "remaining_time": "1:14:37", "throughput": "1099.87", "total_tokens": 855808}
+{"current_steps": 58, "total_steps": 385, "loss": 0.2212, "learning_rate": 4.833333333333334e-07, "epoch": 0.7459807073954984, "percentage": 15.06, "elapsed_time": "0:13:11", "remaining_time": "1:14:23", "throughput": "1100.79", "total_tokens": 871520}
+{"current_steps": 59, "total_steps": 385, "loss": 0.2408, "learning_rate": 4.916666666666667e-07, "epoch": 0.7588424437299035, "percentage": 15.32, "elapsed_time": "0:13:25", "remaining_time": "1:14:09", "throughput": "1100.40", "total_tokens": 886144}
+{"current_steps": 60, "total_steps": 385, "loss": 0.2133, "learning_rate": 5.000000000000001e-07, "epoch": 0.7717041800643086, "percentage": 15.58, "elapsed_time": "0:13:38", "remaining_time": "1:13:55", "throughput": "1100.76", "total_tokens": 901408}
+{"current_steps": 61, "total_steps": 385, "loss": 0.2319, "learning_rate": 5.083333333333334e-07, "epoch": 0.7845659163987139, "percentage": 15.84, "elapsed_time": "0:13:52", "remaining_time": "1:13:41", "throughput": "1101.14", "total_tokens": 916672}
+{"current_steps": 62, "total_steps": 385, "loss": 0.2703, "learning_rate": 5.166666666666667e-07, "epoch": 0.797427652733119, "percentage": 16.1, "elapsed_time": "0:14:06", "remaining_time": "1:13:27", "throughput": "1100.64", "total_tokens": 931168}
+{"current_steps": 63, "total_steps": 385, "loss": 0.2647, "learning_rate": 5.250000000000001e-07, "epoch": 0.8102893890675241, "percentage": 16.36, "elapsed_time": "0:14:19", "remaining_time": "1:13:13", "throughput": "1101.50", "total_tokens": 946816}
+{"current_steps": 64, "total_steps": 385, "loss": 0.2288, "learning_rate": 5.333333333333335e-07, "epoch": 0.8231511254019293, "percentage": 16.62, "elapsed_time": "0:14:33", "remaining_time": "1:12:59", "throughput": "1100.95", "total_tokens": 961248}
+{"current_steps": 65, "total_steps": 385, "loss": 0.2426, "learning_rate": 5.416666666666667e-07, "epoch": 0.8360128617363344, "percentage": 16.88, "elapsed_time": "0:14:46", "remaining_time": "1:12:45", "throughput": "1101.61", "total_tokens": 976736}
+{"current_steps": 66, "total_steps": 385, "loss": 0.1936, "learning_rate": 5.5e-07, "epoch": 0.8488745980707395, "percentage": 17.14, "elapsed_time": "0:15:00", "remaining_time": "1:12:31", "throughput": "1102.11", "total_tokens": 992192}
+{"current_steps": 67, "total_steps": 385, "loss": 0.192, "learning_rate": 5.583333333333333e-07, "epoch": 0.8617363344051447, "percentage": 17.4, "elapsed_time": "0:15:13", "remaining_time": "1:12:17", "throughput": "1102.83", "total_tokens": 1007808}
+{"current_steps": 68, "total_steps": 385, "loss": 0.1956, "learning_rate": 5.666666666666667e-07, "epoch": 0.8745980707395499, "percentage": 17.66, "elapsed_time": "0:15:27", "remaining_time": "1:12:03", "throughput": "1102.10", "total_tokens": 1022176}
+{"current_steps": 69, "total_steps": 385, "loss": 0.1862, "learning_rate": 5.750000000000001e-07, "epoch": 0.887459807073955, "percentage": 17.92, "elapsed_time": "0:15:41", "remaining_time": "1:11:50", "throughput": "1102.18", "total_tokens": 1037280}
+{"current_steps": 70, "total_steps": 385, "loss": 0.195, "learning_rate": 5.833333333333334e-07, "epoch": 0.9003215434083601, "percentage": 18.18, "elapsed_time": "0:15:54", "remaining_time": "1:11:35", "throughput": "1101.34", "total_tokens": 1051392}
+{"current_steps": 71, "total_steps": 385, "loss": 0.1971, "learning_rate": 5.916666666666667e-07, "epoch": 0.9131832797427653, "percentage": 18.44, "elapsed_time": "0:16:08", "remaining_time": "1:11:21", "throughput": "1101.47", "total_tokens": 1066432}
+{"current_steps": 72, "total_steps": 385, "loss": 0.1683, "learning_rate": 6.000000000000001e-07, "epoch": 0.9260450160771704, "percentage": 18.7, "elapsed_time": "0:16:21", "remaining_time": "1:11:07", "throughput": "1101.79", "total_tokens": 1081696}
+{"current_steps": 73, "total_steps": 385, "loss": 0.2003, "learning_rate": 6.083333333333334e-07, "epoch": 0.9389067524115756, "percentage": 18.96, "elapsed_time": "0:16:35", "remaining_time": "1:10:54", "throughput": "1101.80", "total_tokens": 1096672}
+{"current_steps": 74, "total_steps": 385, "loss": 0.1543, "learning_rate": 6.166666666666668e-07, "epoch": 0.9517684887459807, "percentage": 19.22, "elapsed_time": "0:16:48", "remaining_time": "1:10:40", "throughput": "1101.69", "total_tokens": 1111488}
+{"current_steps": 75, "total_steps": 385, "loss": 0.1982, "learning_rate": 6.25e-07, "epoch": 0.9646302250803859, "percentage": 19.48, "elapsed_time": "0:17:02", "remaining_time": "1:10:26", "throughput": "1101.60", "total_tokens": 1126368}
+{"current_steps": 76, "total_steps": 385, "loss": 0.1545, "learning_rate": 6.333333333333334e-07, "epoch": 0.977491961414791, "percentage": 19.74, "elapsed_time": "0:17:16", "remaining_time": "1:10:12", "throughput": "1101.78", "total_tokens": 1141536}
+{"current_steps": 77, "total_steps": 385, "loss": 0.1573, "learning_rate": 6.416666666666667e-07, "epoch": 0.9903536977491961, "percentage": 20.0, "elapsed_time": "0:17:29", "remaining_time": "1:09:58", "throughput": "1101.66", "total_tokens": 1156320}
+{"current_steps": 78, "total_steps": 385, "loss": 0.1788, "learning_rate": 6.5e-07, "epoch": 1.0032154340836013, "percentage": 20.26, "elapsed_time": "0:17:43", "remaining_time": "1:09:44", "throughput": "1102.22", "total_tokens": 1171904}
+{"current_steps": 79, "total_steps": 385, "loss": 0.187, "learning_rate": 6.583333333333333e-07, "epoch": 1.0160771704180065, "percentage": 20.52, "elapsed_time": "0:17:56", "remaining_time": "1:09:31", "throughput": "1102.50", "total_tokens": 1187232}
+{"current_steps": 80, "total_steps": 385, "loss": 0.2009, "learning_rate": 6.666666666666667e-07, "epoch": 1.0289389067524115, "percentage": 20.78, "elapsed_time": "0:18:10", "remaining_time": "1:09:17", "throughput": "1101.97", "total_tokens": 1201632}
+{"current_steps": 81, "total_steps": 385, "loss": 0.2655, "learning_rate": 6.750000000000001e-07, "epoch": 1.0418006430868167, "percentage": 21.04, "elapsed_time": "0:18:24", "remaining_time": "1:09:03", "throughput": "1102.17", "total_tokens": 1216832}
+{"current_steps": 82, "total_steps": 385, "loss": 0.2041, "learning_rate": 6.833333333333334e-07, "epoch": 1.0546623794212218, "percentage": 21.3, "elapsed_time": "0:18:37", "remaining_time": "1:08:49", "throughput": "1102.46", "total_tokens": 1232128}
+{"current_steps": 83, "total_steps": 385, "loss": 0.1675, "learning_rate": 6.916666666666668e-07, "epoch": 1.067524115755627, "percentage": 21.56, "elapsed_time": "0:18:51", "remaining_time": "1:08:35", "throughput": "1102.33", "total_tokens": 1246944}
+{"current_steps": 84, "total_steps": 385, "loss": 0.2097, "learning_rate": 7.000000000000001e-07, "epoch": 1.0803858520900322, "percentage": 21.82, "elapsed_time": "0:19:04", "remaining_time": "1:08:22", "throughput": "1102.93", "total_tokens": 1262624}
+{"current_steps": 85, "total_steps": 385, "loss": 0.2171, "learning_rate": 7.083333333333334e-07, "epoch": 1.0932475884244373, "percentage": 22.08, "elapsed_time": "0:19:18", "remaining_time": "1:08:08", "throughput": "1103.25", "total_tokens": 1278080}
+{"current_steps": 86, "total_steps": 385, "loss": 0.1702, "learning_rate": 7.166666666666668e-07, "epoch": 1.1061093247588425, "percentage": 22.34, "elapsed_time": "0:19:32", "remaining_time": "1:07:54", "throughput": "1103.45", "total_tokens": 1293280}
+{"current_steps": 87, "total_steps": 385, "loss": 0.1255, "learning_rate": 7.25e-07, "epoch": 1.1189710610932475, "percentage": 22.6, "elapsed_time": "0:19:45", "remaining_time": "1:07:40", "throughput": "1102.85", "total_tokens": 1307456}
+{"current_steps": 88, "total_steps": 385, "loss": 0.1826, "learning_rate": 7.333333333333334e-07, "epoch": 1.1318327974276527, "percentage": 22.86, "elapsed_time": "0:19:59", "remaining_time": "1:07:26", "throughput": "1103.48", "total_tokens": 1323168}
+{"current_steps": 89, "total_steps": 385, "loss": 0.2039, "learning_rate": 7.416666666666668e-07, "epoch": 1.144694533762058, "percentage": 23.12, "elapsed_time": "0:20:12", "remaining_time": "1:07:13", "throughput": "1103.34", "total_tokens": 1337984}
+{"current_steps": 90, "total_steps": 385, "loss": 0.2337, "learning_rate": 7.5e-07, "epoch": 1.157556270096463, "percentage": 23.38, "elapsed_time": "0:20:26", "remaining_time": "1:06:59", "throughput": "1103.79", "total_tokens": 1353568}
+{"current_steps": 91, "total_steps": 385, "loss": 0.1466, "learning_rate": 7.583333333333334e-07, "epoch": 1.1704180064308682, "percentage": 23.64, "elapsed_time": "0:20:39", "remaining_time": "1:06:45", "throughput": "1103.97", "total_tokens": 1368800}
+{"current_steps": 92, "total_steps": 385, "loss": 0.1295, "learning_rate": 7.666666666666667e-07, "epoch": 1.1832797427652733, "percentage": 23.9, "elapsed_time": "0:20:53", "remaining_time": "1:06:32", "throughput": "1104.29", "total_tokens": 1384192}
+{"current_steps": 93, "total_steps": 385, "loss": 0.1525, "learning_rate": 7.750000000000001e-07, "epoch": 1.1961414790996785, "percentage": 24.16, "elapsed_time": "0:21:06", "remaining_time": "1:06:18", "throughput": "1104.03", "total_tokens": 1398784}
+{"current_steps": 94, "total_steps": 385, "loss": 0.1735, "learning_rate": 7.833333333333335e-07, "epoch": 1.2090032154340835, "percentage": 24.42, "elapsed_time": "0:21:20", "remaining_time": "1:06:04", "throughput": "1104.42", "total_tokens": 1414304}
+{"current_steps": 95, "total_steps": 385, "loss": 0.1484, "learning_rate": 7.916666666666667e-07, "epoch": 1.2218649517684887, "percentage": 24.68, "elapsed_time": "0:21:34", "remaining_time": "1:05:50", "throughput": "1105.59", "total_tokens": 1430880}
+{"current_steps": 96, "total_steps": 385, "loss": 0.1517, "learning_rate": 8.000000000000001e-07, "epoch": 1.234726688102894, "percentage": 24.94, "elapsed_time": "0:21:47", "remaining_time": "1:05:37", "throughput": "1105.48", "total_tokens": 1445760}
+{"current_steps": 97, "total_steps": 385, "loss": 0.1429, "learning_rate": 8.083333333333334e-07, "epoch": 1.247588424437299, "percentage": 25.19, "elapsed_time": "0:22:01", "remaining_time": "1:05:23", "throughput": "1105.30", "total_tokens": 1460576}
+{"current_steps": 98, "total_steps": 385, "loss": 0.151, "learning_rate": 8.166666666666668e-07, "epoch": 1.2604501607717042, "percentage": 25.45, "elapsed_time": "0:22:15", "remaining_time": "1:05:09", "throughput": "1105.57", "total_tokens": 1476032}
+{"current_steps": 99, "total_steps": 385, "loss": 0.1217, "learning_rate": 8.250000000000001e-07, "epoch": 1.2733118971061093, "percentage": 25.71, "elapsed_time": "0:22:28", "remaining_time": "1:04:56", "throughput": "1105.56", "total_tokens": 1491008}
+{"current_steps": 100, "total_steps": 385, "loss": 0.1366, "learning_rate": 8.333333333333333e-07, "epoch": 1.2861736334405145, "percentage": 25.97, "elapsed_time": "0:22:42", "remaining_time": "1:04:42", "throughput": "1104.97", "total_tokens": 1505120}
+{"current_steps": 101, "total_steps": 385, "loss": 0.1534, "learning_rate": 8.416666666666667e-07, "epoch": 1.2990353697749195, "percentage": 26.23, "elapsed_time": "0:22:55", "remaining_time": "1:04:28", "throughput": "1105.43", "total_tokens": 1520736}
+{"current_steps": 102, "total_steps": 385, "loss": 0.141, "learning_rate": 8.500000000000001e-07, "epoch": 1.3118971061093248, "percentage": 26.49, "elapsed_time": "0:23:09", "remaining_time": "1:04:14", "throughput": "1105.25", "total_tokens": 1535424}
+{"current_steps": 103, "total_steps": 385, "loss": 0.1238, "learning_rate": 8.583333333333334e-07, "epoch": 1.32475884244373, "percentage": 26.75, "elapsed_time": "0:23:22", "remaining_time": "1:04:00", "throughput": "1104.87", "total_tokens": 1549856}
+{"current_steps": 104, "total_steps": 385, "loss": 0.1241, "learning_rate": 8.666666666666668e-07, "epoch": 1.337620578778135, "percentage": 27.01, "elapsed_time": "0:23:36", "remaining_time": "1:03:46", "throughput": "1104.63", "total_tokens": 1564512}
+{"current_steps": 105, "total_steps": 385, "loss": 0.1414, "learning_rate": 8.75e-07, "epoch": 1.3504823151125402, "percentage": 27.27, "elapsed_time": "0:23:49", "remaining_time": "1:03:33", "throughput": "1104.27", "total_tokens": 1578976}
+{"current_steps": 106, "total_steps": 385, "loss": 0.1296, "learning_rate": 8.833333333333334e-07, "epoch": 1.3633440514469453, "percentage": 27.53, "elapsed_time": "0:24:03", "remaining_time": "1:03:19", "throughput": "1104.25", "total_tokens": 1593952}
+{"current_steps": 107, "total_steps": 385, "loss": 0.1232, "learning_rate": 8.916666666666668e-07, "epoch": 1.3762057877813505, "percentage": 27.79, "elapsed_time": "0:24:17", "remaining_time": "1:03:05", "throughput": "1104.12", "total_tokens": 1608704}
+{"current_steps": 108, "total_steps": 385, "loss": 0.1625, "learning_rate": 9.000000000000001e-07, "epoch": 1.3890675241157555, "percentage": 28.05, "elapsed_time": "0:24:30", "remaining_time": "1:02:51", "throughput": "1104.23", "total_tokens": 1623936}
+{"current_steps": 109, "total_steps": 385, "loss": 0.1509, "learning_rate": 9.083333333333335e-07, "epoch": 1.4019292604501608, "percentage": 28.31, "elapsed_time": "0:24:44", "remaining_time": "1:02:38", "throughput": "1104.50", "total_tokens": 1639264}
+{"current_steps": 110, "total_steps": 385, "loss": 0.1416, "learning_rate": 9.166666666666666e-07, "epoch": 1.414790996784566, "percentage": 28.57, "elapsed_time": "0:24:57", "remaining_time": "1:02:24", "throughput": "1104.70", "total_tokens": 1654528}
+{"current_steps": 111, "total_steps": 385, "loss": 0.1481, "learning_rate": 9.25e-07, "epoch": 1.427652733118971, "percentage": 28.83, "elapsed_time": "0:25:11", "remaining_time": "1:02:10", "throughput": "1104.91", "total_tokens": 1669824}
+{"current_steps": 112, "total_steps": 385, "loss": 0.1303, "learning_rate": 9.333333333333334e-07, "epoch": 1.4405144694533762, "percentage": 29.09, "elapsed_time": "0:25:24", "remaining_time": "1:01:56", "throughput": "1104.87", "total_tokens": 1684800}
+{"current_steps": 113, "total_steps": 385, "loss": 0.116, "learning_rate": 9.416666666666667e-07, "epoch": 1.4533762057877815, "percentage": 29.35, "elapsed_time": "0:25:38", "remaining_time": "1:01:43", "throughput": "1104.77", "total_tokens": 1699712}
+{"current_steps": 114, "total_steps": 385, "loss": 0.0981, "learning_rate": 9.500000000000001e-07, "epoch": 1.4662379421221865, "percentage": 29.61, "elapsed_time": "0:25:52", "remaining_time": "1:01:29", "throughput": "1104.48", "total_tokens": 1714208}
+{"current_steps": 115, "total_steps": 385, "loss": 0.1174, "learning_rate": 9.583333333333334e-07, "epoch": 1.4790996784565915, "percentage": 29.87, "elapsed_time": "0:26:05", "remaining_time": "1:01:15", "throughput": "1104.16", "total_tokens": 1728672}
+{"current_steps": 116, "total_steps": 385, "loss": 0.1458, "learning_rate": 9.666666666666668e-07, "epoch": 1.4919614147909968, "percentage": 30.13, "elapsed_time": "0:26:19", "remaining_time": "1:01:02", "throughput": "1103.74", "total_tokens": 1743008}
+{"current_steps": 117, "total_steps": 385, "loss": 0.0952, "learning_rate": 9.750000000000002e-07, "epoch": 1.504823151125402, "percentage": 30.39, "elapsed_time": "0:26:32", "remaining_time": "1:00:48", "throughput": "1103.76", "total_tokens": 1758016}
+{"current_steps": 118, "total_steps": 385, "loss": 0.1233, "learning_rate": 9.833333333333334e-07, "epoch": 1.517684887459807, "percentage": 30.65, "elapsed_time": "0:26:46", "remaining_time": "1:00:34", "throughput": "1103.43", "total_tokens": 1772512}
+{"current_steps": 119, "total_steps": 385, "loss": 0.127, "learning_rate": 9.916666666666668e-07, "epoch": 1.5305466237942122, "percentage": 30.91, "elapsed_time": "0:26:59", "remaining_time": "1:00:21", "throughput": "1103.60", "total_tokens": 1787808}
+{"current_steps": 120, "total_steps": 385, "loss": 0.1121, "learning_rate": 1.0000000000000002e-06, "epoch": 1.5434083601286175, "percentage": 31.17, "elapsed_time": "0:27:13", "remaining_time": "1:00:07", "throughput": "1103.79", "total_tokens": 1803104}
+{"current_steps": 121, "total_steps": 385, "loss": 0.1432, "learning_rate": 1.0083333333333333e-06, "epoch": 1.5562700964630225, "percentage": 31.43, "elapsed_time": "0:27:27", "remaining_time": "0:59:53", "throughput": "1103.55", "total_tokens": 1817632}
+{"current_steps": 122, "total_steps": 385, "loss": 0.1446, "learning_rate": 1.0166666666666667e-06, "epoch": 1.5691318327974275, "percentage": 31.69, "elapsed_time": "0:27:40", "remaining_time": "0:59:39", "throughput": "1103.53", "total_tokens": 1832576}
+{"current_steps": 123, "total_steps": 385, "loss": 0.1056, "learning_rate": 1.025e-06, "epoch": 1.5819935691318328, "percentage": 31.95, "elapsed_time": "0:27:54", "remaining_time": "0:59:26", "throughput": "1103.73", "total_tokens": 1847872}
+{"current_steps": 124, "total_steps": 385, "loss": 0.1193, "learning_rate": 1.0333333333333333e-06, "epoch": 1.594855305466238, "percentage": 32.21, "elapsed_time": "0:28:07", "remaining_time": "0:59:12", "throughput": "1103.80", "total_tokens": 1863008}
+{"current_steps": 125, "total_steps": 385, "loss": 0.1409, "learning_rate": 1.0416666666666667e-06, "epoch": 1.607717041800643, "percentage": 32.47, "elapsed_time": "0:28:21", "remaining_time": "0:58:58", "throughput": "1104.18", "total_tokens": 1878688}
+{"current_steps": 126, "total_steps": 385, "loss": 0.1154, "learning_rate": 1.0500000000000001e-06, "epoch": 1.6205787781350482, "percentage": 32.73, "elapsed_time": "0:28:35", "remaining_time": "0:58:45", "throughput": "1104.44", "total_tokens": 1894144}
+{"current_steps": 127, "total_steps": 385, "loss": 0.1046, "learning_rate": 1.0583333333333335e-06, "epoch": 1.6334405144694535, "percentage": 32.99, "elapsed_time": "0:28:48", "remaining_time": "0:58:31", "throughput": "1104.66", "total_tokens": 1909504}
+{"current_steps": 128, "total_steps": 385, "loss": 0.09, "learning_rate": 1.066666666666667e-06, "epoch": 1.6463022508038585, "percentage": 33.25, "elapsed_time": "0:29:02", "remaining_time": "0:58:17", "throughput": "1104.90", "total_tokens": 1924896}
+{"current_steps": 129, "total_steps": 385, "loss": 0.0858, "learning_rate": 1.075e-06, "epoch": 1.6591639871382635, "percentage": 33.51, "elapsed_time": "0:29:15", "remaining_time": "0:58:04", "throughput": "1104.55", "total_tokens": 1939232}
+{"current_steps": 130, "total_steps": 385, "loss": 0.0782, "learning_rate": 1.0833333333333335e-06, "epoch": 1.6720257234726688, "percentage": 33.77, "elapsed_time": "0:29:29", "remaining_time": "0:57:50", "throughput": "1104.82", "total_tokens": 1954720}
+{"current_steps": 131, "total_steps": 385, "loss": 0.1429, "learning_rate": 1.0916666666666667e-06, "epoch": 1.684887459807074, "percentage": 34.03, "elapsed_time": "0:29:42", "remaining_time": "0:57:36", "throughput": "1104.80", "total_tokens": 1969632}
+{"current_steps": 132, "total_steps": 385, "loss": 0.1121, "learning_rate": 1.1e-06, "epoch": 1.697749196141479, "percentage": 34.29, "elapsed_time": "0:29:56", "remaining_time": "0:57:23", "throughput": "1104.72", "total_tokens": 1984544}
+{"current_steps": 133, "total_steps": 385, "loss": 0.0458, "learning_rate": 1.1083333333333335e-06, "epoch": 1.7106109324758842, "percentage": 34.55, "elapsed_time": "0:30:09", "remaining_time": "0:57:09", "throughput": "1104.76", "total_tokens": 1999584}
+{"current_steps": 134, "total_steps": 385, "loss": 0.1217, "learning_rate": 1.1166666666666666e-06, "epoch": 1.7234726688102895, "percentage": 34.81, "elapsed_time": "0:30:23", "remaining_time": "0:56:55", "throughput": "1104.43", "total_tokens": 2014048}
+{"current_steps": 135, "total_steps": 385, "loss": 0.1253, "learning_rate": 1.125e-06, "epoch": 1.7363344051446945, "percentage": 35.06, "elapsed_time": "0:30:37", "remaining_time": "0:56:42", "throughput": "1104.59", "total_tokens": 2029312}
+{"current_steps": 136, "total_steps": 385, "loss": 0.077, "learning_rate": 1.1333333333333334e-06, "epoch": 1.7491961414790995, "percentage": 35.32, "elapsed_time": "0:30:50", "remaining_time": "0:56:28", "throughput": "1104.56", "total_tokens": 2044192}
+{"current_steps": 137, "total_steps": 385, "loss": 0.0719, "learning_rate": 1.1416666666666668e-06, "epoch": 1.762057877813505, "percentage": 35.58, "elapsed_time": "0:31:04", "remaining_time": "0:56:14", "throughput": "1104.75", "total_tokens": 2059552}
+{"current_steps": 138, "total_steps": 385, "loss": 0.0916, "learning_rate": 1.1500000000000002e-06, "epoch": 1.77491961414791, "percentage": 35.84, "elapsed_time": "0:31:17", "remaining_time": "0:56:01", "throughput": "1104.65", "total_tokens": 2074336}
+{"current_steps": 139, "total_steps": 385, "loss": 0.0812, "learning_rate": 1.1583333333333334e-06, "epoch": 1.787781350482315, "percentage": 36.1, "elapsed_time": "0:31:31", "remaining_time": "0:55:47", "throughput": "1104.64", "total_tokens": 2089280}
+{"current_steps": 140, "total_steps": 385, "loss": 0.1176, "learning_rate": 1.1666666666666668e-06, "epoch": 1.8006430868167203, "percentage": 36.36, "elapsed_time": "0:31:44", "remaining_time": "0:55:33", "throughput": "1104.64", "total_tokens": 2104256}
+{"current_steps": 141, "total_steps": 385, "loss": 0.0631, "learning_rate": 1.175e-06, "epoch": 1.8135048231511255, "percentage": 36.62, "elapsed_time": "0:31:58", "remaining_time": "0:55:19", "throughput": "1104.63", "total_tokens": 2119232}
+{"current_steps": 142, "total_steps": 385, "loss": 0.1137, "learning_rate": 1.1833333333333334e-06, "epoch": 1.8263665594855305, "percentage": 36.88, "elapsed_time": "0:32:12", "remaining_time": "0:55:06", "throughput": "1104.44", "total_tokens": 2133824}
+{"current_steps": 143, "total_steps": 385, "loss": 0.0958, "learning_rate": 1.1916666666666668e-06, "epoch": 1.8392282958199357, "percentage": 37.14, "elapsed_time": "0:32:25", "remaining_time": "0:54:52", "throughput": "1104.29", "total_tokens": 2148480}
+{"current_steps": 144, "total_steps": 385, "loss": 0.1343, "learning_rate": 1.2000000000000002e-06, "epoch": 1.852090032154341, "percentage": 37.4, "elapsed_time": "0:32:39", "remaining_time": "0:54:38", "throughput": "1104.23", "total_tokens": 2163328}
+{"current_steps": 145, "total_steps": 385, "loss": 0.1101, "learning_rate": 1.2083333333333333e-06, "epoch": 1.864951768488746, "percentage": 37.66, "elapsed_time": "0:32:52", "remaining_time": "0:54:25", "throughput": "1103.88", "total_tokens": 2177568}
+{"current_steps": 146, "total_steps": 385, "loss": 0.0914, "learning_rate": 1.2166666666666667e-06, "epoch": 1.877813504823151, "percentage": 37.92, "elapsed_time": "0:33:06", "remaining_time": "0:54:11", "throughput": "1103.82", "total_tokens": 2192384}
+{"current_steps": 147, "total_steps": 385, "loss": 0.1114, "learning_rate": 1.2250000000000001e-06, "epoch": 1.8906752411575563, "percentage": 38.18, "elapsed_time": "0:33:19", "remaining_time": "0:53:57", "throughput": "1103.91", "total_tokens": 2207584}
+{"current_steps": 148, "total_steps": 385, "loss": 0.083, "learning_rate": 1.2333333333333335e-06, "epoch": 1.9035369774919615, "percentage": 38.44, "elapsed_time": "0:33:33", "remaining_time": "0:53:44", "throughput": "1103.76", "total_tokens": 2222208}
+{"current_steps": 149, "total_steps": 385, "loss": 0.1095, "learning_rate": 1.2416666666666667e-06, "epoch": 1.9163987138263665, "percentage": 38.7, "elapsed_time": "0:33:46", "remaining_time": "0:53:30", "throughput": "1104.03", "total_tokens": 2237760}
+{"current_steps": 150, "total_steps": 385, "loss": 0.0662, "learning_rate": 1.25e-06, "epoch": 1.9292604501607717, "percentage": 38.96, "elapsed_time": "0:34:00", "remaining_time": "0:53:16", "throughput": "1104.06", "total_tokens": 2252768}
+{"current_steps": 151, "total_steps": 385, "loss": 0.0979, "learning_rate": 1.2583333333333333e-06, "epoch": 1.942122186495177, "percentage": 39.22, "elapsed_time": "0:34:14", "remaining_time": "0:53:03", "throughput": "1104.54", "total_tokens": 2268736}
+{"current_steps": 152, "total_steps": 385, "loss": 0.0847, "learning_rate": 1.2666666666666669e-06, "epoch": 1.954983922829582, "percentage": 39.48, "elapsed_time": "0:34:27", "remaining_time": "0:52:49", "throughput": "1104.58", "total_tokens": 2283840}
+{"current_steps": 153, "total_steps": 385, "loss": 0.0949, "learning_rate": 1.275e-06, "epoch": 1.967845659163987, "percentage": 39.74, "elapsed_time": "0:34:41", "remaining_time": "0:52:35", "throughput": "1104.06", "total_tokens": 2297696}
+{"current_steps": 154, "total_steps": 385, "loss": 0.1206, "learning_rate": 1.2833333333333335e-06, "epoch": 1.9807073954983923, "percentage": 40.0, "elapsed_time": "0:34:54", "remaining_time": "0:52:22", "throughput": "1103.69", "total_tokens": 2311936}
+{"current_steps": 155, "total_steps": 385, "loss": 0.139, "learning_rate": 1.2916666666666669e-06, "epoch": 1.9935691318327975, "percentage": 40.26, "elapsed_time": "0:35:08", "remaining_time": "0:52:08", "throughput": "1103.92", "total_tokens": 2327392}
+{"current_steps": 156, "total_steps": 385, "loss": 0.07, "learning_rate": 1.3e-06, "epoch": 2.0064308681672025, "percentage": 40.52, "elapsed_time": "0:35:21", "remaining_time": "0:51:54", "throughput": "1103.71", "total_tokens": 2341952}
+{"current_steps": 157, "total_steps": 385, "loss": 0.0562, "learning_rate": 1.3083333333333334e-06, "epoch": 2.0192926045016075, "percentage": 40.78, "elapsed_time": "0:35:35", "remaining_time": "0:51:41", "throughput": "1103.62", "total_tokens": 2356768}
+{"current_steps": 158, "total_steps": 385, "loss": 0.0456, "learning_rate": 1.3166666666666666e-06, "epoch": 2.032154340836013, "percentage": 41.04, "elapsed_time": "0:35:49", "remaining_time": "0:51:27", "throughput": "1103.51", "total_tokens": 2371584}
+{"current_steps": 159, "total_steps": 385, "loss": 0.0582, "learning_rate": 1.3250000000000002e-06, "epoch": 2.045016077170418, "percentage": 41.3, "elapsed_time": "0:36:02", "remaining_time": "0:51:14", "throughput": "1103.66", "total_tokens": 2386880}
+{"current_steps": 160, "total_steps": 385, "loss": 0.0452, "learning_rate": 1.3333333333333334e-06, "epoch": 2.057877813504823, "percentage": 41.56, "elapsed_time": "0:36:16", "remaining_time": "0:51:00", "throughput": "1103.37", "total_tokens": 2401248}
+{"current_steps": 161, "total_steps": 385, "loss": 0.0553, "learning_rate": 1.3416666666666666e-06, "epoch": 2.0707395498392285, "percentage": 41.82, "elapsed_time": "0:36:29", "remaining_time": "0:50:46", "throughput": "1103.25", "total_tokens": 2415968}
+{"current_steps": 162, "total_steps": 385, "loss": 0.1108, "learning_rate": 1.3500000000000002e-06, "epoch": 2.0836012861736335, "percentage": 42.08, "elapsed_time": "0:36:43", "remaining_time": "0:50:33", "throughput": "1103.01", "total_tokens": 2430368}
+{"current_steps": 163, "total_steps": 385, "loss": 0.0791, "learning_rate": 1.3583333333333334e-06, "epoch": 2.0964630225080385, "percentage": 42.34, "elapsed_time": "0:36:56", "remaining_time": "0:50:19", "throughput": "1102.88", "total_tokens": 2445056}
+{"current_steps": 164, "total_steps": 385, "loss": 0.0637, "learning_rate": 1.3666666666666668e-06, "epoch": 2.1093247588424435, "percentage": 42.6, "elapsed_time": "0:37:10", "remaining_time": "0:50:05", "throughput": "1102.77", "total_tokens": 2459744}
+{"current_steps": 165, "total_steps": 385, "loss": 0.0404, "learning_rate": 1.3750000000000002e-06, "epoch": 2.122186495176849, "percentage": 42.86, "elapsed_time": "0:37:24", "remaining_time": "0:49:52", "throughput": "1102.66", "total_tokens": 2474400}
+{"current_steps": 166, "total_steps": 385, "loss": 0.0372, "learning_rate": 1.3833333333333336e-06, "epoch": 2.135048231511254, "percentage": 43.12, "elapsed_time": "0:37:37", "remaining_time": "0:49:38", "throughput": "1102.62", "total_tokens": 2489280}
+{"current_steps": 167, "total_steps": 385, "loss": 0.045, "learning_rate": 1.3916666666666668e-06, "epoch": 2.147909967845659, "percentage": 43.38, "elapsed_time": "0:37:51", "remaining_time": "0:49:24", "throughput": "1102.43", "total_tokens": 2503776}
+{"current_steps": 168, "total_steps": 385, "loss": 0.0964, "learning_rate": 1.4000000000000001e-06, "epoch": 2.1607717041800645, "percentage": 43.64, "elapsed_time": "0:38:04", "remaining_time": "0:49:11", "throughput": "1102.42", "total_tokens": 2518752}
+{"current_steps": 169, "total_steps": 385, "loss": 0.0543, "learning_rate": 1.4083333333333335e-06, "epoch": 2.1736334405144695, "percentage": 43.9, "elapsed_time": "0:38:18", "remaining_time": "0:48:57", "throughput": "1102.35", "total_tokens": 2533568}
+{"current_steps": 170, "total_steps": 385, "loss": 0.071, "learning_rate": 1.4166666666666667e-06, "epoch": 2.1864951768488745, "percentage": 44.16, "elapsed_time": "0:38:31", "remaining_time": "0:48:43", "throughput": "1102.39", "total_tokens": 2548640}
+{"current_steps": 171, "total_steps": 385, "loss": 0.0285, "learning_rate": 1.425e-06, "epoch": 2.19935691318328, "percentage": 44.42, "elapsed_time": "0:38:45", "remaining_time": "0:48:30", "throughput": "1102.26", "total_tokens": 2563232}
+{"current_steps": 172, "total_steps": 385, "loss": 0.0399, "learning_rate": 1.4333333333333335e-06, "epoch": 2.212218649517685, "percentage": 44.68, "elapsed_time": "0:38:59", "remaining_time": "0:48:16", "throughput": "1102.32", "total_tokens": 2578336}
+{"current_steps": 173, "total_steps": 385, "loss": 0.067, "learning_rate": 1.4416666666666667e-06, "epoch": 2.22508038585209, "percentage": 44.94, "elapsed_time": "0:39:12", "remaining_time": "0:48:02", "throughput": "1102.74", "total_tokens": 2594336}
+{"current_steps": 174, "total_steps": 385, "loss": 0.0436, "learning_rate": 1.45e-06, "epoch": 2.237942122186495, "percentage": 45.19, "elapsed_time": "0:39:26", "remaining_time": "0:47:49", "throughput": "1102.78", "total_tokens": 2609408}
+{"current_steps": 175, "total_steps": 385, "loss": 0.0522, "learning_rate": 1.4583333333333335e-06, "epoch": 2.2508038585209005, "percentage": 45.45, "elapsed_time": "0:39:39", "remaining_time": "0:47:35", "throughput": "1102.75", "total_tokens": 2624288}
+{"current_steps": 176, "total_steps": 385, "loss": 0.0521, "learning_rate": 1.4666666666666669e-06, "epoch": 2.2636655948553055, "percentage": 45.71, "elapsed_time": "0:39:53", "remaining_time": "0:47:22", "throughput": "1102.78", "total_tokens": 2639296}
+{"current_steps": 177, "total_steps": 385, "loss": 0.0446, "learning_rate": 1.475e-06, "epoch": 2.2765273311897105, "percentage": 45.97, "elapsed_time": "0:40:06", "remaining_time": "0:47:08", "throughput": "1102.60", "total_tokens": 2653856}
+{"current_steps": 178, "total_steps": 385, "loss": 0.0378, "learning_rate": 1.4833333333333337e-06, "epoch": 2.289389067524116, "percentage": 46.23, "elapsed_time": "0:40:20", "remaining_time": "0:46:54", "throughput": "1102.76", "total_tokens": 2669216}
+{"current_steps": 179, "total_steps": 385, "loss": 0.0387, "learning_rate": 1.4916666666666669e-06, "epoch": 2.302250803858521, "percentage": 46.49, "elapsed_time": "0:40:34", "remaining_time": "0:46:41", "throughput": "1102.37", "total_tokens": 2683264}
+{"current_steps": 180, "total_steps": 385, "loss": 0.036, "learning_rate": 1.5e-06, "epoch": 2.315112540192926, "percentage": 46.75, "elapsed_time": "0:40:47", "remaining_time": "0:46:27", "throughput": "1102.33", "total_tokens": 2698112}
+{"current_steps": 181, "total_steps": 385, "loss": 0.0765, "learning_rate": 1.5083333333333336e-06, "epoch": 2.327974276527331, "percentage": 47.01, "elapsed_time": "0:41:01", "remaining_time": "0:46:13", "throughput": "1102.23", "total_tokens": 2712800}
+{"current_steps": 182, "total_steps": 385, "loss": 0.0884, "learning_rate": 1.5166666666666668e-06, "epoch": 2.3408360128617365, "percentage": 47.27, "elapsed_time": "0:41:14", "remaining_time": "0:46:00", "throughput": "1102.34", "total_tokens": 2728032}
+{"current_steps": 183, "total_steps": 385, "loss": 0.0801, "learning_rate": 1.525e-06, "epoch": 2.3536977491961415, "percentage": 47.53, "elapsed_time": "0:41:28", "remaining_time": "0:45:46", "throughput": "1102.13", "total_tokens": 2742432}
+{"current_steps": 184, "total_steps": 385, "loss": 0.0276, "learning_rate": 1.5333333333333334e-06, "epoch": 2.3665594855305465, "percentage": 47.79, "elapsed_time": "0:41:41", "remaining_time": "0:45:33", "throughput": "1102.42", "total_tokens": 2758176}
+{"current_steps": 185, "total_steps": 385, "loss": 0.0778, "learning_rate": 1.5416666666666668e-06, "epoch": 2.379421221864952, "percentage": 48.05, "elapsed_time": "0:41:55", "remaining_time": "0:45:19", "throughput": "1102.27", "total_tokens": 2772768}
+{"current_steps": 186, "total_steps": 385, "loss": 0.0726, "learning_rate": 1.5500000000000002e-06, "epoch": 2.392282958199357, "percentage": 48.31, "elapsed_time": "0:42:09", "remaining_time": "0:45:05", "throughput": "1102.38", "total_tokens": 2787968}
+{"current_steps": 187, "total_steps": 385, "loss": 0.1381, "learning_rate": 1.5583333333333334e-06, "epoch": 2.405144694533762, "percentage": 48.57, "elapsed_time": "0:42:22", "remaining_time": "0:44:52", "throughput": "1102.23", "total_tokens": 2802560}
+{"current_steps": 188, "total_steps": 385, "loss": 0.0408, "learning_rate": 1.566666666666667e-06, "epoch": 2.418006430868167, "percentage": 48.83, "elapsed_time": "0:42:36", "remaining_time": "0:44:38", "throughput": "1102.37", "total_tokens": 2817856}
+{"current_steps": 189, "total_steps": 385, "loss": 0.1066, "learning_rate": 1.5750000000000002e-06, "epoch": 2.4308681672025725, "percentage": 49.09, "elapsed_time": "0:42:49", "remaining_time": "0:44:24", "throughput": "1102.60", "total_tokens": 2833408}
+{"current_steps": 190, "total_steps": 385, "loss": 0.0686, "learning_rate": 1.5833333333333333e-06, "epoch": 2.4437299035369775, "percentage": 49.35, "elapsed_time": "0:43:03", "remaining_time": "0:44:11", "throughput": "1102.78", "total_tokens": 2848832}
+{"current_steps": 191, "total_steps": 385, "loss": 0.0428, "learning_rate": 1.591666666666667e-06, "epoch": 2.4565916398713825, "percentage": 49.61, "elapsed_time": "0:43:16", "remaining_time": "0:43:57", "throughput": "1102.86", "total_tokens": 2863968}
+{"current_steps": 192, "total_steps": 385, "loss": 0.0387, "learning_rate": 1.6000000000000001e-06, "epoch": 2.469453376205788, "percentage": 49.87, "elapsed_time": "0:43:30", "remaining_time": "0:43:44", "throughput": "1102.85", "total_tokens": 2878944}
+{"current_steps": 193, "total_steps": 385, "loss": 0.0489, "learning_rate": 1.6083333333333333e-06, "epoch": 2.482315112540193, "percentage": 50.13, "elapsed_time": "0:43:44", "remaining_time": "0:43:30", "throughput": "1102.92", "total_tokens": 2894112}
+{"current_steps": 194, "total_steps": 385, "loss": 0.0621, "learning_rate": 1.6166666666666667e-06, "epoch": 2.495176848874598, "percentage": 50.39, "elapsed_time": "0:43:57", "remaining_time": "0:43:16", "throughput": "1102.86", "total_tokens": 2908928}
+{"current_steps": 195, "total_steps": 385, "loss": 0.0651, "learning_rate": 1.6250000000000001e-06, "epoch": 2.508038585209003, "percentage": 50.65, "elapsed_time": "0:44:11", "remaining_time": "0:43:03", "throughput": "1102.76", "total_tokens": 2923680}
+{"current_steps": 196, "total_steps": 385, "loss": 0.0398, "learning_rate": 1.6333333333333335e-06, "epoch": 2.5209003215434085, "percentage": 50.91, "elapsed_time": "0:44:24", "remaining_time": "0:42:49", "throughput": "1102.95", "total_tokens": 2939136}
+{"current_steps": 197, "total_steps": 385, "loss": 0.0369, "learning_rate": 1.6416666666666667e-06, "epoch": 2.5337620578778135, "percentage": 51.17, "elapsed_time": "0:44:38", "remaining_time": "0:42:35", "throughput": "1102.84", "total_tokens": 2953760}
+{"current_steps": 198, "total_steps": 385, "loss": 0.0582, "learning_rate": 1.6500000000000003e-06, "epoch": 2.5466237942122185, "percentage": 51.43, "elapsed_time": "0:44:51", "remaining_time": "0:42:22", "throughput": "1102.81", "total_tokens": 2968640}
+{"current_steps": 199, "total_steps": 385, "loss": 0.0479, "learning_rate": 1.6583333333333335e-06, "epoch": 2.559485530546624, "percentage": 51.69, "elapsed_time": "0:45:05", "remaining_time": "0:42:08", "throughput": "1102.76", "total_tokens": 2983456}
+{"current_steps": 200, "total_steps": 385, "loss": 0.0561, "learning_rate": 1.6666666666666667e-06, "epoch": 2.572347266881029, "percentage": 51.95, "elapsed_time": "0:45:19", "remaining_time": "0:41:55", "throughput": "1102.65", "total_tokens": 2998144}
+{"current_steps": 201, "total_steps": 385, "loss": 0.0497, "learning_rate": 1.6750000000000003e-06, "epoch": 2.585209003215434, "percentage": 52.21, "elapsed_time": "0:45:32", "remaining_time": "0:41:41", "throughput": "1102.71", "total_tokens": 3013248}
+{"current_steps": 202, "total_steps": 385, "loss": 0.063, "learning_rate": 1.6833333333333335e-06, "epoch": 2.598070739549839, "percentage": 52.47, "elapsed_time": "0:45:46", "remaining_time": "0:41:27", "throughput": "1102.74", "total_tokens": 3028320}
+{"current_steps": 203, "total_steps": 385, "loss": 0.054, "learning_rate": 1.6916666666666666e-06, "epoch": 2.6109324758842445, "percentage": 52.73, "elapsed_time": "0:45:59", "remaining_time": "0:41:14", "throughput": "1102.72", "total_tokens": 3043232}
+{"current_steps": 204, "total_steps": 385, "loss": 0.0729, "learning_rate": 1.7000000000000002e-06, "epoch": 2.6237942122186495, "percentage": 52.99, "elapsed_time": "0:46:13", "remaining_time": "0:41:00", "throughput": "1102.94", "total_tokens": 3058848}
+{"current_steps": 205, "total_steps": 385, "loss": 0.0685, "learning_rate": 1.7083333333333334e-06, "epoch": 2.6366559485530545, "percentage": 53.25, "elapsed_time": "0:46:26", "remaining_time": "0:40:47", "throughput": "1103.02", "total_tokens": 3074080}
+{"current_steps": 206, "total_steps": 385, "loss": 0.0823, "learning_rate": 1.7166666666666668e-06, "epoch": 2.64951768488746, "percentage": 53.51, "elapsed_time": "0:46:40", "remaining_time": "0:40:33", "throughput": "1103.02", "total_tokens": 3089152}
+{"current_steps": 207, "total_steps": 385, "loss": 0.0322, "learning_rate": 1.725e-06, "epoch": 2.662379421221865, "percentage": 53.77, "elapsed_time": "0:46:54", "remaining_time": "0:40:19", "throughput": "1102.86", "total_tokens": 3103616}
+{"current_steps": 208, "total_steps": 385, "loss": 0.0821, "learning_rate": 1.7333333333333336e-06, "epoch": 2.67524115755627, "percentage": 54.03, "elapsed_time": "0:47:07", "remaining_time": "0:40:06", "throughput": "1103.07", "total_tokens": 3119200}
+{"current_steps": 209, "total_steps": 385, "loss": 0.0561, "learning_rate": 1.7416666666666668e-06, "epoch": 2.688102893890675, "percentage": 54.29, "elapsed_time": "0:47:21", "remaining_time": "0:39:52", "throughput": "1103.13", "total_tokens": 3134400}
+{"current_steps": 210, "total_steps": 385, "loss": 0.0468, "learning_rate": 1.75e-06, "epoch": 2.7009646302250805, "percentage": 54.55, "elapsed_time": "0:47:34", "remaining_time": "0:39:39", "throughput": "1103.09", "total_tokens": 3149248}
+{"current_steps": 211, "total_steps": 385, "loss": 0.0593, "learning_rate": 1.7583333333333336e-06, "epoch": 2.7138263665594855, "percentage": 54.81, "elapsed_time": "0:47:48", "remaining_time": "0:39:25", "throughput": "1103.31", "total_tokens": 3164800}
+{"current_steps": 212, "total_steps": 385, "loss": 0.0403, "learning_rate": 1.7666666666666668e-06, "epoch": 2.7266881028938905, "percentage": 55.06, "elapsed_time": "0:48:02", "remaining_time": "0:39:11", "throughput": "1103.22", "total_tokens": 3179520}
+{"current_steps": 213, "total_steps": 385, "loss": 0.0459, "learning_rate": 1.7750000000000002e-06, "epoch": 2.739549839228296, "percentage": 55.32, "elapsed_time": "0:48:15", "remaining_time": "0:38:58", "throughput": "1103.05", "total_tokens": 3194016}
+{"current_steps": 214, "total_steps": 385, "loss": 0.0509, "learning_rate": 1.7833333333333336e-06, "epoch": 2.752411575562701, "percentage": 55.58, "elapsed_time": "0:48:29", "remaining_time": "0:38:44", "throughput": "1102.91", "total_tokens": 3208608}
+{"current_steps": 215, "total_steps": 385, "loss": 0.0873, "learning_rate": 1.7916666666666667e-06, "epoch": 2.765273311897106, "percentage": 55.84, "elapsed_time": "0:48:42", "remaining_time": "0:38:31", "throughput": "1103.07", "total_tokens": 3224096}
+{"current_steps": 216, "total_steps": 385, "loss": 0.0618, "learning_rate": 1.8000000000000001e-06, "epoch": 2.778135048231511, "percentage": 56.1, "elapsed_time": "0:48:56", "remaining_time": "0:38:17", "throughput": "1103.10", "total_tokens": 3239136}
+{"current_steps": 217, "total_steps": 385, "loss": 0.1025, "learning_rate": 1.8083333333333335e-06, "epoch": 2.7909967845659165, "percentage": 56.36, "elapsed_time": "0:49:09", "remaining_time": "0:38:03", "throughput": "1102.80", "total_tokens": 3253184}
+{"current_steps": 218, "total_steps": 385, "loss": 0.0669, "learning_rate": 1.816666666666667e-06, "epoch": 2.8038585209003215, "percentage": 56.62, "elapsed_time": "0:49:23", "remaining_time": "0:37:50", "throughput": "1102.74", "total_tokens": 3268000}
+{"current_steps": 219, "total_steps": 385, "loss": 0.0447, "learning_rate": 1.825e-06, "epoch": 2.816720257234727, "percentage": 56.88, "elapsed_time": "0:49:37", "remaining_time": "0:37:36", "throughput": "1102.58", "total_tokens": 3282528}
+{"current_steps": 220, "total_steps": 385, "loss": 0.0635, "learning_rate": 1.8333333333333333e-06, "epoch": 2.829581993569132, "percentage": 57.14, "elapsed_time": "0:49:50", "remaining_time": "0:37:23", "throughput": "1102.68", "total_tokens": 3297824}
+{"current_steps": 221, "total_steps": 385, "loss": 0.0614, "learning_rate": 1.8416666666666669e-06, "epoch": 2.842443729903537, "percentage": 57.4, "elapsed_time": "0:50:04", "remaining_time": "0:37:09", "throughput": "1102.57", "total_tokens": 3312480}
+{"current_steps": 222, "total_steps": 385, "loss": 0.0646, "learning_rate": 1.85e-06, "epoch": 2.855305466237942, "percentage": 57.66, "elapsed_time": "0:50:17", "remaining_time": "0:36:55", "throughput": "1102.56", "total_tokens": 3327392}
+{"current_steps": 223, "total_steps": 385, "loss": 0.0893, "learning_rate": 1.8583333333333335e-06, "epoch": 2.868167202572347, "percentage": 57.92, "elapsed_time": "0:50:31", "remaining_time": "0:36:42", "throughput": "1102.76", "total_tokens": 3342944}
+{"current_steps": 224, "total_steps": 385, "loss": 0.0653, "learning_rate": 1.8666666666666669e-06, "epoch": 2.8810289389067525, "percentage": 58.18, "elapsed_time": "0:50:44", "remaining_time": "0:36:28", "throughput": "1102.83", "total_tokens": 3358112}
+{"current_steps": 225, "total_steps": 385, "loss": 0.0402, "learning_rate": 1.8750000000000003e-06, "epoch": 2.8938906752411575, "percentage": 58.44, "elapsed_time": "0:50:58", "remaining_time": "0:36:14", "throughput": "1102.96", "total_tokens": 3373472}
+{"current_steps": 226, "total_steps": 385, "loss": 0.0407, "learning_rate": 1.8833333333333334e-06, "epoch": 2.906752411575563, "percentage": 58.7, "elapsed_time": "0:51:12", "remaining_time": "0:36:01", "throughput": "1103.17", "total_tokens": 3389088}
+{"current_steps": 227, "total_steps": 385, "loss": 0.0949, "learning_rate": 1.8916666666666668e-06, "epoch": 2.919614147909968, "percentage": 58.96, "elapsed_time": "0:51:25", "remaining_time": "0:35:47", "throughput": "1103.04", "total_tokens": 3403616}
+{"current_steps": 228, "total_steps": 385, "loss": 0.0789, "learning_rate": 1.9000000000000002e-06, "epoch": 2.932475884244373, "percentage": 59.22, "elapsed_time": "0:51:39", "remaining_time": "0:35:34", "throughput": "1103.14", "total_tokens": 3418944}
+{"current_steps": 229, "total_steps": 385, "loss": 0.0438, "learning_rate": 1.9083333333333334e-06, "epoch": 2.945337620578778, "percentage": 59.48, "elapsed_time": "0:51:52", "remaining_time": "0:35:20", "throughput": "1103.18", "total_tokens": 3434048}
+{"current_steps": 230, "total_steps": 385, "loss": 0.0905, "learning_rate": 1.916666666666667e-06, "epoch": 2.958199356913183, "percentage": 59.74, "elapsed_time": "0:52:06", "remaining_time": "0:35:06", "throughput": "1103.48", "total_tokens": 3449952}
+{"current_steps": 231, "total_steps": 385, "loss": 0.0495, "learning_rate": 1.925e-06, "epoch": 2.9710610932475885, "percentage": 60.0, "elapsed_time": "0:52:19", "remaining_time": "0:34:53", "throughput": "1103.43", "total_tokens": 3464736}
+{"current_steps": 232, "total_steps": 385, "loss": 0.0642, "learning_rate": 1.9333333333333336e-06, "epoch": 2.9839228295819935, "percentage": 60.26, "elapsed_time": "0:52:33", "remaining_time": "0:34:39", "throughput": "1103.64", "total_tokens": 3480416}
+{"current_steps": 233, "total_steps": 385, "loss": 0.0607, "learning_rate": 1.9416666666666666e-06, "epoch": 2.996784565916399, "percentage": 60.52, "elapsed_time": "0:52:47", "remaining_time": "0:34:26", "throughput": "1103.60", "total_tokens": 3495264}
+{"current_steps": 234, "total_steps": 385, "loss": 0.0363, "learning_rate": 1.9500000000000004e-06, "epoch": 3.009646302250804, "percentage": 60.78, "elapsed_time": "0:53:00", "remaining_time": "0:34:12", "throughput": "1103.54", "total_tokens": 3510080}
+{"current_steps": 235, "total_steps": 385, "loss": 0.0492, "learning_rate": 1.9583333333333334e-06, "epoch": 3.022508038585209, "percentage": 61.04, "elapsed_time": "0:53:14", "remaining_time": "0:33:58", "throughput": "1103.42", "total_tokens": 3524736}
+{"current_steps": 236, "total_steps": 385, "loss": 0.0373, "learning_rate": 1.9666666666666668e-06, "epoch": 3.035369774919614, "percentage": 61.3, "elapsed_time": "0:53:27", "remaining_time": "0:33:45", "throughput": "1103.50", "total_tokens": 3539968}
+{"current_steps": 237, "total_steps": 385, "loss": 0.0203, "learning_rate": 1.975e-06, "epoch": 3.0482315112540195, "percentage": 61.56, "elapsed_time": "0:53:41", "remaining_time": "0:33:31", "throughput": "1103.35", "total_tokens": 3554464}
+{"current_steps": 238, "total_steps": 385, "loss": 0.0175, "learning_rate": 1.9833333333333335e-06, "epoch": 3.0610932475884245, "percentage": 61.82, "elapsed_time": "0:53:55", "remaining_time": "0:33:18", "throughput": "1103.44", "total_tokens": 3569664}
+{"current_steps": 239, "total_steps": 385, "loss": 0.0497, "learning_rate": 1.991666666666667e-06, "epoch": 3.0739549839228295, "percentage": 62.08, "elapsed_time": "0:54:08", "remaining_time": "0:33:04", "throughput": "1103.62", "total_tokens": 3585248}
+{"current_steps": 240, "total_steps": 385, "loss": 0.0361, "learning_rate": 2.0000000000000003e-06, "epoch": 3.0868167202572345, "percentage": 62.34, "elapsed_time": "0:54:22", "remaining_time": "0:32:50", "throughput": "1103.61", "total_tokens": 3600160}
+{"current_steps": 241, "total_steps": 385, "loss": 0.0193, "learning_rate": 2.0083333333333337e-06, "epoch": 3.09967845659164, "percentage": 62.6, "elapsed_time": "0:54:35", "remaining_time": "0:32:37", "throughput": "1103.63", "total_tokens": 3615232}
+{"current_steps": 242, "total_steps": 385, "loss": 0.0142, "learning_rate": 2.0166666666666667e-06, "epoch": 3.112540192926045, "percentage": 62.86, "elapsed_time": "0:54:49", "remaining_time": "0:32:23", "throughput": "1103.37", "total_tokens": 3629312}
+{"current_steps": 243, "total_steps": 385, "loss": 0.0415, "learning_rate": 2.025e-06, "epoch": 3.12540192926045, "percentage": 63.12, "elapsed_time": "0:55:02", "remaining_time": "0:32:10", "throughput": "1103.30", "total_tokens": 3644064}
+{"current_steps": 244, "total_steps": 385, "loss": 0.0178, "learning_rate": 2.0333333333333335e-06, "epoch": 3.1382636655948555, "percentage": 63.38, "elapsed_time": "0:55:16", "remaining_time": "0:31:56", "throughput": "1103.37", "total_tokens": 3659328}
+{"current_steps": 245, "total_steps": 385, "loss": 0.0166, "learning_rate": 2.041666666666667e-06, "epoch": 3.1511254019292605, "percentage": 63.64, "elapsed_time": "0:55:30", "remaining_time": "0:31:42", "throughput": "1103.51", "total_tokens": 3674720}
+{"current_steps": 246, "total_steps": 385, "loss": 0.0424, "learning_rate": 2.05e-06, "epoch": 3.1639871382636655, "percentage": 63.9, "elapsed_time": "0:55:43", "remaining_time": "0:31:29", "throughput": "1103.59", "total_tokens": 3689984}
+{"current_steps": 247, "total_steps": 385, "loss": 0.0464, "learning_rate": 2.0583333333333337e-06, "epoch": 3.176848874598071, "percentage": 64.16, "elapsed_time": "0:55:57", "remaining_time": "0:31:15", "throughput": "1103.67", "total_tokens": 3705216}
+{"current_steps": 248, "total_steps": 385, "loss": 0.0235, "learning_rate": 2.0666666666666666e-06, "epoch": 3.189710610932476, "percentage": 64.42, "elapsed_time": "0:56:10", "remaining_time": "0:31:02", "throughput": "1103.53", "total_tokens": 3719712}
+{"current_steps": 249, "total_steps": 385, "loss": 0.0128, "learning_rate": 2.075e-06, "epoch": 3.202572347266881, "percentage": 64.68, "elapsed_time": "0:56:24", "remaining_time": "0:30:48", "throughput": "1103.41", "total_tokens": 3734272}
+{"current_steps": 250, "total_steps": 385, "loss": 0.0319, "learning_rate": 2.0833333333333334e-06, "epoch": 3.215434083601286, "percentage": 64.94, "elapsed_time": "0:56:37", "remaining_time": "0:30:34", "throughput": "1103.70", "total_tokens": 3750240}
+{"current_steps": 251, "total_steps": 385, "loss": 0.0196, "learning_rate": 2.091666666666667e-06, "epoch": 3.2282958199356915, "percentage": 65.19, "elapsed_time": "0:56:51", "remaining_time": "0:30:21", "throughput": "1103.57", "total_tokens": 3764736}
+{"current_steps": 252, "total_steps": 385, "loss": 0.0326, "learning_rate": 2.1000000000000002e-06, "epoch": 3.2411575562700965, "percentage": 65.45, "elapsed_time": "0:57:04", "remaining_time": "0:30:07", "throughput": "1103.58", "total_tokens": 3779744}
+{"current_steps": 253, "total_steps": 385, "loss": 0.017, "learning_rate": 2.1083333333333336e-06, "epoch": 3.2540192926045015, "percentage": 65.71, "elapsed_time": "0:57:18", "remaining_time": "0:29:54", "throughput": "1103.43", "total_tokens": 3794208}
+{"current_steps": 254, "total_steps": 385, "loss": 0.0372, "learning_rate": 2.116666666666667e-06, "epoch": 3.266881028938907, "percentage": 65.97, "elapsed_time": "0:57:32", "remaining_time": "0:29:40", "throughput": "1103.48", "total_tokens": 3809376}
+{"current_steps": 255, "total_steps": 385, "loss": 0.0165, "learning_rate": 2.125e-06, "epoch": 3.279742765273312, "percentage": 66.23, "elapsed_time": "0:57:45", "remaining_time": "0:29:26", "throughput": "1103.48", "total_tokens": 3824384}
+{"current_steps": 256, "total_steps": 385, "loss": 0.0142, "learning_rate": 2.133333333333334e-06, "epoch": 3.292604501607717, "percentage": 66.49, "elapsed_time": "0:57:59", "remaining_time": "0:29:13", "throughput": "1103.67", "total_tokens": 3840000}
+{"current_steps": 257, "total_steps": 385, "loss": 0.0117, "learning_rate": 2.1416666666666668e-06, "epoch": 3.305466237942122, "percentage": 66.75, "elapsed_time": "0:58:12", "remaining_time": "0:28:59", "throughput": "1103.45", "total_tokens": 3854240}
+{"current_steps": 258, "total_steps": 385, "loss": 0.0264, "learning_rate": 2.15e-06, "epoch": 3.3183279742765275, "percentage": 67.01, "elapsed_time": "0:58:26", "remaining_time": "0:28:46", "throughput": "1103.60", "total_tokens": 3869792}
+{"current_steps": 259, "total_steps": 385, "loss": 0.034, "learning_rate": 2.1583333333333336e-06, "epoch": 3.3311897106109325, "percentage": 67.27, "elapsed_time": "0:58:40", "remaining_time": "0:28:32", "throughput": "1103.69", "total_tokens": 3885152}
+{"current_steps": 260, "total_steps": 385, "loss": 0.031, "learning_rate": 2.166666666666667e-06, "epoch": 3.3440514469453375, "percentage": 67.53, "elapsed_time": "0:58:53", "remaining_time": "0:28:18", "throughput": "1103.68", "total_tokens": 3900032}
+{"current_steps": 261, "total_steps": 385, "loss": 0.018, "learning_rate": 2.1750000000000004e-06, "epoch": 3.356913183279743, "percentage": 67.79, "elapsed_time": "0:59:07", "remaining_time": "0:28:05", "throughput": "1103.52", "total_tokens": 3914432}
+{"current_steps": 262, "total_steps": 385, "loss": 0.0461, "learning_rate": 2.1833333333333333e-06, "epoch": 3.369774919614148, "percentage": 68.05, "elapsed_time": "0:59:20", "remaining_time": "0:27:51", "throughput": "1103.57", "total_tokens": 3929600}
+{"current_steps": 263, "total_steps": 385, "loss": 0.0372, "learning_rate": 2.191666666666667e-06, "epoch": 3.382636655948553, "percentage": 68.31, "elapsed_time": "0:59:34", "remaining_time": "0:27:38", "throughput": "1103.44", "total_tokens": 3944032}
+{"current_steps": 264, "total_steps": 385, "loss": 0.0471, "learning_rate": 2.2e-06, "epoch": 3.395498392282958, "percentage": 68.57, "elapsed_time": "0:59:47", "remaining_time": "0:27:24", "throughput": "1103.29", "total_tokens": 3958464}
+{"current_steps": 265, "total_steps": 385, "loss": 0.037, "learning_rate": 2.2083333333333335e-06, "epoch": 3.4083601286173635, "percentage": 68.83, "elapsed_time": "1:00:01", "remaining_time": "0:27:10", "throughput": "1103.17", "total_tokens": 3973024}
+{"current_steps": 266, "total_steps": 385, "loss": 0.0245, "learning_rate": 2.216666666666667e-06, "epoch": 3.4212218649517685, "percentage": 69.09, "elapsed_time": "1:00:15", "remaining_time": "0:26:57", "throughput": "1103.04", "total_tokens": 3987520}
+{"current_steps": 267, "total_steps": 385, "loss": 0.0233, "learning_rate": 2.2250000000000003e-06, "epoch": 3.4340836012861735, "percentage": 69.35, "elapsed_time": "1:00:28", "remaining_time": "0:26:43", "throughput": "1103.10", "total_tokens": 4002656}
+{"current_steps": 268, "total_steps": 385, "loss": 0.0256, "learning_rate": 2.2333333333333333e-06, "epoch": 3.446945337620579, "percentage": 69.61, "elapsed_time": "1:00:42", "remaining_time": "0:26:30", "throughput": "1103.15", "total_tokens": 4017824}
+{"current_steps": 269, "total_steps": 385, "loss": 0.0345, "learning_rate": 2.2416666666666667e-06, "epoch": 3.459807073954984, "percentage": 69.87, "elapsed_time": "1:00:55", "remaining_time": "0:26:16", "throughput": "1103.06", "total_tokens": 4032480}
+{"current_steps": 270, "total_steps": 385, "loss": 0.0656, "learning_rate": 2.25e-06, "epoch": 3.472668810289389, "percentage": 70.13, "elapsed_time": "1:01:09", "remaining_time": "0:26:02", "throughput": "1103.11", "total_tokens": 4047648}
+{"current_steps": 271, "total_steps": 385, "loss": 0.0247, "learning_rate": 2.2583333333333335e-06, "epoch": 3.485530546623794, "percentage": 70.39, "elapsed_time": "1:01:22", "remaining_time": "0:25:49", "throughput": "1103.12", "total_tokens": 4062688}
+{"current_steps": 272, "total_steps": 385, "loss": 0.051, "learning_rate": 2.266666666666667e-06, "epoch": 3.4983922829581995, "percentage": 70.65, "elapsed_time": "1:01:36", "remaining_time": "0:25:35", "throughput": "1103.18", "total_tokens": 4077856}
+{"current_steps": 273, "total_steps": 385, "loss": 0.0417, "learning_rate": 2.2750000000000002e-06, "epoch": 3.5112540192926045, "percentage": 70.91, "elapsed_time": "1:01:50", "remaining_time": "0:25:22", "throughput": "1103.19", "total_tokens": 4092864}
+{"current_steps": 274, "total_steps": 385, "loss": 0.0176, "learning_rate": 2.2833333333333336e-06, "epoch": 3.5241157556270095, "percentage": 71.17, "elapsed_time": "1:02:03", "remaining_time": "0:25:08", "throughput": "1103.12", "total_tokens": 4107584}
+{"current_steps": 275, "total_steps": 385, "loss": 0.0155, "learning_rate": 2.2916666666666666e-06, "epoch": 3.536977491961415, "percentage": 71.43, "elapsed_time": "1:02:17", "remaining_time": "0:24:54", "throughput": "1102.99", "total_tokens": 4122080}
+{"current_steps": 276, "total_steps": 385, "loss": 0.0135, "learning_rate": 2.3000000000000004e-06, "epoch": 3.54983922829582, "percentage": 71.69, "elapsed_time": "1:02:30", "remaining_time": "0:24:41", "throughput": "1103.13", "total_tokens": 4137600}
+{"current_steps": 277, "total_steps": 385, "loss": 0.0222, "learning_rate": 2.3083333333333334e-06, "epoch": 3.562700964630225, "percentage": 71.95, "elapsed_time": "1:02:44", "remaining_time": "0:24:27", "throughput": "1103.23", "total_tokens": 4152960}
+{"current_steps": 278, "total_steps": 385, "loss": 0.0289, "learning_rate": 2.316666666666667e-06, "epoch": 3.57556270096463, "percentage": 72.21, "elapsed_time": "1:02:57", "remaining_time": "0:24:14", "throughput": "1103.41", "total_tokens": 4168640}
+{"current_steps": 279, "total_steps": 385, "loss": 0.0314, "learning_rate": 2.325e-06, "epoch": 3.5884244372990355, "percentage": 72.47, "elapsed_time": "1:03:11", "remaining_time": "0:24:00", "throughput": "1103.39", "total_tokens": 4183616}
+{"current_steps": 280, "total_steps": 385, "loss": 0.0582, "learning_rate": 2.3333333333333336e-06, "epoch": 3.6012861736334405, "percentage": 72.73, "elapsed_time": "1:03:25", "remaining_time": "0:23:46", "throughput": "1103.12", "total_tokens": 4197536}
+{"current_steps": 281, "total_steps": 385, "loss": 0.0339, "learning_rate": 2.341666666666667e-06, "epoch": 3.6141479099678455, "percentage": 72.99, "elapsed_time": "1:03:38", "remaining_time": "0:23:33", "throughput": "1103.28", "total_tokens": 4213152}
+{"current_steps": 282, "total_steps": 385, "loss": 0.0656, "learning_rate": 2.35e-06, "epoch": 3.627009646302251, "percentage": 73.25, "elapsed_time": "1:03:52", "remaining_time": "0:23:19", "throughput": "1103.32", "total_tokens": 4228320}
+{"current_steps": 283, "total_steps": 385, "loss": 0.0355, "learning_rate": 2.3583333333333338e-06, "epoch": 3.639871382636656, "percentage": 73.51, "elapsed_time": "1:04:05", "remaining_time": "0:23:06", "throughput": "1103.41", "total_tokens": 4243680}
+{"current_steps": 284, "total_steps": 385, "loss": 0.037, "learning_rate": 2.3666666666666667e-06, "epoch": 3.652733118971061, "percentage": 73.77, "elapsed_time": "1:04:19", "remaining_time": "0:22:52", "throughput": "1103.36", "total_tokens": 4258464}
+{"current_steps": 285, "total_steps": 385, "loss": 0.0206, "learning_rate": 2.375e-06, "epoch": 3.665594855305466, "percentage": 74.03, "elapsed_time": "1:04:33", "remaining_time": "0:22:38", "throughput": "1103.45", "total_tokens": 4273728}
+{"current_steps": 286, "total_steps": 385, "loss": 0.0286, "learning_rate": 2.3833333333333335e-06, "epoch": 3.6784565916398715, "percentage": 74.29, "elapsed_time": "1:04:46", "remaining_time": "0:22:25", "throughput": "1103.36", "total_tokens": 4288416}
+{"current_steps": 287, "total_steps": 385, "loss": 0.0316, "learning_rate": 2.391666666666667e-06, "epoch": 3.6913183279742765, "percentage": 74.55, "elapsed_time": "1:05:00", "remaining_time": "0:22:11", "throughput": "1103.44", "total_tokens": 4303712}
+{"current_steps": 288, "total_steps": 385, "loss": 0.0263, "learning_rate": 2.4000000000000003e-06, "epoch": 3.7041800643086815, "percentage": 74.81, "elapsed_time": "1:05:13", "remaining_time": "0:21:58", "throughput": "1103.55", "total_tokens": 4319104}
+{"current_steps": 289, "total_steps": 385, "loss": 0.0265, "learning_rate": 2.4083333333333337e-06, "epoch": 3.717041800643087, "percentage": 75.06, "elapsed_time": "1:05:27", "remaining_time": "0:21:44", "throughput": "1103.75", "total_tokens": 4334912}
+{"current_steps": 290, "total_steps": 385, "loss": 0.0251, "learning_rate": 2.4166666666666667e-06, "epoch": 3.729903536977492, "percentage": 75.32, "elapsed_time": "1:05:40", "remaining_time": "0:21:31", "throughput": "1103.60", "total_tokens": 4349280}
+{"current_steps": 291, "total_steps": 385, "loss": 0.0293, "learning_rate": 2.425e-06, "epoch": 3.742765273311897, "percentage": 75.58, "elapsed_time": "1:05:54", "remaining_time": "0:21:17", "throughput": "1103.51", "total_tokens": 4363872}
+{"current_steps": 292, "total_steps": 385, "loss": 0.0278, "learning_rate": 2.4333333333333335e-06, "epoch": 3.755627009646302, "percentage": 75.84, "elapsed_time": "1:06:08", "remaining_time": "0:21:03", "throughput": "1103.50", "total_tokens": 4378752}
+{"current_steps": 293, "total_steps": 385, "loss": 0.035, "learning_rate": 2.441666666666667e-06, "epoch": 3.7684887459807075, "percentage": 76.1, "elapsed_time": "1:06:21", "remaining_time": "0:20:50", "throughput": "1103.52", "total_tokens": 4393792}
+{"current_steps": 294, "total_steps": 385, "loss": 0.0455, "learning_rate": 2.4500000000000003e-06, "epoch": 3.7813504823151125, "percentage": 76.36, "elapsed_time": "1:06:35", "remaining_time": "0:20:36", "throughput": "1103.38", "total_tokens": 4408192}
+{"current_steps": 295, "total_steps": 385, "loss": 0.0429, "learning_rate": 2.4583333333333332e-06, "epoch": 3.7942122186495175, "percentage": 76.62, "elapsed_time": "1:06:48", "remaining_time": "0:20:23", "throughput": "1103.20", "total_tokens": 4422432}
+{"current_steps": 296, "total_steps": 385, "loss": 0.056, "learning_rate": 2.466666666666667e-06, "epoch": 3.807073954983923, "percentage": 76.88, "elapsed_time": "1:07:02", "remaining_time": "0:20:09", "throughput": "1103.28", "total_tokens": 4437728}
+{"current_steps": 297, "total_steps": 385, "loss": 0.0325, "learning_rate": 2.475e-06, "epoch": 3.819935691318328, "percentage": 77.14, "elapsed_time": "1:07:15", "remaining_time": "0:19:55", "throughput": "1103.35", "total_tokens": 4452928}
+{"current_steps": 298, "total_steps": 385, "loss": 0.0401, "learning_rate": 2.4833333333333334e-06, "epoch": 3.832797427652733, "percentage": 77.4, "elapsed_time": "1:07:29", "remaining_time": "0:19:42", "throughput": "1103.39", "total_tokens": 4468096}
+{"current_steps": 299, "total_steps": 385, "loss": 0.0182, "learning_rate": 2.491666666666667e-06, "epoch": 3.845659163987138, "percentage": 77.66, "elapsed_time": "1:07:42", "remaining_time": "0:19:28", "throughput": "1103.37", "total_tokens": 4482976}
+{"current_steps": 300, "total_steps": 385, "loss": 0.0344, "learning_rate": 2.5e-06, "epoch": 3.8585209003215435, "percentage": 77.92, "elapsed_time": "1:07:56", "remaining_time": "0:19:15", "throughput": "1103.37", "total_tokens": 4497952}
+{"current_steps": 301, "total_steps": 385, "loss": 0.0111, "learning_rate": 2.5083333333333336e-06, "epoch": 3.8713826366559485, "percentage": 78.18, "elapsed_time": "1:08:10", "remaining_time": "0:19:01", "throughput": "1103.37", "total_tokens": 4512896}
+{"current_steps": 302, "total_steps": 385, "loss": 0.0088, "learning_rate": 2.5166666666666666e-06, "epoch": 3.884244372990354, "percentage": 78.44, "elapsed_time": "1:08:23", "remaining_time": "0:18:47", "throughput": "1103.52", "total_tokens": 4528448}
+{"current_steps": 303, "total_steps": 385, "loss": 0.013, "learning_rate": 2.5250000000000004e-06, "epoch": 3.897106109324759, "percentage": 78.7, "elapsed_time": "1:08:37", "remaining_time": "0:18:34", "throughput": "1103.56", "total_tokens": 4543712}
+{"current_steps": 304, "total_steps": 385, "loss": 0.0433, "learning_rate": 2.5333333333333338e-06, "epoch": 3.909967845659164, "percentage": 78.96, "elapsed_time": "1:08:50", "remaining_time": "0:18:20", "throughput": "1103.64", "total_tokens": 4559008}
+{"current_steps": 305, "total_steps": 385, "loss": 0.0214, "learning_rate": 2.5416666666666668e-06, "epoch": 3.922829581993569, "percentage": 79.22, "elapsed_time": "1:09:04", "remaining_time": "0:18:07", "throughput": "1103.57", "total_tokens": 4573696}
+{"current_steps": 306, "total_steps": 385, "loss": 0.0416, "learning_rate": 2.55e-06, "epoch": 3.935691318327974, "percentage": 79.48, "elapsed_time": "1:09:18", "remaining_time": "0:17:53", "throughput": "1103.50", "total_tokens": 4588384}
+{"current_steps": 307, "total_steps": 385, "loss": 0.0321, "learning_rate": 2.558333333333334e-06, "epoch": 3.9485530546623795, "percentage": 79.74, "elapsed_time": "1:09:31", "remaining_time": "0:17:39", "throughput": "1103.41", "total_tokens": 4602944}
+{"current_steps": 308, "total_steps": 385, "loss": 0.0526, "learning_rate": 2.566666666666667e-06, "epoch": 3.9614147909967845, "percentage": 80.0, "elapsed_time": "1:09:45", "remaining_time": "0:17:26", "throughput": "1103.50", "total_tokens": 4618336}
+{"current_steps": 309, "total_steps": 385, "loss": 0.025, "learning_rate": 2.5750000000000003e-06, "epoch": 3.97427652733119, "percentage": 80.26, "elapsed_time": "1:09:58", "remaining_time": "0:17:12", "throughput": "1103.53", "total_tokens": 4633472}
+{"current_steps": 310, "total_steps": 385, "loss": 0.0787, "learning_rate": 2.5833333333333337e-06, "epoch": 3.987138263665595, "percentage": 80.52, "elapsed_time": "1:10:12", "remaining_time": "0:16:59", "throughput": "1103.77", "total_tokens": 4649472}
+{"current_steps": 311, "total_steps": 385, "loss": 0.0559, "learning_rate": 2.5916666666666667e-06, "epoch": 4.0, "percentage": 80.78, "elapsed_time": "1:10:25", "remaining_time": "0:16:45", "throughput": "1103.76", "total_tokens": 4664384}
+{"current_steps": 312, "total_steps": 385, "loss": 0.0259, "learning_rate": 2.6e-06, "epoch": 4.012861736334405, "percentage": 81.04, "elapsed_time": "1:10:39", "remaining_time": "0:16:31", "throughput": "1103.73", "total_tokens": 4679200}
+{"current_steps": 313, "total_steps": 385, "loss": 0.037, "learning_rate": 2.608333333333333e-06, "epoch": 4.02572347266881, "percentage": 81.3, "elapsed_time": "1:10:53", "remaining_time": "0:16:18", "throughput": "1103.60", "total_tokens": 4693696}
+{"current_steps": 314, "total_steps": 385, "loss": 0.0293, "learning_rate": 2.616666666666667e-06, "epoch": 4.038585209003215, "percentage": 81.56, "elapsed_time": "1:11:06", "remaining_time": "0:16:04", "throughput": "1103.59", "total_tokens": 4708608}
+{"current_steps": 315, "total_steps": 385, "loss": 0.0153, "learning_rate": 2.6250000000000003e-06, "epoch": 4.051446945337621, "percentage": 81.82, "elapsed_time": "1:11:20", "remaining_time": "0:15:51", "throughput": "1103.45", "total_tokens": 4722944}
+{"current_steps": 316, "total_steps": 385, "loss": 0.0234, "learning_rate": 2.6333333333333332e-06, "epoch": 4.064308681672026, "percentage": 82.08, "elapsed_time": "1:11:33", "remaining_time": "0:15:37", "throughput": "1103.75", "total_tokens": 4739296}
+{"current_steps": 317, "total_steps": 385, "loss": 0.0212, "learning_rate": 2.6416666666666666e-06, "epoch": 4.077170418006431, "percentage": 82.34, "elapsed_time": "1:11:47", "remaining_time": "0:15:23", "throughput": "1103.71", "total_tokens": 4754048}
+{"current_steps": 318, "total_steps": 385, "loss": 0.0187, "learning_rate": 2.6500000000000005e-06, "epoch": 4.090032154340836, "percentage": 82.6, "elapsed_time": "1:12:00", "remaining_time": "0:15:10", "throughput": "1103.79", "total_tokens": 4769376}
+{"current_steps": 319, "total_steps": 385, "loss": 0.0245, "learning_rate": 2.6583333333333334e-06, "epoch": 4.102893890675241, "percentage": 82.86, "elapsed_time": "1:12:14", "remaining_time": "0:14:56", "throughput": "1103.86", "total_tokens": 4784640}
+{"current_steps": 320, "total_steps": 385, "loss": 0.0032, "learning_rate": 2.666666666666667e-06, "epoch": 4.115755627009646, "percentage": 83.12, "elapsed_time": "1:12:28", "remaining_time": "0:14:43", "throughput": "1103.80", "total_tokens": 4799392}
+{"current_steps": 321, "total_steps": 385, "loss": 0.0367, "learning_rate": 2.6750000000000002e-06, "epoch": 4.128617363344051, "percentage": 83.38, "elapsed_time": "1:12:41", "remaining_time": "0:14:29", "throughput": "1103.65", "total_tokens": 4813696}
+{"current_steps": 322, "total_steps": 385, "loss": 0.038, "learning_rate": 2.683333333333333e-06, "epoch": 4.141479099678457, "percentage": 83.64, "elapsed_time": "1:12:55", "remaining_time": "0:14:16", "throughput": "1103.71", "total_tokens": 4828928}
+{"current_steps": 323, "total_steps": 385, "loss": 0.0194, "learning_rate": 2.691666666666667e-06, "epoch": 4.154340836012862, "percentage": 83.9, "elapsed_time": "1:13:08", "remaining_time": "0:14:02", "throughput": "1103.55", "total_tokens": 4843232}
+{"current_steps": 324, "total_steps": 385, "loss": 0.0262, "learning_rate": 2.7000000000000004e-06, "epoch": 4.167202572347267, "percentage": 84.16, "elapsed_time": "1:13:22", "remaining_time": "0:13:48", "throughput": "1103.64", "total_tokens": 4858592}
+{"current_steps": 325, "total_steps": 385, "loss": 0.0151, "learning_rate": 2.7083333333333334e-06, "epoch": 4.180064308681672, "percentage": 84.42, "elapsed_time": "1:13:35", "remaining_time": "0:13:35", "throughput": "1103.50", "total_tokens": 4872928}
+{"current_steps": 326, "total_steps": 385, "loss": 0.0267, "learning_rate": 2.7166666666666668e-06, "epoch": 4.192926045016077, "percentage": 84.68, "elapsed_time": "1:13:49", "remaining_time": "0:13:21", "throughput": "1103.52", "total_tokens": 4887968}
+{"current_steps": 327, "total_steps": 385, "loss": 0.0041, "learning_rate": 2.7250000000000006e-06, "epoch": 4.205787781350482, "percentage": 84.94, "elapsed_time": "1:14:02", "remaining_time": "0:13:08", "throughput": "1103.49", "total_tokens": 4902752}
+{"current_steps": 328, "total_steps": 385, "loss": 0.0365, "learning_rate": 2.7333333333333336e-06, "epoch": 4.218649517684887, "percentage": 85.19, "elapsed_time": "1:14:16", "remaining_time": "0:12:54", "throughput": "1103.40", "total_tokens": 4917280}
+{"current_steps": 329, "total_steps": 385, "loss": 0.0151, "learning_rate": 2.741666666666667e-06, "epoch": 4.231511254019293, "percentage": 85.45, "elapsed_time": "1:14:30", "remaining_time": "0:12:40", "throughput": "1103.57", "total_tokens": 4933088}
+{"current_steps": 330, "total_steps": 385, "loss": 0.006, "learning_rate": 2.7500000000000004e-06, "epoch": 4.244372990353698, "percentage": 85.71, "elapsed_time": "1:14:43", "remaining_time": "0:12:27", "throughput": "1103.59", "total_tokens": 4948128}
+{"current_steps": 331, "total_steps": 385, "loss": 0.0247, "learning_rate": 2.7583333333333333e-06, "epoch": 4.257234726688103, "percentage": 85.97, "elapsed_time": "1:14:57", "remaining_time": "0:12:13", "throughput": "1103.51", "total_tokens": 4962752}
+{"current_steps": 332, "total_steps": 385, "loss": 0.0147, "learning_rate": 2.766666666666667e-06, "epoch": 4.270096463022508, "percentage": 86.23, "elapsed_time": "1:15:10", "remaining_time": "0:12:00", "throughput": "1103.61", "total_tokens": 4978272}
+{"current_steps": 333, "total_steps": 385, "loss": 0.0413, "learning_rate": 2.7750000000000005e-06, "epoch": 4.282958199356913, "percentage": 86.49, "elapsed_time": "1:15:24", "remaining_time": "0:11:46", "throughput": "1103.45", "total_tokens": 4992384}
+{"current_steps": 334, "total_steps": 385, "loss": 0.0318, "learning_rate": 2.7833333333333335e-06, "epoch": 4.295819935691318, "percentage": 86.75, "elapsed_time": "1:15:37", "remaining_time": "0:11:32", "throughput": "1103.40", "total_tokens": 5007136}
+{"current_steps": 335, "total_steps": 385, "loss": 0.0513, "learning_rate": 2.791666666666667e-06, "epoch": 4.308681672025724, "percentage": 87.01, "elapsed_time": "1:15:51", "remaining_time": "0:11:19", "throughput": "1103.30", "total_tokens": 5021600}
+{"current_steps": 336, "total_steps": 385, "loss": 0.0165, "learning_rate": 2.8000000000000003e-06, "epoch": 4.321543408360129, "percentage": 87.27, "elapsed_time": "1:16:04", "remaining_time": "0:11:05", "throughput": "1103.44", "total_tokens": 5037216}
+{"current_steps": 337, "total_steps": 385, "loss": 0.0131, "learning_rate": 2.8083333333333333e-06, "epoch": 4.334405144694534, "percentage": 87.53, "elapsed_time": "1:16:18", "remaining_time": "0:10:52", "throughput": "1103.48", "total_tokens": 5052320}
+{"current_steps": 338, "total_steps": 385, "loss": 0.0246, "learning_rate": 2.816666666666667e-06, "epoch": 4.347266881028939, "percentage": 87.79, "elapsed_time": "1:16:32", "remaining_time": "0:10:38", "throughput": "1103.52", "total_tokens": 5067552}
+{"current_steps": 339, "total_steps": 385, "loss": 0.0465, "learning_rate": 2.825e-06, "epoch": 4.360128617363344, "percentage": 88.05, "elapsed_time": "1:16:45", "remaining_time": "0:10:24", "throughput": "1103.46", "total_tokens": 5082240}
+{"current_steps": 340, "total_steps": 385, "loss": 0.0084, "learning_rate": 2.8333333333333335e-06, "epoch": 4.372990353697749, "percentage": 88.31, "elapsed_time": "1:16:59", "remaining_time": "0:10:11", "throughput": "1103.49", "total_tokens": 5097408}
+{"current_steps": 341, "total_steps": 385, "loss": 0.0105, "learning_rate": 2.841666666666667e-06, "epoch": 4.385852090032154, "percentage": 88.57, "elapsed_time": "1:17:12", "remaining_time": "0:09:57", "throughput": "1103.35", "total_tokens": 5111680}
+{"current_steps": 342, "total_steps": 385, "loss": 0.0388, "learning_rate": 2.85e-06, "epoch": 4.39871382636656, "percentage": 88.83, "elapsed_time": "1:17:26", "remaining_time": "0:09:44", "throughput": "1103.48", "total_tokens": 5127264}
+{"current_steps": 343, "total_steps": 385, "loss": 0.0486, "learning_rate": 2.8583333333333336e-06, "epoch": 4.411575562700965, "percentage": 89.09, "elapsed_time": "1:17:39", "remaining_time": "0:09:30", "throughput": "1103.54", "total_tokens": 5142464}
+{"current_steps": 344, "total_steps": 385, "loss": 0.0354, "learning_rate": 2.866666666666667e-06, "epoch": 4.42443729903537, "percentage": 89.35, "elapsed_time": "1:17:53", "remaining_time": "0:09:17", "throughput": "1103.53", "total_tokens": 5157376}
+{"current_steps": 345, "total_steps": 385, "loss": 0.0464, "learning_rate": 2.875e-06, "epoch": 4.437299035369775, "percentage": 89.61, "elapsed_time": "1:18:07", "remaining_time": "0:09:03", "throughput": "1103.38", "total_tokens": 5171648}
+{"current_steps": 346, "total_steps": 385, "loss": 0.0312, "learning_rate": 2.8833333333333334e-06, "epoch": 4.45016077170418, "percentage": 89.87, "elapsed_time": "1:18:20", "remaining_time": "0:08:49", "throughput": "1103.41", "total_tokens": 5186752}
+{"current_steps": 347, "total_steps": 385, "loss": 0.0352, "learning_rate": 2.8916666666666672e-06, "epoch": 4.463022508038585, "percentage": 90.13, "elapsed_time": "1:18:34", "remaining_time": "0:08:36", "throughput": "1103.50", "total_tokens": 5202176}
+{"current_steps": 348, "total_steps": 385, "loss": 0.0116, "learning_rate": 2.9e-06, "epoch": 4.47588424437299, "percentage": 90.39, "elapsed_time": "1:18:47", "remaining_time": "0:08:22", "throughput": "1103.46", "total_tokens": 5216896}
+{"current_steps": 349, "total_steps": 385, "loss": 0.0118, "learning_rate": 2.9083333333333336e-06, "epoch": 4.488745980707396, "percentage": 90.65, "elapsed_time": "1:19:01", "remaining_time": "0:08:09", "throughput": "1103.57", "total_tokens": 5232448}
+{"current_steps": 350, "total_steps": 385, "loss": 0.0138, "learning_rate": 2.916666666666667e-06, "epoch": 4.501607717041801, "percentage": 90.91, "elapsed_time": "1:19:14", "remaining_time": "0:07:55", "throughput": "1103.54", "total_tokens": 5247296}
+{"current_steps": 351, "total_steps": 385, "loss": 0.0344, "learning_rate": 2.925e-06, "epoch": 4.514469453376206, "percentage": 91.17, "elapsed_time": "1:19:28", "remaining_time": "0:07:41", "throughput": "1103.54", "total_tokens": 5262304}
+{"current_steps": 352, "total_steps": 385, "loss": 0.0231, "learning_rate": 2.9333333333333338e-06, "epoch": 4.527331189710611, "percentage": 91.43, "elapsed_time": "1:19:42", "remaining_time": "0:07:28", "throughput": "1103.62", "total_tokens": 5277664}
+{"current_steps": 353, "total_steps": 385, "loss": 0.0267, "learning_rate": 2.941666666666667e-06, "epoch": 4.540192926045016, "percentage": 91.69, "elapsed_time": "1:19:55", "remaining_time": "0:07:14", "throughput": "1103.62", "total_tokens": 5292576}
+{"current_steps": 354, "total_steps": 385, "loss": 0.0109, "learning_rate": 2.95e-06, "epoch": 4.553054662379421, "percentage": 91.95, "elapsed_time": "1:20:09", "remaining_time": "0:07:01", "throughput": "1103.38", "total_tokens": 5306304}
+{"current_steps": 355, "total_steps": 385, "loss": 0.0444, "learning_rate": 2.9583333333333335e-06, "epoch": 4.565916398713826, "percentage": 92.21, "elapsed_time": "1:20:22", "remaining_time": "0:06:47", "throughput": "1103.26", "total_tokens": 5320768}
+{"current_steps": 356, "total_steps": 385, "loss": 0.0142, "learning_rate": 2.9666666666666673e-06, "epoch": 4.578778135048232, "percentage": 92.47, "elapsed_time": "1:20:36", "remaining_time": "0:06:33", "throughput": "1103.35", "total_tokens": 5336128}
+{"current_steps": 357, "total_steps": 385, "loss": 0.0361, "learning_rate": 2.9750000000000003e-06, "epoch": 4.591639871382637, "percentage": 92.73, "elapsed_time": "1:20:49", "remaining_time": "0:06:20", "throughput": "1103.37", "total_tokens": 5351168}
+{"current_steps": 358, "total_steps": 385, "loss": 0.0197, "learning_rate": 2.9833333333333337e-06, "epoch": 4.604501607717042, "percentage": 92.99, "elapsed_time": "1:21:03", "remaining_time": "0:06:06", "throughput": "1103.36", "total_tokens": 5366112}
+{"current_steps": 359, "total_steps": 385, "loss": 0.0373, "learning_rate": 2.991666666666667e-06, "epoch": 4.617363344051447, "percentage": 93.25, "elapsed_time": "1:21:16", "remaining_time": "0:05:53", "throughput": "1103.44", "total_tokens": 5381440}
+{"current_steps": 360, "total_steps": 385, "loss": 0.0469, "learning_rate": 3e-06, "epoch": 4.630225080385852, "percentage": 93.51, "elapsed_time": "1:21:30", "remaining_time": "0:05:39", "throughput": "1103.59", "total_tokens": 5397152}
+{"current_steps": 361, "total_steps": 385, "loss": 0.0283, "learning_rate": 3.0083333333333335e-06, "epoch": 4.643086816720257, "percentage": 93.77, "elapsed_time": "1:21:44", "remaining_time": "0:05:26", "throughput": "1103.61", "total_tokens": 5412128}
+{"current_steps": 362, "total_steps": 385, "loss": 0.0271, "learning_rate": 3.0166666666666673e-06, "epoch": 4.655948553054662, "percentage": 94.03, "elapsed_time": "1:21:57", "remaining_time": "0:05:12", "throughput": "1103.55", "total_tokens": 5426816}
+{"current_steps": 363, "total_steps": 385, "loss": 0.0303, "learning_rate": 3.0250000000000003e-06, "epoch": 4.668810289389068, "percentage": 94.29, "elapsed_time": "1:22:11", "remaining_time": "0:04:58", "throughput": "1103.54", "total_tokens": 5441760}
+{"current_steps": 364, "total_steps": 385, "loss": 0.0207, "learning_rate": 3.0333333333333337e-06, "epoch": 4.681672025723473, "percentage": 94.55, "elapsed_time": "1:22:24", "remaining_time": "0:04:45", "throughput": "1103.69", "total_tokens": 5457504}
+{"current_steps": 365, "total_steps": 385, "loss": 0.0157, "learning_rate": 3.0416666666666666e-06, "epoch": 4.694533762057878, "percentage": 94.81, "elapsed_time": "1:22:38", "remaining_time": "0:04:31", "throughput": "1103.80", "total_tokens": 5473088}
+{"current_steps": 366, "total_steps": 385, "loss": 0.0128, "learning_rate": 3.05e-06, "epoch": 4.707395498392283, "percentage": 95.06, "elapsed_time": "1:22:52", "remaining_time": "0:04:18", "throughput": "1103.71", "total_tokens": 5487712}
+{"current_steps": 367, "total_steps": 385, "loss": 0.0293, "learning_rate": 3.058333333333334e-06, "epoch": 4.720257234726688, "percentage": 95.32, "elapsed_time": "1:23:05", "remaining_time": "0:04:04", "throughput": "1103.92", "total_tokens": 5503680}
+{"current_steps": 368, "total_steps": 385, "loss": 0.0211, "learning_rate": 3.066666666666667e-06, "epoch": 4.733118971061093, "percentage": 95.58, "elapsed_time": "1:23:19", "remaining_time": "0:03:50", "throughput": "1103.87", "total_tokens": 5518464}
+{"current_steps": 369, "total_steps": 385, "loss": 0.0929, "learning_rate": 3.075e-06, "epoch": 4.745980707395498, "percentage": 95.84, "elapsed_time": "1:23:32", "remaining_time": "0:03:37", "throughput": "1103.90", "total_tokens": 5533504}
+{"current_steps": 370, "total_steps": 385, "loss": 0.0195, "learning_rate": 3.0833333333333336e-06, "epoch": 4.758842443729904, "percentage": 96.1, "elapsed_time": "1:23:46", "remaining_time": "0:03:23", "throughput": "1103.97", "total_tokens": 5548832}
+{"current_steps": 371, "total_steps": 385, "loss": 0.0114, "learning_rate": 3.0916666666666666e-06, "epoch": 4.771704180064309, "percentage": 96.36, "elapsed_time": "1:23:59", "remaining_time": "0:03:10", "throughput": "1104.05", "total_tokens": 5564256}
+{"current_steps": 372, "total_steps": 385, "loss": 0.0207, "learning_rate": 3.1000000000000004e-06, "epoch": 4.784565916398714, "percentage": 96.62, "elapsed_time": "1:24:13", "remaining_time": "0:02:56", "throughput": "1104.23", "total_tokens": 5580064}
+{"current_steps": 373, "total_steps": 385, "loss": 0.0144, "learning_rate": 3.1083333333333338e-06, "epoch": 4.797427652733119, "percentage": 96.88, "elapsed_time": "1:24:26", "remaining_time": "0:02:43", "throughput": "1104.22", "total_tokens": 5595008}
+{"current_steps": 374, "total_steps": 385, "loss": 0.0197, "learning_rate": 3.1166666666666668e-06, "epoch": 4.810289389067524, "percentage": 97.14, "elapsed_time": "1:24:40", "remaining_time": "0:02:29", "throughput": "1104.27", "total_tokens": 5610304}
+{"current_steps": 375, "total_steps": 385, "loss": 0.0669, "learning_rate": 3.125e-06, "epoch": 4.823151125401929, "percentage": 97.4, "elapsed_time": "1:24:54", "remaining_time": "0:02:15", "throughput": "1104.49", "total_tokens": 5626464}
+{"current_steps": 376, "total_steps": 385, "loss": 0.0156, "learning_rate": 3.133333333333334e-06, "epoch": 4.836012861736334, "percentage": 97.66, "elapsed_time": "1:25:07", "remaining_time": "0:02:02", "throughput": "1104.45", "total_tokens": 5641216}
+{"current_steps": 377, "total_steps": 385, "loss": 0.018, "learning_rate": 3.141666666666667e-06, "epoch": 4.84887459807074, "percentage": 97.92, "elapsed_time": "1:25:21", "remaining_time": "0:01:48", "throughput": "1104.57", "total_tokens": 5656832}
+{"current_steps": 378, "total_steps": 385, "loss": 0.064, "learning_rate": 3.1500000000000003e-06, "epoch": 4.861736334405145, "percentage": 98.18, "elapsed_time": "1:25:34", "remaining_time": "0:01:35", "throughput": "1104.56", "total_tokens": 5671744}
+{"current_steps": 379, "total_steps": 385, "loss": 0.0263, "learning_rate": 3.1583333333333337e-06, "epoch": 4.87459807073955, "percentage": 98.44, "elapsed_time": "1:25:48", "remaining_time": "0:01:21", "throughput": "1104.51", "total_tokens": 5686496}
+{"current_steps": 380, "total_steps": 385, "loss": 0.0256, "learning_rate": 3.1666666666666667e-06, "epoch": 4.887459807073955, "percentage": 98.7, "elapsed_time": "1:26:02", "remaining_time": "0:01:07", "throughput": "1104.46", "total_tokens": 5701216}
+{"current_steps": 381, "total_steps": 385, "loss": 0.0269, "learning_rate": 3.175e-06, "epoch": 4.90032154340836, "percentage": 98.96, "elapsed_time": "1:26:15", "remaining_time": "0:00:54", "throughput": "1104.40", "total_tokens": 5715904}
+{"current_steps": 382, "total_steps": 385, "loss": 0.0281, "learning_rate": 3.183333333333334e-06, "epoch": 4.913183279742765, "percentage": 99.22, "elapsed_time": "1:26:29", "remaining_time": "0:00:40", "throughput": "1104.41", "total_tokens": 5730976}
+{"current_steps": 383, "total_steps": 385, "loss": 0.0199, "learning_rate": 3.191666666666667e-06, "epoch": 4.92604501607717, "percentage": 99.48, "elapsed_time": "1:26:42", "remaining_time": "0:00:27", "throughput": "1104.39", "total_tokens": 5745824}
+{"current_steps": 384, "total_steps": 385, "loss": 0.0277, "learning_rate": 3.2000000000000003e-06, "epoch": 4.938906752411576, "percentage": 99.74, "elapsed_time": "1:26:56", "remaining_time": "0:00:13", "throughput": "1104.28", "total_tokens": 5760160}
+{"current_steps": 385, "total_steps": 385, "loss": 0.0158, "learning_rate": 3.2083333333333337e-06, "epoch": 4.951768488745981, "percentage": 100.0, "elapsed_time": "1:27:09", "remaining_time": "0:00:00", "throughput": "1104.47", "total_tokens": 5776160}
+{"current_steps": 385, "total_steps": 385, "epoch": 4.951768488745981, "percentage": 100.0, "elapsed_time": "1:28:10", "remaining_time": "0:00:00", "throughput": "1091.86", "total_tokens": 5776160}

trainer_state.json

@@ -0,0 +1,3123 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 4.951768488745981,
+  "eval_steps": 500,
+  "global_step": 385,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.012861736334405145,
+      "grad_norm": 577.1012573242188,
+      "learning_rate": 8.333333333333335e-09,
+      "loss": 13.9619,
+      "num_input_tokens_seen": 15136,
+      "step": 1
+    },
+    {
+      "epoch": 0.02572347266881029,
+      "grad_norm": 552.5532836914062,
+      "learning_rate": 1.666666666666667e-08,
+      "loss": 13.7129,
+      "num_input_tokens_seen": 30208,
+      "step": 2
+    },
+    {
+      "epoch": 0.03858520900321544,
+      "grad_norm": 562.699462890625,
+      "learning_rate": 2.5000000000000002e-08,
+      "loss": 13.8474,
+      "num_input_tokens_seen": 45376,
+      "step": 3
+    },
+    {
+      "epoch": 0.05144694533762058,
+      "grad_norm": 585.796630859375,
+      "learning_rate": 3.333333333333334e-08,
+      "loss": 13.8844,
+      "num_input_tokens_seen": 59968,
+      "step": 4
+    },
+    {
+      "epoch": 0.06430868167202572,
+      "grad_norm": 592.56689453125,
+      "learning_rate": 4.166666666666667e-08,
+      "loss": 14.138,
+      "num_input_tokens_seen": 75136,
+      "step": 5
+    },
+    {
+      "epoch": 0.07717041800643087,
+      "grad_norm": 569.5825805664062,
+      "learning_rate": 5.0000000000000004e-08,
+      "loss": 13.9077,
+      "num_input_tokens_seen": 90816,
+      "step": 6
+    },
+    {
+      "epoch": 0.09003215434083602,
+      "grad_norm": 578.7579956054688,
+      "learning_rate": 5.833333333333334e-08,
+      "loss": 13.8396,
+      "num_input_tokens_seen": 105600,
+      "step": 7
+    },
+    {
+      "epoch": 0.10289389067524116,
+      "grad_norm": 577.330322265625,
+      "learning_rate": 6.666666666666668e-08,
+      "loss": 13.9828,
+      "num_input_tokens_seen": 119872,
+      "step": 8
+    },
+    {
+      "epoch": 0.1157556270096463,
+      "grad_norm": 575.2134399414062,
+      "learning_rate": 7.500000000000001e-08,
+      "loss": 14.0361,
+      "num_input_tokens_seen": 135104,
+      "step": 9
+    },
+    {
+      "epoch": 0.12861736334405144,
+      "grad_norm": 579.0157470703125,
+      "learning_rate": 8.333333333333334e-08,
+      "loss": 13.9392,
+      "num_input_tokens_seen": 150016,
+      "step": 10
+    },
+    {
+      "epoch": 0.1414790996784566,
+      "grad_norm": 585.5687255859375,
+      "learning_rate": 9.166666666666668e-08,
+      "loss": 14.0256,
+      "num_input_tokens_seen": 165216,
+      "step": 11
+    },
+    {
+      "epoch": 0.15434083601286175,
+      "grad_norm": 566.5042114257812,
+      "learning_rate": 1.0000000000000001e-07,
+      "loss": 13.6693,
+      "num_input_tokens_seen": 179296,
+      "step": 12
+    },
+    {
+      "epoch": 0.16720257234726688,
+      "grad_norm": 574.4198608398438,
+      "learning_rate": 1.0833333333333335e-07,
+      "loss": 13.9031,
+      "num_input_tokens_seen": 193312,
+      "step": 13
+    },
+    {
+      "epoch": 0.18006430868167203,
+      "grad_norm": 588.1041259765625,
+      "learning_rate": 1.1666666666666668e-07,
+      "loss": 13.8575,
+      "num_input_tokens_seen": 208768,
+      "step": 14
+    },
+    {
+      "epoch": 0.19292604501607716,
+      "grad_norm": 573.4750366210938,
+      "learning_rate": 1.2500000000000002e-07,
+      "loss": 13.8366,
+      "num_input_tokens_seen": 223456,
+      "step": 15
+    },
+    {
+      "epoch": 0.2057877813504823,
+      "grad_norm": 572.26220703125,
+      "learning_rate": 1.3333333333333336e-07,
+      "loss": 13.8705,
+      "num_input_tokens_seen": 238656,
+      "step": 16
+    },
+    {
+      "epoch": 0.21864951768488747,
+      "grad_norm": 566.3518676757812,
+      "learning_rate": 1.4166666666666668e-07,
+      "loss": 13.1816,
+      "num_input_tokens_seen": 253312,
+      "step": 17
+    },
+    {
+      "epoch": 0.2315112540192926,
+      "grad_norm": 560.5255126953125,
+      "learning_rate": 1.5000000000000002e-07,
+      "loss": 13.2292,
+      "num_input_tokens_seen": 268352,
+      "step": 18
+    },
+    {
+      "epoch": 0.24437299035369775,
+      "grad_norm": 583.0404663085938,
+      "learning_rate": 1.5833333333333336e-07,
+      "loss": 13.4366,
+      "num_input_tokens_seen": 282560,
+      "step": 19
+    },
+    {
+      "epoch": 0.2572347266881029,
+      "grad_norm": 555.8986206054688,
+      "learning_rate": 1.6666666666666668e-07,
+      "loss": 12.9904,
+      "num_input_tokens_seen": 297120,
+      "step": 20
+    },
+    {
+      "epoch": 0.27009646302250806,
+      "grad_norm": 557.931396484375,
+      "learning_rate": 1.7500000000000002e-07,
+      "loss": 12.878,
+      "num_input_tokens_seen": 312416,
+      "step": 21
+    },
+    {
+      "epoch": 0.2829581993569132,
+      "grad_norm": 562.4014282226562,
+      "learning_rate": 1.8333333333333336e-07,
+      "loss": 12.7794,
+      "num_input_tokens_seen": 327968,
+      "step": 22
+    },
+    {
+      "epoch": 0.2958199356913183,
+      "grad_norm": 548.7069702148438,
+      "learning_rate": 1.9166666666666668e-07,
+      "loss": 11.3144,
+      "num_input_tokens_seen": 342880,
+      "step": 23
+    },
+    {
+      "epoch": 0.3086816720257235,
+      "grad_norm": 511.2892761230469,
+      "learning_rate": 2.0000000000000002e-07,
+      "loss": 10.8531,
+      "num_input_tokens_seen": 358848,
+      "step": 24
+    },
+    {
+      "epoch": 0.3215434083601286,
+      "grad_norm": 496.71868896484375,
+      "learning_rate": 2.0833333333333333e-07,
+      "loss": 10.7149,
+      "num_input_tokens_seen": 373792,
+      "step": 25
+    },
+    {
+      "epoch": 0.33440514469453375,
+      "grad_norm": 504.7909851074219,
+      "learning_rate": 2.166666666666667e-07,
+      "loss": 10.5802,
+      "num_input_tokens_seen": 389248,
+      "step": 26
+    },
+    {
+      "epoch": 0.34726688102893893,
+      "grad_norm": 495.0346984863281,
+      "learning_rate": 2.2500000000000002e-07,
+      "loss": 10.3671,
+      "num_input_tokens_seen": 404320,
+      "step": 27
+    },
+    {
+      "epoch": 0.36012861736334406,
+      "grad_norm": 482.26318359375,
+      "learning_rate": 2.3333333333333336e-07,
+      "loss": 10.1751,
+      "num_input_tokens_seen": 419296,
+      "step": 28
+    },
+    {
+      "epoch": 0.3729903536977492,
+      "grad_norm": 480.48193359375,
+      "learning_rate": 2.416666666666667e-07,
+      "loss": 9.7707,
+      "num_input_tokens_seen": 433632,
+      "step": 29
+    },
+    {
+      "epoch": 0.3858520900321543,
+      "grad_norm": 467.7600402832031,
+      "learning_rate": 2.5000000000000004e-07,
+      "loss": 9.6489,
+      "num_input_tokens_seen": 448736,
+      "step": 30
+    },
+    {
+      "epoch": 0.3987138263665595,
+      "grad_norm": 468.9175720214844,
+      "learning_rate": 2.5833333333333333e-07,
+      "loss": 8.6805,
+      "num_input_tokens_seen": 463840,
+      "step": 31
+    },
+    {
+      "epoch": 0.4115755627009646,
+      "grad_norm": 394.77691650390625,
+      "learning_rate": 2.666666666666667e-07,
+      "loss": 5.9207,
+      "num_input_tokens_seen": 478816,
+      "step": 32
+    },
+    {
+      "epoch": 0.42443729903536975,
+      "grad_norm": 440.14697265625,
+      "learning_rate": 2.75e-07,
+      "loss": 5.7661,
+      "num_input_tokens_seen": 494464,
+      "step": 33
+    },
+    {
+      "epoch": 0.43729903536977494,
+      "grad_norm": 436.6060791015625,
+      "learning_rate": 2.8333333333333336e-07,
+      "loss": 5.6168,
+      "num_input_tokens_seen": 510080,
+      "step": 34
+    },
+    {
+      "epoch": 0.45016077170418006,
+      "grad_norm": 388.576416015625,
+      "learning_rate": 2.916666666666667e-07,
+      "loss": 5.3367,
+      "num_input_tokens_seen": 525216,
+      "step": 35
+    },
+    {
+      "epoch": 0.4630225080385852,
+      "grad_norm": 327.9267272949219,
+      "learning_rate": 3.0000000000000004e-07,
+      "loss": 4.9751,
+      "num_input_tokens_seen": 539648,
+      "step": 36
+    },
+    {
+      "epoch": 0.4758842443729904,
+      "grad_norm": 322.6181335449219,
+      "learning_rate": 3.083333333333334e-07,
+      "loss": 4.7041,
+      "num_input_tokens_seen": 554080,
+      "step": 37
+    },
+    {
+      "epoch": 0.4887459807073955,
+      "grad_norm": 313.6160888671875,
+      "learning_rate": 3.166666666666667e-07,
+      "loss": 4.4631,
+      "num_input_tokens_seen": 568960,
+      "step": 38
+    },
+    {
+      "epoch": 0.5016077170418006,
+      "grad_norm": 299.7319641113281,
+      "learning_rate": 3.25e-07,
+      "loss": 4.1912,
+      "num_input_tokens_seen": 583904,
+      "step": 39
+    },
+    {
+      "epoch": 0.5144694533762058,
+      "grad_norm": 308.4530944824219,
+      "learning_rate": 3.3333333333333335e-07,
+      "loss": 3.9146,
+      "num_input_tokens_seen": 598848,
+      "step": 40
+    },
+    {
+      "epoch": 0.5273311897106109,
+      "grad_norm": 315.37396240234375,
+      "learning_rate": 3.416666666666667e-07,
+      "loss": 3.053,
+      "num_input_tokens_seen": 614208,
+      "step": 41
+    },
+    {
+      "epoch": 0.5401929260450161,
+      "grad_norm": 263.33258056640625,
+      "learning_rate": 3.5000000000000004e-07,
+      "loss": 1.5544,
+      "num_input_tokens_seen": 629280,
+      "step": 42
+    },
+    {
+      "epoch": 0.5530546623794212,
+      "grad_norm": 177.95458984375,
+      "learning_rate": 3.583333333333334e-07,
+      "loss": 1.0549,
+      "num_input_tokens_seen": 643968,
+      "step": 43
+    },
+    {
+      "epoch": 0.5659163987138264,
+      "grad_norm": 142.09307861328125,
+      "learning_rate": 3.666666666666667e-07,
+      "loss": 0.711,
+      "num_input_tokens_seen": 659040,
+      "step": 44
+    },
+    {
+      "epoch": 0.5787781350482315,
+      "grad_norm": 93.70928192138672,
+      "learning_rate": 3.75e-07,
+      "loss": 0.5127,
+      "num_input_tokens_seen": 674464,
+      "step": 45
+    },
+    {
+      "epoch": 0.5916398713826366,
+      "grad_norm": 65.4582290649414,
+      "learning_rate": 3.8333333333333335e-07,
+      "loss": 0.4143,
+      "num_input_tokens_seen": 689216,
+      "step": 46
+    },
+    {
+      "epoch": 0.6045016077170418,
+      "grad_norm": 51.321414947509766,
+      "learning_rate": 3.9166666666666675e-07,
+      "loss": 0.4014,
+      "num_input_tokens_seen": 704448,
+      "step": 47
+    },
+    {
+      "epoch": 0.617363344051447,
+      "grad_norm": 38.20650100708008,
+      "learning_rate": 4.0000000000000003e-07,
+      "loss": 0.384,
+      "num_input_tokens_seen": 719520,
+      "step": 48
+    },
+    {
+      "epoch": 0.6302250803858521,
+      "grad_norm": 70.43089294433594,
+      "learning_rate": 4.083333333333334e-07,
+      "loss": 0.3017,
+      "num_input_tokens_seen": 735520,
+      "step": 49
+    },
+    {
+      "epoch": 0.6430868167202572,
+      "grad_norm": 47.982505798339844,
+      "learning_rate": 4.1666666666666667e-07,
+      "loss": 0.3031,
+      "num_input_tokens_seen": 750976,
+      "step": 50
+    },
+    {
+      "epoch": 0.6559485530546624,
+      "grad_norm": 27.063344955444336,
+      "learning_rate": 4.2500000000000006e-07,
+      "loss": 0.2818,
+      "num_input_tokens_seen": 765728,
+      "step": 51
+    },
+    {
+      "epoch": 0.6688102893890675,
+      "grad_norm": 60.75875473022461,
+      "learning_rate": 4.333333333333334e-07,
+      "loss": 0.2881,
+      "num_input_tokens_seen": 780608,
+      "step": 52
+    },
+    {
+      "epoch": 0.6816720257234726,
+      "grad_norm": 56.81061935424805,
+      "learning_rate": 4.416666666666667e-07,
+      "loss": 0.2943,
+      "num_input_tokens_seen": 796192,
+      "step": 53
+    },
+    {
+      "epoch": 0.6945337620578779,
+      "grad_norm": 31.529571533203125,
+      "learning_rate": 4.5000000000000003e-07,
+      "loss": 0.2781,
+      "num_input_tokens_seen": 810496,
+      "step": 54
+    },
+    {
+      "epoch": 0.707395498392283,
+      "grad_norm": 33.71590805053711,
+      "learning_rate": 4.583333333333333e-07,
+      "loss": 0.2724,
+      "num_input_tokens_seen": 826048,
+      "step": 55
+    },
+    {
+      "epoch": 0.7202572347266881,
+      "grad_norm": 26.352554321289062,
+      "learning_rate": 4.666666666666667e-07,
+      "loss": 0.252,
+      "num_input_tokens_seen": 840512,
+      "step": 56
+    },
+    {
+      "epoch": 0.7331189710610932,
+      "grad_norm": 15.606438636779785,
+      "learning_rate": 4.7500000000000006e-07,
+      "loss": 0.2482,
+      "num_input_tokens_seen": 855808,
+      "step": 57
+    },
+    {
+      "epoch": 0.7459807073954984,
+      "grad_norm": 31.445789337158203,
+      "learning_rate": 4.833333333333334e-07,
+      "loss": 0.2212,
+      "num_input_tokens_seen": 871520,
+      "step": 58
+    },
+    {
+      "epoch": 0.7588424437299035,
+      "grad_norm": 9.967268943786621,
+      "learning_rate": 4.916666666666667e-07,
+      "loss": 0.2408,
+      "num_input_tokens_seen": 886144,
+      "step": 59
+    },
+    {
+      "epoch": 0.7717041800643086,
+      "grad_norm": 23.841659545898438,
+      "learning_rate": 5.000000000000001e-07,
+      "loss": 0.2133,
+      "num_input_tokens_seen": 901408,
+      "step": 60
+    },
+    {
+      "epoch": 0.7845659163987139,
+      "grad_norm": 16.51668357849121,
+      "learning_rate": 5.083333333333334e-07,
+      "loss": 0.2319,
+      "num_input_tokens_seen": 916672,
+      "step": 61
+    },
+    {
+      "epoch": 0.797427652733119,
+      "grad_norm": 51.05732345581055,
+      "learning_rate": 5.166666666666667e-07,
+      "loss": 0.2703,
+      "num_input_tokens_seen": 931168,
+      "step": 62
+    },
+    {
+      "epoch": 0.8102893890675241,
+      "grad_norm": 61.92572784423828,
+      "learning_rate": 5.250000000000001e-07,
+      "loss": 0.2647,
+      "num_input_tokens_seen": 946816,
+      "step": 63
+    },
+    {
+      "epoch": 0.8231511254019293,
+      "grad_norm": 10.283011436462402,
+      "learning_rate": 5.333333333333335e-07,
+      "loss": 0.2288,
+      "num_input_tokens_seen": 961248,
+      "step": 64
+    },
+    {
+      "epoch": 0.8360128617363344,
+      "grad_norm": 45.712772369384766,
+      "learning_rate": 5.416666666666667e-07,
+      "loss": 0.2426,
+      "num_input_tokens_seen": 976736,
+      "step": 65
+    },
+    {
+      "epoch": 0.8488745980707395,
+      "grad_norm": 22.493711471557617,
+      "learning_rate": 5.5e-07,
+      "loss": 0.1936,
+      "num_input_tokens_seen": 992192,
+      "step": 66
+    },
+    {
+      "epoch": 0.8617363344051447,
+      "grad_norm": 10.771052360534668,
+      "learning_rate": 5.583333333333333e-07,
+      "loss": 0.192,
+      "num_input_tokens_seen": 1007808,
+      "step": 67
+    },
+    {
+      "epoch": 0.8745980707395499,
+      "grad_norm": 11.903575897216797,
+      "learning_rate": 5.666666666666667e-07,
+      "loss": 0.1956,
+      "num_input_tokens_seen": 1022176,
+      "step": 68
+    },
+    {
+      "epoch": 0.887459807073955,
+      "grad_norm": 23.1299991607666,
+      "learning_rate": 5.750000000000001e-07,
+      "loss": 0.1862,
+      "num_input_tokens_seen": 1037280,
+      "step": 69
+    },
+    {
+      "epoch": 0.9003215434083601,
+      "grad_norm": 11.018325805664062,
+      "learning_rate": 5.833333333333334e-07,
+      "loss": 0.195,
+      "num_input_tokens_seen": 1051392,
+      "step": 70
+    },
+    {
+      "epoch": 0.9131832797427653,
+      "grad_norm": 27.96196174621582,
+      "learning_rate": 5.916666666666667e-07,
+      "loss": 0.1971,
+      "num_input_tokens_seen": 1066432,
+      "step": 71
+    },
+    {
+      "epoch": 0.9260450160771704,
+      "grad_norm": 14.441658020019531,
+      "learning_rate": 6.000000000000001e-07,
+      "loss": 0.1683,
+      "num_input_tokens_seen": 1081696,
+      "step": 72
+    },
+    {
+      "epoch": 0.9389067524115756,
+      "grad_norm": 17.725786209106445,
+      "learning_rate": 6.083333333333334e-07,
+      "loss": 0.2003,
+      "num_input_tokens_seen": 1096672,
+      "step": 73
+    },
+    {
+      "epoch": 0.9517684887459807,
+      "grad_norm": 26.286319732666016,
+      "learning_rate": 6.166666666666668e-07,
+      "loss": 0.1543,
+      "num_input_tokens_seen": 1111488,
+      "step": 74
+    },
+    {
+      "epoch": 0.9646302250803859,
+      "grad_norm": 38.698097229003906,
+      "learning_rate": 6.25e-07,
+      "loss": 0.1982,
+      "num_input_tokens_seen": 1126368,
+      "step": 75
+    },
+    {
+      "epoch": 0.977491961414791,
+      "grad_norm": 8.287293434143066,
+      "learning_rate": 6.333333333333334e-07,
+      "loss": 0.1545,
+      "num_input_tokens_seen": 1141536,
+      "step": 76
+    },
+    {
+      "epoch": 0.9903536977491961,
+      "grad_norm": 44.253807067871094,
+      "learning_rate": 6.416666666666667e-07,
+      "loss": 0.1573,
+      "num_input_tokens_seen": 1156320,
+      "step": 77
+    },
+    {
+      "epoch": 1.0032154340836013,
+      "grad_norm": 57.11432647705078,
+      "learning_rate": 6.5e-07,
+      "loss": 0.1788,
+      "num_input_tokens_seen": 1171904,
+      "step": 78
+    },
+    {
+      "epoch": 1.0160771704180065,
+      "grad_norm": 21.049449920654297,
+      "learning_rate": 6.583333333333333e-07,
+      "loss": 0.187,
+      "num_input_tokens_seen": 1187232,
+      "step": 79
+    },
+    {
+      "epoch": 1.0289389067524115,
+      "grad_norm": 67.61766052246094,
+      "learning_rate": 6.666666666666667e-07,
+      "loss": 0.2009,
+      "num_input_tokens_seen": 1201632,
+      "step": 80
+    },
+    {
+      "epoch": 1.0418006430868167,
+      "grad_norm": 91.35673522949219,
+      "learning_rate": 6.750000000000001e-07,
+      "loss": 0.2655,
+      "num_input_tokens_seen": 1216832,
+      "step": 81
+    },
+    {
+      "epoch": 1.0546623794212218,
+      "grad_norm": 54.41016387939453,
+      "learning_rate": 6.833333333333334e-07,
+      "loss": 0.2041,
+      "num_input_tokens_seen": 1232128,
+      "step": 82
+    },
+    {
+      "epoch": 1.067524115755627,
+      "grad_norm": 10.082817077636719,
+      "learning_rate": 6.916666666666668e-07,
+      "loss": 0.1675,
+      "num_input_tokens_seen": 1246944,
+      "step": 83
+    },
+    {
+      "epoch": 1.0803858520900322,
+      "grad_norm": 43.82609558105469,
+      "learning_rate": 7.000000000000001e-07,
+      "loss": 0.2097,
+      "num_input_tokens_seen": 1262624,
+      "step": 84
+    },
+    {
+      "epoch": 1.0932475884244373,
+      "grad_norm": 51.51130676269531,
+      "learning_rate": 7.083333333333334e-07,
+      "loss": 0.2171,
+      "num_input_tokens_seen": 1278080,
+      "step": 85
+    },
+    {
+      "epoch": 1.1061093247588425,
+      "grad_norm": 35.92145538330078,
+      "learning_rate": 7.166666666666668e-07,
+      "loss": 0.1702,
+      "num_input_tokens_seen": 1293280,
+      "step": 86
+    },
+    {
+      "epoch": 1.1189710610932475,
+      "grad_norm": 8.067242622375488,
+      "learning_rate": 7.25e-07,
+      "loss": 0.1255,
+      "num_input_tokens_seen": 1307456,
+      "step": 87
+    },
+    {
+      "epoch": 1.1318327974276527,
+      "grad_norm": 28.8727970123291,
+      "learning_rate": 7.333333333333334e-07,
+      "loss": 0.1826,
+      "num_input_tokens_seen": 1323168,
+      "step": 88
+    },
+    {
+      "epoch": 1.144694533762058,
+      "grad_norm": 45.813865661621094,
+      "learning_rate": 7.416666666666668e-07,
+      "loss": 0.2039,
+      "num_input_tokens_seen": 1337984,
+      "step": 89
+    },
+    {
+      "epoch": 1.157556270096463,
+      "grad_norm": 53.024105072021484,
+      "learning_rate": 7.5e-07,
+      "loss": 0.2337,
+      "num_input_tokens_seen": 1353568,
+      "step": 90
+    },
+    {
+      "epoch": 1.1704180064308682,
+      "grad_norm": 17.642776489257812,
+      "learning_rate": 7.583333333333334e-07,
+      "loss": 0.1466,
+      "num_input_tokens_seen": 1368800,
+      "step": 91
+    },
+    {
+      "epoch": 1.1832797427652733,
+      "grad_norm": 17.741811752319336,
+      "learning_rate": 7.666666666666667e-07,
+      "loss": 0.1295,
+      "num_input_tokens_seen": 1384192,
+      "step": 92
+    },
+    {
+      "epoch": 1.1961414790996785,
+      "grad_norm": 24.114030838012695,
+      "learning_rate": 7.750000000000001e-07,
+      "loss": 0.1525,
+      "num_input_tokens_seen": 1398784,
+      "step": 93
+    },
+    {
+      "epoch": 1.2090032154340835,
+      "grad_norm": 27.668102264404297,
+      "learning_rate": 7.833333333333335e-07,
+      "loss": 0.1735,
+      "num_input_tokens_seen": 1414304,
+      "step": 94
+    },
+    {
+      "epoch": 1.2218649517684887,
+      "grad_norm": 7.892695426940918,
+      "learning_rate": 7.916666666666667e-07,
+      "loss": 0.1484,
+      "num_input_tokens_seen": 1430880,
+      "step": 95
+    },
+    {
+      "epoch": 1.234726688102894,
+      "grad_norm": 12.985011100769043,
+      "learning_rate": 8.000000000000001e-07,
+      "loss": 0.1517,
+      "num_input_tokens_seen": 1445760,
+      "step": 96
+    },
+    {
+      "epoch": 1.247588424437299,
+      "grad_norm": 20.373502731323242,
+      "learning_rate": 8.083333333333334e-07,
+      "loss": 0.1429,
+      "num_input_tokens_seen": 1460576,
+      "step": 97
+    },
+    {
+      "epoch": 1.2604501607717042,
+      "grad_norm": 8.579370498657227,
+      "learning_rate": 8.166666666666668e-07,
+      "loss": 0.151,
+      "num_input_tokens_seen": 1476032,
+      "step": 98
+    },
+    {
+      "epoch": 1.2733118971061093,
+      "grad_norm": 6.366987705230713,
+      "learning_rate": 8.250000000000001e-07,
+      "loss": 0.1217,
+      "num_input_tokens_seen": 1491008,
+      "step": 99
+    },
+    {
+      "epoch": 1.2861736334405145,
+      "grad_norm": 19.89562225341797,
+      "learning_rate": 8.333333333333333e-07,
+      "loss": 0.1366,
+      "num_input_tokens_seen": 1505120,
+      "step": 100
+    },
+    {
+      "epoch": 1.2990353697749195,
+      "grad_norm": 10.997241020202637,
+      "learning_rate": 8.416666666666667e-07,
+      "loss": 0.1534,
+      "num_input_tokens_seen": 1520736,
+      "step": 101
+    },
+    {
+      "epoch": 1.3118971061093248,
+      "grad_norm": 17.20572280883789,
+      "learning_rate": 8.500000000000001e-07,
+      "loss": 0.141,
+      "num_input_tokens_seen": 1535424,
+      "step": 102
+    },
+    {
+      "epoch": 1.32475884244373,
+      "grad_norm": 10.88858413696289,
+      "learning_rate": 8.583333333333334e-07,
+      "loss": 0.1238,
+      "num_input_tokens_seen": 1549856,
+      "step": 103
+    },
+    {
+      "epoch": 1.337620578778135,
+      "grad_norm": 6.872950077056885,
+      "learning_rate": 8.666666666666668e-07,
+      "loss": 0.1241,
+      "num_input_tokens_seen": 1564512,
+      "step": 104
+    },
+    {
+      "epoch": 1.3504823151125402,
+      "grad_norm": 6.931344509124756,
+      "learning_rate": 8.75e-07,
+      "loss": 0.1414,
+      "num_input_tokens_seen": 1578976,
+      "step": 105
+    },
+    {
+      "epoch": 1.3633440514469453,
+      "grad_norm": 12.237205505371094,
+      "learning_rate": 8.833333333333334e-07,
+      "loss": 0.1296,
+      "num_input_tokens_seen": 1593952,
+      "step": 106
+    },
+    {
+      "epoch": 1.3762057877813505,
+      "grad_norm": 13.147500991821289,
+      "learning_rate": 8.916666666666668e-07,
+      "loss": 0.1232,
+      "num_input_tokens_seen": 1608704,
+      "step": 107
+    },
+    {
+      "epoch": 1.3890675241157555,
+      "grad_norm": 23.525880813598633,
+      "learning_rate": 9.000000000000001e-07,
+      "loss": 0.1625,
+      "num_input_tokens_seen": 1623936,
+      "step": 108
+    },
+    {
+      "epoch": 1.4019292604501608,
+      "grad_norm": 25.576210021972656,
+      "learning_rate": 9.083333333333335e-07,
+      "loss": 0.1509,
+      "num_input_tokens_seen": 1639264,
+      "step": 109
+    },
+    {
+      "epoch": 1.414790996784566,
+      "grad_norm": 18.12908935546875,
+      "learning_rate": 9.166666666666666e-07,
+      "loss": 0.1416,
+      "num_input_tokens_seen": 1654528,
+      "step": 110
+    },
+    {
+      "epoch": 1.427652733118971,
+      "grad_norm": 14.664992332458496,
+      "learning_rate": 9.25e-07,
+      "loss": 0.1481,
+      "num_input_tokens_seen": 1669824,
+      "step": 111
+    },
+    {
+      "epoch": 1.4405144694533762,
+      "grad_norm": 17.804119110107422,
+      "learning_rate": 9.333333333333334e-07,
+      "loss": 0.1303,
+      "num_input_tokens_seen": 1684800,
+      "step": 112
+    },
+    {
+      "epoch": 1.4533762057877815,
+      "grad_norm": 10.886981964111328,
+      "learning_rate": 9.416666666666667e-07,
+      "loss": 0.116,
+      "num_input_tokens_seen": 1699712,
+      "step": 113
+    },
+    {
+      "epoch": 1.4662379421221865,
+      "grad_norm": 14.883463859558105,
+      "learning_rate": 9.500000000000001e-07,
+      "loss": 0.0981,
+      "num_input_tokens_seen": 1714208,
+      "step": 114
+    },
+    {
+      "epoch": 1.4790996784565915,
+      "grad_norm": 9.800952911376953,
+      "learning_rate": 9.583333333333334e-07,
+      "loss": 0.1174,
+      "num_input_tokens_seen": 1728672,
+      "step": 115
+    },
+    {
+      "epoch": 1.4919614147909968,
+      "grad_norm": 8.804801940917969,
+      "learning_rate": 9.666666666666668e-07,
+      "loss": 0.1458,
+      "num_input_tokens_seen": 1743008,
+      "step": 116
+    },
+    {
+      "epoch": 1.504823151125402,
+      "grad_norm": 9.836427688598633,
+      "learning_rate": 9.750000000000002e-07,
+      "loss": 0.0952,
+      "num_input_tokens_seen": 1758016,
+      "step": 117
+    },
+    {
+      "epoch": 1.517684887459807,
+      "grad_norm": 7.373986721038818,
+      "learning_rate": 9.833333333333334e-07,
+      "loss": 0.1233,
+      "num_input_tokens_seen": 1772512,
+      "step": 118
+    },
+    {
+      "epoch": 1.5305466237942122,
+      "grad_norm": 16.394031524658203,
+      "learning_rate": 9.916666666666668e-07,
+      "loss": 0.127,
+      "num_input_tokens_seen": 1787808,
+      "step": 119
+    },
+    {
+      "epoch": 1.5434083601286175,
+      "grad_norm": 8.336946487426758,
+      "learning_rate": 1.0000000000000002e-06,
+      "loss": 0.1121,
+      "num_input_tokens_seen": 1803104,
+      "step": 120
+    },
+    {
+      "epoch": 1.5562700964630225,
+      "grad_norm": 28.298938751220703,
+      "learning_rate": 1.0083333333333333e-06,
+      "loss": 0.1432,
+      "num_input_tokens_seen": 1817632,
+      "step": 121
+    },
+    {
+      "epoch": 1.5691318327974275,
+      "grad_norm": 25.474973678588867,
+      "learning_rate": 1.0166666666666667e-06,
+      "loss": 0.1446,
+      "num_input_tokens_seen": 1832576,
+      "step": 122
+    },
+    {
+      "epoch": 1.5819935691318328,
+      "grad_norm": 12.33470630645752,
+      "learning_rate": 1.025e-06,
+      "loss": 0.1056,
+      "num_input_tokens_seen": 1847872,
+      "step": 123
+    },
+    {
+      "epoch": 1.594855305466238,
+      "grad_norm": 16.17420768737793,
+      "learning_rate": 1.0333333333333333e-06,
+      "loss": 0.1193,
+      "num_input_tokens_seen": 1863008,
+      "step": 124
+    },
+    {
+      "epoch": 1.607717041800643,
+      "grad_norm": 23.109798431396484,
+      "learning_rate": 1.0416666666666667e-06,
+      "loss": 0.1409,
+      "num_input_tokens_seen": 1878688,
+      "step": 125
+    },
+    {
+      "epoch": 1.6205787781350482,
+      "grad_norm": 15.091007232666016,
+      "learning_rate": 1.0500000000000001e-06,
+      "loss": 0.1154,
+      "num_input_tokens_seen": 1894144,
+      "step": 126
+    },
+    {
+      "epoch": 1.6334405144694535,
+      "grad_norm": 7.409396171569824,
+      "learning_rate": 1.0583333333333335e-06,
+      "loss": 0.1046,
+      "num_input_tokens_seen": 1909504,
+      "step": 127
+    },
+    {
+      "epoch": 1.6463022508038585,
+      "grad_norm": 12.553366661071777,
+      "learning_rate": 1.066666666666667e-06,
+      "loss": 0.09,
+      "num_input_tokens_seen": 1924896,
+      "step": 128
+    },
+    {
+      "epoch": 1.6591639871382635,
+      "grad_norm": 10.23271369934082,
+      "learning_rate": 1.075e-06,
+      "loss": 0.0858,
+      "num_input_tokens_seen": 1939232,
+      "step": 129
+    },
+    {
+      "epoch": 1.6720257234726688,
+      "grad_norm": 6.091500282287598,
+      "learning_rate": 1.0833333333333335e-06,
+      "loss": 0.0782,
+      "num_input_tokens_seen": 1954720,
+      "step": 130
+    },
+    {
+      "epoch": 1.684887459807074,
+      "grad_norm": 12.670690536499023,
+      "learning_rate": 1.0916666666666667e-06,
+      "loss": 0.1429,
+      "num_input_tokens_seen": 1969632,
+      "step": 131
+    },
+    {
+      "epoch": 1.697749196141479,
+      "grad_norm": 12.42375659942627,
+      "learning_rate": 1.1e-06,
+      "loss": 0.1121,
+      "num_input_tokens_seen": 1984544,
+      "step": 132
+    },
+    {
+      "epoch": 1.7106109324758842,
+      "grad_norm": 4.910019874572754,
+      "learning_rate": 1.1083333333333335e-06,
+      "loss": 0.0458,
+      "num_input_tokens_seen": 1999584,
+      "step": 133
+    },
+    {
+      "epoch": 1.7234726688102895,
+      "grad_norm": 8.262114524841309,
+      "learning_rate": 1.1166666666666666e-06,
+      "loss": 0.1217,
+      "num_input_tokens_seen": 2014048,
+      "step": 134
+    },
+    {
+      "epoch": 1.7363344051446945,
+      "grad_norm": 21.927522659301758,
+      "learning_rate": 1.125e-06,
+      "loss": 0.1253,
+      "num_input_tokens_seen": 2029312,
+      "step": 135
+    },
+    {
+      "epoch": 1.7491961414790995,
+      "grad_norm": 8.816388130187988,
+      "learning_rate": 1.1333333333333334e-06,
+      "loss": 0.077,
+      "num_input_tokens_seen": 2044192,
+      "step": 136
+    },
+    {
+      "epoch": 1.762057877813505,
+      "grad_norm": 6.877133846282959,
+      "learning_rate": 1.1416666666666668e-06,
+      "loss": 0.0719,
+      "num_input_tokens_seen": 2059552,
+      "step": 137
+    },
+    {
+      "epoch": 1.77491961414791,
+      "grad_norm": 5.565485000610352,
+      "learning_rate": 1.1500000000000002e-06,
+      "loss": 0.0916,
+      "num_input_tokens_seen": 2074336,
+      "step": 138
+    },
+    {
+      "epoch": 1.787781350482315,
+      "grad_norm": 11.287223815917969,
+      "learning_rate": 1.1583333333333334e-06,
+      "loss": 0.0812,
+      "num_input_tokens_seen": 2089280,
+      "step": 139
+    },
+    {
+      "epoch": 1.8006430868167203,
+      "grad_norm": 7.971100807189941,
+      "learning_rate": 1.1666666666666668e-06,
+      "loss": 0.1176,
+      "num_input_tokens_seen": 2104256,
+      "step": 140
+    },
+    {
+      "epoch": 1.8135048231511255,
+      "grad_norm": 7.78786563873291,
+      "learning_rate": 1.175e-06,
+      "loss": 0.0631,
+      "num_input_tokens_seen": 2119232,
+      "step": 141
+    },
+    {
+      "epoch": 1.8263665594855305,
+      "grad_norm": 8.365852355957031,
+      "learning_rate": 1.1833333333333334e-06,
+      "loss": 0.1137,
+      "num_input_tokens_seen": 2133824,
+      "step": 142
+    },
+    {
+      "epoch": 1.8392282958199357,
+      "grad_norm": 16.2335262298584,
+      "learning_rate": 1.1916666666666668e-06,
+      "loss": 0.0958,
+      "num_input_tokens_seen": 2148480,
+      "step": 143
+    },
+    {
+      "epoch": 1.852090032154341,
+      "grad_norm": 8.455872535705566,
+      "learning_rate": 1.2000000000000002e-06,
+      "loss": 0.1343,
+      "num_input_tokens_seen": 2163328,
+      "step": 144
+    },
+    {
+      "epoch": 1.864951768488746,
+      "grad_norm": 13.241278648376465,
+      "learning_rate": 1.2083333333333333e-06,
+      "loss": 0.1101,
+      "num_input_tokens_seen": 2177568,
+      "step": 145
+    },
+    {
+      "epoch": 1.877813504823151,
+      "grad_norm": 13.46755313873291,
+      "learning_rate": 1.2166666666666667e-06,
+      "loss": 0.0914,
+      "num_input_tokens_seen": 2192384,
+      "step": 146
+    },
+    {
+      "epoch": 1.8906752411575563,
+      "grad_norm": 7.21525764465332,
+      "learning_rate": 1.2250000000000001e-06,
+      "loss": 0.1114,
+      "num_input_tokens_seen": 2207584,
+      "step": 147
+    },
+    {
+      "epoch": 1.9035369774919615,
+      "grad_norm": 5.000830173492432,
+      "learning_rate": 1.2333333333333335e-06,
+      "loss": 0.083,
+      "num_input_tokens_seen": 2222208,
+      "step": 148
+    },
+    {
+      "epoch": 1.9163987138263665,
+      "grad_norm": 8.995044708251953,
+      "learning_rate": 1.2416666666666667e-06,
+      "loss": 0.1095,
+      "num_input_tokens_seen": 2237760,
+      "step": 149
+    },
+    {
+      "epoch": 1.9292604501607717,
+      "grad_norm": 7.872910976409912,
+      "learning_rate": 1.25e-06,
+      "loss": 0.0662,
+      "num_input_tokens_seen": 2252768,
+      "step": 150
+    },
+    {
+      "epoch": 1.942122186495177,
+      "grad_norm": 11.594476699829102,
+      "learning_rate": 1.2583333333333333e-06,
+      "loss": 0.0979,
+      "num_input_tokens_seen": 2268736,
+      "step": 151
+    },
+    {
+      "epoch": 1.954983922829582,
+      "grad_norm": 14.245850563049316,
+      "learning_rate": 1.2666666666666669e-06,
+      "loss": 0.0847,
+      "num_input_tokens_seen": 2283840,
+      "step": 152
+    },
+    {
+      "epoch": 1.967845659163987,
+      "grad_norm": 12.000141143798828,
+      "learning_rate": 1.275e-06,
+      "loss": 0.0949,
+      "num_input_tokens_seen": 2297696,
+      "step": 153
+    },
+    {
+      "epoch": 1.9807073954983923,
+      "grad_norm": 16.763721466064453,
+      "learning_rate": 1.2833333333333335e-06,
+      "loss": 0.1206,
+      "num_input_tokens_seen": 2311936,
+      "step": 154
+    },
+    {
+      "epoch": 1.9935691318327975,
+      "grad_norm": 22.926712036132812,
+      "learning_rate": 1.2916666666666669e-06,
+      "loss": 0.139,
+      "num_input_tokens_seen": 2327392,
+      "step": 155
+    },
+    {
+      "epoch": 2.0064308681672025,
+      "grad_norm": 7.715733528137207,
+      "learning_rate": 1.3e-06,
+      "loss": 0.07,
+      "num_input_tokens_seen": 2341952,
+      "step": 156
+    },
+    {
+      "epoch": 2.0192926045016075,
+      "grad_norm": 5.047181129455566,
+      "learning_rate": 1.3083333333333334e-06,
+      "loss": 0.0562,
+      "num_input_tokens_seen": 2356768,
+      "step": 157
+    },
+    {
+      "epoch": 2.032154340836013,
+      "grad_norm": 10.79956340789795,
+      "learning_rate": 1.3166666666666666e-06,
+      "loss": 0.0456,
+      "num_input_tokens_seen": 2371584,
+      "step": 158
+    },
+    {
+      "epoch": 2.045016077170418,
+      "grad_norm": 13.864229202270508,
+      "learning_rate": 1.3250000000000002e-06,
+      "loss": 0.0582,
+      "num_input_tokens_seen": 2386880,
+      "step": 159
+    },
+    {
+      "epoch": 2.057877813504823,
+      "grad_norm": 7.294124603271484,
+      "learning_rate": 1.3333333333333334e-06,
+      "loss": 0.0452,
+      "num_input_tokens_seen": 2401248,
+      "step": 160
+    },
+    {
+      "epoch": 2.0707395498392285,
+      "grad_norm": 5.838871479034424,
+      "learning_rate": 1.3416666666666666e-06,
+      "loss": 0.0553,
+      "num_input_tokens_seen": 2415968,
+      "step": 161
+    },
+    {
+      "epoch": 2.0836012861736335,
+      "grad_norm": 20.286243438720703,
+      "learning_rate": 1.3500000000000002e-06,
+      "loss": 0.1108,
+      "num_input_tokens_seen": 2430368,
+      "step": 162
+    },
+    {
+      "epoch": 2.0964630225080385,
+      "grad_norm": 12.720942497253418,
+      "learning_rate": 1.3583333333333334e-06,
+      "loss": 0.0791,
+      "num_input_tokens_seen": 2445056,
+      "step": 163
+    },
+    {
+      "epoch": 2.1093247588424435,
+      "grad_norm": 9.107832908630371,
+      "learning_rate": 1.3666666666666668e-06,
+      "loss": 0.0637,
+      "num_input_tokens_seen": 2459744,
+      "step": 164
+    },
+    {
+      "epoch": 2.122186495176849,
+      "grad_norm": 6.473385334014893,
+      "learning_rate": 1.3750000000000002e-06,
+      "loss": 0.0404,
+      "num_input_tokens_seen": 2474400,
+      "step": 165
+    },
+    {
+      "epoch": 2.135048231511254,
+      "grad_norm": 6.510437488555908,
+      "learning_rate": 1.3833333333333336e-06,
+      "loss": 0.0372,
+      "num_input_tokens_seen": 2489280,
+      "step": 166
+    },
+    {
+      "epoch": 2.147909967845659,
+      "grad_norm": 8.90833854675293,
+      "learning_rate": 1.3916666666666668e-06,
+      "loss": 0.045,
+      "num_input_tokens_seen": 2503776,
+      "step": 167
+    },
+    {
+      "epoch": 2.1607717041800645,
+      "grad_norm": 11.84021282196045,
+      "learning_rate": 1.4000000000000001e-06,
+      "loss": 0.0964,
+      "num_input_tokens_seen": 2518752,
+      "step": 168
+    },
+    {
+      "epoch": 2.1736334405144695,
+      "grad_norm": 5.904385566711426,
+      "learning_rate": 1.4083333333333335e-06,
+      "loss": 0.0543,
+      "num_input_tokens_seen": 2533568,
+      "step": 169
+    },
+    {
+      "epoch": 2.1864951768488745,
+      "grad_norm": 6.935682773590088,
+      "learning_rate": 1.4166666666666667e-06,
+      "loss": 0.071,
+      "num_input_tokens_seen": 2548640,
+      "step": 170
+    },
+    {
+      "epoch": 2.19935691318328,
+      "grad_norm": 3.85262131690979,
+      "learning_rate": 1.425e-06,
+      "loss": 0.0285,
+      "num_input_tokens_seen": 2563232,
+      "step": 171
+    },
+    {
+      "epoch": 2.212218649517685,
+      "grad_norm": 5.231224060058594,
+      "learning_rate": 1.4333333333333335e-06,
+      "loss": 0.0399,
+      "num_input_tokens_seen": 2578336,
+      "step": 172
+    },
+    {
+      "epoch": 2.22508038585209,
+      "grad_norm": 6.751733779907227,
+      "learning_rate": 1.4416666666666667e-06,
+      "loss": 0.067,
+      "num_input_tokens_seen": 2594336,
+      "step": 173
+    },
+    {
+      "epoch": 2.237942122186495,
+      "grad_norm": 4.619236469268799,
+      "learning_rate": 1.45e-06,
+      "loss": 0.0436,
+      "num_input_tokens_seen": 2609408,
+      "step": 174
+    },
+    {
+      "epoch": 2.2508038585209005,
+      "grad_norm": 4.799732685089111,
+      "learning_rate": 1.4583333333333335e-06,
+      "loss": 0.0522,
+      "num_input_tokens_seen": 2624288,
+      "step": 175
+    },
+    {
+      "epoch": 2.2636655948553055,
+      "grad_norm": 7.050292015075684,
+      "learning_rate": 1.4666666666666669e-06,
+      "loss": 0.0521,
+      "num_input_tokens_seen": 2639296,
+      "step": 176
+    },
+    {
+      "epoch": 2.2765273311897105,
+      "grad_norm": 6.788993835449219,
+      "learning_rate": 1.475e-06,
+      "loss": 0.0446,
+      "num_input_tokens_seen": 2653856,
+      "step": 177
+    },
+    {
+      "epoch": 2.289389067524116,
+      "grad_norm": 7.368555545806885,
+      "learning_rate": 1.4833333333333337e-06,
+      "loss": 0.0378,
+      "num_input_tokens_seen": 2669216,
+      "step": 178
+    },
+    {
+      "epoch": 2.302250803858521,
+      "grad_norm": 6.674912452697754,
+      "learning_rate": 1.4916666666666669e-06,
+      "loss": 0.0387,
+      "num_input_tokens_seen": 2683264,
+      "step": 179
+    },
+    {
+      "epoch": 2.315112540192926,
+      "grad_norm": 4.754708766937256,
+      "learning_rate": 1.5e-06,
+      "loss": 0.036,
+      "num_input_tokens_seen": 2698112,
+      "step": 180
+    },
+    {
+      "epoch": 2.327974276527331,
+      "grad_norm": 11.09841251373291,
+      "learning_rate": 1.5083333333333336e-06,
+      "loss": 0.0765,
+      "num_input_tokens_seen": 2712800,
+      "step": 181
+    },
+    {
+      "epoch": 2.3408360128617365,
+      "grad_norm": 13.097764015197754,
+      "learning_rate": 1.5166666666666668e-06,
+      "loss": 0.0884,
+      "num_input_tokens_seen": 2728032,
+      "step": 182
+    },
+    {
+      "epoch": 2.3536977491961415,
+      "grad_norm": 14.366617202758789,
+      "learning_rate": 1.525e-06,
+      "loss": 0.0801,
+      "num_input_tokens_seen": 2742432,
+      "step": 183
+    },
+    {
+      "epoch": 2.3665594855305465,
+      "grad_norm": 6.947451591491699,
+      "learning_rate": 1.5333333333333334e-06,
+      "loss": 0.0276,
+      "num_input_tokens_seen": 2758176,
+      "step": 184
+    },
+    {
+      "epoch": 2.379421221864952,
+      "grad_norm": 11.02580738067627,
+      "learning_rate": 1.5416666666666668e-06,
+      "loss": 0.0778,
+      "num_input_tokens_seen": 2772768,
+      "step": 185
+    },
+    {
+      "epoch": 2.392282958199357,
+      "grad_norm": 15.404431343078613,
+      "learning_rate": 1.5500000000000002e-06,
+      "loss": 0.0726,
+      "num_input_tokens_seen": 2787968,
+      "step": 186
+    },
+    {
+      "epoch": 2.405144694533762,
+      "grad_norm": 19.688905715942383,
+      "learning_rate": 1.5583333333333334e-06,
+      "loss": 0.1381,
+      "num_input_tokens_seen": 2802560,
+      "step": 187
+    },
+    {
+      "epoch": 2.418006430868167,
+      "grad_norm": 5.732620716094971,
+      "learning_rate": 1.566666666666667e-06,
+      "loss": 0.0408,
+      "num_input_tokens_seen": 2817856,
+      "step": 188
+    },
+    {
+      "epoch": 2.4308681672025725,
+      "grad_norm": 9.736536026000977,
+      "learning_rate": 1.5750000000000002e-06,
+      "loss": 0.1066,
+      "num_input_tokens_seen": 2833408,
+      "step": 189
+    },
+    {
+      "epoch": 2.4437299035369775,
+      "grad_norm": 9.312838554382324,
+      "learning_rate": 1.5833333333333333e-06,
+      "loss": 0.0686,
+      "num_input_tokens_seen": 2848832,
+      "step": 190
+    },
+    {
+      "epoch": 2.4565916398713825,
+      "grad_norm": 7.711926460266113,
+      "learning_rate": 1.591666666666667e-06,
+      "loss": 0.0428,
+      "num_input_tokens_seen": 2863968,
+      "step": 191
+    },
+    {
+      "epoch": 2.469453376205788,
+      "grad_norm": 6.855398654937744,
+      "learning_rate": 1.6000000000000001e-06,
+      "loss": 0.0387,
+      "num_input_tokens_seen": 2878944,
+      "step": 192
+    },
+    {
+      "epoch": 2.482315112540193,
+      "grad_norm": 3.374630928039551,
+      "learning_rate": 1.6083333333333333e-06,
+      "loss": 0.0489,
+      "num_input_tokens_seen": 2894112,
+      "step": 193
+    },
+    {
+      "epoch": 2.495176848874598,
+      "grad_norm": 7.452945709228516,
+      "learning_rate": 1.6166666666666667e-06,
+      "loss": 0.0621,
+      "num_input_tokens_seen": 2908928,
+      "step": 194
+    },
+    {
+      "epoch": 2.508038585209003,
+      "grad_norm": 12.478065490722656,
+      "learning_rate": 1.6250000000000001e-06,
+      "loss": 0.0651,
+      "num_input_tokens_seen": 2923680,
+      "step": 195
+    },
+    {
+      "epoch": 2.5209003215434085,
+      "grad_norm": 4.181528091430664,
+      "learning_rate": 1.6333333333333335e-06,
+      "loss": 0.0398,
+      "num_input_tokens_seen": 2939136,
+      "step": 196
+    },
+    {
+      "epoch": 2.5337620578778135,
+      "grad_norm": 5.443283557891846,
+      "learning_rate": 1.6416666666666667e-06,
+      "loss": 0.0369,
+      "num_input_tokens_seen": 2953760,
+      "step": 197
+    },
+    {
+      "epoch": 2.5466237942122185,
+      "grad_norm": 7.586729526519775,
+      "learning_rate": 1.6500000000000003e-06,
+      "loss": 0.0582,
+      "num_input_tokens_seen": 2968640,
+      "step": 198
+    },
+    {
+      "epoch": 2.559485530546624,
+      "grad_norm": 6.357609272003174,
+      "learning_rate": 1.6583333333333335e-06,
+      "loss": 0.0479,
+      "num_input_tokens_seen": 2983456,
+      "step": 199
+    },
+    {
+      "epoch": 2.572347266881029,
+      "grad_norm": 8.761418342590332,
+      "learning_rate": 1.6666666666666667e-06,
+      "loss": 0.0561,
+      "num_input_tokens_seen": 2998144,
+      "step": 200
+    },
+    {
+      "epoch": 2.585209003215434,
+      "grad_norm": 7.058523654937744,
+      "learning_rate": 1.6750000000000003e-06,
+      "loss": 0.0497,
+      "num_input_tokens_seen": 3013248,
+      "step": 201
+    },
+    {
+      "epoch": 2.598070739549839,
+      "grad_norm": 8.091582298278809,
+      "learning_rate": 1.6833333333333335e-06,
+      "loss": 0.063,
+      "num_input_tokens_seen": 3028320,
+      "step": 202
+    },
+    {
+      "epoch": 2.6109324758842445,
+      "grad_norm": 9.102255821228027,
+      "learning_rate": 1.6916666666666666e-06,
+      "loss": 0.054,
+      "num_input_tokens_seen": 3043232,
+      "step": 203
+    },
+    {
+      "epoch": 2.6237942122186495,
+      "grad_norm": 9.521942138671875,
+      "learning_rate": 1.7000000000000002e-06,
+      "loss": 0.0729,
+      "num_input_tokens_seen": 3058848,
+      "step": 204
+    },
+    {
+      "epoch": 2.6366559485530545,
+      "grad_norm": 7.026569843292236,
+      "learning_rate": 1.7083333333333334e-06,
+      "loss": 0.0685,
+      "num_input_tokens_seen": 3074080,
+      "step": 205
+    },
+    {
+      "epoch": 2.64951768488746,
+      "grad_norm": 13.492867469787598,
+      "learning_rate": 1.7166666666666668e-06,
+      "loss": 0.0823,
+      "num_input_tokens_seen": 3089152,
+      "step": 206
+    },
+    {
+      "epoch": 2.662379421221865,
+      "grad_norm": 6.880157947540283,
+      "learning_rate": 1.725e-06,
+      "loss": 0.0322,
+      "num_input_tokens_seen": 3103616,
+      "step": 207
+    },
+    {
+      "epoch": 2.67524115755627,
+      "grad_norm": 8.232446670532227,
+      "learning_rate": 1.7333333333333336e-06,
+      "loss": 0.0821,
+      "num_input_tokens_seen": 3119200,
+      "step": 208
+    },
+    {
+      "epoch": 2.688102893890675,
+      "grad_norm": 7.365771293640137,
+      "learning_rate": 1.7416666666666668e-06,
+      "loss": 0.0561,
+      "num_input_tokens_seen": 3134400,
+      "step": 209
+    },
+    {
+      "epoch": 2.7009646302250805,
+      "grad_norm": 9.879140853881836,
+      "learning_rate": 1.75e-06,
+      "loss": 0.0468,
+      "num_input_tokens_seen": 3149248,
+      "step": 210
+    },
+    {
+      "epoch": 2.7138263665594855,
+      "grad_norm": 8.636083602905273,
+      "learning_rate": 1.7583333333333336e-06,
+      "loss": 0.0593,
+      "num_input_tokens_seen": 3164800,
+      "step": 211
+    },
+    {
+      "epoch": 2.7266881028938905,
+      "grad_norm": 5.015254497528076,
+      "learning_rate": 1.7666666666666668e-06,
+      "loss": 0.0403,
+      "num_input_tokens_seen": 3179520,
+      "step": 212
+    },
+    {
+      "epoch": 2.739549839228296,
+      "grad_norm": 7.730808258056641,
+      "learning_rate": 1.7750000000000002e-06,
+      "loss": 0.0459,
+      "num_input_tokens_seen": 3194016,
+      "step": 213
+    },
+    {
+      "epoch": 2.752411575562701,
+      "grad_norm": 6.433826923370361,
+      "learning_rate": 1.7833333333333336e-06,
+      "loss": 0.0509,
+      "num_input_tokens_seen": 3208608,
+      "step": 214
+    },
+    {
+      "epoch": 2.765273311897106,
+      "grad_norm": 7.32877779006958,
+      "learning_rate": 1.7916666666666667e-06,
+      "loss": 0.0873,
+      "num_input_tokens_seen": 3224096,
+      "step": 215
+    },
+    {
+      "epoch": 2.778135048231511,
+      "grad_norm": 7.283654689788818,
+      "learning_rate": 1.8000000000000001e-06,
+      "loss": 0.0618,
+      "num_input_tokens_seen": 3239136,
+      "step": 216
+    },
+    {
+      "epoch": 2.7909967845659165,
+      "grad_norm": 11.044650077819824,
+      "learning_rate": 1.8083333333333335e-06,
+      "loss": 0.1025,
+      "num_input_tokens_seen": 3253184,
+      "step": 217
+    },
+    {
+      "epoch": 2.8038585209003215,
+      "grad_norm": 7.287780284881592,
+      "learning_rate": 1.816666666666667e-06,
+      "loss": 0.0669,
+      "num_input_tokens_seen": 3268000,
+      "step": 218
+    },
+    {
+      "epoch": 2.816720257234727,
+      "grad_norm": 7.542795658111572,
+      "learning_rate": 1.825e-06,
+      "loss": 0.0447,
+      "num_input_tokens_seen": 3282528,
+      "step": 219
+    },
+    {
+      "epoch": 2.829581993569132,
+      "grad_norm": 9.812615394592285,
+      "learning_rate": 1.8333333333333333e-06,
+      "loss": 0.0635,
+      "num_input_tokens_seen": 3297824,
+      "step": 220
+    },
+    {
+      "epoch": 2.842443729903537,
+      "grad_norm": 5.7050580978393555,
+      "learning_rate": 1.8416666666666669e-06,
+      "loss": 0.0614,
+      "num_input_tokens_seen": 3312480,
+      "step": 221
+    },
+    {
+      "epoch": 2.855305466237942,
+      "grad_norm": 13.298068046569824,
+      "learning_rate": 1.85e-06,
+      "loss": 0.0646,
+      "num_input_tokens_seen": 3327392,
+      "step": 222
+    },
+    {
+      "epoch": 2.868167202572347,
+      "grad_norm": 13.815607070922852,
+      "learning_rate": 1.8583333333333335e-06,
+      "loss": 0.0893,
+      "num_input_tokens_seen": 3342944,
+      "step": 223
+    },
+    {
+      "epoch": 2.8810289389067525,
+      "grad_norm": 6.954331398010254,
+      "learning_rate": 1.8666666666666669e-06,
+      "loss": 0.0653,
+      "num_input_tokens_seen": 3358112,
+      "step": 224
+    },
+    {
+      "epoch": 2.8938906752411575,
+      "grad_norm": 5.133541584014893,
+      "learning_rate": 1.8750000000000003e-06,
+      "loss": 0.0402,
+      "num_input_tokens_seen": 3373472,
+      "step": 225
+    },
+    {
+      "epoch": 2.906752411575563,
+      "grad_norm": 7.754403114318848,
+      "learning_rate": 1.8833333333333334e-06,
+      "loss": 0.0407,
+      "num_input_tokens_seen": 3389088,
+      "step": 226
+    },
+    {
+      "epoch": 2.919614147909968,
+      "grad_norm": 9.570775985717773,
+      "learning_rate": 1.8916666666666668e-06,
+      "loss": 0.0949,
+      "num_input_tokens_seen": 3403616,
+      "step": 227
+    },
+    {
+      "epoch": 2.932475884244373,
+      "grad_norm": 9.21279525756836,
+      "learning_rate": 1.9000000000000002e-06,
+      "loss": 0.0789,
+      "num_input_tokens_seen": 3418944,
+      "step": 228
+    },
+    {
+      "epoch": 2.945337620578778,
+      "grad_norm": 4.255781650543213,
+      "learning_rate": 1.9083333333333334e-06,
+      "loss": 0.0438,
+      "num_input_tokens_seen": 3434048,
+      "step": 229
+    },
+    {
+      "epoch": 2.958199356913183,
+      "grad_norm": 10.596036911010742,
+      "learning_rate": 1.916666666666667e-06,
+      "loss": 0.0905,
+      "num_input_tokens_seen": 3449952,
+      "step": 230
+    },
+    {
+      "epoch": 2.9710610932475885,
+      "grad_norm": 9.656909942626953,
+      "learning_rate": 1.925e-06,
+      "loss": 0.0495,
+      "num_input_tokens_seen": 3464736,
+      "step": 231
+    },
+    {
+      "epoch": 2.9839228295819935,
+      "grad_norm": 9.526041984558105,
+      "learning_rate": 1.9333333333333336e-06,
+      "loss": 0.0642,
+      "num_input_tokens_seen": 3480416,
+      "step": 232
+    },
+    {
+      "epoch": 2.996784565916399,
+      "grad_norm": 5.788015365600586,
+      "learning_rate": 1.9416666666666666e-06,
+      "loss": 0.0607,
+      "num_input_tokens_seen": 3495264,
+      "step": 233
+    },
+    {
+      "epoch": 3.009646302250804,
+      "grad_norm": 6.213415145874023,
+      "learning_rate": 1.9500000000000004e-06,
+      "loss": 0.0363,
+      "num_input_tokens_seen": 3510080,
+      "step": 234
+    },
+    {
+      "epoch": 3.022508038585209,
+      "grad_norm": 7.382534980773926,
+      "learning_rate": 1.9583333333333334e-06,
+      "loss": 0.0492,
+      "num_input_tokens_seen": 3524736,
+      "step": 235
+    },
+    {
+      "epoch": 3.035369774919614,
+      "grad_norm": 9.54261302947998,
+      "learning_rate": 1.9666666666666668e-06,
+      "loss": 0.0373,
+      "num_input_tokens_seen": 3539968,
+      "step": 236
+    },
+    {
+      "epoch": 3.0482315112540195,
+      "grad_norm": 3.8869571685791016,
+      "learning_rate": 1.975e-06,
+      "loss": 0.0203,
+      "num_input_tokens_seen": 3554464,
+      "step": 237
+    },
+    {
+      "epoch": 3.0610932475884245,
+      "grad_norm": 2.183758497238159,
+      "learning_rate": 1.9833333333333335e-06,
+      "loss": 0.0175,
+      "num_input_tokens_seen": 3569664,
+      "step": 238
+    },
+    {
+      "epoch": 3.0739549839228295,
+      "grad_norm": 3.851379871368408,
+      "learning_rate": 1.991666666666667e-06,
+      "loss": 0.0497,
+      "num_input_tokens_seen": 3585248,
+      "step": 239
+    },
+    {
+      "epoch": 3.0868167202572345,
+      "grad_norm": 6.622572898864746,
+      "learning_rate": 2.0000000000000003e-06,
+      "loss": 0.0361,
+      "num_input_tokens_seen": 3600160,
+      "step": 240
+    },
+    {
+      "epoch": 3.09967845659164,
+      "grad_norm": 3.4328978061676025,
+      "learning_rate": 2.0083333333333337e-06,
+      "loss": 0.0193,
+      "num_input_tokens_seen": 3615232,
+      "step": 241
+    },
+    {
+      "epoch": 3.112540192926045,
+      "grad_norm": 8.150415420532227,
+      "learning_rate": 2.0166666666666667e-06,
+      "loss": 0.0142,
+      "num_input_tokens_seen": 3629312,
+      "step": 242
+    },
+    {
+      "epoch": 3.12540192926045,
+      "grad_norm": 5.3073835372924805,
+      "learning_rate": 2.025e-06,
+      "loss": 0.0415,
+      "num_input_tokens_seen": 3644064,
+      "step": 243
+    },
+    {
+      "epoch": 3.1382636655948555,
+      "grad_norm": 4.33547830581665,
+      "learning_rate": 2.0333333333333335e-06,
+      "loss": 0.0178,
+      "num_input_tokens_seen": 3659328,
+      "step": 244
+    },
+    {
+      "epoch": 3.1511254019292605,
+      "grad_norm": 2.770535469055176,
+      "learning_rate": 2.041666666666667e-06,
+      "loss": 0.0166,
+      "num_input_tokens_seen": 3674720,
+      "step": 245
+    },
+    {
+      "epoch": 3.1639871382636655,
+      "grad_norm": 8.939215660095215,
+      "learning_rate": 2.05e-06,
+      "loss": 0.0424,
+      "num_input_tokens_seen": 3689984,
+      "step": 246
+    },
+    {
+      "epoch": 3.176848874598071,
+      "grad_norm": 4.35013484954834,
+      "learning_rate": 2.0583333333333337e-06,
+      "loss": 0.0464,
+      "num_input_tokens_seen": 3705216,
+      "step": 247
+    },
+    {
+      "epoch": 3.189710610932476,
+      "grad_norm": 4.8758368492126465,
+      "learning_rate": 2.0666666666666666e-06,
+      "loss": 0.0235,
+      "num_input_tokens_seen": 3719712,
+      "step": 248
+    },
+    {
+      "epoch": 3.202572347266881,
+      "grad_norm": 2.769258499145508,
+      "learning_rate": 2.075e-06,
+      "loss": 0.0128,
+      "num_input_tokens_seen": 3734272,
+      "step": 249
+    },
+    {
+      "epoch": 3.215434083601286,
+      "grad_norm": 4.699342727661133,
+      "learning_rate": 2.0833333333333334e-06,
+      "loss": 0.0319,
+      "num_input_tokens_seen": 3750240,
+      "step": 250
+    },
+    {
+      "epoch": 3.2282958199356915,
+      "grad_norm": 3.377748966217041,
+      "learning_rate": 2.091666666666667e-06,
+      "loss": 0.0196,
+      "num_input_tokens_seen": 3764736,
+      "step": 251
+    },
+    {
+      "epoch": 3.2411575562700965,
+      "grad_norm": 2.9469077587127686,
+      "learning_rate": 2.1000000000000002e-06,
+      "loss": 0.0326,
+      "num_input_tokens_seen": 3779744,
+      "step": 252
+    },
+    {
+      "epoch": 3.2540192926045015,
+      "grad_norm": 3.4729647636413574,
+      "learning_rate": 2.1083333333333336e-06,
+      "loss": 0.017,
+      "num_input_tokens_seen": 3794208,
+      "step": 253
+    },
+    {
+      "epoch": 3.266881028938907,
+      "grad_norm": 3.1130521297454834,
+      "learning_rate": 2.116666666666667e-06,
+      "loss": 0.0372,
+      "num_input_tokens_seen": 3809376,
+      "step": 254
+    },
+    {
+      "epoch": 3.279742765273312,
+      "grad_norm": 3.0094053745269775,
+      "learning_rate": 2.125e-06,
+      "loss": 0.0165,
+      "num_input_tokens_seen": 3824384,
+      "step": 255
+    },
+    {
+      "epoch": 3.292604501607717,
+      "grad_norm": 2.0758721828460693,
+      "learning_rate": 2.133333333333334e-06,
+      "loss": 0.0142,
+      "num_input_tokens_seen": 3840000,
+      "step": 256
+    },
+    {
+      "epoch": 3.305466237942122,
+      "grad_norm": 2.0254931449890137,
+      "learning_rate": 2.1416666666666668e-06,
+      "loss": 0.0117,
+      "num_input_tokens_seen": 3854240,
+      "step": 257
+    },
+    {
+      "epoch": 3.3183279742765275,
+      "grad_norm": 5.01600980758667,
+      "learning_rate": 2.15e-06,
+      "loss": 0.0264,
+      "num_input_tokens_seen": 3869792,
+      "step": 258
+    },
+    {
+      "epoch": 3.3311897106109325,
+      "grad_norm": 7.374160289764404,
+      "learning_rate": 2.1583333333333336e-06,
+      "loss": 0.034,
+      "num_input_tokens_seen": 3885152,
+      "step": 259
+    },
+    {
+      "epoch": 3.3440514469453375,
+      "grad_norm": 7.036656856536865,
+      "learning_rate": 2.166666666666667e-06,
+      "loss": 0.031,
+      "num_input_tokens_seen": 3900032,
+      "step": 260
+    },
+    {
+      "epoch": 3.356913183279743,
+      "grad_norm": 4.439852237701416,
+      "learning_rate": 2.1750000000000004e-06,
+      "loss": 0.018,
+      "num_input_tokens_seen": 3914432,
+      "step": 261
+    },
+    {
+      "epoch": 3.369774919614148,
+      "grad_norm": 8.757259368896484,
+      "learning_rate": 2.1833333333333333e-06,
+      "loss": 0.0461,
+      "num_input_tokens_seen": 3929600,
+      "step": 262
+    },
+    {
+      "epoch": 3.382636655948553,
+      "grad_norm": 4.993197917938232,
+      "learning_rate": 2.191666666666667e-06,
+      "loss": 0.0372,
+      "num_input_tokens_seen": 3944032,
+      "step": 263
+    },
+    {
+      "epoch": 3.395498392282958,
+      "grad_norm": 5.627339839935303,
+      "learning_rate": 2.2e-06,
+      "loss": 0.0471,
+      "num_input_tokens_seen": 3958464,
+      "step": 264
+    },
+    {
+      "epoch": 3.4083601286173635,
+      "grad_norm": 6.644600868225098,
+      "learning_rate": 2.2083333333333335e-06,
+      "loss": 0.037,
+      "num_input_tokens_seen": 3973024,
+      "step": 265
+    },
+    {
+      "epoch": 3.4212218649517685,
+      "grad_norm": 4.708102703094482,
+      "learning_rate": 2.216666666666667e-06,
+      "loss": 0.0245,
+      "num_input_tokens_seen": 3987520,
+      "step": 266
+    },
+    {
+      "epoch": 3.4340836012861735,
+      "grad_norm": 3.4480528831481934,
+      "learning_rate": 2.2250000000000003e-06,
+      "loss": 0.0233,
+      "num_input_tokens_seen": 4002656,
+      "step": 267
+    },
+    {
+      "epoch": 3.446945337620579,
+      "grad_norm": 5.0224175453186035,
+      "learning_rate": 2.2333333333333333e-06,
+      "loss": 0.0256,
+      "num_input_tokens_seen": 4017824,
+      "step": 268
+    },
+    {
+      "epoch": 3.459807073954984,
+      "grad_norm": 9.12070369720459,
+      "learning_rate": 2.2416666666666667e-06,
+      "loss": 0.0345,
+      "num_input_tokens_seen": 4032480,
+      "step": 269
+    },
+    {
+      "epoch": 3.472668810289389,
+      "grad_norm": 5.336618900299072,
+      "learning_rate": 2.25e-06,
+      "loss": 0.0656,
+      "num_input_tokens_seen": 4047648,
+      "step": 270
+    },
+    {
+      "epoch": 3.485530546623794,
+      "grad_norm": 4.003684997558594,
+      "learning_rate": 2.2583333333333335e-06,
+      "loss": 0.0247,
+      "num_input_tokens_seen": 4062688,
+      "step": 271
+    },
+    {
+      "epoch": 3.4983922829581995,
+      "grad_norm": 5.631625175476074,
+      "learning_rate": 2.266666666666667e-06,
+      "loss": 0.051,
+      "num_input_tokens_seen": 4077856,
+      "step": 272
+    },
+    {
+      "epoch": 3.5112540192926045,
+      "grad_norm": 4.748563766479492,
+      "learning_rate": 2.2750000000000002e-06,
+      "loss": 0.0417,
+      "num_input_tokens_seen": 4092864,
+      "step": 273
+    },
+    {
+      "epoch": 3.5241157556270095,
+      "grad_norm": 3.368995189666748,
+      "learning_rate": 2.2833333333333336e-06,
+      "loss": 0.0176,
+      "num_input_tokens_seen": 4107584,
+      "step": 274
+    },
+    {
+      "epoch": 3.536977491961415,
+      "grad_norm": 3.463752508163452,
+      "learning_rate": 2.2916666666666666e-06,
+      "loss": 0.0155,
+      "num_input_tokens_seen": 4122080,
+      "step": 275
+    },
+    {
+      "epoch": 3.54983922829582,
+      "grad_norm": 4.7195234298706055,
+      "learning_rate": 2.3000000000000004e-06,
+      "loss": 0.0135,
+      "num_input_tokens_seen": 4137600,
+      "step": 276
+    },
+    {
+      "epoch": 3.562700964630225,
+      "grad_norm": 6.3303680419921875,
+      "learning_rate": 2.3083333333333334e-06,
+      "loss": 0.0222,
+      "num_input_tokens_seen": 4152960,
+      "step": 277
+    },
+    {
+      "epoch": 3.57556270096463,
+      "grad_norm": 3.4051268100738525,
+      "learning_rate": 2.316666666666667e-06,
+      "loss": 0.0289,
+      "num_input_tokens_seen": 4168640,
+      "step": 278
+    },
+    {
+      "epoch": 3.5884244372990355,
+      "grad_norm": 3.971222162246704,
+      "learning_rate": 2.325e-06,
+      "loss": 0.0314,
+      "num_input_tokens_seen": 4183616,
+      "step": 279
+    },
+    {
+      "epoch": 3.6012861736334405,
+      "grad_norm": 5.509104251861572,
+      "learning_rate": 2.3333333333333336e-06,
+      "loss": 0.0582,
+      "num_input_tokens_seen": 4197536,
+      "step": 280
+    },
+    {
+      "epoch": 3.6141479099678455,
+      "grad_norm": 4.121878623962402,
+      "learning_rate": 2.341666666666667e-06,
+      "loss": 0.0339,
+      "num_input_tokens_seen": 4213152,
+      "step": 281
+    },
+    {
+      "epoch": 3.627009646302251,
+      "grad_norm": 6.977634429931641,
+      "learning_rate": 2.35e-06,
+      "loss": 0.0656,
+      "num_input_tokens_seen": 4228320,
+      "step": 282
+    },
+    {
+      "epoch": 3.639871382636656,
+      "grad_norm": 3.6625123023986816,
+      "learning_rate": 2.3583333333333338e-06,
+      "loss": 0.0355,
+      "num_input_tokens_seen": 4243680,
+      "step": 283
+    },
+    {
+      "epoch": 3.652733118971061,
+      "grad_norm": 4.691339492797852,
+      "learning_rate": 2.3666666666666667e-06,
+      "loss": 0.037,
+      "num_input_tokens_seen": 4258464,
+      "step": 284
+    },
+    {
+      "epoch": 3.665594855305466,
+      "grad_norm": 3.1770055294036865,
+      "learning_rate": 2.375e-06,
+      "loss": 0.0206,
+      "num_input_tokens_seen": 4273728,
+      "step": 285
+    },
+    {
+      "epoch": 3.6784565916398715,
+      "grad_norm": 2.424788475036621,
+      "learning_rate": 2.3833333333333335e-06,
+      "loss": 0.0286,
+      "num_input_tokens_seen": 4288416,
+      "step": 286
+    },
+    {
+      "epoch": 3.6913183279742765,
+      "grad_norm": 5.731730937957764,
+      "learning_rate": 2.391666666666667e-06,
+      "loss": 0.0316,
+      "num_input_tokens_seen": 4303712,
+      "step": 287
+    },
+    {
+      "epoch": 3.7041800643086815,
+      "grad_norm": 4.939082622528076,
+      "learning_rate": 2.4000000000000003e-06,
+      "loss": 0.0263,
+      "num_input_tokens_seen": 4319104,
+      "step": 288
+    },
+    {
+      "epoch": 3.717041800643087,
+      "grad_norm": 2.817589521408081,
+      "learning_rate": 2.4083333333333337e-06,
+      "loss": 0.0265,
+      "num_input_tokens_seen": 4334912,
+      "step": 289
+    },
+    {
+      "epoch": 3.729903536977492,
+      "grad_norm": 2.6983306407928467,
+      "learning_rate": 2.4166666666666667e-06,
+      "loss": 0.0251,
+      "num_input_tokens_seen": 4349280,
+      "step": 290
+    },
+    {
+      "epoch": 3.742765273311897,
+      "grad_norm": 5.260397434234619,
+      "learning_rate": 2.425e-06,
+      "loss": 0.0293,
+      "num_input_tokens_seen": 4363872,
+      "step": 291
+    },
+    {
+      "epoch": 3.755627009646302,
+      "grad_norm": 4.8456034660339355,
+      "learning_rate": 2.4333333333333335e-06,
+      "loss": 0.0278,
+      "num_input_tokens_seen": 4378752,
+      "step": 292
+    },
+    {
+      "epoch": 3.7684887459807075,
+      "grad_norm": 4.560141563415527,
+      "learning_rate": 2.441666666666667e-06,
+      "loss": 0.035,
+      "num_input_tokens_seen": 4393792,
+      "step": 293
+    },
+    {
+      "epoch": 3.7813504823151125,
+      "grad_norm": 6.035860538482666,
+      "learning_rate": 2.4500000000000003e-06,
+      "loss": 0.0455,
+      "num_input_tokens_seen": 4408192,
+      "step": 294
+    },
+    {
+      "epoch": 3.7942122186495175,
+      "grad_norm": 6.634167194366455,
+      "learning_rate": 2.4583333333333332e-06,
+      "loss": 0.0429,
+      "num_input_tokens_seen": 4422432,
+      "step": 295
+    },
+    {
+      "epoch": 3.807073954983923,
+      "grad_norm": 9.78260326385498,
+      "learning_rate": 2.466666666666667e-06,
+      "loss": 0.056,
+      "num_input_tokens_seen": 4437728,
+      "step": 296
+    },
+    {
+      "epoch": 3.819935691318328,
+      "grad_norm": 6.473577499389648,
+      "learning_rate": 2.475e-06,
+      "loss": 0.0325,
+      "num_input_tokens_seen": 4452928,
+      "step": 297
+    },
+    {
+      "epoch": 3.832797427652733,
+      "grad_norm": 8.688652992248535,
+      "learning_rate": 2.4833333333333334e-06,
+      "loss": 0.0401,
+      "num_input_tokens_seen": 4468096,
+      "step": 298
+    },
+    {
+      "epoch": 3.845659163987138,
+      "grad_norm": 4.41790771484375,
+      "learning_rate": 2.491666666666667e-06,
+      "loss": 0.0182,
+      "num_input_tokens_seen": 4482976,
+      "step": 299
+    },
+    {
+      "epoch": 3.8585209003215435,
+      "grad_norm": 3.7182106971740723,
+      "learning_rate": 2.5e-06,
+      "loss": 0.0344,
+      "num_input_tokens_seen": 4497952,
+      "step": 300
+    },
+    {
+      "epoch": 3.8713826366559485,
+      "grad_norm": 4.0538225173950195,
+      "learning_rate": 2.5083333333333336e-06,
+      "loss": 0.0111,
+      "num_input_tokens_seen": 4512896,
+      "step": 301
+    },
+    {
+      "epoch": 3.884244372990354,
+      "grad_norm": 3.3492507934570312,
+      "learning_rate": 2.5166666666666666e-06,
+      "loss": 0.0088,
+      "num_input_tokens_seen": 4528448,
+      "step": 302
+    },
+    {
+      "epoch": 3.897106109324759,
+      "grad_norm": 3.518641710281372,
+      "learning_rate": 2.5250000000000004e-06,
+      "loss": 0.013,
+      "num_input_tokens_seen": 4543712,
+      "step": 303
+    },
+    {
+      "epoch": 3.909967845659164,
+      "grad_norm": 5.015272617340088,
+      "learning_rate": 2.5333333333333338e-06,
+      "loss": 0.0433,
+      "num_input_tokens_seen": 4559008,
+      "step": 304
+    },
+    {
+      "epoch": 3.922829581993569,
+      "grad_norm": 3.2924585342407227,
+      "learning_rate": 2.5416666666666668e-06,
+      "loss": 0.0214,
+      "num_input_tokens_seen": 4573696,
+      "step": 305
+    },
+    {
+      "epoch": 3.935691318327974,
+      "grad_norm": 8.486410140991211,
+      "learning_rate": 2.55e-06,
+      "loss": 0.0416,
+      "num_input_tokens_seen": 4588384,
+      "step": 306
+    },
+    {
+      "epoch": 3.9485530546623795,
+      "grad_norm": 6.4795002937316895,
+      "learning_rate": 2.558333333333334e-06,
+      "loss": 0.0321,
+      "num_input_tokens_seen": 4602944,
+      "step": 307
+    },
+    {
+      "epoch": 3.9614147909967845,
+      "grad_norm": 6.211554050445557,
+      "learning_rate": 2.566666666666667e-06,
+      "loss": 0.0526,
+      "num_input_tokens_seen": 4618336,
+      "step": 308
+    },
+    {
+      "epoch": 3.97427652733119,
+      "grad_norm": 5.602302074432373,
+      "learning_rate": 2.5750000000000003e-06,
+      "loss": 0.025,
+      "num_input_tokens_seen": 4633472,
+      "step": 309
+    },
+    {
+      "epoch": 3.987138263665595,
+      "grad_norm": 9.136455535888672,
+      "learning_rate": 2.5833333333333337e-06,
+      "loss": 0.0787,
+      "num_input_tokens_seen": 4649472,
+      "step": 310
+    },
+    {
+      "epoch": 4.0,
+      "grad_norm": 8.398818969726562,
+      "learning_rate": 2.5916666666666667e-06,
+      "loss": 0.0559,
+      "num_input_tokens_seen": 4664384,
+      "step": 311
+    },
+    {
+      "epoch": 4.012861736334405,
+      "grad_norm": 2.286210536956787,
+      "learning_rate": 2.6e-06,
+      "loss": 0.0259,
+      "num_input_tokens_seen": 4679200,
+      "step": 312
+    },
+    {
+      "epoch": 4.02572347266881,
+      "grad_norm": 6.477627754211426,
+      "learning_rate": 2.608333333333333e-06,
+      "loss": 0.037,
+      "num_input_tokens_seen": 4693696,
+      "step": 313
+    },
+    {
+      "epoch": 4.038585209003215,
+      "grad_norm": 4.953974723815918,
+      "learning_rate": 2.616666666666667e-06,
+      "loss": 0.0293,
+      "num_input_tokens_seen": 4708608,
+      "step": 314
+    },
+    {
+      "epoch": 4.051446945337621,
+      "grad_norm": 2.7035064697265625,
+      "learning_rate": 2.6250000000000003e-06,
+      "loss": 0.0153,
+      "num_input_tokens_seen": 4722944,
+      "step": 315
+    },
+    {
+      "epoch": 4.064308681672026,
+      "grad_norm": 3.8599040508270264,
+      "learning_rate": 2.6333333333333332e-06,
+      "loss": 0.0234,
+      "num_input_tokens_seen": 4739296,
+      "step": 316
+    },
+    {
+      "epoch": 4.077170418006431,
+      "grad_norm": 5.4720072746276855,
+      "learning_rate": 2.6416666666666666e-06,
+      "loss": 0.0212,
+      "num_input_tokens_seen": 4754048,
+      "step": 317
+    },
+    {
+      "epoch": 4.090032154340836,
+      "grad_norm": 3.8060128688812256,
+      "learning_rate": 2.6500000000000005e-06,
+      "loss": 0.0187,
+      "num_input_tokens_seen": 4769376,
+      "step": 318
+    },
+    {
+      "epoch": 4.102893890675241,
+      "grad_norm": 5.436901092529297,
+      "learning_rate": 2.6583333333333334e-06,
+      "loss": 0.0245,
+      "num_input_tokens_seen": 4784640,
+      "step": 319
+    },
+    {
+      "epoch": 4.115755627009646,
+      "grad_norm": 1.5260390043258667,
+      "learning_rate": 2.666666666666667e-06,
+      "loss": 0.0032,
+      "num_input_tokens_seen": 4799392,
+      "step": 320
+    },
+    {
+      "epoch": 4.128617363344051,
+      "grad_norm": 11.145827293395996,
+      "learning_rate": 2.6750000000000002e-06,
+      "loss": 0.0367,
+      "num_input_tokens_seen": 4813696,
+      "step": 321
+    },
+    {
+      "epoch": 4.141479099678457,
+      "grad_norm": 5.737987518310547,
+      "learning_rate": 2.683333333333333e-06,
+      "loss": 0.038,
+      "num_input_tokens_seen": 4828928,
+      "step": 322
+    },
+    {
+      "epoch": 4.154340836012862,
+      "grad_norm": 12.955948829650879,
+      "learning_rate": 2.691666666666667e-06,
+      "loss": 0.0194,
+      "num_input_tokens_seen": 4843232,
+      "step": 323
+    },
+    {
+      "epoch": 4.167202572347267,
+      "grad_norm": 2.663821220397949,
+      "learning_rate": 2.7000000000000004e-06,
+      "loss": 0.0262,
+      "num_input_tokens_seen": 4858592,
+      "step": 324
+    },
+    {
+      "epoch": 4.180064308681672,
+      "grad_norm": 7.183210372924805,
+      "learning_rate": 2.7083333333333334e-06,
+      "loss": 0.0151,
+      "num_input_tokens_seen": 4872928,
+      "step": 325
+    },
+    {
+      "epoch": 4.192926045016077,
+      "grad_norm": 5.024378776550293,
+      "learning_rate": 2.7166666666666668e-06,
+      "loss": 0.0267,
+      "num_input_tokens_seen": 4887968,
+      "step": 326
+    },
+    {
+      "epoch": 4.205787781350482,
+      "grad_norm": 1.0667839050292969,
+      "learning_rate": 2.7250000000000006e-06,
+      "loss": 0.0041,
+      "num_input_tokens_seen": 4902752,
+      "step": 327
+    },
+    {
+      "epoch": 4.218649517684887,
+      "grad_norm": 6.243584632873535,
+      "learning_rate": 2.7333333333333336e-06,
+      "loss": 0.0365,
+      "num_input_tokens_seen": 4917280,
+      "step": 328
+    },
+    {
+      "epoch": 4.231511254019293,
+      "grad_norm": 3.255237579345703,
+      "learning_rate": 2.741666666666667e-06,
+      "loss": 0.0151,
+      "num_input_tokens_seen": 4933088,
+      "step": 329
+    },
+    {
+      "epoch": 4.244372990353698,
+      "grad_norm": 1.9820326566696167,
+      "learning_rate": 2.7500000000000004e-06,
+      "loss": 0.006,
+      "num_input_tokens_seen": 4948128,
+      "step": 330
+    },
+    {
+      "epoch": 4.257234726688103,
+      "grad_norm": 4.553264141082764,
+      "learning_rate": 2.7583333333333333e-06,
+      "loss": 0.0247,
+      "num_input_tokens_seen": 4962752,
+      "step": 331
+    },
+    {
+      "epoch": 4.270096463022508,
+      "grad_norm": 3.139723062515259,
+      "learning_rate": 2.766666666666667e-06,
+      "loss": 0.0147,
+      "num_input_tokens_seen": 4978272,
+      "step": 332
+    },
+    {
+      "epoch": 4.282958199356913,
+      "grad_norm": 9.011144638061523,
+      "learning_rate": 2.7750000000000005e-06,
+      "loss": 0.0413,
+      "num_input_tokens_seen": 4992384,
+      "step": 333
+    },
+    {
+      "epoch": 4.295819935691318,
+      "grad_norm": 11.36248779296875,
+      "learning_rate": 2.7833333333333335e-06,
+      "loss": 0.0318,
+      "num_input_tokens_seen": 5007136,
+      "step": 334
+    },
+    {
+      "epoch": 4.308681672025724,
+      "grad_norm": 7.9140305519104,
+      "learning_rate": 2.791666666666667e-06,
+      "loss": 0.0513,
+      "num_input_tokens_seen": 5021600,
+      "step": 335
+    },
+    {
+      "epoch": 4.321543408360129,
+      "grad_norm": 4.12211799621582,
+      "learning_rate": 2.8000000000000003e-06,
+      "loss": 0.0165,
+      "num_input_tokens_seen": 5037216,
+      "step": 336
+    },
+    {
+      "epoch": 4.334405144694534,
+      "grad_norm": 4.724823951721191,
+      "learning_rate": 2.8083333333333333e-06,
+      "loss": 0.0131,
+      "num_input_tokens_seen": 5052320,
+      "step": 337
+    },
+    {
+      "epoch": 4.347266881028939,
+      "grad_norm": 6.9196085929870605,
+      "learning_rate": 2.816666666666667e-06,
+      "loss": 0.0246,
+      "num_input_tokens_seen": 5067552,
+      "step": 338
+    },
+    {
+      "epoch": 4.360128617363344,
+      "grad_norm": 5.785276412963867,
+      "learning_rate": 2.825e-06,
+      "loss": 0.0465,
+      "num_input_tokens_seen": 5082240,
+      "step": 339
+    },
+    {
+      "epoch": 4.372990353697749,
+      "grad_norm": 1.6201478242874146,
+      "learning_rate": 2.8333333333333335e-06,
+      "loss": 0.0084,
+      "num_input_tokens_seen": 5097408,
+      "step": 340
+    },
+    {
+      "epoch": 4.385852090032154,
+      "grad_norm": 5.511898994445801,
+      "learning_rate": 2.841666666666667e-06,
+      "loss": 0.0105,
+      "num_input_tokens_seen": 5111680,
+      "step": 341
+    },
+    {
+      "epoch": 4.39871382636656,
+      "grad_norm": 8.025418281555176,
+      "learning_rate": 2.85e-06,
+      "loss": 0.0388,
+      "num_input_tokens_seen": 5127264,
+      "step": 342
+    },
+    {
+      "epoch": 4.411575562700965,
+      "grad_norm": 12.538291931152344,
+      "learning_rate": 2.8583333333333336e-06,
+      "loss": 0.0486,
+      "num_input_tokens_seen": 5142464,
+      "step": 343
+    },
+    {
+      "epoch": 4.42443729903537,
+      "grad_norm": 10.100396156311035,
+      "learning_rate": 2.866666666666667e-06,
+      "loss": 0.0354,
+      "num_input_tokens_seen": 5157376,
+      "step": 344
+    },
+    {
+      "epoch": 4.437299035369775,
+      "grad_norm": 4.218758583068848,
+      "learning_rate": 2.875e-06,
+      "loss": 0.0464,
+      "num_input_tokens_seen": 5171648,
+      "step": 345
+    },
+    {
+      "epoch": 4.45016077170418,
+      "grad_norm": 5.751765251159668,
+      "learning_rate": 2.8833333333333334e-06,
+      "loss": 0.0312,
+      "num_input_tokens_seen": 5186752,
+      "step": 346
+    },
+    {
+      "epoch": 4.463022508038585,
+      "grad_norm": 6.100085258483887,
+      "learning_rate": 2.8916666666666672e-06,
+      "loss": 0.0352,
+      "num_input_tokens_seen": 5202176,
+      "step": 347
+    },
+    {
+      "epoch": 4.47588424437299,
+      "grad_norm": 6.4870991706848145,
+      "learning_rate": 2.9e-06,
+      "loss": 0.0116,
+      "num_input_tokens_seen": 5216896,
+      "step": 348
+    },
+    {
+      "epoch": 4.488745980707396,
+      "grad_norm": 1.8258070945739746,
+      "learning_rate": 2.9083333333333336e-06,
+      "loss": 0.0118,
+      "num_input_tokens_seen": 5232448,
+      "step": 349
+    },
+    {
+      "epoch": 4.501607717041801,
+      "grad_norm": 2.6930766105651855,
+      "learning_rate": 2.916666666666667e-06,
+      "loss": 0.0138,
+      "num_input_tokens_seen": 5247296,
+      "step": 350
+    },
+    {
+      "epoch": 4.514469453376206,
+      "grad_norm": 5.420289993286133,
+      "learning_rate": 2.925e-06,
+      "loss": 0.0344,
+      "num_input_tokens_seen": 5262304,
+      "step": 351
+    },
+    {
+      "epoch": 4.527331189710611,
+      "grad_norm": 4.682693958282471,
+      "learning_rate": 2.9333333333333338e-06,
+      "loss": 0.0231,
+      "num_input_tokens_seen": 5277664,
+      "step": 352
+    },
+    {
+      "epoch": 4.540192926045016,
+      "grad_norm": 4.194357872009277,
+      "learning_rate": 2.941666666666667e-06,
+      "loss": 0.0267,
+      "num_input_tokens_seen": 5292576,
+      "step": 353
+    },
+    {
+      "epoch": 4.553054662379421,
+      "grad_norm": 1.0653674602508545,
+      "learning_rate": 2.95e-06,
+      "loss": 0.0109,
+      "num_input_tokens_seen": 5306304,
+      "step": 354
+    },
+    {
+      "epoch": 4.565916398713826,
+      "grad_norm": 1.8720684051513672,
+      "learning_rate": 2.9583333333333335e-06,
+      "loss": 0.0444,
+      "num_input_tokens_seen": 5320768,
+      "step": 355
+    },
+    {
+      "epoch": 4.578778135048232,
+      "grad_norm": 4.005693435668945,
+      "learning_rate": 2.9666666666666673e-06,
+      "loss": 0.0142,
+      "num_input_tokens_seen": 5336128,
+      "step": 356
+    },
+    {
+      "epoch": 4.591639871382637,
+      "grad_norm": 5.997867107391357,
+      "learning_rate": 2.9750000000000003e-06,
+      "loss": 0.0361,
+      "num_input_tokens_seen": 5351168,
+      "step": 357
+    },
+    {
+      "epoch": 4.604501607717042,
+      "grad_norm": 3.8871259689331055,
+      "learning_rate": 2.9833333333333337e-06,
+      "loss": 0.0197,
+      "num_input_tokens_seen": 5366112,
+      "step": 358
+    },
+    {
+      "epoch": 4.617363344051447,
+      "grad_norm": 4.269320011138916,
+      "learning_rate": 2.991666666666667e-06,
+      "loss": 0.0373,
+      "num_input_tokens_seen": 5381440,
+      "step": 359
+    },
+    {
+      "epoch": 4.630225080385852,
+      "grad_norm": 3.6487338542938232,
+      "learning_rate": 3e-06,
+      "loss": 0.0469,
+      "num_input_tokens_seen": 5397152,
+      "step": 360
+    },
+    {
+      "epoch": 4.643086816720257,
+      "grad_norm": 3.9794583320617676,
+      "learning_rate": 3.0083333333333335e-06,
+      "loss": 0.0283,
+      "num_input_tokens_seen": 5412128,
+      "step": 361
+    },
+    {
+      "epoch": 4.655948553054662,
+      "grad_norm": 5.071774482727051,
+      "learning_rate": 3.0166666666666673e-06,
+      "loss": 0.0271,
+      "num_input_tokens_seen": 5426816,
+      "step": 362
+    },
+    {
+      "epoch": 4.668810289389068,
+      "grad_norm": 3.651871681213379,
+      "learning_rate": 3.0250000000000003e-06,
+      "loss": 0.0303,
+      "num_input_tokens_seen": 5441760,
+      "step": 363
+    },
+    {
+      "epoch": 4.681672025723473,
+      "grad_norm": 3.1399481296539307,
+      "learning_rate": 3.0333333333333337e-06,
+      "loss": 0.0207,
+      "num_input_tokens_seen": 5457504,
+      "step": 364
+    },
+    {
+      "epoch": 4.694533762057878,
+      "grad_norm": 3.089629888534546,
+      "learning_rate": 3.0416666666666666e-06,
+      "loss": 0.0157,
+      "num_input_tokens_seen": 5473088,
+      "step": 365
+    },
+    {
+      "epoch": 4.707395498392283,
+      "grad_norm": 1.4184813499450684,
+      "learning_rate": 3.05e-06,
+      "loss": 0.0128,
+      "num_input_tokens_seen": 5487712,
+      "step": 366
+    },
+    {
+      "epoch": 4.720257234726688,
+      "grad_norm": 5.21925687789917,
+      "learning_rate": 3.058333333333334e-06,
+      "loss": 0.0293,
+      "num_input_tokens_seen": 5503680,
+      "step": 367
+    },
+    {
+      "epoch": 4.733118971061093,
+      "grad_norm": 3.7801499366760254,
+      "learning_rate": 3.066666666666667e-06,
+      "loss": 0.0211,
+      "num_input_tokens_seen": 5518464,
+      "step": 368
+    },
+    {
+      "epoch": 4.745980707395498,
+      "grad_norm": 9.715234756469727,
+      "learning_rate": 3.075e-06,
+      "loss": 0.0929,
+      "num_input_tokens_seen": 5533504,
+      "step": 369
+    },
+    {
+      "epoch": 4.758842443729904,
+      "grad_norm": 6.241518974304199,
+      "learning_rate": 3.0833333333333336e-06,
+      "loss": 0.0195,
+      "num_input_tokens_seen": 5548832,
+      "step": 370
+    },
+    {
+      "epoch": 4.771704180064309,
+      "grad_norm": 2.6658706665039062,
+      "learning_rate": 3.0916666666666666e-06,
+      "loss": 0.0114,
+      "num_input_tokens_seen": 5564256,
+      "step": 371
+    },
+    {
+      "epoch": 4.784565916398714,
+      "grad_norm": 7.38095235824585,
+      "learning_rate": 3.1000000000000004e-06,
+      "loss": 0.0207,
+      "num_input_tokens_seen": 5580064,
+      "step": 372
+    },
+    {
+      "epoch": 4.797427652733119,
+      "grad_norm": 5.605347156524658,
+      "learning_rate": 3.1083333333333338e-06,
+      "loss": 0.0144,
+      "num_input_tokens_seen": 5595008,
+      "step": 373
+    },
+    {
+      "epoch": 4.810289389067524,
+      "grad_norm": 4.410486221313477,
+      "learning_rate": 3.1166666666666668e-06,
+      "loss": 0.0197,
+      "num_input_tokens_seen": 5610304,
+      "step": 374
+    },
+    {
+      "epoch": 4.823151125401929,
+      "grad_norm": 5.259794235229492,
+      "learning_rate": 3.125e-06,
+      "loss": 0.0669,
+      "num_input_tokens_seen": 5626464,
+      "step": 375
+    },
+    {
+      "epoch": 4.836012861736334,
+      "grad_norm": 5.21687650680542,
+      "learning_rate": 3.133333333333334e-06,
+      "loss": 0.0156,
+      "num_input_tokens_seen": 5641216,
+      "step": 376
+    },
+    {
+      "epoch": 4.84887459807074,
+      "grad_norm": 3.3515796661376953,
+      "learning_rate": 3.141666666666667e-06,
+      "loss": 0.018,
+      "num_input_tokens_seen": 5656832,
+      "step": 377
+    },
+    {
+      "epoch": 4.861736334405145,
+      "grad_norm": 7.488304138183594,
+      "learning_rate": 3.1500000000000003e-06,
+      "loss": 0.064,
+      "num_input_tokens_seen": 5671744,
+      "step": 378
+    },
+    {
+      "epoch": 4.87459807073955,
+      "grad_norm": 2.625629186630249,
+      "learning_rate": 3.1583333333333337e-06,
+      "loss": 0.0263,
+      "num_input_tokens_seen": 5686496,
+      "step": 379
+    },
+    {
+      "epoch": 4.887459807073955,
+      "grad_norm": 3.379544496536255,
+      "learning_rate": 3.1666666666666667e-06,
+      "loss": 0.0256,
+      "num_input_tokens_seen": 5701216,
+      "step": 380
+    },
+    {
+      "epoch": 4.90032154340836,
+      "grad_norm": 2.0447638034820557,
+      "learning_rate": 3.175e-06,
+      "loss": 0.0269,
+      "num_input_tokens_seen": 5715904,
+      "step": 381
+    },
+    {
+      "epoch": 4.913183279742765,
+      "grad_norm": 3.5290844440460205,
+      "learning_rate": 3.183333333333334e-06,
+      "loss": 0.0281,
+      "num_input_tokens_seen": 5730976,
+      "step": 382
+    },
+    {
+      "epoch": 4.92604501607717,
+      "grad_norm": 2.183229923248291,
+      "learning_rate": 3.191666666666667e-06,
+      "loss": 0.0199,
+      "num_input_tokens_seen": 5745824,
+      "step": 383
+    },
+    {
+      "epoch": 4.938906752411576,
+      "grad_norm": 1.9661577939987183,
+      "learning_rate": 3.2000000000000003e-06,
+      "loss": 0.0277,
+      "num_input_tokens_seen": 5760160,
+      "step": 384
+    },
+    {
+      "epoch": 4.951768488745981,
+      "grad_norm": 5.09304666519165,
+      "learning_rate": 3.2083333333333337e-06,
+      "loss": 0.0158,
+      "num_input_tokens_seen": 5776160,
+      "step": 385
+    },
+    {
+      "epoch": 4.951768488745981,
+      "num_input_tokens_seen": 5776160,
+      "step": 385,
+      "total_flos": 2.6009787341943603e+17,
+      "train_loss": 1.2260490821160424,
+      "train_runtime": 5290.2175,
+      "train_samples_per_second": 18.789,
+      "train_steps_per_second": 0.073
+    }
+  ],
+  "logging_steps": 1,
+  "max_steps": 385,
+  "num_input_tokens_seen": 5776160,
+  "num_train_epochs": 5,
+  "save_steps": 1000,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 2.6009787341943603e+17,
+  "train_batch_size": 4,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f37864624d2abe9284ede821087a8ef2503abe21bd8ee765f3fbb1875521922c
+size 6584