Training in progress, step 200, checkpoint

last-checkpoint/config.json

@@ -1,25 +1,26 @@
 {
-  "_name_or_path": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
+  "_name_or_path": "trl-internal-testing/tiny-random-LlamaForCausalLM",
   "architectures": [
     "LlamaForCausalLM"
   ],
   "attention_bias": false,
   "attention_dropout": 0.0,
-  "bos_token_id": 1,
+  "bos_token_id": 0,
   "eos_token_id": 2,
-  "head_dim": 64,
+  "head_dim": 4,
   "hidden_act": "silu",
-  "hidden_size": 2048,
+  "hidden_size": 16,
   "initializer_range": 0.02,
-  "intermediate_size": 5632,
+  "intermediate_size": 64,
   "max_position_embeddings": 2048,
   "mlp_bias": false,
   "model_type": "llama",
-  "num_attention_heads": 32,
-  "num_hidden_layers": 22,
+  "num_attention_heads": 4,
+  "num_hidden_layers": 2,
   "num_key_value_heads": 4,
+  "pad_token_id": -1,
   "pretraining_tp": 1,
-  "rms_norm_eps": 1e-05,
+  "rms_norm_eps": 1e-06,
   "rope_scaling": null,
   "rope_theta": 10000.0,
   "tie_word_embeddings": false,

last-checkpoint/generation_config.json

@@ -1,8 +1,8 @@
 {
-  "bos_token_id": 1,
+  "_from_model_config": true,
+  "bos_token_id": 0,
   "do_sample": true,
-  "eos_token_id": 2,
-  "max_length": 2048,
-  "pad_token_id": 0,
+  "eos_token_id": 1,
+  "pad_token_id": 2,
   "transformers_version": "4.48.1"
 }

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9f733e9041aab06f212569d2073af6e6c0acf30fcc84bec18f30feab19bc086f
-size 2200119864
+oid sha256:23835aa3baa0dcf892abf69863f1f0c1ea20a2c3908d99240a722cf7177381d0
+size 2066752

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2e7f6e0595a900a9079a5615b8fdd966aefa419bcc59d072afe009c68faa40b1
-size 2235234810
+oid sha256:5de0b4166267d002b9869980fe6745c1cfa3e9331d120a3e69c74cbfb28d2bac
+size 2162798

last-checkpoint/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5e139d403fa01e97c7f0b1fb6f366f88e99b3ef2d1090658accfac79d4e4bf7a
+oid sha256:9927ec66b801c0f4855203d926f0323830f03e7abef37344059118107768879f
 size 15984

last-checkpoint/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:163db0385f3f2c6cc1cdde3fd1262ef8c70bc95cb0742071833b93242de76dd2
+oid sha256:319005a7acd12212946f28795d824f4aa3bb9ae8b8a8fa8426acf8d1683c7711
 size 15984

last-checkpoint/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a3f2223c650d0a4e2baa05aba88577beb8cf1341342d6523706f17972a2f481c
+oid sha256:c6e15e0019059647c2b2120160188600af8acd95ba49388e85735382c6fec110
 size 15984

last-checkpoint/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f06016ff924c2dbb5f6bd90ff3c51224f11735ecac5d42e1b0d6eb800acc7b67
+oid sha256:ee6bd9483dc526698ba668e9cd1c38d218e9960bb3776fb46ce2d46403967e58
 size 15984

last-checkpoint/rng_state_4.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7099cb24dc8750eaaa6a5df6cead1dfa156e46f7a3a478fb7ea1f45a9599d5c1
+oid sha256:6e3c33d4b9177e702fbc3b7fe348e5bc53c0acdd468c8fa1d8d47c41439445bb
 size 15984

last-checkpoint/rng_state_5.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:19562befb91856d315c73b5e1fe6c1554bd4ffd5d803c7f9588b2b2a18b10eb7
+oid sha256:befcb54359f53141f634f3331a650f261e163fb42877249c8517ea83d36a7124
 size 15984

last-checkpoint/rng_state_6.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b055646d6b89d5f656bbcc0f7774bff592c8649c36beead05e7d98a901842815
+oid sha256:4d14477440f631aa977c46e3038b7d41c1e28d50b31065840e8a8470d3bdfe80
 size 15984

last-checkpoint/rng_state_7.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6d80f9631b5929172f6acf476e00b42058e6cff03360fb2e0c295ebe290c9cd6
+oid sha256:4a246be5cf8c2f6460956f9737bacb1b56ffc6195bb80d3c3d20f45b6c2226d5
 size 15984

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3a60c7d771c1fd156acee762fba03c724cb41829a3f71df370ecd1d20b134982
+oid sha256:5197657cfe0474e002b5e477499687f2485548beb0dd654c8444eec350c0acb8
 size 1064

last-checkpoint/special_tokens_map.json

@@ -14,7 +14,7 @@
     "single_word": false
   },
   "pad_token": {
-    "content": "</s>",
+    "content": "<unk>",
     "lstrip": false,
     "normalized": false,
     "rstrip": false,

last-checkpoint/tokenizer.json

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c88bda6bdd84543eadebdf4bd2ec325ae43b71a3ff1fead6a766562c00b29bd8
-size 3619016
+oid sha256:d606f46a8aa6f29f0a0abdec7c3ffddefc9f9bfe26919532d209a0a850e25029
+size 3619013

last-checkpoint/tokenizer_config.json

@@ -29,17 +29,15 @@
     }
   },
   "bos_token": "<s>",
-  "chat_template": "{% for message in messages %}\n{% if message['role'] == 'user' %}\n{{ '<|user|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'system' %}\n{{ '<|system|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'assistant' %}\n{{ '<|assistant|>\n'  + message['content'] + eos_token }}\n{% endif %}\n{% if loop.last and add_generation_prompt %}\n{{ '<|assistant|>' }}\n{% endif %}\n{% endfor %}",
+  "chat_template": "{% for message in messages %}{% if message['role'] == 'user' %}{{ '### Instruction: ' + message['content'] + '\n\n' }}{% elif message['role'] == 'assistant' %}{{ '### Response: ' + message['content'] + eos_token}}{% endif %}{% endfor %}",
   "clean_up_tokenization_spaces": false,
   "eos_token": "</s>",
   "extra_special_tokens": {},
-  "legacy": false,
+  "legacy": true,
   "model_max_length": 2048,
-  "pad_token": "</s>",
-  "padding_side": "right",
-  "sp_model_kwargs": {},
+  "pad_token": "<unk>",
   "tokenizer_class": "LlamaTokenizer",
   "unk_token": "<unk>",
-  "use_default_system_prompt": false,
+  "use_default_system_prompt": true,
   "use_fast": true
 }

last-checkpoint/trainer_state.json

@@ -1,96 +1,174 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 12.5,
+  "epoch": 66.66666666666667,
   "eval_steps": 200,
-  "global_step": 100,
+  "global_step": 200,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 0.125,
-      "eval_loss": 2.0389645099639893,
-      "eval_runtime": 3.3031,
-      "eval_samples_per_second": 454.427,
-      "eval_steps_per_second": 4.844,
+      "epoch": 0.3333333333333333,
+      "eval_loss": 10.375551223754883,
+      "eval_runtime": 1.0096,
+      "eval_samples_per_second": 1486.793,
+      "eval_steps_per_second": 5.943,
       "step": 1
     },
     {
-      "epoch": 1.25,
-      "grad_norm": 2.234375,
-      "learning_rate": 0.00019863613034027224,
-      "loss": 1.8385,
+      "epoch": 3.3333333333333335,
+      "grad_norm": 0.1015625,
+      "learning_rate": 0.00019945218953682734,
+      "loss": 10.3718,
       "step": 10
     },
     {
-      "epoch": 2.5,
-      "grad_norm": 1.90625,
-      "learning_rate": 0.0001879473751206489,
-      "loss": 1.1862,
+      "epoch": 6.666666666666667,
+      "grad_norm": 0.162109375,
+      "learning_rate": 0.00019781476007338058,
+      "loss": 10.3553,
       "step": 20
     },
     {
-      "epoch": 3.75,
-      "grad_norm": 1.375,
-      "learning_rate": 0.00016772815716257412,
-      "loss": 0.615,
+      "epoch": 10.0,
+      "grad_norm": 0.291015625,
+      "learning_rate": 0.00019510565162951537,
+      "loss": 10.3251,
       "step": 30
     },
     {
-      "epoch": 5.0,
-      "grad_norm": 0.82421875,
-      "learning_rate": 0.00014016954246529696,
-      "loss": 0.2473,
+      "epoch": 13.333333333333334,
+      "grad_norm": 0.330078125,
+      "learning_rate": 0.0001913545457642601,
+      "loss": 10.2723,
       "step": 40
     },
     {
-      "epoch": 6.25,
-      "grad_norm": 0.498046875,
-      "learning_rate": 0.00010825793454723325,
-      "loss": 0.0731,
+      "epoch": 16.666666666666668,
+      "grad_norm": 0.328125,
+      "learning_rate": 0.00018660254037844388,
+      "loss": 10.2096,
       "step": 50
     },
     {
-      "epoch": 7.5,
-      "grad_norm": 0.29296875,
-      "learning_rate": 7.54514512859201e-05,
-      "loss": 0.0275,
+      "epoch": 20.0,
+      "grad_norm": 0.328125,
+      "learning_rate": 0.00018090169943749476,
+      "loss": 10.1499,
       "step": 60
     },
     {
-      "epoch": 8.75,
-      "grad_norm": 0.11962890625,
-      "learning_rate": 4.530518418775733e-05,
-      "loss": 0.0112,
+      "epoch": 23.333333333333332,
+      "grad_norm": 0.326171875,
+      "learning_rate": 0.00017431448254773944,
+      "loss": 10.0935,
       "step": 70
     },
     {
-      "epoch": 10.0,
-      "grad_norm": 0.06787109375,
-      "learning_rate": 2.1085949060360654e-05,
-      "loss": 0.0078,
+      "epoch": 26.666666666666668,
+      "grad_norm": 0.333984375,
+      "learning_rate": 0.00016691306063588583,
+      "loss": 10.0398,
       "step": 80
     },
     {
-      "epoch": 11.25,
-      "grad_norm": 0.061767578125,
-      "learning_rate": 5.418275829936537e-06,
-      "loss": 0.0072,
+      "epoch": 30.0,
+      "grad_norm": 0.337890625,
+      "learning_rate": 0.00015877852522924732,
+      "loss": 9.9895,
       "step": 90
     },
     {
-      "epoch": 12.5,
-      "grad_norm": 0.0615234375,
-      "learning_rate": 0.0,
-      "loss": 0.007,
+      "epoch": 33.333333333333336,
+      "grad_norm": 0.341796875,
+      "learning_rate": 0.00015000000000000001,
+      "loss": 9.9424,
       "step": 100
+    },
+    {
+      "epoch": 36.666666666666664,
+      "grad_norm": 0.34375,
+      "learning_rate": 0.00014067366430758004,
+      "loss": 9.8995,
+      "step": 110
+    },
+    {
+      "epoch": 40.0,
+      "grad_norm": 0.345703125,
+      "learning_rate": 0.00013090169943749476,
+      "loss": 9.859,
+      "step": 120
+    },
+    {
+      "epoch": 43.333333333333336,
+      "grad_norm": 0.34765625,
+      "learning_rate": 0.00012079116908177593,
+      "loss": 9.8216,
+      "step": 130
+    },
+    {
+      "epoch": 46.666666666666664,
+      "grad_norm": 0.3515625,
+      "learning_rate": 0.00011045284632676536,
+      "loss": 9.7872,
+      "step": 140
+    },
+    {
+      "epoch": 50.0,
+      "grad_norm": 0.357421875,
+      "learning_rate": 0.0001,
+      "loss": 9.7569,
+      "step": 150
+    },
+    {
+      "epoch": 53.333333333333336,
+      "grad_norm": 0.35546875,
+      "learning_rate": 8.954715367323468e-05,
+      "loss": 9.7325,
+      "step": 160
+    },
+    {
+      "epoch": 56.666666666666664,
+      "grad_norm": 0.359375,
+      "learning_rate": 7.920883091822408e-05,
+      "loss": 9.712,
+      "step": 170
+    },
+    {
+      "epoch": 60.0,
+      "grad_norm": 0.359375,
+      "learning_rate": 6.909830056250527e-05,
+      "loss": 9.697,
+      "step": 180
+    },
+    {
+      "epoch": 63.333333333333336,
+      "grad_norm": 0.361328125,
+      "learning_rate": 5.9326335692419995e-05,
+      "loss": 9.6841,
+      "step": 190
+    },
+    {
+      "epoch": 66.66666666666667,
+      "grad_norm": 0.361328125,
+      "learning_rate": 5.000000000000002e-05,
+      "loss": 9.6746,
+      "step": 200
+    },
+    {
+      "epoch": 66.66666666666667,
+      "eval_loss": 9.681697845458984,
+      "eval_runtime": 0.9466,
+      "eval_samples_per_second": 1585.661,
+      "eval_steps_per_second": 6.338,
+      "step": 200
     }
   ],
   "logging_steps": 10,
-  "max_steps": 100,
+  "max_steps": 300,
   "num_input_tokens_seen": 0,
-  "num_train_epochs": 13,
+  "num_train_epochs": 100,
   "save_steps": 200,
   "stateful_callbacks": {
     "TrainerControl": {
@@ -99,13 +177,13 @@
         "should_evaluate": false,
         "should_log": false,
         "should_save": true,
-        "should_training_stop": true
+        "should_training_stop": false
       },
       "attributes": {}
     }
   },
-  "total_flos": 1.2203604110737408e+17,
-  "train_batch_size": 12,
+  "total_flos": 327326839603200.0,
+  "train_batch_size": 32,
   "trial_name": null,
   "trial_params": null
 }

last-checkpoint/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:917005e54e6a74be4d5bc3c0309ff36483f7719e494b126c6cee70ee375d0421
-size 7032
+oid sha256:c9e9fca0c23d9cd44121007d8469411a44af500190e6fe6533d8ba7e8ebaeafa
+size 6840