Training in progress, step 240, checkpoint

last-checkpoint/config.json

@@ -0,0 +1,31 @@
+{
+  "_name_or_path": "fxmarty/small-llama-testing",
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 0,
+  "eos_token_id": 2,
+  "head_dim": 64,
+  "hidden_act": "silu",
+  "hidden_size": 256,
+  "initializer_range": 0.02,
+  "intermediate_size": 128,
+  "max_position_embeddings": 2048,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 4,
+  "num_hidden_layers": 2,
+  "num_key_value_heads": 4,
+  "pad_token_id": -1,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 10000.0,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.46.0",
+  "use_cache": false,
+  "vocab_size": 32000
+}

last-checkpoint/generation_config.json

@@ -0,0 +1,8 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 0,
+  "do_sample": true,
+  "eos_token_id": 1,
+  "pad_token_id": 0,
+  "transformers_version": "4.46.0"
+}

last-checkpoint/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7e549d7179adb64a4e2a52cedddf05230be52485658b853b03c7aba1ecaf7fd6
+size 34214640

last-checkpoint/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e228394c879980cd76d721f6b9b8b132a11327f05a28a55508272667527de7dc
+size 34779218

last-checkpoint/rng_state_0.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4e71309e46df46520f6d6dcb919f310beb5e69a8318763750f4860490798dc01
+size 15984

last-checkpoint/rng_state_1.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:47bf81a1a7f7f8a5252eefb5e61cc6ec9f8600fedcf94091c6a6d9c24b75f0ba
+size 15984

last-checkpoint/rng_state_2.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:833527739976a7e9917579f8b306bb81e03a4c1f076c430695aa6a78cc587298
+size 15984

last-checkpoint/rng_state_3.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:25d3c20ffc631c18c400be1343209759c5df8250f79de9778ea66fd7008fe971
+size 15984

last-checkpoint/rng_state_4.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b75cebf2ce909ee45cd32d01d1ae2f911a3c091be907116570ad8be18a24cebf
+size 15984

last-checkpoint/rng_state_5.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7165b023a2d24791cd3729f05676124b0fa0a5fe3f624747f2f1e635e7d0bc1d
+size 15984

last-checkpoint/rng_state_6.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:97c412892e442c8878e58cb8c2a58709482d5ce63e98fc6e206c84da23b629a6
+size 15984

last-checkpoint/rng_state_7.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5c4e6180ac8c290bca56d1099b4cf1ee81f37d096f191c10922952a0ea2fa530
+size 15984

last-checkpoint/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1120064e5b190a1eb7774c9c06c05ec078e2185e35c67b16678cffbc124bc7d7
+size 1064

last-checkpoint/special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  }
+}

last-checkpoint/tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347
+size 499723

last-checkpoint/tokenizer_config.json

@@ -0,0 +1,43 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "add_prefix_space": null,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<s>",
+  "chat_template": "{% for message in messages %}{% if message['role'] == 'user' %}{{ '### Instruction: ' + message['content'] + '\n\n' }}{% elif message['role'] == 'assistant' %}{{ '### Response: ' + message['content'] + eos_token}}{% endif %}{% endfor %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "legacy": true,
+  "model_max_length": 2048,
+  "pad_token": "</s>",
+  "sp_model_kwargs": {},
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false,
+  "use_fast": true
+}

last-checkpoint/trainer_state.json

@@ -0,0 +1,305 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 2.696629213483146,
+  "eval_steps": 20,
+  "global_step": 240,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.011235955056179775,
+      "eval_loss": 10.422812461853027,
+      "eval_runtime": 1.3009,
+      "eval_samples_per_second": 1154.612,
+      "eval_steps_per_second": 7.687,
+      "step": 1
+    },
+    {
+      "epoch": 0.11235955056179775,
+      "grad_norm": 1.5390625,
+      "learning_rate": 7.692307692307693e-05,
+      "loss": 10.4033,
+      "step": 10
+    },
+    {
+      "epoch": 0.2247191011235955,
+      "grad_norm": 1.46875,
+      "learning_rate": 0.00015384615384615385,
+      "loss": 10.127,
+      "step": 20
+    },
+    {
+      "epoch": 0.2247191011235955,
+      "eval_loss": 9.863175392150879,
+      "eval_runtime": 1.4324,
+      "eval_samples_per_second": 1048.618,
+      "eval_steps_per_second": 6.981,
+      "step": 20
+    },
+    {
+      "epoch": 0.33707865168539325,
+      "grad_norm": 1.5234375,
+      "learning_rate": 0.00019998942319271077,
+      "loss": 9.616,
+      "step": 30
+    },
+    {
+      "epoch": 0.449438202247191,
+      "grad_norm": 1.3046875,
+      "learning_rate": 0.00019987045980408906,
+      "loss": 9.0393,
+      "step": 40
+    },
+    {
+      "epoch": 0.449438202247191,
+      "eval_loss": 8.740312576293945,
+      "eval_runtime": 1.4689,
+      "eval_samples_per_second": 1022.538,
+      "eval_steps_per_second": 6.808,
+      "step": 40
+    },
+    {
+      "epoch": 0.5617977528089888,
+      "grad_norm": 1.9296875,
+      "learning_rate": 0.00019961946980917456,
+      "loss": 8.5271,
+      "step": 50
+    },
+    {
+      "epoch": 0.6741573033707865,
+      "grad_norm": 1.15625,
+      "learning_rate": 0.00019923678501135848,
+      "loss": 8.1127,
+      "step": 60
+    },
+    {
+      "epoch": 0.6741573033707865,
+      "eval_loss": 7.918918132781982,
+      "eval_runtime": 1.4635,
+      "eval_samples_per_second": 1026.272,
+      "eval_steps_per_second": 6.833,
+      "step": 60
+    },
+    {
+      "epoch": 0.7865168539325843,
+      "grad_norm": 1.015625,
+      "learning_rate": 0.00019872291131173742,
+      "loss": 7.7733,
+      "step": 70
+    },
+    {
+      "epoch": 0.898876404494382,
+      "grad_norm": 0.9765625,
+      "learning_rate": 0.00019807852804032305,
+      "loss": 7.5513,
+      "step": 80
+    },
+    {
+      "epoch": 0.898876404494382,
+      "eval_loss": 7.457934856414795,
+      "eval_runtime": 1.5378,
+      "eval_samples_per_second": 976.69,
+      "eval_steps_per_second": 6.503,
+      "step": 80
+    },
+    {
+      "epoch": 1.0112359550561798,
+      "grad_norm": 0.625,
+      "learning_rate": 0.00019730448705798239,
+      "loss": 7.4017,
+      "step": 90
+    },
+    {
+      "epoch": 1.1235955056179776,
+      "grad_norm": 0.515625,
+      "learning_rate": 0.00019640181163029595,
+      "loss": 7.2769,
+      "step": 100
+    },
+    {
+      "epoch": 1.1235955056179776,
+      "eval_loss": 7.276979446411133,
+      "eval_runtime": 1.6975,
+      "eval_samples_per_second": 884.842,
+      "eval_steps_per_second": 5.891,
+      "step": 100
+    },
+    {
+      "epoch": 1.2359550561797752,
+      "grad_norm": 0.470703125,
+      "learning_rate": 0.0001953716950748227,
+      "loss": 7.2175,
+      "step": 110
+    },
+    {
+      "epoch": 1.348314606741573,
+      "grad_norm": 0.953125,
+      "learning_rate": 0.00019421549918355966,
+      "loss": 7.1384,
+      "step": 120
+    },
+    {
+      "epoch": 1.348314606741573,
+      "eval_loss": 7.176700115203857,
+      "eval_runtime": 1.4059,
+      "eval_samples_per_second": 1068.345,
+      "eval_steps_per_second": 7.113,
+      "step": 120
+    },
+    {
+      "epoch": 1.4606741573033708,
+      "grad_norm": 0.52734375,
+      "learning_rate": 0.00019293475242268223,
+      "loss": 7.1637,
+      "step": 130
+    },
+    {
+      "epoch": 1.5730337078651684,
+      "grad_norm": 0.49609375,
+      "learning_rate": 0.00019153114791194473,
+      "loss": 7.0576,
+      "step": 140
+    },
+    {
+      "epoch": 1.5730337078651684,
+      "eval_loss": 7.057478904724121,
+      "eval_runtime": 1.5081,
+      "eval_samples_per_second": 995.943,
+      "eval_steps_per_second": 6.631,
+      "step": 140
+    },
+    {
+      "epoch": 1.6853932584269664,
+      "grad_norm": 0.7265625,
+      "learning_rate": 0.00019000654118641211,
+      "loss": 7.0339,
+      "step": 150
+    },
+    {
+      "epoch": 1.797752808988764,
+      "grad_norm": 0.494140625,
+      "learning_rate": 0.00018836294774348278,
+      "loss": 6.9564,
+      "step": 160
+    },
+    {
+      "epoch": 1.797752808988764,
+      "eval_loss": 6.937857627868652,
+      "eval_runtime": 1.4808,
+      "eval_samples_per_second": 1014.296,
+      "eval_steps_per_second": 6.753,
+      "step": 160
+    },
+    {
+      "epoch": 1.9101123595505618,
+      "grad_norm": 1.1171875,
+      "learning_rate": 0.00018660254037844388,
+      "loss": 6.8863,
+      "step": 170
+    },
+    {
+      "epoch": 2.0224719101123596,
+      "grad_norm": 0.78125,
+      "learning_rate": 0.0001847276463120828,
+      "loss": 6.8785,
+      "step": 180
+    },
+    {
+      "epoch": 2.0224719101123596,
+      "eval_loss": 6.820836067199707,
+      "eval_runtime": 1.5264,
+      "eval_samples_per_second": 984.014,
+      "eval_steps_per_second": 6.551,
+      "step": 180
+    },
+    {
+      "epoch": 2.134831460674157,
+      "grad_norm": 1.2265625,
+      "learning_rate": 0.00018274074411415105,
+      "loss": 6.7019,
+      "step": 190
+    },
+    {
+      "epoch": 2.247191011235955,
+      "grad_norm": 0.78515625,
+      "learning_rate": 0.00018064446042674828,
+      "loss": 6.7027,
+      "step": 200
+    },
+    {
+      "epoch": 2.247191011235955,
+      "eval_loss": 6.721198558807373,
+      "eval_runtime": 1.4329,
+      "eval_samples_per_second": 1048.203,
+      "eval_steps_per_second": 6.979,
+      "step": 200
+    },
+    {
+      "epoch": 2.359550561797753,
+      "grad_norm": 1.0078125,
+      "learning_rate": 0.00017844156649195759,
+      "loss": 6.6148,
+      "step": 210
+    },
+    {
+      "epoch": 2.4719101123595504,
+      "grad_norm": 0.7109375,
+      "learning_rate": 0.00017613497448832312,
+      "loss": 6.5913,
+      "step": 220
+    },
+    {
+      "epoch": 2.4719101123595504,
+      "eval_loss": 6.636184215545654,
+      "eval_runtime": 1.5174,
+      "eval_samples_per_second": 989.862,
+      "eval_steps_per_second": 6.59,
+      "step": 220
+    },
+    {
+      "epoch": 2.5842696629213484,
+      "grad_norm": 0.734375,
+      "learning_rate": 0.0001737277336810124,
+      "loss": 6.5713,
+      "step": 230
+    },
+    {
+      "epoch": 2.696629213483146,
+      "grad_norm": 1.1875,
+      "learning_rate": 0.0001712230263907531,
+      "loss": 6.498,
+      "step": 240
+    },
+    {
+      "epoch": 2.696629213483146,
+      "eval_loss": 6.557174205780029,
+      "eval_runtime": 1.526,
+      "eval_samples_per_second": 984.272,
+      "eval_steps_per_second": 6.553,
+      "step": 240
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 890,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 10,
+  "save_steps": 40,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 4047954180571136.0,
+  "train_batch_size": 19,
+  "trial_name": null,
+  "trial_params": null
+}

last-checkpoint/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:88983683f78772e6fd62c9a97937ef6f408b0ee554cb1a96936c237405b3f78a
+size 6520