Model save

README.md

@@ -0,0 +1,60 @@
+---
+license: apache-2.0
+base_model: TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T
+tags:
+- trl
+- sft
+- generated_from_trainer
+datasets:
+- generator
+model-index:
+- name: tinyllama-1.1b-mt-sft-full_sardine2
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# tinyllama-1.1b-mt-sft-full_sardine2
+
+This model is a fine-tuned version of [TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T](https://huggingface.co/TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T) on the generator dataset.
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 1e-05
+- train_batch_size: 1
+- eval_batch_size: 8
+- seed: 42
+- distributed_type: multi-GPU
+- gradient_accumulation_steps: 16
+- total_train_batch_size: 16
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: cosine
+- lr_scheduler_warmup_ratio: 0.1
+- num_epochs: 1
+
+### Training results
+
+
+
+### Framework versions
+
+- Transformers 4.41.2
+- Pytorch 2.1.2
+- Datasets 2.20.0
+- Tokenizers 0.19.1

all_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 0.9954337899543378,
+    "total_flos": 2.216988177648845e+16,
+    "train_loss": 1.2956732773999555,
+    "train_runtime": 586.8184,
+    "train_samples": 15798,
+    "train_samples_per_second": 2.986,
+    "train_steps_per_second": 0.186
+}

generation_config.json

@@ -0,0 +1,7 @@
+{
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "max_length": 2048,
+  "pad_token_id": 0,
+  "transformers_version": "4.41.2"
+}

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6c37632110d28f2330d4fc59def9e6d8b7e5ff01169a77c558b70b2121a313ff
+oid sha256:0ddfd792263871bb1b36bcc54af0efd51698dc47e35b8c024ad0b37c2761aabb
 size 2200119864

runs/Jul23_12-36-58_poseidon/events.out.tfevents.1721738230.poseidon.3118508.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:578e3a04a9718914c53be1870e7a699d617f99c17a07f5adb9ef3bda6f59e070
-size 7337
+oid sha256:6f3a371e626fac1a15b4997d3a3e2b5cc7dfc992ba706c442571e4281fc46326
+size 7685

train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 0.9954337899543378,
+    "total_flos": 2.216988177648845e+16,
+    "train_loss": 1.2956732773999555,
+    "train_runtime": 586.8184,
+    "train_samples": 15798,
+    "train_samples_per_second": 2.986,
+    "train_steps_per_second": 0.186
+}

trainer_state.json

@@ -0,0 +1,119 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.9954337899543378,
+  "eval_steps": 500,
+  "global_step": 109,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0091324200913242,
+      "grad_norm": 22.5,
+      "learning_rate": 9.090909090909091e-07,
+      "loss": 1.6621,
+      "step": 1
+    },
+    {
+      "epoch": 0.091324200913242,
+      "grad_norm": 3.671875,
+      "learning_rate": 9.090909090909091e-06,
+      "loss": 1.5917,
+      "step": 10
+    },
+    {
+      "epoch": 0.182648401826484,
+      "grad_norm": 2.390625,
+      "learning_rate": 9.793339265183303e-06,
+      "loss": 1.4031,
+      "step": 20
+    },
+    {
+      "epoch": 0.273972602739726,
+      "grad_norm": 1.984375,
+      "learning_rate": 9.10086127298478e-06,
+      "loss": 1.3219,
+      "step": 30
+    },
+    {
+      "epoch": 0.365296803652968,
+      "grad_norm": 1.8125,
+      "learning_rate": 7.99055265245608e-06,
+      "loss": 1.2743,
+      "step": 40
+    },
+    {
+      "epoch": 0.45662100456621,
+      "grad_norm": 1.71875,
+      "learning_rate": 6.575541090118105e-06,
+      "loss": 1.2351,
+      "step": 50
+    },
+    {
+      "epoch": 0.547945205479452,
+      "grad_norm": 1.703125,
+      "learning_rate": 5e-06,
+      "loss": 1.2429,
+      "step": 60
+    },
+    {
+      "epoch": 0.639269406392694,
+      "grad_norm": 1.640625,
+      "learning_rate": 3.424458909881897e-06,
+      "loss": 1.2251,
+      "step": 70
+    },
+    {
+      "epoch": 0.730593607305936,
+      "grad_norm": 1.734375,
+      "learning_rate": 2.00944734754392e-06,
+      "loss": 1.2427,
+      "step": 80
+    },
+    {
+      "epoch": 0.821917808219178,
+      "grad_norm": 1.7734375,
+      "learning_rate": 8.991387270152202e-07,
+      "loss": 1.2333,
+      "step": 90
+    },
+    {
+      "epoch": 0.91324200913242,
+      "grad_norm": 1.7734375,
+      "learning_rate": 2.0666073481669714e-07,
+      "loss": 1.2286,
+      "step": 100
+    },
+    {
+      "epoch": 0.9954337899543378,
+      "step": 109,
+      "total_flos": 2.216988177648845e+16,
+      "train_loss": 1.2956732773999555,
+      "train_runtime": 586.8184,
+      "train_samples_per_second": 2.986,
+      "train_steps_per_second": 0.186
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 109,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 100,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 2.216988177648845e+16,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}