Model save

README.md

@@ -0,0 +1,60 @@
+---
+library_name: transformers
+license: apache-2.0
+base_model: pkarypis/gpt2-sft-dutch
+tags:
+- trl
+- dpo
+- generated_from_trainer
+model-index:
+- name: gpt2-dpo-dutch
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# gpt2-dpo-dutch
+
+This model is a fine-tuned version of [pkarypis/gpt2-sft-dutch](https://huggingface.co/pkarypis/gpt2-sft-dutch) on an unknown dataset.
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 5e-07
+- train_batch_size: 2
+- eval_batch_size: 2
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 4
+- total_train_batch_size: 8
+- total_eval_batch_size: 8
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: cosine
+- lr_scheduler_warmup_ratio: 0.1
+- num_epochs: 1
+
+### Training results
+
+
+
+### Framework versions
+
+- Transformers 4.44.2
+- Pytorch 2.1.2
+- Datasets 3.0.0
+- Tokenizers 0.19.1

all_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 1.0,
+    "total_flos": 0.0,
+    "train_loss": 0.6930974917333634,
+    "train_runtime": 41.8675,
+    "train_samples": 482,
+    "train_samples_per_second": 11.513,
+    "train_steps_per_second": 1.457
+}

generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 0,
+  "eos_token_id": 2,
+  "pad_token_id": 1,
+  "transformers_version": "4.44.2"
+}

train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 1.0,
+    "total_flos": 0.0,
+    "train_loss": 0.6930974917333634,
+    "train_runtime": 41.8675,
+    "train_samples": 482,
+    "train_samples_per_second": 11.513,
+    "train_steps_per_second": 1.457
+}

trainer_state.json

@@ -0,0 +1,147 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "eval_steps": 100,
+  "global_step": 61,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.01639344262295082,
+      "grad_norm": 135.0,
+      "learning_rate": 7.142857142857142e-08,
+      "logits/chosen": -3.7637007236480713,
+      "logits/rejected": -3.5916824340820312,
+      "logps/chosen": -828.8677978515625,
+      "logps/rejected": -847.9320068359375,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 1
+    },
+    {
+      "epoch": 0.16393442622950818,
+      "grad_norm": 117.5,
+      "learning_rate": 4.96201938253052e-07,
+      "logits/chosen": -3.4565608501434326,
+      "logits/rejected": -3.833305835723877,
+      "logps/chosen": -1986.59619140625,
+      "logps/rejected": -1251.6014404296875,
+      "loss": 0.6976,
+      "rewards/accuracies": 0.3333333432674408,
+      "rewards/chosen": -0.02148149348795414,
+      "rewards/margins": -0.028772609308362007,
+      "rewards/rejected": 0.007291116751730442,
+      "step": 10
+    },
+    {
+      "epoch": 0.32786885245901637,
+      "grad_norm": 130.0,
+      "learning_rate": 4.3184341039326217e-07,
+      "logits/chosen": -3.440230131149292,
+      "logits/rejected": -3.7806174755096436,
+      "logps/chosen": -1971.769775390625,
+      "logps/rejected": -1173.9190673828125,
+      "loss": 0.697,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.005861644633114338,
+      "rewards/margins": 0.024966660887002945,
+      "rewards/rejected": -0.019105013459920883,
+      "step": 20
+    },
+    {
+      "epoch": 0.4918032786885246,
+      "grad_norm": 139.0,
+      "learning_rate": 3.0765396768561004e-07,
+      "logits/chosen": -3.6158289909362793,
+      "logits/rejected": -3.936098575592041,
+      "logps/chosen": -1709.213134765625,
+      "logps/rejected": -1273.736572265625,
+      "loss": 0.6991,
+      "rewards/accuracies": 0.4000000059604645,
+      "rewards/chosen": -0.0283970944583416,
+      "rewards/margins": -0.022606659680604935,
+      "rewards/rejected": -0.0057904343120753765,
+      "step": 30
+    },
+    {
+      "epoch": 0.6557377049180327,
+      "grad_norm": 107.5,
+      "learning_rate": 1.6449496416858282e-07,
+      "logits/chosen": -3.4193108081817627,
+      "logits/rejected": -3.7654635906219482,
+      "logps/chosen": -1581.18798828125,
+      "logps/rejected": -937.7156372070312,
+      "loss": 0.6866,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.02223411574959755,
+      "rewards/margins": 0.020158350467681885,
+      "rewards/rejected": 0.002075767144560814,
+      "step": 40
+    },
+    {
+      "epoch": 0.819672131147541,
+      "grad_norm": 123.0,
+      "learning_rate": 4.9469201811239035e-08,
+      "logits/chosen": -3.4550349712371826,
+      "logits/rejected": -3.6855628490448,
+      "logps/chosen": -2060.73046875,
+      "logps/rejected": -1470.384033203125,
+      "loss": 0.6853,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.01815984770655632,
+      "rewards/margins": 0.01882082223892212,
+      "rewards/rejected": -0.0006609730189666152,
+      "step": 50
+    },
+    {
+      "epoch": 0.9836065573770492,
+      "grad_norm": 93.0,
+      "learning_rate": 4.2296043218295606e-10,
+      "logits/chosen": -3.4209322929382324,
+      "logits/rejected": -3.847364902496338,
+      "logps/chosen": -1751.2679443359375,
+      "logps/rejected": -1242.93359375,
+      "loss": 0.6941,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.019947882741689682,
+      "rewards/margins": 0.00426260894164443,
+      "rewards/rejected": 0.015685271471738815,
+      "step": 60
+    },
+    {
+      "epoch": 1.0,
+      "step": 61,
+      "total_flos": 0.0,
+      "train_loss": 0.6930974917333634,
+      "train_runtime": 41.8675,
+      "train_samples_per_second": 11.513,
+      "train_steps_per_second": 1.457
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 61,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 100,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}