Model save

README.md

@@ -0,0 +1,58 @@
+---
+license: apache-2.0
+base_model: Qwen/Qwen2.5-7B
+tags:
+- generated_from_trainer
+model-index:
+- name: Qwen2.5-7B-dpo-10k
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# Qwen2.5-7B-dpo-10k
+
+This model is a fine-tuned version of [Qwen/Qwen2.5-7B](https://huggingface.co/Qwen/Qwen2.5-7B) on an unknown dataset.
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 5e-07
+- train_batch_size: 4
+- eval_batch_size: 4
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 4
+- gradient_accumulation_steps: 2
+- total_train_batch_size: 32
+- total_eval_batch_size: 16
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: linear
+- lr_scheduler_warmup_ratio: 0.1
+- num_epochs: 1
+
+### Training results
+
+
+
+### Framework versions
+
+- Transformers 4.43.3
+- Pytorch 2.2.2+cu121
+- Datasets 2.20.0
+- Tokenizers 0.19.1

all_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 1.0,
+    "total_flos": 0.0,
+    "train_loss": 0.8670984277358422,
+    "train_runtime": 947.0697,
+    "train_samples": 2484,
+    "train_samples_per_second": 2.623,
+    "train_steps_per_second": 0.082
+}

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "bos_token_id": 151643,
+  "eos_token_id": 151643,
+  "max_new_tokens": 2048,
+  "transformers_version": "4.43.3"
+}

train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 1.0,
+    "total_flos": 0.0,
+    "train_loss": 0.8670984277358422,
+    "train_runtime": 947.0697,
+    "train_samples": 2484,
+    "train_samples_per_second": 2.623,
+    "train_steps_per_second": 0.082
+}

trainer_state.json

@@ -0,0 +1,162 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "eval_steps": 200,
+  "global_step": 78,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.01282051282051282,
+      "grad_norm": 85.6017268244877,
+      "learning_rate": 6.25e-08,
+      "logits/generated": -1.035823106765747,
+      "logits/real": -0.7845579385757446,
+      "logps/generated": -221.43978881835938,
+      "logps/real": -290.43585205078125,
+      "loss": 0.9091,
+      "rewards/accuracies": 0.0,
+      "rewards/generated": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/real": 0.0,
+      "step": 1
+    },
+    {
+      "epoch": 0.1282051282051282,
+      "grad_norm": 60.394755772470546,
+      "learning_rate": 4.857142857142857e-07,
+      "logits/generated": -1.0380977392196655,
+      "logits/real": -0.9431765675544739,
+      "logps/generated": -217.96856689453125,
+      "logps/real": -227.99403381347656,
+      "loss": 0.9082,
+      "rewards/accuracies": 0.5694444179534912,
+      "rewards/generated": 0.11325030773878098,
+      "rewards/margins": 0.022136474028229713,
+      "rewards/real": 0.13538677990436554,
+      "step": 10
+    },
+    {
+      "epoch": 0.2564102564102564,
+      "grad_norm": 51.89266128280823,
+      "learning_rate": 4.142857142857143e-07,
+      "logits/generated": -1.076645016670227,
+      "logits/real": -1.0400656461715698,
+      "logps/generated": -203.994140625,
+      "logps/real": -197.03224182128906,
+      "loss": 0.8977,
+      "rewards/accuracies": 0.5,
+      "rewards/generated": 0.5957759618759155,
+      "rewards/margins": 0.02595127746462822,
+      "rewards/real": 0.6217272877693176,
+      "step": 20
+    },
+    {
+      "epoch": 0.38461538461538464,
+      "grad_norm": 61.44237094531356,
+      "learning_rate": 3.4285714285714286e-07,
+      "logits/generated": -1.0620293617248535,
+      "logits/real": -0.9935398101806641,
+      "logps/generated": -198.7886199951172,
+      "logps/real": -196.684814453125,
+      "loss": 0.8674,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/generated": 1.4610140323638916,
+      "rewards/margins": 0.0977163091301918,
+      "rewards/real": 1.5587302446365356,
+      "step": 30
+    },
+    {
+      "epoch": 0.5128205128205128,
+      "grad_norm": 54.65869490615913,
+      "learning_rate": 2.714285714285714e-07,
+      "logits/generated": -0.9623354077339172,
+      "logits/real": -0.8568245768547058,
+      "logps/generated": -176.11087036132812,
+      "logps/real": -208.48501586914062,
+      "loss": 0.8731,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/generated": 2.0073351860046387,
+      "rewards/margins": 0.06225720793008804,
+      "rewards/real": 2.0695924758911133,
+      "step": 40
+    },
+    {
+      "epoch": 0.6410256410256411,
+      "grad_norm": 47.595811149692466,
+      "learning_rate": 2e-07,
+      "logits/generated": -1.1244269609451294,
+      "logits/real": -0.9023059010505676,
+      "logps/generated": -168.40548706054688,
+      "logps/real": -198.1172332763672,
+      "loss": 0.8405,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/generated": 2.258265733718872,
+      "rewards/margins": 0.20940211415290833,
+      "rewards/real": 2.467667818069458,
+      "step": 50
+    },
+    {
+      "epoch": 0.7692307692307693,
+      "grad_norm": 58.1930213116206,
+      "learning_rate": 1.2857142857142855e-07,
+      "logits/generated": -1.0248687267303467,
+      "logits/real": -0.8792899250984192,
+      "logps/generated": -177.2668914794922,
+      "logps/real": -185.9062042236328,
+      "loss": 0.8628,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/generated": 2.466890811920166,
+      "rewards/margins": 0.10949119180440903,
+      "rewards/real": 2.5763819217681885,
+      "step": 60
+    },
+    {
+      "epoch": 0.8974358974358975,
+      "grad_norm": 56.77078246951395,
+      "learning_rate": 5.714285714285714e-08,
+      "logits/generated": -0.9708479642868042,
+      "logits/real": -0.8643589019775391,
+      "logps/generated": -182.65744018554688,
+      "logps/real": -199.50527954101562,
+      "loss": 0.8634,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/generated": 2.7308669090270996,
+      "rewards/margins": 0.10345491021871567,
+      "rewards/real": 2.834322214126587,
+      "step": 70
+    },
+    {
+      "epoch": 1.0,
+      "step": 78,
+      "total_flos": 0.0,
+      "train_loss": 0.8670984277358422,
+      "train_runtime": 947.0697,
+      "train_samples_per_second": 2.623,
+      "train_steps_per_second": 0.082
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 78,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 200,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 4,
+  "trial_name": null,
+  "trial_params": null
+}