Model save

README.md

@@ -27,7 +27,7 @@ print(output["generated_text"])
 
 ## Training procedure
 
-[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/llm-m_wandb-weblab/Qwen2.5-7B-Instruct-GRPO/runs/07i0f523) 
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/llm-m_wandb-weblab/Qwen2.5-7B-Instruct-GRPO/runs/potnc7q9) 
 
 
 This model was trained with GRPO, a method introduced in [DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models](https://huggingface.co/papers/2402.03300).
@@ -35,9 +35,9 @@ This model was trained with GRPO, a method introduced in [DeepSeekMath: Pushing
 ### Framework versions
 
 - TRL: 0.16.0.dev0
-- Transformers: 4.50.0.dev0
+- Transformers: 4.49.0
 - Pytorch: 2.5.1
-- Datasets: 3.3.1
+- Datasets: 3.3.2
 - Tokenizers: 0.21.0
 
 ## Citations

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.02936485406525176,
-    "train_runtime": 6394.9086,
+    "train_loss": 0.020674003962555837,
+    "train_runtime": 6417.2756,
     "train_samples": 7500,
-    "train_samples_per_second": 1.173,
+    "train_samples_per_second": 1.169,
     "train_steps_per_second": 0.009
 }

config.json

@@ -22,7 +22,7 @@
   "sliding_window": 131072,
   "tie_word_embeddings": false,
   "torch_dtype": "bfloat16",
-  "transformers_version": "4.50.0.dev0",
+  "transformers_version": "4.49.0",
   "use_cache": false,
   "use_sliding_window": false,
   "vocab_size": 152064

generation_config.json

@@ -10,5 +10,5 @@
   "temperature": 0.7,
   "top_k": 20,
   "top_p": 0.8,
-  "transformers_version": "4.50.0.dev0"
+  "transformers_version": "4.49.0"
 }

model-00001-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:dc60d9a7495d0aa48dc90505ebf3d104535956d4f3458df65220089242913819
+oid sha256:247735adddbbd9944bf2dc1cc35ccc9b4cfd5e79ef3d8c3ab7340c7f26a0955e
 size 4877660776

model-00002-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:303ef5188ca1847519cc09888a6cf60e6d29c993c31beb5f9bc46192297c602a
+oid sha256:3b9ef086092554c80dab98bd0bfbaa398b21a3d2e367bf94ad86f1e05c6ce509
 size 4932751008

model-00003-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:39e887afa83ae21afba2f860b89e41bfdeb3836eaa05be27e78109801f5dea72
+oid sha256:e80ee5064fdd08d096850130cfbb55ce4521b8c8f8f019740bac43298665148f
 size 4330865200

model-00004-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:01566002591b0833a84065e44ef7d3477dde51cb6125ca752f17d1877f864e26
+oid sha256:fe67c93ef45e03e5b117ec79ebbbcc459cf2fe2e878531f0a57c2da06c2fc0ef
 size 1089994880

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.02936485406525176,
-    "train_runtime": 6394.9086,
+    "train_loss": 0.020674003962555837,
+    "train_runtime": 6417.2756,
     "train_samples": 7500,
-    "train_samples_per_second": 1.173,
+    "train_samples_per_second": 1.169,
     "train_steps_per_second": 0.009
 }

trainer_state.json

@@ -9,174 +9,187 @@
   "is_world_process_zero": true,
   "log_history": [
     {
+      "clip_ratio": 0.0,
       "completion_length": 498.510066986084,
       "epoch": 0.017057569296375266,
-      "grad_norm": 1.1288529634475708,
+      "grad_norm": 1.1215301752090454,
       "kl": 0.0,
       "learning_rate": 5e-07,
-      "loss": 0.0113,
-      "reward": 0.6216518096625805,
-      "reward_std": 0.43382299318909645,
+      "loss": 0.0115,
+      "reward": 0.6171875260770321,
+      "reward_std": 0.437016986310482,
       "rewards/accuracy_reward": 0.17633929289877415,
-      "rewards/format_reward": 0.4453125223517418,
+      "rewards/format_reward": 0.4408482313156128,
       "step": 1
     },
     {
-      "completion_length": 489.7723445892334,
+      "clip_ratio": 0.0,
+      "completion_length": 487.4813299179077,
       "epoch": 0.08528784648187633,
-      "grad_norm": 434.209228515625,
-      "kl": 1.8089315593242645,
+      "grad_norm": 476.3103332519531,
+      "kl": 1.7723130583763123,
       "learning_rate": 2.5e-06,
-      "loss": 0.0766,
-      "reward": 0.7890625391155481,
-      "reward_std": 0.38019732665270567,
-      "rewards/accuracy_reward": 0.18470982904545963,
-      "rewards/format_reward": 0.6043527061119676,
+      "loss": 0.0822,
+      "reward": 0.7912946743890643,
+      "reward_std": 0.3657265743240714,
+      "rewards/accuracy_reward": 0.1872209922876209,
+      "rewards/format_reward": 0.6040736874565482,
       "step": 5
     },
     {
-      "completion_length": 477.6071632385254,
+      "clip_ratio": 0.0,
+      "completion_length": 466.69912643432616,
       "epoch": 0.17057569296375266,
-      "grad_norm": 0.20538534224033356,
-      "kl": 1.3172725677490233,
+      "grad_norm": 0.41791579127311707,
+      "kl": 0.012205886840820312,
       "learning_rate": 2.956412726139078e-06,
-      "loss": 0.0959,
-      "reward": 1.1899554073810577,
-      "reward_std": 0.323445713147521,
-      "rewards/accuracy_reward": 0.2645089406520128,
-      "rewards/format_reward": 0.9254464700818061,
+      "loss": 0.0203,
+      "reward": 1.2589286297559739,
+      "reward_std": 0.32850122936069964,
+      "rewards/accuracy_reward": 0.3185267999768257,
+      "rewards/format_reward": 0.9404018238186836,
       "step": 10
     },
     {
-      "completion_length": 438.728816986084,
+      "clip_ratio": 0.0,
+      "completion_length": 444.567431640625,
       "epoch": 0.255863539445629,
-      "grad_norm": 0.4574572443962097,
-      "kl": 0.0261322021484375,
+      "grad_norm": 0.6771596670150757,
+      "kl": 0.02176055908203125,
       "learning_rate": 2.7836719084521715e-06,
-      "loss": 0.0128,
-      "reward": 1.4294643551111221,
-      "reward_std": 0.346218079701066,
-      "rewards/accuracy_reward": 0.47678573802113533,
-      "rewards/format_reward": 0.9526786103844642,
+      "loss": 0.0059,
+      "reward": 1.4082589864730835,
+      "reward_std": 0.3335796441882849,
+      "rewards/accuracy_reward": 0.441294664144516,
+      "rewards/format_reward": 0.9669643238186836,
       "step": 15
     },
     {
-      "completion_length": 410.66720657348634,
+      "clip_ratio": 0.0,
+      "completion_length": 422.7493499755859,
       "epoch": 0.3411513859275053,
-      "grad_norm": 0.21916256844997406,
-      "kl": 0.0268157958984375,
+      "grad_norm": 0.23769104480743408,
+      "kl": 0.026959228515625,
       "learning_rate": 2.4946839873611927e-06,
-      "loss": 0.0144,
-      "reward": 1.516741144657135,
-      "reward_std": 0.3288130540400743,
-      "rewards/accuracy_reward": 0.5453125208616256,
-      "rewards/format_reward": 0.9714286103844643,
+      "loss": 0.01,
+      "reward": 1.4937500715255738,
+      "reward_std": 0.3376178216189146,
+      "rewards/accuracy_reward": 0.5156250238418579,
+      "rewards/format_reward": 0.9781250357627869,
       "step": 20
     },
     {
-      "completion_length": 440.30113372802737,
+      "clip_ratio": 0.0,
+      "completion_length": 433.67725372314453,
       "epoch": 0.42643923240938164,
-      "grad_norm": 0.19857414066791534,
-      "kl": 0.043634033203125,
+      "grad_norm": 0.1763259768486023,
+      "kl": 0.0300628662109375,
       "learning_rate": 2.1156192081791355e-06,
-      "loss": 0.0152,
-      "reward": 1.6386161416769027,
-      "reward_std": 0.2348614836111665,
-      "rewards/accuracy_reward": 0.6629464566707611,
-      "rewards/format_reward": 0.9756696790456771,
+      "loss": 0.0157,
+      "reward": 1.5979911386966705,
+      "reward_std": 0.2915887963026762,
+      "rewards/accuracy_reward": 0.6209821671247482,
+      "rewards/format_reward": 0.9770089671015739,
       "step": 25
     },
     {
-      "completion_length": 435.9283676147461,
+      "clip_ratio": 0.0,
+      "completion_length": 461.1984573364258,
       "epoch": 0.511727078891258,
-      "grad_norm": 4.214118003845215,
-      "kl": 0.04940185546875,
+      "grad_norm": 0.1527547538280487,
+      "kl": 0.0370269775390625,
       "learning_rate": 1.6808050203829845e-06,
-      "loss": 0.0198,
-      "reward": 1.6723215013742447,
-      "reward_std": 0.2099373336881399,
-      "rewards/accuracy_reward": 0.7004464611411094,
-      "rewards/format_reward": 0.971875037252903,
+      "loss": 0.0155,
+      "reward": 1.6714286535978318,
+      "reward_std": 0.2001216158270836,
+      "rewards/accuracy_reward": 0.6986607477068901,
+      "rewards/format_reward": 0.9727678924798966,
       "step": 30
     },
     {
-      "completion_length": 431.46564331054685,
+      "clip_ratio": 0.0,
+      "completion_length": 458.88328018188474,
       "epoch": 0.5970149253731343,
-      "grad_norm": 0.245810866355896,
-      "kl": 0.0542022705078125,
+      "grad_norm": 0.13492096960544586,
+      "kl": 0.03984375,
       "learning_rate": 1.2296174432791415e-06,
-      "loss": 0.014,
-      "reward": 1.6790179312229156,
-      "reward_std": 0.20684626493602992,
-      "rewards/accuracy_reward": 0.7033482506871224,
-      "rewards/format_reward": 0.9756696790456771,
+      "loss": 0.0205,
+      "reward": 1.6863840162754058,
+      "reward_std": 0.19958442291244866,
+      "rewards/accuracy_reward": 0.7189732484519482,
+      "rewards/format_reward": 0.9674107521772385,
       "step": 35
     },
     {
-      "completion_length": 422.8857360839844,
+      "clip_ratio": 0.0,
+      "completion_length": 435.2419822692871,
       "epoch": 0.6823027718550106,
-      "grad_norm": 0.2607899606227875,
-      "kl": 0.0524322509765625,
+      "grad_norm": 1.2669559717178345,
+      "kl": 0.0412078857421875,
       "learning_rate": 8.029152419343472e-07,
-      "loss": 0.018,
-      "reward": 1.6703125715255738,
-      "reward_std": 0.21576487701386213,
-      "rewards/accuracy_reward": 0.6926339581608772,
-      "rewards/format_reward": 0.9776786059141159,
+      "loss": 0.0147,
+      "reward": 1.7008929342031478,
+      "reward_std": 0.18920395569875836,
+      "rewards/accuracy_reward": 0.7238839611411094,
+      "rewards/format_reward": 0.9770089611411095,
       "step": 40
     },
     {
-      "completion_length": 441.6977897644043,
+      "clip_ratio": 0.0,
+      "completion_length": 436.82925872802736,
       "epoch": 0.767590618336887,
-      "grad_norm": 0.6231416463851929,
-      "kl": 0.07574462890625,
+      "grad_norm": 0.38188719749450684,
+      "kl": 0.047900390625,
       "learning_rate": 4.3933982822017883e-07,
-      "loss": 0.0199,
-      "reward": 1.656696504354477,
-      "reward_std": 0.2240034222602844,
-      "rewards/accuracy_reward": 0.675892886519432,
-      "rewards/format_reward": 0.9808036059141159,
+      "loss": 0.0174,
+      "reward": 1.6915179312229156,
+      "reward_std": 0.19770997650921346,
+      "rewards/accuracy_reward": 0.712276816368103,
+      "rewards/format_reward": 0.9792410984635354,
       "step": 45
     },
     {
-      "completion_length": 432.76631317138674,
+      "clip_ratio": 0.0,
+      "completion_length": 426.6221176147461,
       "epoch": 0.8528784648187633,
-      "grad_norm": 1.0140599012374878,
-      "kl": 0.046148681640625,
+      "grad_norm": 1.376158356666565,
+      "kl": 0.202972412109375,
       "learning_rate": 1.718159615201853e-07,
-      "loss": 0.0211,
-      "reward": 1.656919714808464,
-      "reward_std": 0.2316950935870409,
-      "rewards/accuracy_reward": 0.6727678880095482,
-      "rewards/format_reward": 0.9841518223285675,
+      "loss": 0.0264,
+      "reward": 1.6868304401636123,
+      "reward_std": 0.19814990404993296,
+      "rewards/accuracy_reward": 0.7000000312924385,
+      "rewards/format_reward": 0.986830385029316,
       "step": 50
     },
     {
-      "completion_length": 437.499796295166,
+      "clip_ratio": 0.0,
+      "completion_length": 429.37256622314453,
       "epoch": 0.9381663113006397,
-      "grad_norm": 3.5078225135803223,
-      "kl": 0.1864105224609375,
+      "grad_norm": 2.8599460124969482,
+      "kl": 0.0670989990234375,
       "learning_rate": 2.4570139579284723e-08,
-      "loss": 0.0283,
-      "reward": 1.6799107879400252,
-      "reward_std": 0.23468854520469903,
-      "rewards/accuracy_reward": 0.6982143193483352,
-      "rewards/format_reward": 0.9816964596509934,
+      "loss": 0.0186,
+      "reward": 1.722991144657135,
+      "reward_std": 0.19592140736058355,
+      "rewards/accuracy_reward": 0.7395089641213417,
+      "rewards/format_reward": 0.9834821745753288,
       "step": 55
     },
     {
-      "completion_length": 428.84972254435223,
+      "clip_ratio": 0.0,
+      "completion_length": 415.95802815755206,
       "epoch": 0.9893390191897654,
-      "kl": 0.059397379557291664,
-      "reward": 1.661458412806193,
-      "reward_std": 0.2265977036828796,
-      "rewards/accuracy_reward": 0.6741071691115698,
-      "rewards/format_reward": 0.9873512263099352,
+      "kl": 0.0960235595703125,
+      "reward": 1.7020090073347092,
+      "reward_std": 0.19463430003573498,
+      "rewards/accuracy_reward": 0.7127976529300213,
+      "rewards/format_reward": 0.9892113382617632,
       "step": 58,
       "total_flos": 0.0,
-      "train_loss": 0.02936485406525176,
-      "train_runtime": 6394.9086,
-      "train_samples_per_second": 1.173,
+      "train_loss": 0.020674003962555837,
+      "train_runtime": 6417.2756,
+      "train_samples_per_second": 1.169,
       "train_steps_per_second": 0.009
     }
   ],

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6c10d85dded8cb7e681e8c38934d69c85d8616a0994fe725143d71a834a82197
-size 7928
+oid sha256:d2b52bc508eabf11f85b23f9777010f62ee53072d9454f224c22b6871df1b509
+size 7992