Model save

README.md

@@ -26,7 +26,7 @@ print(output["generated_text"])
 
 ## Training procedure
 
-[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/build_r1/huggingface/runs/y15c7y2k) 
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/build_r1/huggingface/runs/eibr831y) 
 
 
 This model was trained with GRPO, a method introduced in [DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models](https://huggingface.co/papers/2402.03300).

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.04879110743259561,
-    "train_runtime": 6811.9096,
+    "train_loss": 0.04595949243882607,
+    "train_runtime": 10042.601,
     "train_samples": 7500,
-    "train_samples_per_second": 1.101,
+    "train_samples_per_second": 1.12,
     "train_steps_per_second": 0.009
 }

model-00001-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b05a200f8eff0c9682bda7c7ca0f6ddac58dde5aa48aaa20e78afa6ff4084c60
+oid sha256:ec0ec6e8eae6ed6c09cf51b16e85e04243a866447b02fd0aadfa764e0dbfb51e
 size 4877660776

model-00002-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9ca1931bf7bd1f87ef7b9749b513f80aff098394cce6b093f9b309b8ec7a2248
+oid sha256:c9d05531ab5b327e85a930b6f9a3164925a7c7fa275db557f7022043140fc6d0
 size 4932751008

model-00003-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:12579595285e46a2919c30c9ae0a7d44a9ac942b5b225cb9d358277069c23511
+oid sha256:99b713bd7f2bcca87a5bdb24fa6ad0cb08df843ae4eee8950838cd184393a843
 size 4330865200

model-00004-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4f79b7232dbf5bb9d05f6a881e8a84efac44f77459ac5a2a57ee9348b14c522d
+oid sha256:93690863728eaa6161feaede8869275fe5f72ce777b7875e1cfee955625b9a14
 size 1089994880

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.04879110743259561,
-    "train_runtime": 6811.9096,
+    "train_loss": 0.04595949243882607,
+    "train_runtime": 10042.601,
     "train_samples": 7500,
-    "train_samples_per_second": 1.101,
+    "train_samples_per_second": 1.12,
     "train_steps_per_second": 0.009
 }

trainer_state.json

@@ -1,189 +1,267 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.9893390191897654,
-  "eval_steps": 100,
-  "global_step": 58,
+  "epoch": 1.4946695095948828,
+  "eval_steps": 500,
+  "global_step": 87,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 361.4732322692871,
+      "completion_length": 383.43863677978516,
       "epoch": 0.017057569296375266,
-      "grad_norm": 1.274139404296875,
+      "grad_norm": 0.7056748867034912,
       "kl": 0.0,
-      "learning_rate": 5e-07,
-      "loss": 0.0472,
-      "reward": 0.7053571790456772,
-      "reward_std": 0.4220927283167839,
-      "rewards/accuracy_reward": 0.7020089626312256,
-      "rewards/format_reward": 0.00334821455180645,
+      "learning_rate": 3.333333333333333e-07,
+      "loss": 0.1106,
+      "reward": 0.712053619325161,
+      "reward_std": 0.39283478632569313,
+      "rewards/accuracy_reward": 0.7087053954601288,
+      "rewards/format_reward": 0.003348214435391128,
       "step": 1
     },
     {
-      "completion_length": 417.75085735321045,
+      "completion_length": 417.2608995437622,
       "epoch": 0.08528784648187633,
-      "grad_norm": 0.5048803687095642,
-      "kl": 0.0004904866218566895,
-      "learning_rate": 2.5e-06,
-      "loss": 0.0592,
-      "reward": 0.7407924458384514,
-      "reward_std": 0.36367086321115494,
-      "rewards/accuracy_reward": 0.738560302183032,
-      "rewards/format_reward": 0.0022321429569274187,
+      "grad_norm": 10.000102996826172,
+      "kl": 0.0006268918514251709,
+      "learning_rate": 1.6666666666666669e-06,
+      "loss": 0.0585,
+      "reward": 0.7357701230794191,
+      "reward_std": 0.37447383999824524,
+      "rewards/accuracy_reward": 0.7313058394938707,
+      "rewards/format_reward": 0.004464285942958668,
       "step": 5
     },
     {
-      "completion_length": 505.70426177978516,
+      "completion_length": 497.51051940917966,
       "epoch": 0.17057569296375266,
-      "grad_norm": 0.24235394597053528,
-      "kl": 0.004541778564453125,
-      "learning_rate": 2.956412726139078e-06,
-      "loss": 0.0926,
-      "reward": 0.8145089685916901,
-      "reward_std": 0.2785459361970425,
-      "rewards/accuracy_reward": 0.8138393238186836,
-      "rewards/format_reward": 0.0006696428870782256,
+      "grad_norm": 0.3063008487224579,
+      "kl": 0.0017390489578247071,
+      "learning_rate": 2.9987834972573546e-06,
+      "loss": 0.0784,
+      "reward": 0.7703125357627869,
+      "reward_std": 0.3321108963340521,
+      "rewards/accuracy_reward": 0.7689732491970063,
+      "rewards/format_reward": 0.0013392857974395156,
       "step": 10
     },
     {
-      "completion_length": 480.2105125427246,
+      "completion_length": 500.5500213623047,
       "epoch": 0.255863539445629,
-      "grad_norm": 0.11897137761116028,
-      "kl": 0.011313629150390626,
-      "learning_rate": 2.7836719084521715e-06,
-      "loss": 0.0924,
-      "reward": 0.87388396859169,
-      "reward_std": 0.17128493105992676,
-      "rewards/accuracy_reward": 0.8736607551574707,
+      "grad_norm": 2.702301263809204,
+      "kl": 0.010482597351074218,
+      "learning_rate": 2.956412726139078e-06,
+      "loss": 0.1039,
+      "reward": 0.8607143312692642,
+      "reward_std": 0.2081299727782607,
+      "rewards/accuracy_reward": 0.8604911163449287,
       "rewards/format_reward": 0.00022321429569274187,
       "step": 15
     },
     {
-      "completion_length": 491.12703094482424,
+      "completion_length": 579.4114112854004,
       "epoch": 0.3411513859275053,
-      "grad_norm": 0.11417115479707718,
-      "kl": 0.004793930053710938,
-      "learning_rate": 2.4946839873611927e-06,
-      "loss": 0.0477,
-      "reward": 0.9104911133646965,
-      "reward_std": 0.13314618533477188,
-      "rewards/accuracy_reward": 0.9104911133646965,
+      "grad_norm": 0.1238444522023201,
+      "kl": 0.002919578552246094,
+      "learning_rate": 2.8551756519155732e-06,
+      "loss": 0.0479,
+      "reward": 0.8312500357627869,
+      "reward_std": 0.18435155414044857,
+      "rewards/accuracy_reward": 0.8312500357627869,
       "rewards/format_reward": 0.0,
       "step": 20
     },
     {
-      "completion_length": 585.9723449707031,
+      "completion_length": 549.7174324035644,
       "epoch": 0.42643923240938164,
-      "grad_norm": 0.36804813146591187,
-      "kl": 0.0033447265625,
-      "learning_rate": 2.1156192081791355e-06,
-      "loss": 0.0454,
-      "reward": 0.8185268267989159,
-      "reward_std": 0.19493254837580026,
-      "rewards/accuracy_reward": 0.8185268267989159,
+      "grad_norm": 0.8831402659416199,
+      "kl": 0.004978370666503906,
+      "learning_rate": 2.699164145105252e-06,
+      "loss": 0.049,
+      "reward": 0.8495536148548126,
+      "reward_std": 0.16553675523027778,
+      "rewards/accuracy_reward": 0.8495536148548126,
       "rewards/format_reward": 0.0,
       "step": 25
     },
     {
-      "completion_length": 576.9643096923828,
+      "completion_length": 592.7422164916992,
       "epoch": 0.511727078891258,
-      "grad_norm": 0.20736576616764069,
-      "kl": 0.004184150695800781,
-      "learning_rate": 1.6808050203829845e-06,
+      "grad_norm": 0.1488523930311203,
+      "kl": 0.0038507461547851564,
+      "learning_rate": 2.4946839873611927e-06,
       "loss": 0.0382,
-      "reward": 0.8551339626312255,
-      "reward_std": 0.16432261737063528,
-      "rewards/accuracy_reward": 0.8549107477068901,
-      "rewards/format_reward": 0.00022321429569274187,
+      "reward": 0.8165178954601288,
+      "reward_std": 0.17473283503204584,
+      "rewards/accuracy_reward": 0.8165178954601288,
+      "rewards/format_reward": 0.0,
       "step": 30
     },
     {
-      "completion_length": 667.760962677002,
+      "completion_length": 622.1555999755859,
       "epoch": 0.5970149253731343,
-      "grad_norm": 0.1233765110373497,
-      "kl": 0.006726646423339843,
-      "learning_rate": 1.2296174432791415e-06,
-      "loss": 0.0361,
-      "reward": 0.7089286044239997,
-      "reward_std": 0.2383435256779194,
-      "rewards/accuracy_reward": 0.7089286044239997,
+      "grad_norm": 0.15904287993907928,
+      "kl": 0.003652000427246094,
+      "learning_rate": 2.25e-06,
+      "loss": 0.0396,
+      "reward": 0.7752232491970062,
+      "reward_std": 0.2052750363945961,
+      "rewards/accuracy_reward": 0.7752232491970062,
       "rewards/format_reward": 0.0,
       "step": 35
     },
     {
-      "completion_length": 596.525032043457,
+      "completion_length": 623.6841766357422,
       "epoch": 0.6823027718550106,
-      "grad_norm": 0.15342573821544647,
-      "kl": 0.004629325866699219,
-      "learning_rate": 8.029152419343472e-07,
-      "loss": 0.0368,
-      "reward": 0.7676339626312256,
-      "reward_std": 0.19298281758092345,
-      "rewards/accuracy_reward": 0.7671875342726707,
-      "rewards/format_reward": 0.00044642859138548373,
+      "grad_norm": 0.11450017243623734,
+      "kl": 0.004055023193359375,
+      "learning_rate": 1.975001990702209e-06,
+      "loss": 0.0393,
+      "reward": 0.7602678909897804,
+      "reward_std": 0.2107524886727333,
+      "rewards/accuracy_reward": 0.7602678909897804,
+      "rewards/format_reward": 0.0,
       "step": 40
     },
     {
-      "completion_length": 672.100700378418,
+      "completion_length": 709.4147674560547,
       "epoch": 0.767590618336887,
-      "grad_norm": 0.17312845587730408,
-      "kl": 0.004273223876953125,
-      "learning_rate": 4.3933982822017883e-07,
-      "loss": 0.0322,
-      "reward": 0.7252232484519482,
-      "reward_std": 0.22731470372527837,
-      "rewards/accuracy_reward": 0.7252232484519482,
+      "grad_norm": 0.08205878734588623,
+      "kl": 0.011851119995117187,
+      "learning_rate": 1.6808050203829845e-06,
+      "loss": 0.0359,
+      "reward": 0.5767857410013676,
+      "reward_std": 0.23020651061087846,
+      "rewards/accuracy_reward": 0.5767857410013676,
       "rewards/format_reward": 0.0,
       "step": 45
     },
     {
-      "completion_length": 787.2594100952149,
+      "completion_length": 716.3203491210937,
       "epoch": 0.8528784648187633,
-      "grad_norm": 0.09071672707796097,
-      "kl": 0.005226516723632812,
-      "learning_rate": 1.718159615201853e-07,
-      "loss": 0.0297,
-      "reward": 0.39843751713633535,
-      "reward_std": 0.24290567636489868,
-      "rewards/accuracy_reward": 0.398214303702116,
-      "rewards/format_reward": 0.00022321429569274187,
+      "grad_norm": 0.11587467789649963,
+      "kl": 0.00437164306640625,
+      "learning_rate": 1.3793001469249112e-06,
+      "loss": 0.0316,
+      "reward": 0.574553594738245,
+      "reward_std": 0.24477371014654636,
+      "rewards/accuracy_reward": 0.574553594738245,
+      "rewards/format_reward": 0.0,
       "step": 50
     },
     {
-      "completion_length": 777.8308364868165,
+      "completion_length": 724.092886352539,
       "epoch": 0.9381663113006397,
-      "grad_norm": 0.16111528873443604,
-      "kl": 0.0036163330078125,
-      "learning_rate": 2.4570139579284723e-08,
-      "loss": 0.0458,
-      "reward": 0.5151785917580127,
-      "reward_std": 0.3003951694816351,
-      "rewards/accuracy_reward": 0.5147321626543999,
-      "rewards/format_reward": 0.00044642859138548373,
+      "grad_norm": 0.09563813358545303,
+      "kl": 0.003982162475585938,
+      "learning_rate": 1.0826738041253211e-06,
+      "loss": 0.0343,
+      "reward": 0.586830385029316,
+      "reward_std": 0.25132143292576076,
+      "rewards/accuracy_reward": 0.586830385029316,
+      "rewards/format_reward": 0.0,
       "step": 55
     },
     {
-      "completion_length": 620.0777829488119,
-      "epoch": 0.9893390191897654,
-      "kl": 0.0057366689046223955,
-      "reward": 0.6614583618938923,
-      "reward_std": 0.20894569593171278,
-      "rewards/accuracy_reward": 0.6614583618938923,
+      "completion_length": 572.2068037553267,
+      "epoch": 1.0341151385927505,
+      "grad_norm": 2.0511133670806885,
+      "kl": 0.006678494540127841,
+      "learning_rate": 8.029152419343472e-07,
+      "loss": 0.0443,
+      "reward": 0.7159091213887389,
+      "reward_std": 0.1854671297002245,
+      "rewards/accuracy_reward": 0.7159091213887389,
+      "rewards/format_reward": 0.0,
+      "step": 60
+    },
+    {
+      "completion_length": 398.4535873413086,
+      "epoch": 1.1194029850746268,
+      "grad_norm": 0.10762660950422287,
+      "kl": 0.007398223876953125,
+      "learning_rate": 5.513319366069343e-07,
+      "loss": 0.0402,
+      "reward": 0.9424107536673546,
+      "reward_std": 0.08303849077783525,
+      "rewards/accuracy_reward": 0.9424107536673546,
+      "rewards/format_reward": 0.0,
+      "step": 65
+    },
+    {
+      "completion_length": 435.78461837768555,
+      "epoch": 1.2046908315565032,
+      "grad_norm": 0.2290273904800415,
+      "kl": 0.022534942626953124,
+      "learning_rate": 3.380925572585183e-07,
+      "loss": 0.0382,
+      "reward": 0.9220982596278191,
+      "reward_std": 0.10309098572470247,
+      "rewards/accuracy_reward": 0.9220982596278191,
+      "rewards/format_reward": 0.0,
+      "step": 70
+    },
+    {
+      "completion_length": 480.5415397644043,
+      "epoch": 1.2899786780383795,
+      "grad_norm": 0.15823833644390106,
+      "kl": 0.005892181396484375,
+      "learning_rate": 1.718159615201853e-07,
+      "loss": 0.043,
+      "reward": 0.8839286133646965,
+      "reward_std": 0.1372868578415364,
+      "rewards/accuracy_reward": 0.883705398440361,
+      "rewards/format_reward": 0.00022321429569274187,
+      "step": 75
+    },
+    {
+      "completion_length": 517.8933265686035,
+      "epoch": 1.375266524520256,
+      "grad_norm": 0.12397471815347672,
+      "kl": 0.005947113037109375,
+      "learning_rate": 5.922283255294164e-08,
+      "loss": 0.0319,
+      "reward": 0.8825893267989159,
+      "reward_std": 0.14030799297615887,
+      "rewards/accuracy_reward": 0.8825893267989159,
+      "rewards/format_reward": 0.0,
+      "step": 80
+    },
+    {
+      "completion_length": 516.631275177002,
+      "epoch": 1.4605543710021323,
+      "grad_norm": 0.13666246831417084,
+      "kl": 0.006908416748046875,
+      "learning_rate": 4.864037798685106e-09,
+      "loss": 0.0268,
+      "reward": 0.8665178909897804,
+      "reward_std": 0.14118066830560566,
+      "rewards/accuracy_reward": 0.8662946775555611,
+      "rewards/format_reward": 0.00022321429569274187,
+      "step": 85
+    },
+    {
+      "completion_length": 589.9185562133789,
+      "epoch": 1.4946695095948828,
+      "kl": 0.005696296691894531,
+      "reward": 0.7940848618745804,
+      "reward_std": 0.18597882147878408,
+      "rewards/accuracy_reward": 0.7940848618745804,
       "rewards/format_reward": 0.0,
-      "step": 58,
+      "step": 87,
       "total_flos": 0.0,
-      "train_loss": 0.04879110743259561,
-      "train_runtime": 6811.9096,
-      "train_samples_per_second": 1.101,
+      "train_loss": 0.04595949243882607,
+      "train_runtime": 10042.601,
+      "train_samples_per_second": 1.12,
       "train_steps_per_second": 0.009
     }
   ],
   "logging_steps": 5,
-  "max_steps": 58,
+  "max_steps": 87,
   "num_input_tokens_seen": 0,
-  "num_train_epochs": 1,
+  "num_train_epochs": 2,
   "save_steps": 500,
   "stateful_callbacks": {
     "TrainerControl": {

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:75f96607a86012bc08709eab2696fa27643d75215f49c3ecc44d0bf2695b8003
+oid sha256:ce58edffad6a7ecf0fb5e39a0693546167339a25bbe859eff254f86d634dbac1
 size 7928