End of training

README.md

@@ -40,7 +40,7 @@ This model was trained with GRPO, a method introduced in [DeepSeekMath: Pushing
 - TRL: 0.15.2
 - Transformers: 4.51.3
 - Pytorch: 2.5.1
-- Datasets: 3.5.0
+- Datasets: 3.5.1
 - Tokenizers: 0.21.1
 
 ## Citations

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 9.83700156211853e-06,
-    "train_runtime": 243.7306,
-    "train_samples": 28,
-    "train_samples_per_second": 1.313,
-    "train_steps_per_second": 0.082
+    "train_loss": 3.151223813802062e-06,
+    "train_runtime": 518.4281,
+    "train_samples": 84,
+    "train_samples_per_second": 0.617,
+    "train_steps_per_second": 0.039
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:55fed104c18030771caa633c9c425b23de0d5b8293a4075fb67c397427263b37
+oid sha256:d9a701aeb54101ce4e42a9633ea798be4a0e3fe48627a8452bd09b42a9b7e3ae
 size 1976163472

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 9.83700156211853e-06,
-    "train_runtime": 243.7306,
-    "train_samples": 28,
-    "train_samples_per_second": 1.313,
-    "train_steps_per_second": 0.082
+    "train_loss": 3.151223813802062e-06,
+    "train_runtime": 518.4281,
+    "train_samples": 84,
+    "train_samples_per_second": 0.617,
+    "train_steps_per_second": 0.039
 }

trainer_state.json

@@ -2,7 +2,7 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 5.0,
+  "epoch": 1.8571428571428572,
   "eval_steps": 500,
   "global_step": 20,
   "is_hyper_param_search": false,
@@ -10,209 +10,209 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 329.9375,
-      "epoch": 0.5714285714285714,
-      "grad_norm": 6.7969136238098145,
+      "completion_length": 343.84375,
+      "epoch": 0.19047619047619047,
+      "grad_norm": 6.41450834274292,
       "kl": 0.0,
       "learning_rate": 5e-07,
-      "loss": -0.0,
-      "reward": 1.3629008643329144,
-      "reward_std": 1.512424037558958,
-      "rewards/concensus_correctness_reward_func": 0.6731250006705523,
+      "loss": 0.0,
+      "reward": 1.3421010066522285,
+      "reward_std": 1.406665334085119,
+      "rewards/concensus_correctness_reward_func": 0.6851250007748604,
       "rewards/consensus_reward_func": 0.0625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.35540085507091135,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.32650725438725203,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.2718750031199306,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.1273437519557774,
       "step": 2
     },
     {
-      "completion_length": 345.375,
-      "epoch": 1.0,
-      "grad_norm": 5.820635795593262,
-      "kl": 0.0012729961405663441,
+      "completion_length": 339.53125,
+      "epoch": 0.38095238095238093,
+      "grad_norm": 6.3102498054504395,
+      "kl": 0.0012349215921858558,
       "learning_rate": 4.864543104251586e-07,
       "loss": 0.0,
-      "reward": 0.47224158281460404,
-      "reward_std": 0.5307725191354015,
-      "rewards/concensus_correctness_reward_func": 0.04791666567325592,
-      "rewards/consensus_reward_func": 0.0,
+      "reward": 0.3265813000034541,
+      "reward_std": 0.4629429771230207,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.22349158317471543,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.13289380201604217,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.20083333489795527,
+      "rewards/xmlcount_reward_func": 0.06868749670684338,
       "step": 4
     },
     {
-      "completion_length": 411.90625,
-      "epoch": 1.5714285714285714,
-      "grad_norm": 5.046466827392578,
-      "kl": 0.002466292877215892,
+      "completion_length": 296.1875,
+      "epoch": 0.5714285714285714,
+      "grad_norm": 61.84025573730469,
+      "kl": 0.005961592213679978,
       "learning_rate": 4.472851273490984e-07,
       "loss": 0.0,
-      "reward": 0.33047787082614377,
-      "reward_std": 0.5196221396327019,
-      "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.0,
+      "reward": 0.5405496190069243,
+      "reward_std": 0.5028865041094832,
+      "rewards/concensus_correctness_reward_func": 0.059812501072883606,
+      "rewards/consensus_reward_func": 0.125,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.2953216189634986,
+      "rewards/question_recreation_reward_func": 0.27689335064496845,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.03515625186264515,
+      "rewards/xmlcount_reward_func": 0.07884375378489494,
       "step": 6
     },
     {
-      "completion_length": 520.5416666666666,
-      "epoch": 2.0,
-      "grad_norm": 140.4196014404297,
-      "kl": 0.021952209106530063,
+      "completion_length": 249.1875,
+      "epoch": 0.7619047619047619,
+      "grad_norm": 56.74770736694336,
+      "kl": 0.002046526838967111,
       "learning_rate": 3.867370395306068e-07,
       "loss": 0.0,
-      "reward": 0.1229500036376218,
-      "reward_std": 0.5561859489729007,
+      "reward": 0.37825396441621706,
+      "reward_std": 0.37752981589437695,
       "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.3202416733838618,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.18716021499130875,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": -0.19729167285064855,
+      "rewards/xmlcount_reward_func": 0.06609375029802322,
       "step": 8
     },
     {
-      "completion_length": 395.84375,
-      "epoch": 2.571428571428571,
-      "grad_norm": 5.094494342803955,
-      "kl": 0.007353434473770903,
+      "completion_length": 285.75,
+      "epoch": 0.9523809523809523,
+      "grad_norm": 35.71620178222656,
+      "kl": 0.0014948487314541126,
       "learning_rate": 3.1137137178519977e-07,
       "loss": 0.0,
-      "reward": 0.6689693744992837,
-      "reward_std": 0.6049241637811065,
-      "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.0,
+      "reward": 0.6967899320879951,
+      "reward_std": 0.5912124498172489,
+      "rewards/concensus_correctness_reward_func": 0.06024999916553497,
+      "rewards/consensus_reward_func": 0.125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.39621935749892145,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.2623211840982549,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.2102499958127737,
+      "rewards/xmlcount_reward_func": 0.12421874701976776,
       "step": 10
     },
     {
-      "completion_length": 274.5416666666667,
-      "epoch": 3.0,
-      "grad_norm": 9.407255172729492,
-      "kl": 0.0037853789932948225,
+      "completion_length": 347.2916666666667,
+      "epoch": 1.0952380952380953,
+      "grad_norm": 5.5424346923828125,
+      "kl": 0.001293833294766955,
       "learning_rate": 2.2935516363191693e-07,
       "loss": 0.0,
-      "reward": 0.30114452727138996,
-      "reward_std": 0.36317425857608515,
+      "reward": 0.1296683083055541,
+      "reward_std": 0.42109029918598634,
       "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.08333333333333333,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.2888111875702937,
+      "rewards/question_recreation_reward_func": 0.13004330720286816,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.012333341253300508,
+      "rewards/xmlcount_reward_func": -0.08370832974712054,
       "step": 12
     },
     {
-      "completion_length": 431.28125,
-      "epoch": 3.571428571428571,
-      "grad_norm": 11.95157527923584,
-      "kl": 0.01949344891909277,
+      "completion_length": 373.09375,
+      "epoch": 1.2857142857142856,
+      "grad_norm": 17.043167114257812,
+      "kl": 0.0012364095746306702,
       "learning_rate": 1.4957614383675767e-07,
       "loss": 0.0,
-      "reward": 0.2974823787808418,
-      "reward_std": 0.6576210716739297,
-      "rewards/concensus_correctness_reward_func": 0.0078125,
+      "reward": 0.3166529495501891,
+      "reward_std": 0.406706450904494,
+      "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.26476360601373017,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.16937170409073588,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": -0.10009373724460602,
+      "rewards/xmlcount_reward_func": 0.08478125440888107,
       "step": 14
     },
     {
-      "completion_length": 474.25,
-      "epoch": 4.0,
-      "grad_norm": 7.882875919342041,
-      "kl": 0.005596131251271193,
+      "completion_length": 245.1875,
+      "epoch": 1.4761904761904763,
+      "grad_norm": 27.399072647094727,
+      "kl": 0.0028427706747606862,
       "learning_rate": 8.067960709356478e-08,
       "loss": 0.0,
-      "reward": 0.6628411809603373,
-      "reward_std": 0.5658263303339481,
-      "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.0,
+      "reward": 0.6848066907841712,
+      "reward_std": 0.6611888102197554,
+      "rewards/concensus_correctness_reward_func": 0.06012500077486038,
+      "rewards/consensus_reward_func": 0.125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.36009117122739553,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.25196293997578323,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.3027500019719203,
+      "rewards/xmlcount_reward_func": 0.12271874817088246,
       "step": 16
     },
     {
-      "completion_length": 425.71875,
-      "epoch": 4.571428571428571,
-      "grad_norm": 83.45905303955078,
-      "kl": 0.035086556898022536,
+      "completion_length": 264.25,
+      "epoch": 1.6666666666666665,
+      "grad_norm": 42.305606842041016,
+      "kl": 0.0022985513096500654,
       "learning_rate": 3.013156219837776e-08,
       "loss": 0.0,
-      "reward": 0.4946426055394113,
-      "reward_std": 0.4243872029474005,
-      "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.0,
+      "reward": 1.2035528365522623,
+      "reward_std": 1.0453436647367198,
+      "rewards/concensus_correctness_reward_func": 0.15699999779462814,
+      "rewards/consensus_reward_func": 0.25,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.3207676014280878,
+      "rewards/final_correctness_reward_func": 0.1875,
+      "rewards/question_recreation_reward_func": 0.26874033408239484,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.17387500149197876,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.3246875018812716,
       "step": 18
     },
     {
-      "completion_length": 440.2083333333333,
-      "epoch": 5.0,
-      "grad_norm": 3.990732192993164,
-      "kl": 0.012679507781285793,
+      "completion_length": 378.5625,
+      "epoch": 1.8571428571428572,
+      "grad_norm": 64.05872344970703,
+      "kl": 0.013361819263081998,
       "learning_rate": 3.4096741493194193e-09,
       "loss": 0.0,
-      "reward": 1.0740113233526547,
-      "reward_std": 1.404585100710392,
-      "rewards/concensus_correctness_reward_func": 0.11616666397700708,
+      "reward": 0.8212161803094205,
+      "reward_std": 1.0907838449347764,
+      "rewards/concensus_correctness_reward_func": 0.06012500077486038,
       "rewards/consensus_reward_func": 0.25,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.42130298881481093,
+      "rewards/final_correctness_reward_func": 0.1875,
+      "rewards/question_recreation_reward_func": 0.24912242317805067,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.2865416711817185,
+      "rewards/xmlcount_reward_func": 0.07446875050663948,
       "step": 20
     },
     {
-      "epoch": 5.0,
+      "epoch": 1.8571428571428572,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 9.83700156211853e-06,
-      "train_runtime": 243.7306,
-      "train_samples_per_second": 1.313,
-      "train_steps_per_second": 0.082
+      "train_loss": 3.151223813802062e-06,
+      "train_runtime": 518.4281,
+      "train_samples_per_second": 0.617,
+      "train_steps_per_second": 0.039
     }
   ],
   "logging_steps": 2,
   "max_steps": 20,
   "num_input_tokens_seen": 0,
-  "num_train_epochs": 7,
+  "num_train_epochs": 2,
   "save_steps": 25,
   "stateful_callbacks": {
     "TrainerControl": {

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:75ca64635e7cd097ae36824928d29cbad5ce061f20204409a84dc9ab6d50771c
+oid sha256:78268932364dfe195fcfba8eaf3212b6eedd2ad354c3defdcbbb8bcdd22dd702
 size 5944