End of training

README.md

@@ -38,7 +38,7 @@ This model was trained with GRPO, a method introduced in [DeepSeekMath: Pushing
 ### Framework versions
 
 - TRL: 0.15.2
-- Transformers: 4.50.3
+- Transformers: 4.51.3
 - Pytorch: 2.5.1
 - Datasets: 3.5.0
 - Tokenizers: 0.21.1

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 6.368570029735565e-06,
-    "train_runtime": 420.747,
-    "train_samples": 160,
-    "train_samples_per_second": 0.761,
-    "train_steps_per_second": 0.048
+    "train_loss": 3.901487593793717e-06,
+    "train_runtime": 857.4804,
+    "train_samples": 53,
+    "train_samples_per_second": 0.373,
+    "train_steps_per_second": 0.023
 }

config.json

@@ -21,7 +21,7 @@
   "sliding_window": 32768,
   "tie_word_embeddings": true,
   "torch_dtype": "float32",
-  "transformers_version": "4.50.3",
+  "transformers_version": "4.51.3",
   "use_cache": true,
   "use_sliding_window": false,
   "vocab_size": 151936

generation_config.json

@@ -10,5 +10,5 @@
   "temperature": 0.7,
   "top_k": 20,
   "top_p": 0.8,
-  "transformers_version": "4.50.3"
+  "transformers_version": "4.51.3"
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0cb01c1e29f73d69b5571593082ee448ce8e393e1f33baca2fc4c73fa7534328
+oid sha256:4d1db651d0fa62ae23fbb79e00dd79f109407f69c7733eb5354ad05b6468bf97
 size 1976163472

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 6.368570029735565e-06,
-    "train_runtime": 420.747,
-    "train_samples": 160,
-    "train_samples_per_second": 0.761,
-    "train_steps_per_second": 0.048
+    "train_loss": 3.901487593793717e-06,
+    "train_runtime": 857.4804,
+    "train_samples": 53,
+    "train_samples_per_second": 0.373,
+    "train_steps_per_second": 0.023
 }

trainer_state.json

@@ -2,7 +2,7 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 1.0,
+  "epoch": 2.9056603773584904,
   "eval_steps": 500,
   "global_step": 20,
   "is_hyper_param_search": false,
@@ -10,209 +10,209 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 350.8125,
-      "epoch": 0.1,
-      "grad_norm": 39.23478698730469,
+      "completion_length": 378.625,
+      "epoch": 0.3018867924528302,
+      "grad_norm": 9.093167304992676,
       "kl": 0.0,
-      "learning_rate": 4.965903258506806e-07,
+      "learning_rate": 5e-07,
       "loss": -0.0,
-      "reward": 1.2133730966597795,
-      "reward_std": 0.5402863117633387,
-      "rewards/concensus_correctness_reward_func": 0.0599374994635582,
-      "rewards/consensus_reward_func": 0.125,
+      "reward": 1.0540178306400776,
+      "reward_std": 1.0569077435939107,
+      "rewards/concensus_correctness_reward_func": 0.178312499076128,
+      "rewards/consensus_reward_func": 0.25,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.4502793492283672,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.34108033566735685,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.5781562521588057,
+      "rewards/xmlcount_reward_func": 0.15962500142632052,
       "step": 2
     },
     {
-      "completion_length": 278.4375,
-      "epoch": 0.2,
-      "grad_norm": 6.766746997833252,
-      "kl": 0.001203565305331722,
-      "learning_rate": 4.698684378016222e-07,
+      "completion_length": 362.0,
+      "epoch": 0.6037735849056604,
+      "grad_norm": 10.739517211914062,
+      "kl": 0.0017382375299348496,
+      "learning_rate": 4.864543104251586e-07,
       "loss": 0.0,
-      "reward": 1.6612788662314415,
-      "reward_std": 0.6323463647277094,
-      "rewards/concensus_correctness_reward_func": 0.06006250157952309,
-      "rewards/consensus_reward_func": 0.0625,
+      "reward": 0.5339535549283028,
+      "reward_std": 0.2954929779516533,
+      "rewards/concensus_correctness_reward_func": 0.05387499928474426,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.6065601068548858,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.19642229308374226,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.8071562610566616,
+      "rewards/xmlcount_reward_func": 0.22115624975413084,
       "step": 4
     },
     {
-      "completion_length": 312.84375,
-      "epoch": 0.3,
-      "grad_norm": 7.9881415367126465,
-      "kl": 0.002273020745633403,
-      "learning_rate": 4.193203929064353e-07,
+      "completion_length": 299.90625,
+      "epoch": 0.9056603773584906,
+      "grad_norm": 10.603949546813965,
+      "kl": 0.0025668062080512755,
+      "learning_rate": 4.472851273490984e-07,
       "loss": 0.0,
-      "reward": 1.397591108456254,
-      "reward_std": 0.8458844633860281,
-      "rewards/concensus_correctness_reward_func": 0.05868750065565109,
-      "rewards/consensus_reward_func": 0.0625,
+      "reward": 0.7476452244445682,
+      "reward_std": 1.0407738303765655,
+      "rewards/concensus_correctness_reward_func": 0.0253749992698431,
+      "rewards/consensus_reward_func": 0.125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.1875,
-      "rewards/question_recreation_reward_func": 0.39743484029895626,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.30070771696045995,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.6914687640964985,
+      "rewards/xmlcount_reward_func": 0.2340625070501119,
       "step": 6
     },
     {
-      "completion_length": 375.46875,
-      "epoch": 0.4,
-      "grad_norm": 5.538845062255859,
-      "kl": 0.0035255207076261286,
-      "learning_rate": 3.5042385616324236e-07,
+      "completion_length": 254.80769230769232,
+      "epoch": 1.150943396226415,
+      "grad_norm": 21.24864959716797,
+      "kl": 0.002797715992738421,
+      "learning_rate": 3.867370395306068e-07,
       "loss": 0.0,
-      "reward": 1.8364141806960106,
-      "reward_std": 0.7308946952689439,
-      "rewards/concensus_correctness_reward_func": 0.1797499991953373,
-      "rewards/consensus_reward_func": 0.3125,
+      "reward": 0.3774323563736219,
+      "reward_std": 0.6041523932665138,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.5836954447440803,
+      "rewards/final_correctness_reward_func": 0.07692307692307693,
+      "rewards/question_recreation_reward_func": 0.22254774518884146,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.7604687586426735,
+      "rewards/strict_format_reward_func": 0.019230769230769232,
+      "rewards/xmlcount_reward_func": 0.058730767896542184,
       "step": 8
     },
     {
-      "completion_length": 301.3125,
-      "epoch": 0.5,
-      "grad_norm": 8.966378211975098,
-      "kl": 0.006074020639061928,
-      "learning_rate": 2.706448363680831e-07,
+      "completion_length": 371.21875,
+      "epoch": 1.4528301886792452,
+      "grad_norm": 6.234846591949463,
+      "kl": 0.0040573262340330984,
+      "learning_rate": 3.1137137178519977e-07,
       "loss": 0.0,
-      "reward": 1.8557742610573769,
-      "reward_std": 0.673103028557307,
-      "rewards/concensus_correctness_reward_func": 0.05999999865889549,
+      "reward": 0.4840935148531571,
+      "reward_std": 1.0272551258221938,
+      "rewards/concensus_correctness_reward_func": 0.051750000566244125,
       "rewards/consensus_reward_func": 0.0625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.583211736753583,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.3480935108382255,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.9000625386834145,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.006125002168118954,
       "step": 10
     },
     {
-      "completion_length": 513.09375,
-      "epoch": 0.6,
-      "grad_norm": 5.500680923461914,
-      "kl": 0.0054717185339541174,
-      "learning_rate": 1.886286282148002e-07,
+      "completion_length": 409.0,
+      "epoch": 1.7547169811320755,
+      "grad_norm": 8.263574600219727,
+      "kl": 0.01272706116833433,
+      "learning_rate": 2.2935516363191693e-07,
       "loss": 0.0,
-      "reward": 1.3633840680122375,
-      "reward_std": 1.3847295328741893,
-      "rewards/concensus_correctness_reward_func": 0.22243749722838402,
-      "rewards/consensus_reward_func": 0.1875,
+      "reward": 1.0864606173709035,
+      "reward_std": 1.7664562326390296,
+      "rewards/concensus_correctness_reward_func": 0.7353124991059303,
+      "rewards/consensus_reward_func": 0.125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.1875,
-      "rewards/question_recreation_reward_func": 0.5454153548926115,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.19583564292406663,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.22053123265504837,
+      "rewards/xmlcount_reward_func": -0.03218750189989805,
       "step": 12
     },
     {
-      "completion_length": 355.75,
-      "epoch": 0.7,
-      "grad_norm": 6.750906944274902,
-      "kl": 0.01095420480123721,
-      "learning_rate": 1.1326296046939333e-07,
+      "completion_length": 405.84615384615387,
+      "epoch": 2.0,
+      "grad_norm": 32.39244842529297,
+      "kl": 0.003084615822165058,
+      "learning_rate": 1.4957614383675767e-07,
       "loss": 0.0,
-      "reward": 2.009573094546795,
-      "reward_std": 1.0107755847275257,
+      "reward": 0.4143683403157271,
+      "reward_std": 0.5211973208934069,
       "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.1875,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.4375,
-      "rewards/question_recreation_reward_func": 0.5910418333951384,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.31379142231666124,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.7935312613844872,
+      "rewards/xmlcount_reward_func": 0.10057691979007079,
       "step": 14
     },
     {
-      "completion_length": 387.34375,
-      "epoch": 0.8,
-      "grad_norm": 7.572938919067383,
-      "kl": 0.009205558279063553,
-      "learning_rate": 5.271487265090163e-08,
+      "completion_length": 365.625,
+      "epoch": 2.30188679245283,
+      "grad_norm": 76.20389556884766,
+      "kl": 0.003164041761920089,
+      "learning_rate": 8.067960709356478e-08,
       "loss": 0.0,
-      "reward": 1.7320444360375404,
-      "reward_std": 0.9637731028778944,
-      "rewards/concensus_correctness_reward_func": 0.12006250023841858,
-      "rewards/consensus_reward_func": 0.3125,
+      "reward": 0.42775836275541224,
+      "reward_std": 0.7112423235084862,
+      "rewards/concensus_correctness_reward_func": 0.028625001199543476,
+      "rewards/consensus_reward_func": 0.125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.1875,
-      "rewards/question_recreation_reward_func": 0.40157566242851317,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.33463336058775894,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.7104062773287296,
+      "rewards/xmlcount_reward_func": -0.06049999222159386,
       "step": 16
     },
     {
-      "completion_length": 418.53125,
-      "epoch": 0.9,
-      "grad_norm": 5.404604434967041,
-      "kl": 0.012272321924683638,
-      "learning_rate": 1.3545689574841341e-08,
+      "completion_length": 240.96875,
+      "epoch": 2.6037735849056602,
+      "grad_norm": 25.190492630004883,
+      "kl": 0.00812290481189848,
+      "learning_rate": 3.013156219837776e-08,
       "loss": 0.0,
-      "reward": 1.2925836332142353,
-      "reward_std": 1.1542230368359014,
-      "rewards/concensus_correctness_reward_func": 0.2397499978542328,
-      "rewards/consensus_reward_func": 0.125,
+      "reward": 0.5747865298762918,
+      "reward_std": 0.5580734928153106,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.5634898480493575,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.31997403455898166,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.3643437512218952,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.17668750206939876,
       "step": 18
     },
     {
-      "completion_length": 382.40625,
-      "epoch": 1.0,
-      "grad_norm": 6.85258674621582,
-      "kl": 0.009818575592362322,
-      "learning_rate": 0.0,
+      "completion_length": 365.375,
+      "epoch": 2.9056603773584904,
+      "grad_norm": 43.69164276123047,
+      "kl": 0.0020233836730767507,
+      "learning_rate": 3.4096741493194193e-09,
       "loss": 0.0,
-      "reward": 1.7785236164927483,
-      "reward_std": 1.1780102625489235,
+      "reward": 0.457977767335251,
+      "reward_std": 0.5906311667640693,
       "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.25,
+      "rewards/consensus_reward_func": 0.0625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.1875,
-      "rewards/question_recreation_reward_func": 0.7126173116266727,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.21113401418551803,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.6284062843769789,
+      "rewards/xmlcount_reward_func": 0.12184375035576522,
       "step": 20
     },
     {
-      "epoch": 1.0,
+      "epoch": 2.9056603773584904,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 6.368570029735565e-06,
-      "train_runtime": 420.747,
-      "train_samples_per_second": 0.761,
-      "train_steps_per_second": 0.048
+      "train_loss": 3.901487593793717e-06,
+      "train_runtime": 857.4804,
+      "train_samples_per_second": 0.373,
+      "train_steps_per_second": 0.023
     }
   ],
   "logging_steps": 2,
   "max_steps": 20,
   "num_input_tokens_seen": 0,
-  "num_train_epochs": 1,
+  "num_train_epochs": 4,
   "save_steps": 25,
   "stateful_callbacks": {
     "TrainerControl": {

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:85467a903ff9de653ec44eb444d45780255b47e00aa1f803bb07b7f753f11efc
-size 6008
+oid sha256:9f2ae03c555cc570ac479d191726452136e516d68f1706d019485ded55311da7
+size 5944