End of training

README.md

@@ -39,7 +39,7 @@ This model was trained with GRPO, a method introduced in [DeepSeekMath: Pushing
 ### Framework versions
 
 - TRL: 0.15.2
-- Transformers: 4.51.3
+- Transformers: 4.48.2
 - Pytorch: 2.5.1
 - Datasets: 3.6.0
 - Tokenizers: 0.21.1

adapter_config.json

@@ -23,13 +23,13 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "k_proj",
-    "down_proj",
     "gate_proj",
+    "k_proj",
+    "up_proj",
+    "v_proj",
     "q_proj",
     "o_proj",
-    "v_proj",
-    "up_proj"
+    "down_proj"
   ],
   "task_type": "CAUSAL_LM",
   "use_dora": false,

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ec94e591006394e5861342585e4aaf983b29a686e71b397bcc3aa5e4b210244d
+oid sha256:9b58c38db8219dc11775b1d8be833eed4496f52cd90b769290c3ca6366928d53
 size 35237104

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 1.1757511778398565e-06,
-    "train_runtime": 2092.798,
-    "train_samples": 31,
-    "train_samples_per_second": 0.459,
-    "train_steps_per_second": 0.029
+    "train_loss": 7.3265395705846e-08,
+    "train_runtime": 1657.2389,
+    "train_samples": 11,
+    "train_samples_per_second": 0.193,
+    "train_steps_per_second": 0.012
 }

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 1.1757511778398565e-06,
-    "train_runtime": 2092.798,
-    "train_samples": 31,
-    "train_samples_per_second": 0.459,
-    "train_steps_per_second": 0.029
+    "train_loss": 7.3265395705846e-08,
+    "train_runtime": 1657.2389,
+    "train_samples": 11,
+    "train_samples_per_second": 0.193,
+    "train_steps_per_second": 0.012
 }

trainer_state.json

@@ -1,598 +1,217 @@
 {
-  "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 7.516129032258064,
+  "epoch": 6.7272727272727275,
   "eval_steps": 500,
-  "global_step": 60,
+  "global_step": 20,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 430.8125,
-      "epoch": 0.25806451612903225,
+      "completion_length": 436.0625,
+      "epoch": 0.7272727272727273,
       "grad_norm": NaN,
-      "kl": 0.00041734772025847633,
-      "learning_rate": 2.5e-07,
+      "kl": 1.7621703932491073e-05,
+      "learning_rate": 5e-07,
       "loss": 0.0,
-      "reward": 0.11391110043041408,
-      "reward_std": 0.41625179536640644,
+      "reward": 0.20653149974532425,
+      "reward_std": 0.27299811027478427,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.20462985103949904,
+      "rewards/question_recreation_reward_func": 0.1650314915459603,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": -0.09071875200606883,
+      "rewards/xmlcount_reward_func": 0.04149999842047691,
       "step": 2
     },
     {
-      "completion_length": 343.34375,
-      "epoch": 0.5161290322580645,
-      "grad_norm": 1.9656559228897095,
-      "kl": 0.00040763827485079673,
-      "learning_rate": 5e-07,
+      "completion_length": 421.64285714285717,
+      "epoch": 1.3636363636363638,
+      "grad_norm": NaN,
+      "kl": 3.5645763091451954e-05,
+      "learning_rate": 4.965903258506806e-07,
       "loss": 0.0,
-      "reward": 0.08423173008486629,
-      "reward_std": 0.1950428435811773,
+      "reward": 0.22060505353978702,
+      "reward_std": 0.3099955760740808,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.10804423363879323,
+      "rewards/question_recreation_reward_func": 0.13264077582529613,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": -0.02381250262260437,
+      "rewards/xmlcount_reward_func": 0.08796428569725581,
       "step": 4
     },
     {
-      "completion_length": 426.71875,
-      "epoch": 0.7741935483870968,
-      "grad_norm": 3.116727590560913,
-      "kl": 0.00013401095452536538,
-      "learning_rate": 4.985344892885899e-07,
+      "completion_length": 341.14285714285717,
+      "epoch": 2.0,
+      "grad_norm": 4.797464370727539,
+      "kl": 3.0444448124658914e-05,
+      "learning_rate": 4.864543104251586e-07,
       "loss": 0.0,
-      "reward": 0.2961328737437725,
-      "reward_std": 0.3188573829829693,
+      "reward": 0.3554938094956534,
+      "reward_std": 0.3407979671444212,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.20203911093994975,
+      "rewards/question_recreation_reward_func": 0.24795808962413243,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.09409375116229057,
+      "rewards/xmlcount_reward_func": 0.10753571561404637,
       "step": 6
     },
     {
-      "completion_length": 363.85714285714283,
-      "epoch": 1.0,
-      "grad_norm": 4.322468280792236,
-      "kl": 0.000584651980066805,
-      "learning_rate": 4.967066306353815e-07,
+      "completion_length": 405.03125,
+      "epoch": 2.7272727272727275,
+      "grad_norm": NaN,
+      "kl": 3.89778874705371e-05,
+      "learning_rate": 4.698684378016222e-07,
       "loss": 0.0,
-      "reward": 0.28172825063977924,
-      "reward_std": 0.2894980385899544,
+      "reward": 0.22569512180052698,
+      "reward_std": 0.3155328747816384,
       "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0625,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.1669425368309021,
+      "rewards/question_recreation_reward_func": 0.13241386436857283,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.11478571407496929,
+      "rewards/xmlcount_reward_func": 0.030781249282881618,
       "step": 8
     },
     {
-      "completion_length": 375.15625,
-      "epoch": 1.2580645161290323,
-      "grad_norm": 4.052526473999023,
-      "kl": 0.00137491720761318,
-      "learning_rate": 4.941551389275217e-07,
+      "completion_length": 366.9642857142857,
+      "epoch": 3.3636363636363638,
+      "grad_norm": 66.79761505126953,
+      "kl": 2.5024709300071534e-05,
+      "learning_rate": 4.193203929064353e-07,
       "loss": 0.0,
-      "reward": 0.1756317124236375,
-      "reward_std": 0.13710087712388486,
+      "reward": 0.18169026502541133,
+      "reward_std": 0.24965994379350118,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.10741296270862222,
+      "rewards/question_recreation_reward_func": 0.08347597638411182,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.06821874901652336,
+      "rewards/xmlcount_reward_func": 0.09821428571428571,
       "step": 10
     },
     {
-      "completion_length": 527.28125,
-      "epoch": 1.5161290322580645,
-      "grad_norm": 1.0876020193099976,
-      "kl": 7.662167183752899e-05,
-      "learning_rate": 4.869132927957006e-07,
+      "completion_length": 383.17857142857144,
+      "epoch": 4.0,
+      "grad_norm": NaN,
+      "kl": 6.10867003030892e-05,
+      "learning_rate": 3.867370395306068e-07,
       "loss": 0.0,
-      "reward": 0.05637967900838703,
-      "reward_std": 0.29673084686510265,
+      "reward": 0.16173036502940313,
+      "reward_std": 0.19714894411819323,
       "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.07142857142857142,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.08856716414447874,
+      "rewards/question_recreation_reward_func": 0.09376607196671623,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": -0.03218749677762389,
+      "rewards/xmlcount_reward_func": -0.003464285284280777,
       "step": 12
     },
     {
-      "completion_length": 420.4375,
-      "epoch": 1.7741935483870968,
-      "grad_norm": 1.930647373199463,
-      "kl": 0.00019077809974987758,
-      "learning_rate": 4.768938549177392e-07,
+      "completion_length": 405.03125,
+      "epoch": 4.7272727272727275,
+      "grad_norm": 10.484304428100586,
+      "kl": 3.165742793953541e-05,
+      "learning_rate": 3.1137137178519977e-07,
       "loss": 0.0,
-      "reward": 0.32431007781997323,
-      "reward_std": 0.3349539686460048,
+      "reward": -0.017911486327648163,
+      "reward_std": 0.3045911204535514,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.19974758382886648,
+      "rewards/question_recreation_reward_func": 0.09355727583169937,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.12456249631941319,
+      "rewards/xmlcount_reward_func": -0.11146875284612179,
       "step": 14
     },
     {
-      "completion_length": 235.67857142857142,
-      "epoch": 2.0,
-      "grad_norm": 25.613550186157227,
-      "kl": 0.001633655218029162,
-      "learning_rate": 4.708780111115057e-07,
+      "completion_length": 487.5357142857143,
+      "epoch": 5.363636363636363,
+      "grad_norm": 66.34976196289062,
+      "kl": 0.00016280047917202216,
+      "learning_rate": 2.2935516363191693e-07,
       "loss": 0.0,
-      "reward": 0.187016135347741,
-      "reward_std": 0.25221954605409075,
+      "reward": 0.40852856050644604,
+      "reward_std": 0.3531542556094272,
       "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.07142857142857142,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.11437327787280083,
+      "rewards/question_recreation_reward_func": 0.23220711361084664,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.07264285454792636,
+      "rewards/xmlcount_reward_func": 0.10489285630839211,
       "step": 16
     },
     {
-      "completion_length": 318.28125,
-      "epoch": 2.258064516129032,
-      "grad_norm": 3.3770768642425537,
-      "kl": 0.0005925071802721504,
-      "learning_rate": 4.5692224953922265e-07,
+      "completion_length": 415.4642857142857,
+      "epoch": 6.0,
+      "grad_norm": 11.347963333129883,
+      "kl": 2.8945602155415274e-05,
+      "learning_rate": 1.4957614383675767e-07,
       "loss": 0.0,
-      "reward": 0.2636642586439848,
-      "reward_std": 0.3803391242399812,
+      "reward": 0.21686194677438056,
+      "reward_std": 0.31533121078142096,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.13447676633950323,
+      "rewards/question_recreation_reward_func": 0.16504052467644215,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.12918749847449362,
+      "rewards/xmlcount_reward_func": 0.05182142768587385,
       "step": 18
     },
     {
-      "completion_length": 371.59375,
-      "epoch": 2.5161290322580645,
-      "grad_norm": 1.974790096282959,
-      "kl": 0.0002036209966718161,
-      "learning_rate": 4.405405137819091e-07,
+      "completion_length": 404.03125,
+      "epoch": 6.7272727272727275,
+      "grad_norm": 2.1316444873809814,
+      "kl": 9.378601941989473e-05,
+      "learning_rate": 8.067960709356478e-08,
       "loss": 0.0,
-      "reward": 0.10157998092472553,
-      "reward_std": 0.10903631476685405,
+      "reward": 0.10079575958661735,
+      "reward_std": 0.08944826829247177,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.06636122916825116,
+      "rewards/question_recreation_reward_func": 0.05104575864970684,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.035218749195337296,
+      "rewards/xmlcount_reward_func": 0.04975000023841858,
       "step": 20
     },
     {
-      "completion_length": 279.71875,
-      "epoch": 2.774193548387097,
-      "grad_norm": 26.12720489501953,
-      "kl": 0.0019963133127021138,
-      "learning_rate": 4.219248647133558e-07,
-      "loss": 0.0,
-      "reward": 0.35939981369301677,
-      "reward_std": 0.5500808532815427,
-      "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.0,
-      "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.25952482083812356,
-      "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.09987499937415123,
-      "step": 22
-    },
-    {
-      "completion_length": 383.64285714285717,
-      "epoch": 3.0,
-      "grad_norm": 0.9543819427490234,
-      "kl": 0.0006048666277170664,
-      "learning_rate": 4.012935537984413e-07,
-      "loss": 0.0,
-      "reward": 0.22724560940904276,
-      "reward_std": 0.20333547570875712,
-      "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.0,
-      "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.18638846703938075,
-      "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.04085714370012283,
-      "step": 24
-    },
-    {
-      "completion_length": 381.96875,
-      "epoch": 3.258064516129032,
-      "grad_norm": 3.211329698562622,
-      "kl": 0.00012533261292446696,
-      "learning_rate": 3.788884642942554e-07,
-      "loss": 0.0,
-      "reward": 0.19361284282058477,
-      "reward_std": 0.24643944506533444,
-      "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.0,
-      "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.17839408991858363,
-      "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.015218751970678568,
-      "step": 26
-    },
-    {
-      "completion_length": 412.5,
-      "epoch": 3.5161290322580645,
-      "grad_norm": 3.394382953643799,
-      "kl": 0.00010090849150401482,
-      "learning_rate": 3.5497227539006614e-07,
-      "loss": 0.0,
-      "reward": 0.2241632086224854,
-      "reward_std": 0.2526557689998299,
-      "rewards/concensus_correctness_reward_func": 0.013875000178813934,
-      "rewards/consensus_reward_func": 0.0,
-      "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.17597571481019258,
-      "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.03431250061839819,
-      "step": 28
-    },
-    {
-      "completion_length": 284.59375,
-      "epoch": 3.774193548387097,
-      "grad_norm": 2.9376416206359863,
-      "kl": 0.0008606196147411538,
-      "learning_rate": 3.2982538253399494e-07,
-      "loss": 0.0,
-      "reward": 0.0774760153144598,
-      "reward_std": 0.1906891551334411,
-      "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.0,
-      "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.09122601104900241,
-      "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": -0.01374999899417162,
-      "step": 30
-    },
-    {
-      "completion_length": 549.1428571428571,
-      "epoch": 4.0,
-      "grad_norm": 0.9819178581237793,
-      "kl": 3.5185036722396035e-05,
-      "learning_rate": 3.0374261005275605e-07,
-      "loss": 0.0,
-      "reward": 0.25383392149316414,
-      "reward_std": 0.30834708123334814,
-      "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.0,
-      "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.11165534212653126,
-      "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.14217857164996012,
-      "step": 32
-    },
-    {
-      "completion_length": 366.59375,
-      "epoch": 4.258064516129032,
-      "grad_norm": 2.5270180702209473,
-      "kl": 0.004282280417555739,
-      "learning_rate": 2.770297546059854e-07,
-      "loss": 0.0,
-      "reward": 0.06346084550023079,
-      "reward_std": 0.28176579938735813,
-      "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.0,
-      "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.12233584048226476,
-      "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": -0.0588750010356307,
-      "step": 34
-    },
-    {
-      "completion_length": 411.375,
-      "epoch": 4.516129032258064,
-      "grad_norm": 4.21773099899292,
-      "kl": 8.090390974757611e-05,
-      "learning_rate": 2.5e-07,
-      "loss": 0.0,
-      "reward": 0.11983968829736114,
-      "reward_std": 0.17383667826652527,
-      "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.0,
-      "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.11633968772366643,
-      "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.0035000010393559933,
-      "step": 36
-    },
-    {
-      "completion_length": 329.28125,
-      "epoch": 4.774193548387097,
-      "grad_norm": 18.660564422607422,
-      "kl": 0.0010729862383414002,
-      "learning_rate": 2.229702453940146e-07,
-      "loss": 0.0,
-      "reward": 0.22536780731752515,
-      "reward_std": 0.22597291774582118,
-      "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.0,
-      "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.13943030545488,
-      "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.0859375,
-      "step": 38
-    },
-    {
-      "completion_length": 466.82142857142856,
-      "epoch": 5.0,
-      "grad_norm": 30.90364646911621,
-      "kl": 0.0050324718344459795,
-      "learning_rate": 1.9625738994724398e-07,
-      "loss": 0.0,
-      "reward": -0.09624448684709412,
-      "reward_std": 0.43322019491876873,
-      "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.0,
-      "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.08754122070968151,
-      "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": -0.18378571208034242,
-      "step": 40
-    },
-    {
-      "completion_length": 434.0625,
-      "epoch": 5.258064516129032,
-      "grad_norm": 39.12836837768555,
-      "kl": 0.005596646230287661,
-      "learning_rate": 1.7017461746600504e-07,
-      "loss": 0.0,
-      "reward": 0.13575120829045773,
-      "reward_std": 0.13530697469832376,
-      "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.0,
-      "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.1172824571840465,
-      "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.01846875064074993,
-      "step": 42
-    },
-    {
-      "completion_length": 394.4375,
-      "epoch": 5.516129032258064,
-      "grad_norm": 20.784292221069336,
-      "kl": 0.0022700289155181963,
-      "learning_rate": 1.4502772460993384e-07,
-      "loss": 0.0,
-      "reward": 0.11197976651601493,
-      "reward_std": 0.5696409860393032,
-      "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.0,
-      "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.13532351609319448,
-      "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": -0.02334374561905861,
-      "step": 44
-    },
-    {
-      "completion_length": 360.03125,
-      "epoch": 5.774193548387097,
-      "grad_norm": 4.866372585296631,
-      "kl": 0.000709792191400993,
-      "learning_rate": 1.2111153570574453e-07,
-      "loss": 0.0,
-      "reward": 0.1314556710422039,
-      "reward_std": 0.27511434676125646,
-      "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.0,
-      "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.1430806666612625,
-      "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": -0.011624999344348907,
-      "step": 46
-    },
-    {
-      "completion_length": 201.39285714285714,
-      "epoch": 6.0,
-      "grad_norm": 2.460200786590576,
-      "kl": 0.0008755299386393744,
-      "learning_rate": 9.870644620155877e-08,
-      "loss": 0.0,
-      "reward": 0.26238812506198883,
-      "reward_std": 0.20772267784923315,
-      "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.0,
-      "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.23092382933412278,
-      "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.031464285616363795,
-      "step": 48
-    },
-    {
-      "completion_length": 346.40625,
-      "epoch": 6.258064516129032,
-      "grad_norm": 21.511545181274414,
-      "kl": 0.0017042956070554283,
-      "learning_rate": 7.807513528664414e-08,
-      "loss": 0.0,
-      "reward": 0.14076933730393648,
-      "reward_std": 0.16657349606975913,
-      "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.0,
-      "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.07820683810859919,
-      "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.06256250012665987,
-      "step": 50
-    },
-    {
-      "completion_length": 325.4375,
-      "epoch": 6.516129032258064,
-      "grad_norm": 1.8207321166992188,
-      "kl": 0.0018291326173311973,
-      "learning_rate": 5.945948621809091e-08,
-      "loss": 0.0,
-      "reward": 0.19795077852904797,
-      "reward_std": 0.22306197369471192,
-      "rewards/concensus_correctness_reward_func": 0.013875000178813934,
-      "rewards/consensus_reward_func": 0.0,
-      "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.16435701958835125,
-      "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.01971874851733446,
-      "step": 52
-    },
-    {
-      "completion_length": 414.5625,
-      "epoch": 6.774193548387097,
-      "grad_norm": 3.8619086742401123,
-      "kl": 0.0004780482821615806,
-      "learning_rate": 4.3077750460777384e-08,
-      "loss": 0.0,
-      "reward": 0.09389630286023021,
-      "reward_std": 0.19037447159644216,
-      "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.0,
-      "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.11933380062691867,
-      "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": -0.02543749986216426,
-      "step": 54
-    },
-    {
-      "completion_length": 342.5,
-      "epoch": 7.0,
-      "grad_norm": 1.9887802600860596,
-      "kl": 0.0015315435006674047,
-      "learning_rate": 2.912198888849429e-08,
-      "loss": 0.0,
-      "reward": 0.1833051379237856,
-      "reward_std": 0.16237848784242356,
-      "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.0,
-      "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.11926942531551633,
-      "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.06403571314045362,
-      "step": 56
-    },
-    {
-      "completion_length": 335.1875,
-      "epoch": 7.258064516129032,
-      "grad_norm": 2.110353469848633,
-      "kl": 0.00010102857913807384,
-      "learning_rate": 1.7755820045802144e-08,
-      "loss": 0.0,
-      "reward": 0.14824581565335393,
-      "reward_std": 0.22165359160862863,
-      "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.0,
-      "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.16133957030251622,
-      "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": -0.013093750923871994,
-      "step": 58
-    },
-    {
-      "completion_length": 431.1875,
-      "epoch": 7.516129032258064,
-      "grad_norm": 2.8052024841308594,
-      "kl": 0.001484481835632323,
-      "learning_rate": 9.112501870194272e-09,
-      "loss": 0.0,
-      "reward": 0.3731462989235297,
-      "reward_std": 0.42140341037884355,
-      "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.0,
-      "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.20230254053603858,
-      "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.17084375163540244,
-      "step": 60
-    },
-    {
-      "epoch": 7.516129032258064,
-      "step": 60,
+      "epoch": 6.7272727272727275,
+      "step": 20,
       "total_flos": 0.0,
-      "train_loss": 1.1757511778398565e-06,
-      "train_runtime": 2092.798,
-      "train_samples_per_second": 0.459,
-      "train_steps_per_second": 0.029
+      "train_loss": 7.3265395705846e-08,
+      "train_runtime": 1657.2389,
+      "train_samples_per_second": 0.193,
+      "train_steps_per_second": 0.012
     }
   ],
   "logging_steps": 2,
-  "max_steps": 60,
+  "max_steps": 20,
   "num_input_tokens_seen": 0,
-  "num_train_epochs": 9,
+  "num_train_epochs": 10,
   "save_steps": 25,
   "stateful_callbacks": {
     "TrainerControl": {

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ba492b83878803b060fdb46b451bc0e8ef320a4e25a944fde55bafb7a4852a22
-size 6008
+oid sha256:240b86098d0c2bf357499ba2bc09c0c1d8d44ef54776e40f3b67b44ac8bdd067
+size 5944