End of training

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:73ce040e47bc2b66aed9044bd62426c52ee86454b3332da4702bbcc78d486395
+oid sha256:ec94e591006394e5861342585e4aaf983b29a686e71b397bcc3aa5e4b210244d
 size 35237104

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 2.2126714176451402e-07,
-    "train_runtime": 2214.7036,
+    "train_loss": 1.1757511778398565e-06,
+    "train_runtime": 2092.798,
     "train_samples": 31,
-    "train_samples_per_second": 0.433,
-    "train_steps_per_second": 0.027
+    "train_samples_per_second": 0.459,
+    "train_steps_per_second": 0.029
 }

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 2.2126714176451402e-07,
-    "train_runtime": 2214.7036,
+    "train_loss": 1.1757511778398565e-06,
+    "train_runtime": 2092.798,
     "train_samples": 31,
-    "train_samples_per_second": 0.433,
-    "train_steps_per_second": 0.027
+    "train_samples_per_second": 0.459,
+    "train_steps_per_second": 0.029
 }

trainer_state.json

@@ -10,583 +10,583 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 468.09375,
+      "completion_length": 430.8125,
       "epoch": 0.25806451612903225,
       "grad_norm": NaN,
-      "kl": 0.00012564308224227716,
+      "kl": 0.00041734772025847633,
       "learning_rate": 2.5e-07,
       "loss": 0.0,
-      "reward": 0.1961599849164486,
-      "reward_std": 0.37325981876347214,
+      "reward": 0.11391110043041408,
+      "reward_std": 0.41625179536640644,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.135066231014207,
+      "rewards/question_recreation_reward_func": 0.20462985103949904,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.06109375273808837,
+      "rewards/xmlcount_reward_func": -0.09071875200606883,
       "step": 2
     },
     {
-      "completion_length": 394.59375,
+      "completion_length": 343.34375,
       "epoch": 0.5161290322580645,
-      "grad_norm": 2.8862016201019287,
-      "kl": 3.452185706009914e-05,
+      "grad_norm": 1.9656559228897095,
+      "kl": 0.00040763827485079673,
       "learning_rate": 5e-07,
       "loss": 0.0,
-      "reward": 0.11793552408926189,
-      "reward_std": 0.16145133040845394,
+      "reward": 0.08423173008486629,
+      "reward_std": 0.1950428435811773,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.08818553062155843,
+      "rewards/question_recreation_reward_func": 0.10804423363879323,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.029749999288469553,
+      "rewards/xmlcount_reward_func": -0.02381250262260437,
       "step": 4
     },
     {
-      "completion_length": 363.40625,
+      "completion_length": 426.71875,
       "epoch": 0.7741935483870968,
-      "grad_norm": 2.1103923320770264,
-      "kl": 2.3821680770197418e-05,
-      "learning_rate": 4.996333534627809e-07,
+      "grad_norm": 3.116727590560913,
+      "kl": 0.00013401095452536538,
+      "learning_rate": 4.985344892885899e-07,
       "loss": 0.0,
-      "reward": 0.09433670341968536,
-      "reward_std": 0.26609469903633,
+      "reward": 0.2961328737437725,
+      "reward_std": 0.3188573829829693,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.1436179461888969,
+      "rewards/question_recreation_reward_func": 0.20203911093994975,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": -0.049281246960163116,
+      "rewards/xmlcount_reward_func": 0.09409375116229057,
       "step": 6
     },
     {
-      "completion_length": 374.57142857142856,
+      "completion_length": 363.85714285714283,
       "epoch": 1.0,
-      "grad_norm": 1.734000325202942,
-      "kl": 1.5630724841945005e-05,
+      "grad_norm": 4.322468280792236,
+      "kl": 0.000584651980066805,
       "learning_rate": 4.967066306353815e-07,
       "loss": 0.0,
-      "reward": 0.2236826547554561,
-      "reward_std": 0.17228200073753083,
+      "reward": 0.28172825063977924,
+      "reward_std": 0.2894980385899544,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.13896837245140756,
+      "rewards/question_recreation_reward_func": 0.1669425368309021,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.08471428602933884,
+      "rewards/xmlcount_reward_func": 0.11478571407496929,
       "step": 8
     },
     {
-      "completion_length": 418.46875,
+      "completion_length": 375.15625,
       "epoch": 1.2580645161290323,
-      "grad_norm": 1.0722944736480713,
-      "kl": 3.292944302302203e-05,
-      "learning_rate": 4.908874981298057e-07,
+      "grad_norm": 4.052526473999023,
+      "kl": 0.00137491720761318,
+      "learning_rate": 4.941551389275217e-07,
       "loss": 0.0,
-      "reward": 0.1162781547755003,
-      "reward_std": 0.6191137628629804,
+      "reward": 0.1756317124236375,
+      "reward_std": 0.13710087712388486,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.2167781526222825,
+      "rewards/question_recreation_reward_func": 0.10741296270862222,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": -0.1004999983124435,
+      "rewards/xmlcount_reward_func": 0.06821874901652336,
       "step": 10
     },
     {
-      "completion_length": 442.96875,
+      "completion_length": 527.28125,
       "epoch": 1.5161290322580645,
-      "grad_norm": 16.9965763092041,
-      "kl": 7.811630212017917e-05,
+      "grad_norm": 1.0876020193099976,
+      "kl": 7.662167183752899e-05,
       "learning_rate": 4.869132927957006e-07,
       "loss": 0.0,
-      "reward": 0.1594895573798567,
-      "reward_std": 0.14911856758408248,
+      "reward": 0.05637967900838703,
+      "reward_std": 0.29673084686510265,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.10480205074418336,
+      "rewards/question_recreation_reward_func": 0.08856716414447874,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.0546875,
+      "rewards/xmlcount_reward_func": -0.03218749677762389,
       "step": 12
     },
     {
-      "completion_length": 383.15625,
+      "completion_length": 420.4375,
       "epoch": 1.7741935483870968,
-      "grad_norm": 3.1423959732055664,
-      "kl": 7.733380130048317e-05,
+      "grad_norm": 1.930647373199463,
+      "kl": 0.00019077809974987758,
       "learning_rate": 4.768938549177392e-07,
       "loss": 0.0,
-      "reward": 0.17886506160721183,
-      "reward_std": 0.34254320128820837,
+      "reward": 0.32431007781997323,
+      "reward_std": 0.3349539686460048,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.1495525580830872,
+      "rewards/question_recreation_reward_func": 0.19974758382886648,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.02931250142864883,
+      "rewards/xmlcount_reward_func": 0.12456249631941319,
       "step": 14
     },
     {
-      "completion_length": 468.2857142857143,
+      "completion_length": 235.67857142857142,
       "epoch": 2.0,
-      "grad_norm": 1.208578109741211,
-      "kl": 1.6717019986702197e-05,
-      "learning_rate": 4.642142940418973e-07,
-      "loss": -0.0,
-      "reward": 0.30199881323746275,
-      "reward_std": 0.3142883365175554,
+      "grad_norm": 25.613550186157227,
+      "kl": 0.001633655218029162,
+      "learning_rate": 4.708780111115057e-07,
+      "loss": 0.0,
+      "reward": 0.187016135347741,
+      "reward_std": 0.25221954605409075,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.16321310134870665,
+      "rewards/question_recreation_reward_func": 0.11437327787280083,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.13878571508186205,
+      "rewards/xmlcount_reward_func": 0.07264285454792636,
       "step": 16
     },
     {
-      "completion_length": 338.84375,
+      "completion_length": 318.28125,
       "epoch": 2.258064516129032,
-      "grad_norm": 1.581721544265747,
-      "kl": 8.628947284705646e-05,
+      "grad_norm": 3.3770768642425537,
+      "kl": 0.0005925071802721504,
       "learning_rate": 4.5692224953922265e-07,
       "loss": 0.0,
-      "reward": 0.2258664125110954,
-      "reward_std": 0.18049531092401594,
+      "reward": 0.2636642586439848,
+      "reward_std": 0.3803391242399812,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.08808515965938568,
+      "rewards/question_recreation_reward_func": 0.13447676633950323,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.13778125029057264,
+      "rewards/xmlcount_reward_func": 0.12918749847449362,
       "step": 18
     },
     {
-      "completion_length": 454.125,
+      "completion_length": 371.59375,
       "epoch": 2.5161290322580645,
-      "grad_norm": 2.681896209716797,
-      "kl": 3.0871672720422794e-05,
+      "grad_norm": 1.974790096282959,
+      "kl": 0.0002036209966718161,
       "learning_rate": 4.405405137819091e-07,
       "loss": 0.0,
-      "reward": -0.02695425506681204,
-      "reward_std": 0.7458917489275336,
+      "reward": 0.10157998092472553,
+      "reward_std": 0.10903631476685405,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.1082957349717617,
+      "rewards/question_recreation_reward_func": 0.06636122916825116,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": -0.1352499956265092,
+      "rewards/xmlcount_reward_func": 0.035218749195337296,
       "step": 20
     },
     {
-      "completion_length": 363.40625,
+      "completion_length": 279.71875,
       "epoch": 2.774193548387097,
-      "grad_norm": 3.3509066104888916,
-      "kl": 0.00016107357282635348,
+      "grad_norm": 26.12720489501953,
+      "kl": 0.0019963133127021138,
       "learning_rate": 4.219248647133558e-07,
       "loss": 0.0,
-      "reward": 0.29551874240860343,
-      "reward_std": 0.26216102205216885,
+      "reward": 0.35939981369301677,
+      "reward_std": 0.5500808532815427,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.16439374536275864,
+      "rewards/question_recreation_reward_func": 0.25952482083812356,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.13112499960698187,
+      "rewards/xmlcount_reward_func": 0.09987499937415123,
       "step": 22
     },
     {
-      "completion_length": 440.82142857142856,
+      "completion_length": 383.64285714285717,
       "epoch": 3.0,
-      "grad_norm": 6.132716178894043,
-      "kl": 2.4475816254770117e-05,
+      "grad_norm": 0.9543819427490234,
+      "kl": 0.0006048666277170664,
       "learning_rate": 4.012935537984413e-07,
       "loss": 0.0,
-      "reward": 0.17054232954978943,
-      "reward_std": 0.742070411997182,
+      "reward": 0.22724560940904276,
+      "reward_std": 0.20333547570875712,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.2551851751548903,
+      "rewards/question_recreation_reward_func": 0.18638846703938075,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": -0.08464285651487964,
+      "rewards/xmlcount_reward_func": 0.04085714370012283,
       "step": 24
     },
     {
-      "completion_length": 376.3125,
+      "completion_length": 381.96875,
       "epoch": 3.258064516129032,
-      "grad_norm": 18.50641632080078,
-      "kl": 0.0002098049449159589,
+      "grad_norm": 3.211329698562622,
+      "kl": 0.00012533261292446696,
       "learning_rate": 3.788884642942554e-07,
       "loss": 0.0,
-      "reward": 0.36438849940896034,
-      "reward_std": 0.43193568650167435,
+      "reward": 0.19361284282058477,
+      "reward_std": 0.24643944506533444,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.2423259963397868,
+      "rewards/question_recreation_reward_func": 0.17839408991858363,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.12206250196322799,
+      "rewards/xmlcount_reward_func": 0.015218751970678568,
       "step": 26
     },
     {
-      "completion_length": 470.875,
+      "completion_length": 412.5,
       "epoch": 3.5161290322580645,
-      "grad_norm": 30.481353759765625,
-      "kl": 0.00012239812804182293,
+      "grad_norm": 3.394382953643799,
+      "kl": 0.00010090849150401482,
       "learning_rate": 3.5497227539006614e-07,
       "loss": 0.0,
-      "reward": 0.2246184330433607,
-      "reward_std": 0.25769937818404287,
-      "rewards/concensus_correctness_reward_func": 0.0,
+      "reward": 0.2241632086224854,
+      "reward_std": 0.2526557689998299,
+      "rewards/concensus_correctness_reward_func": 0.013875000178813934,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.09374343021772802,
+      "rewards/question_recreation_reward_func": 0.17597571481019258,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.1308750007301569,
+      "rewards/xmlcount_reward_func": 0.03431250061839819,
       "step": 28
     },
     {
-      "completion_length": 489.3125,
+      "completion_length": 284.59375,
       "epoch": 3.774193548387097,
-      "grad_norm": 1.8558696508407593,
-      "kl": 7.136619524317211e-05,
+      "grad_norm": 2.9376416206359863,
+      "kl": 0.0008606196147411538,
       "learning_rate": 3.2982538253399494e-07,
       "loss": 0.0,
-      "reward": 0.3110757367685437,
-      "reward_std": 0.3201977536082268,
+      "reward": 0.0774760153144598,
+      "reward_std": 0.1906891551334411,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.21076323743909597,
+      "rewards/question_recreation_reward_func": 0.09122601104900241,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.10031249932944775,
+      "rewards/xmlcount_reward_func": -0.01374999899417162,
       "step": 30
     },
     {
-      "completion_length": 375.7857142857143,
+      "completion_length": 549.1428571428571,
       "epoch": 4.0,
-      "grad_norm": 6.766973495483398,
-      "kl": 6.908967420063814e-05,
+      "grad_norm": 0.9819178581237793,
+      "kl": 3.5185036722396035e-05,
       "learning_rate": 3.0374261005275605e-07,
       "loss": 0.0,
-      "reward": 0.006137807454381671,
-      "reward_std": 0.27292134180398925,
+      "reward": 0.25383392149316414,
+      "reward_std": 0.30834708123334814,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.08913780535970416,
+      "rewards/question_recreation_reward_func": 0.11165534212653126,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": -0.08299999790532249,
+      "rewards/xmlcount_reward_func": 0.14217857164996012,
       "step": 32
     },
     {
-      "completion_length": 415.15625,
+      "completion_length": 366.59375,
       "epoch": 4.258064516129032,
-      "grad_norm": 1.7305625677108765,
-      "kl": 2.464489864451025e-05,
+      "grad_norm": 2.5270180702209473,
+      "kl": 0.004282280417555739,
       "learning_rate": 2.770297546059854e-07,
       "loss": 0.0,
-      "reward": 0.3909291820600629,
-      "reward_std": 0.48921840731054544,
+      "reward": 0.06346084550023079,
+      "reward_std": 0.28176579938735813,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.22102294443175197,
+      "rewards/question_recreation_reward_func": 0.12233584048226476,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.16990624740719795,
+      "rewards/xmlcount_reward_func": -0.0588750010356307,
       "step": 34
     },
     {
-      "completion_length": 403.96875,
+      "completion_length": 411.375,
       "epoch": 4.516129032258064,
-      "grad_norm": 2.19710636138916,
-      "kl": 4.924387462779123e-05,
+      "grad_norm": 4.21773099899292,
+      "kl": 8.090390974757611e-05,
       "learning_rate": 2.5e-07,
       "loss": 0.0,
-      "reward": 0.13032389804720879,
-      "reward_std": 0.2979316939599812,
+      "reward": 0.11983968829736114,
+      "reward_std": 0.17383667826652527,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.11326138750882819,
+      "rewards/question_recreation_reward_func": 0.11633968772366643,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.01706250198185444,
+      "rewards/xmlcount_reward_func": 0.0035000010393559933,
       "step": 36
     },
     {
-      "completion_length": 518.46875,
+      "completion_length": 329.28125,
       "epoch": 4.774193548387097,
-      "grad_norm": 22.284698486328125,
-      "kl": 0.000312702951873689,
+      "grad_norm": 18.660564422607422,
+      "kl": 0.0010729862383414002,
       "learning_rate": 2.229702453940146e-07,
       "loss": 0.0,
-      "reward": 0.16360057145357132,
-      "reward_std": 0.40589657705277205,
+      "reward": 0.22536780731752515,
+      "reward_std": 0.22597291774582118,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.11656931973993778,
+      "rewards/question_recreation_reward_func": 0.13943030545488,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.04703124985098839,
+      "rewards/xmlcount_reward_func": 0.0859375,
       "step": 38
     },
     {
-      "completion_length": 217.14285714285714,
+      "completion_length": 466.82142857142856,
       "epoch": 5.0,
-      "grad_norm": 15.186713218688965,
-      "kl": 0.0013402944018058147,
+      "grad_norm": 30.90364646911621,
+      "kl": 0.0050324718344459795,
       "learning_rate": 1.9625738994724398e-07,
       "loss": 0.0,
-      "reward": 0.14894756089363778,
-      "reward_std": 0.21149546493377006,
+      "reward": -0.09624448684709412,
+      "reward_std": 0.43322019491876873,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.15655470479811942,
+      "rewards/question_recreation_reward_func": 0.08754122070968151,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": -0.007607143372297287,
+      "rewards/xmlcount_reward_func": -0.18378571208034242,
       "step": 40
     },
     {
-      "completion_length": 370.59375,
+      "completion_length": 434.0625,
       "epoch": 5.258064516129032,
-      "grad_norm": 1.900159478187561,
-      "kl": 7.740367414044158e-05,
+      "grad_norm": 39.12836837768555,
+      "kl": 0.005596646230287661,
       "learning_rate": 1.7017461746600504e-07,
       "loss": 0.0,
-      "reward": 0.4386673276312649,
-      "reward_std": 0.3562059886753559,
+      "reward": 0.13575120829045773,
+      "reward_std": 0.13530697469832376,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.2130110774305649,
+      "rewards/question_recreation_reward_func": 0.1172824571840465,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.22565624909475446,
+      "rewards/xmlcount_reward_func": 0.01846875064074993,
       "step": 42
     },
     {
-      "completion_length": 575.25,
+      "completion_length": 394.4375,
       "epoch": 5.516129032258064,
-      "grad_norm": 1.1061934232711792,
-      "kl": 2.8744487849508005e-05,
+      "grad_norm": 20.784292221069336,
+      "kl": 0.0022700289155181963,
       "learning_rate": 1.4502772460993384e-07,
       "loss": 0.0,
-      "reward": 0.04084427189081907,
-      "reward_std": 0.2996016829274595,
+      "reward": 0.11197976651601493,
+      "reward_std": 0.5696409860393032,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.08375052409246564,
+      "rewards/question_recreation_reward_func": 0.13532351609319448,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": -0.04290625033900142,
+      "rewards/xmlcount_reward_func": -0.02334374561905861,
       "step": 44
     },
     {
-      "completion_length": 512.59375,
+      "completion_length": 360.03125,
       "epoch": 5.774193548387097,
-      "grad_norm": 31.07492446899414,
-      "kl": 0.0005856987711467809,
+      "grad_norm": 4.866372585296631,
+      "kl": 0.000709792191400993,
       "learning_rate": 1.2111153570574453e-07,
       "loss": 0.0,
-      "reward": 0.2757360441610217,
-      "reward_std": 0.34823702834546566,
+      "reward": 0.1314556710422039,
+      "reward_std": 0.27511434676125646,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.1400797851383686,
+      "rewards/question_recreation_reward_func": 0.1430806666612625,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.13565625250339508,
+      "rewards/xmlcount_reward_func": -0.011624999344348907,
       "step": 46
     },
     {
-      "completion_length": 368.89285714285717,
+      "completion_length": 201.39285714285714,
       "epoch": 6.0,
-      "grad_norm": 1.8032582998275757,
-      "kl": 7.638505134569382e-05,
+      "grad_norm": 2.460200786590576,
+      "kl": 0.0008755299386393744,
       "learning_rate": 9.870644620155877e-08,
       "loss": 0.0,
-      "reward": 0.19913670314209803,
-      "reward_std": 0.5407656750508717,
-      "rewards/concensus_correctness_reward_func": 0.008928571428571428,
+      "reward": 0.26238812506198883,
+      "reward_std": 0.20772267784923315,
+      "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.24160099548420735,
+      "rewards/question_recreation_reward_func": 0.23092382933412278,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": -0.05139286071062088,
+      "rewards/xmlcount_reward_func": 0.031464285616363795,
       "step": 48
     },
     {
-      "completion_length": 216.34375,
+      "completion_length": 346.40625,
       "epoch": 6.258064516129032,
-      "grad_norm": 3.3991875648498535,
-      "kl": 0.0012653063204197679,
+      "grad_norm": 21.511545181274414,
+      "kl": 0.0017042956070554283,
       "learning_rate": 7.807513528664414e-08,
       "loss": 0.0,
-      "reward": 0.1986216944642365,
-      "reward_std": 0.18251595762558281,
+      "reward": 0.14076933730393648,
+      "reward_std": 0.16657349606975913,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.15384044079110026,
+      "rewards/question_recreation_reward_func": 0.07820683810859919,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.044781249947845936,
+      "rewards/xmlcount_reward_func": 0.06256250012665987,
       "step": 50
     },
     {
-      "completion_length": 449.8125,
+      "completion_length": 325.4375,
       "epoch": 6.516129032258064,
-      "grad_norm": 2.6445517539978027,
-      "kl": 0.00018877460297517246,
+      "grad_norm": 1.8207321166992188,
+      "kl": 0.0018291326173311973,
       "learning_rate": 5.945948621809091e-08,
       "loss": 0.0,
-      "reward": 0.14643485844135284,
-      "reward_std": 0.399348977021873,
-      "rewards/concensus_correctness_reward_func": 0.0,
+      "reward": 0.19795077852904797,
+      "reward_std": 0.22306197369471192,
+      "rewards/concensus_correctness_reward_func": 0.013875000178813934,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.158841117285192,
+      "rewards/question_recreation_reward_func": 0.16435701958835125,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": -0.01240625255741179,
+      "rewards/xmlcount_reward_func": 0.01971874851733446,
       "step": 52
     },
     {
-      "completion_length": 347.28125,
+      "completion_length": 414.5625,
       "epoch": 6.774193548387097,
-      "grad_norm": 1.4666146039962769,
-      "kl": 0.0003983834214977833,
+      "grad_norm": 3.8619086742401123,
+      "kl": 0.0004780482821615806,
       "learning_rate": 4.3077750460777384e-08,
       "loss": 0.0,
-      "reward": 0.19152415730059147,
-      "reward_std": 0.27376102562993765,
+      "reward": 0.09389630286023021,
+      "reward_std": 0.19037447159644216,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.12736790860071778,
+      "rewards/question_recreation_reward_func": 0.11933380062691867,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.06415624869987369,
+      "rewards/xmlcount_reward_func": -0.02543749986216426,
       "step": 54
     },
     {
-      "completion_length": 498.32142857142856,
+      "completion_length": 342.5,
       "epoch": 7.0,
-      "grad_norm": 1.52250337600708,
-      "kl": 0.00023215323692836267,
+      "grad_norm": 1.9887802600860596,
+      "kl": 0.0015315435006674047,
       "learning_rate": 2.912198888849429e-08,
       "loss": 0.0,
-      "reward": 0.12091957139117378,
-      "reward_std": 0.2569341185901846,
+      "reward": 0.1833051379237856,
+      "reward_std": 0.16237848784242356,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.1385624294302293,
+      "rewards/question_recreation_reward_func": 0.11926942531551633,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": -0.017642855311610868,
+      "rewards/xmlcount_reward_func": 0.06403571314045362,
       "step": 56
     },
     {
-      "completion_length": 290.09375,
+      "completion_length": 335.1875,
       "epoch": 7.258064516129032,
-      "grad_norm": 1.6776108741760254,
-      "kl": 0.0008473875428762767,
+      "grad_norm": 2.110353469848633,
+      "kl": 0.00010102857913807384,
       "learning_rate": 1.7755820045802144e-08,
       "loss": 0.0,
-      "reward": 0.3240678207948804,
-      "reward_std": 0.37887709541246295,
+      "reward": 0.14824581565335393,
+      "reward_std": 0.22165359160862863,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.15538030490279198,
+      "rewards/question_recreation_reward_func": 0.16133957030251622,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.16868749679997563,
+      "rewards/xmlcount_reward_func": -0.013093750923871994,
       "step": 58
     },
     {
-      "completion_length": 422.78125,
+      "completion_length": 431.1875,
       "epoch": 7.516129032258064,
-      "grad_norm": 4.0938944816589355,
-      "kl": 7.57059717670927e-05,
+      "grad_norm": 2.8052024841308594,
+      "kl": 0.001484481835632323,
       "learning_rate": 9.112501870194272e-09,
       "loss": 0.0,
-      "reward": 0.16282579209655523,
-      "reward_std": 0.3961476086406037,
+      "reward": 0.3731462989235297,
+      "reward_std": 0.42140341037884355,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.12666953651933,
+      "rewards/question_recreation_reward_func": 0.20230254053603858,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.0361562481848523,
+      "rewards/xmlcount_reward_func": 0.17084375163540244,
       "step": 60
     },
     {
       "epoch": 7.516129032258064,
       "step": 60,
       "total_flos": 0.0,
-      "train_loss": 2.2126714176451402e-07,
-      "train_runtime": 2214.7036,
-      "train_samples_per_second": 0.433,
-      "train_steps_per_second": 0.027
+      "train_loss": 1.1757511778398565e-06,
+      "train_runtime": 2092.798,
+      "train_samples_per_second": 0.459,
+      "train_steps_per_second": 0.029
     }
   ],
   "logging_steps": 2,