End of training

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3281181189a85a5834810266a8cf511d3e5135526961c2c694565613980f82b1
+oid sha256:73ce040e47bc2b66aed9044bd62426c52ee86454b3332da4702bbcc78d486395
 size 35237104

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 5.784175674724186e-08,
-    "train_runtime": 2119.7857,
+    "train_loss": 2.2126714176451402e-07,
+    "train_runtime": 2214.7036,
     "train_samples": 31,
-    "train_samples_per_second": 0.453,
-    "train_steps_per_second": 0.028
+    "train_samples_per_second": 0.433,
+    "train_steps_per_second": 0.027
 }

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 5.784175674724186e-08,
-    "train_runtime": 2119.7857,
+    "train_loss": 2.2126714176451402e-07,
+    "train_runtime": 2214.7036,
     "train_samples": 31,
-    "train_samples_per_second": 0.453,
-    "train_steps_per_second": 0.028
+    "train_samples_per_second": 0.433,
+    "train_steps_per_second": 0.027
 }

trainer_state.json

@@ -10,583 +10,583 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 432.875,
+      "completion_length": 468.09375,
       "epoch": 0.25806451612903225,
       "grad_norm": NaN,
-      "kl": 3.750716791728337e-05,
+      "kl": 0.00012564308224227716,
       "learning_rate": 2.5e-07,
       "loss": 0.0,
-      "reward": 0.22504595573991537,
-      "reward_std": 0.19483413733541965,
+      "reward": 0.1961599849164486,
+      "reward_std": 0.37325981876347214,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.13373345276340842,
+      "rewards/question_recreation_reward_func": 0.135066231014207,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.09131249971687794,
+      "rewards/xmlcount_reward_func": 0.06109375273808837,
       "step": 2
     },
     {
-      "completion_length": 344.125,
+      "completion_length": 394.59375,
       "epoch": 0.5161290322580645,
-      "grad_norm": 2.933063507080078,
-      "kl": 1.8082299504840194e-05,
+      "grad_norm": 2.8862016201019287,
+      "kl": 3.452185706009914e-05,
       "learning_rate": 5e-07,
       "loss": 0.0,
-      "reward": 0.10348843690007925,
-      "reward_std": 0.1289034126093611,
+      "reward": 0.11793552408926189,
+      "reward_std": 0.16145133040845394,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.0874571877066046,
+      "rewards/question_recreation_reward_func": 0.08818553062155843,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.016031250823289156,
+      "rewards/xmlcount_reward_func": 0.029749999288469553,
       "step": 4
     },
     {
-      "completion_length": 283.96875,
+      "completion_length": 363.40625,
       "epoch": 0.7741935483870968,
-      "grad_norm": NaN,
-      "kl": 3.2842143809830304e-05,
+      "grad_norm": 2.1103923320770264,
+      "kl": 2.3821680770197418e-05,
       "learning_rate": 4.996333534627809e-07,
       "loss": 0.0,
-      "reward": 0.09516873746179044,
-      "reward_std": 0.22872156300581992,
+      "reward": 0.09433670341968536,
+      "reward_std": 0.26609469903633,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.13560624094679952,
+      "rewards/question_recreation_reward_func": 0.1436179461888969,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": -0.04043749743141234,
+      "rewards/xmlcount_reward_func": -0.049281246960163116,
       "step": 6
     },
     {
-      "completion_length": 457.14285714285717,
+      "completion_length": 374.57142857142856,
       "epoch": 1.0,
-      "grad_norm": 2.904953956604004,
-      "kl": 1.8724533090010353e-05,
-      "learning_rate": 4.996333534627809e-07,
+      "grad_norm": 1.734000325202942,
+      "kl": 1.5630724841945005e-05,
+      "learning_rate": 4.967066306353815e-07,
       "loss": 0.0,
-      "reward": 0.12228438950010709,
-      "reward_std": 0.06985677113490445,
+      "reward": 0.2236826547554561,
+      "reward_std": 0.17228200073753083,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.09549867734313011,
+      "rewards/question_recreation_reward_func": 0.13896837245140756,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.026785714285714284,
+      "rewards/xmlcount_reward_func": 0.08471428602933884,
       "step": 8
     },
     {
-      "completion_length": 355.53125,
+      "completion_length": 418.46875,
       "epoch": 1.2580645161290323,
-      "grad_norm": 23.755529403686523,
-      "kl": 4.272310263786494e-05,
-      "learning_rate": 4.967066306353815e-07,
+      "grad_norm": 1.0722944736480713,
+      "kl": 3.292944302302203e-05,
+      "learning_rate": 4.908874981298057e-07,
       "loss": 0.0,
-      "reward": 0.18326038727536798,
-      "reward_std": 0.1927749067544937,
+      "reward": 0.1162781547755003,
+      "reward_std": 0.6191137628629804,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.10126038501039147,
+      "rewards/question_recreation_reward_func": 0.2167781526222825,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.08200000133365393,
+      "rewards/xmlcount_reward_func": -0.1004999983124435,
       "step": 10
     },
     {
-      "completion_length": 320.4375,
+      "completion_length": 442.96875,
       "epoch": 1.5161290322580645,
-      "grad_norm": 31.614431381225586,
-      "kl": 2.9514652624129667e-05,
-      "learning_rate": 4.908874981298057e-07,
+      "grad_norm": 16.9965763092041,
+      "kl": 7.811630212017917e-05,
+      "learning_rate": 4.869132927957006e-07,
       "loss": 0.0,
-      "reward": 0.1640878210309893,
-      "reward_std": 0.14025292510632426,
+      "reward": 0.1594895573798567,
+      "reward_std": 0.14911856758408248,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.1364940747153014,
+      "rewards/question_recreation_reward_func": 0.10480205074418336,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.027593749575316906,
+      "rewards/xmlcount_reward_func": 0.0546875,
       "step": 12
     },
     {
-      "completion_length": 383.25,
+      "completion_length": 383.15625,
       "epoch": 1.7741935483870968,
-      "grad_norm": 2.241777181625366,
-      "kl": 2.29040282420101e-05,
-      "learning_rate": 4.822441799541978e-07,
+      "grad_norm": 3.1423959732055664,
+      "kl": 7.733380130048317e-05,
+      "learning_rate": 4.768938549177392e-07,
       "loss": 0.0,
-      "reward": 0.1947793704457581,
-      "reward_std": 0.17583196458872408,
+      "reward": 0.17886506160721183,
+      "reward_std": 0.34254320128820837,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.1243106194306165,
+      "rewards/question_recreation_reward_func": 0.1495525580830872,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.07046875008381903,
+      "rewards/xmlcount_reward_func": 0.02931250142864883,
       "step": 14
     },
     {
-      "completion_length": 477.32142857142856,
+      "completion_length": 468.2857142857143,
       "epoch": 2.0,
-      "grad_norm": 2.594723701477051,
-      "kl": 2.8977556732050808e-05,
-      "learning_rate": 4.708780111115057e-07,
-      "loss": 0.0,
-      "reward": 0.08107536818299975,
-      "reward_std": 0.5913927901004042,
+      "grad_norm": 1.208578109741211,
+      "kl": 1.6717019986702197e-05,
+      "learning_rate": 4.642142940418973e-07,
+      "loss": -0.0,
+      "reward": 0.30199881323746275,
+      "reward_std": 0.3142883365175554,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.16007535771599837,
+      "rewards/question_recreation_reward_func": 0.16321310134870665,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": -0.0789999964513949,
+      "rewards/xmlcount_reward_func": 0.13878571508186205,
       "step": 16
     },
     {
-      "completion_length": 380.96875,
+      "completion_length": 338.84375,
       "epoch": 2.258064516129032,
-      "grad_norm": 1.3737965822219849,
-      "kl": 1.7574639173290052e-05,
+      "grad_norm": 1.581721544265747,
+      "kl": 8.628947284705646e-05,
       "learning_rate": 4.5692224953922265e-07,
       "loss": 0.0,
-      "reward": 0.1724318852648139,
-      "reward_std": 0.14052149164490402,
+      "reward": 0.2258664125110954,
+      "reward_std": 0.18049531092401594,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.13252563844434917,
+      "rewards/question_recreation_reward_func": 0.08808515965938568,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.03990624938160181,
+      "rewards/xmlcount_reward_func": 0.13778125029057264,
       "step": 18
     },
     {
-      "completion_length": 405.25,
+      "completion_length": 454.125,
       "epoch": 2.5161290322580645,
-      "grad_norm": 3.6580724716186523,
-      "kl": 1.5755969684505544e-05,
+      "grad_norm": 2.681896209716797,
+      "kl": 3.0871672720422794e-05,
       "learning_rate": 4.405405137819091e-07,
       "loss": 0.0,
-      "reward": 0.10335489339195192,
-      "reward_std": 0.30938106472603977,
+      "reward": -0.02695425506681204,
+      "reward_std": 0.7458917489275336,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.11972989398054779,
+      "rewards/question_recreation_reward_func": 0.1082957349717617,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": -0.01637500338256359,
+      "rewards/xmlcount_reward_func": -0.1352499956265092,
       "step": 20
     },
     {
-      "completion_length": 299.6875,
+      "completion_length": 363.40625,
       "epoch": 2.774193548387097,
-      "grad_norm": 13.701998710632324,
-      "kl": 3.887651394052227e-05,
+      "grad_norm": 3.3509066104888916,
+      "kl": 0.00016107357282635348,
       "learning_rate": 4.219248647133558e-07,
       "loss": 0.0,
-      "reward": 0.1660241886274889,
-      "reward_std": 0.19414572673849761,
+      "reward": 0.29551874240860343,
+      "reward_std": 0.26216102205216885,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.12224293930921704,
+      "rewards/question_recreation_reward_func": 0.16439374536275864,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.043781247455626726,
+      "rewards/xmlcount_reward_func": 0.13112499960698187,
       "step": 22
     },
     {
-      "completion_length": 444.42857142857144,
+      "completion_length": 440.82142857142856,
       "epoch": 3.0,
-      "grad_norm": 15.90574836730957,
-      "kl": 3.00039647689638e-05,
+      "grad_norm": 6.132716178894043,
+      "kl": 2.4475816254770117e-05,
       "learning_rate": 4.012935537984413e-07,
       "loss": 0.0,
-      "reward": 0.18139142862388066,
-      "reward_std": 0.51617239096335,
+      "reward": 0.17054232954978943,
+      "reward_std": 0.742070411997182,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.15978428348898888,
+      "rewards/question_recreation_reward_func": 0.2551851751548903,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.021607143538338796,
+      "rewards/xmlcount_reward_func": -0.08464285651487964,
       "step": 24
     },
     {
-      "completion_length": 372.5,
+      "completion_length": 376.3125,
       "epoch": 3.258064516129032,
-      "grad_norm": 12.167678833007812,
-      "kl": 4.000507453838509e-05,
+      "grad_norm": 18.50641632080078,
+      "kl": 0.0002098049449159589,
       "learning_rate": 3.788884642942554e-07,
       "loss": 0.0,
-      "reward": 0.12209517322480679,
-      "reward_std": 0.15160708187613636,
+      "reward": 0.36438849940896034,
+      "reward_std": 0.43193568650167435,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.07968892669305205,
+      "rewards/question_recreation_reward_func": 0.2423259963397868,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.04240625002421439,
+      "rewards/xmlcount_reward_func": 0.12206250196322799,
       "step": 26
     },
     {
-      "completion_length": 490.65625,
+      "completion_length": 470.875,
       "epoch": 3.5161290322580645,
-      "grad_norm": 0.8940890431404114,
-      "kl": 4.283679243144434e-05,
+      "grad_norm": 30.481353759765625,
+      "kl": 0.00012239812804182293,
       "learning_rate": 3.5497227539006614e-07,
       "loss": 0.0,
-      "reward": 0.2202528682537377,
-      "reward_std": 0.3431535854469985,
+      "reward": 0.2246184330433607,
+      "reward_std": 0.25769937818404287,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.19440911011770368,
+      "rewards/question_recreation_reward_func": 0.09374343021772802,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.025843751151114702,
+      "rewards/xmlcount_reward_func": 0.1308750007301569,
       "step": 28
     },
     {
-      "completion_length": 482.84375,
+      "completion_length": 489.3125,
       "epoch": 3.774193548387097,
-      "grad_norm": 1.2267228364944458,
-      "kl": 2.3803617409612343e-05,
+      "grad_norm": 1.8558696508407593,
+      "kl": 7.136619524317211e-05,
       "learning_rate": 3.2982538253399494e-07,
       "loss": 0.0,
-      "reward": 0.05619814898818731,
-      "reward_std": 0.34625155106186867,
+      "reward": 0.3110757367685437,
+      "reward_std": 0.3201977536082268,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.08547939965501428,
+      "rewards/question_recreation_reward_func": 0.21076323743909597,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": -0.029281248804181814,
+      "rewards/xmlcount_reward_func": 0.10031249932944775,
       "step": 30
     },
     {
-      "completion_length": 265.17857142857144,
+      "completion_length": 375.7857142857143,
       "epoch": 4.0,
-      "grad_norm": 1.5219106674194336,
-      "kl": 1.8693077631074368e-05,
+      "grad_norm": 6.766973495483398,
+      "kl": 6.908967420063814e-05,
       "learning_rate": 3.0374261005275605e-07,
       "loss": 0.0,
-      "reward": 0.23597935427512443,
-      "reward_std": 0.2642956982765879,
+      "reward": 0.006137807454381671,
+      "reward_std": 0.27292134180398925,
       "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.07142857142857142,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.13537221455148288,
+      "rewards/question_recreation_reward_func": 0.08913780535970416,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.029178571488176073,
+      "rewards/xmlcount_reward_func": -0.08299999790532249,
       "step": 32
     },
     {
-      "completion_length": 321.25,
+      "completion_length": 415.15625,
       "epoch": 4.258064516129032,
-      "grad_norm": 2.6403026580810547,
-      "kl": 0.00013892813103666413,
+      "grad_norm": 1.7305625677108765,
+      "kl": 2.464489864451025e-05,
       "learning_rate": 2.770297546059854e-07,
       "loss": 0.0,
-      "reward": 0.07006022986024618,
-      "reward_std": 0.1371611359063536,
+      "reward": 0.3909291820600629,
+      "reward_std": 0.48921840731054544,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.11359148006886244,
+      "rewards/question_recreation_reward_func": 0.22102294443175197,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": -0.04353124741464853,
+      "rewards/xmlcount_reward_func": 0.16990624740719795,
       "step": 34
     },
     {
-      "completion_length": 425.0,
+      "completion_length": 403.96875,
       "epoch": 4.516129032258064,
-      "grad_norm": 10.667489051818848,
-      "kl": 0.0001304006651707823,
+      "grad_norm": 2.19710636138916,
+      "kl": 4.924387462779123e-05,
       "learning_rate": 2.5e-07,
       "loss": 0.0,
-      "reward": 0.12346046464517713,
-      "reward_std": 0.38260534568689764,
+      "reward": 0.13032389804720879,
+      "reward_std": 0.2979316939599812,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.21452296525239944,
+      "rewards/question_recreation_reward_func": 0.11326138750882819,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": -0.09106250060722232,
+      "rewards/xmlcount_reward_func": 0.01706250198185444,
       "step": 36
     },
     {
-      "completion_length": 522.5625,
+      "completion_length": 518.46875,
       "epoch": 4.774193548387097,
-      "grad_norm": 2.4991886615753174,
-      "kl": 2.2307130848275847e-05,
+      "grad_norm": 22.284698486328125,
+      "kl": 0.000312702951873689,
       "learning_rate": 2.229702453940146e-07,
       "loss": 0.0,
-      "reward": 0.09028978087008,
-      "reward_std": 0.3559774635359645,
+      "reward": 0.16360057145357132,
+      "reward_std": 0.40589657705277205,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.1463522738777101,
+      "rewards/question_recreation_reward_func": 0.11656931973993778,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": -0.056062495801597834,
+      "rewards/xmlcount_reward_func": 0.04703124985098839,
       "step": 38
     },
     {
-      "completion_length": 348.07142857142856,
+      "completion_length": 217.14285714285714,
       "epoch": 5.0,
-      "grad_norm": 21.351070404052734,
-      "kl": 0.00017456527660085288,
+      "grad_norm": 15.186713218688965,
+      "kl": 0.0013402944018058147,
       "learning_rate": 1.9625738994724398e-07,
       "loss": 0.0,
-      "reward": 0.11296253544943673,
-      "reward_std": 0.29939703097833054,
+      "reward": 0.14894756089363778,
+      "reward_std": 0.21149546493377006,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.14192682451435498,
+      "rewards/question_recreation_reward_func": 0.15655470479811942,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": -0.028964285339627947,
+      "rewards/xmlcount_reward_func": -0.007607143372297287,
       "step": 40
     },
     {
-      "completion_length": 307.375,
+      "completion_length": 370.59375,
       "epoch": 5.258064516129032,
-      "grad_norm": 2.1031601428985596,
-      "kl": 9.678845617600018e-05,
+      "grad_norm": 1.900159478187561,
+      "kl": 7.740367414044158e-05,
       "learning_rate": 1.7017461746600504e-07,
       "loss": 0.0,
-      "reward": 0.19189717480912805,
-      "reward_std": 0.30354586499743164,
-      "rewards/concensus_correctness_reward_func": 0.0078125,
+      "reward": 0.4386673276312649,
+      "reward_std": 0.3562059886753559,
+      "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.15374093130230904,
+      "rewards/question_recreation_reward_func": 0.2130110774305649,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.030343750957399607,
+      "rewards/xmlcount_reward_func": 0.22565624909475446,
       "step": 42
     },
     {
-      "completion_length": 481.25,
+      "completion_length": 575.25,
       "epoch": 5.516129032258064,
-      "grad_norm": 1.445552945137024,
-      "kl": 2.5597480430405994e-05,
+      "grad_norm": 1.1061934232711792,
+      "kl": 2.8744487849508005e-05,
       "learning_rate": 1.4502772460993384e-07,
       "loss": 0.0,
-      "reward": 0.09247332578524947,
-      "reward_std": 0.5333574498072267,
-      "rewards/concensus_correctness_reward_func": 0.0078125,
+      "reward": 0.04084427189081907,
+      "reward_std": 0.2996016829274595,
+      "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.1771920770406723,
+      "rewards/question_recreation_reward_func": 0.08375052409246564,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": -0.09253124939277768,
+      "rewards/xmlcount_reward_func": -0.04290625033900142,
       "step": 44
     },
     {
-      "completion_length": 410.1875,
+      "completion_length": 512.59375,
       "epoch": 5.774193548387097,
-      "grad_norm": 11.555668830871582,
-      "kl": 2.4003912358239177e-05,
+      "grad_norm": 31.07492446899414,
+      "kl": 0.0005856987711467809,
       "learning_rate": 1.2111153570574453e-07,
       "loss": 0.0,
-      "reward": 0.08301540045067668,
-      "reward_std": 0.407743769697845,
+      "reward": 0.2757360441610217,
+      "reward_std": 0.34823702834546566,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.11485915258526802,
+      "rewards/question_recreation_reward_func": 0.1400797851383686,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": -0.03184375003911555,
+      "rewards/xmlcount_reward_func": 0.13565625250339508,
       "step": 46
     },
     {
-      "completion_length": 455.92857142857144,
+      "completion_length": 368.89285714285717,
       "epoch": 6.0,
-      "grad_norm": 11.974881172180176,
-      "kl": 3.5962535028894695e-05,
+      "grad_norm": 1.8032582998275757,
+      "kl": 7.638505134569382e-05,
       "learning_rate": 9.870644620155877e-08,
       "loss": 0.0,
-      "reward": 0.31864730907338007,
-      "reward_std": 0.5955199256007161,
-      "rewards/concensus_correctness_reward_func": 0.01685714295932225,
+      "reward": 0.19913670314209803,
+      "reward_std": 0.5407656750508717,
+      "rewards/concensus_correctness_reward_func": 0.008928571428571428,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.24411158582993916,
+      "rewards/question_recreation_reward_func": 0.24160099548420735,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.05767856751169477,
+      "rewards/xmlcount_reward_func": -0.05139286071062088,
       "step": 48
     },
     {
-      "completion_length": 318.375,
+      "completion_length": 216.34375,
       "epoch": 6.258064516129032,
-      "grad_norm": 2.5164284706115723,
-      "kl": 7.310123135084723e-05,
+      "grad_norm": 3.3991875648498535,
+      "kl": 0.0012653063204197679,
       "learning_rate": 7.807513528664414e-08,
       "loss": 0.0,
-      "reward": 0.11656019790098071,
-      "reward_std": 0.10175749775953591,
+      "reward": 0.1986216944642365,
+      "reward_std": 0.18251595762558281,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.08921644371002913,
+      "rewards/question_recreation_reward_func": 0.15384044079110026,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.02734375,
+      "rewards/xmlcount_reward_func": 0.044781249947845936,
       "step": 50
     },
     {
-      "completion_length": 508.75,
+      "completion_length": 449.8125,
       "epoch": 6.516129032258064,
-      "grad_norm": 1.1965211629867554,
-      "kl": 1.5494790318371088e-05,
+      "grad_norm": 2.6445517539978027,
+      "kl": 0.00018877460297517246,
       "learning_rate": 5.945948621809091e-08,
       "loss": 0.0,
-      "reward": 0.3042877707630396,
-      "reward_std": 0.33586700027808547,
+      "reward": 0.14643485844135284,
+      "reward_std": 0.399348977021873,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.1793815130367875,
+      "rewards/question_recreation_reward_func": 0.158841117285192,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.12490625120699406,
+      "rewards/xmlcount_reward_func": -0.01240625255741179,
       "step": 52
     },
     {
-      "completion_length": 319.5,
+      "completion_length": 347.28125,
       "epoch": 6.774193548387097,
-      "grad_norm": 1.9498324394226074,
-      "kl": 2.631079428283556e-05,
+      "grad_norm": 1.4666146039962769,
+      "kl": 0.0003983834214977833,
       "learning_rate": 4.3077750460777384e-08,
       "loss": 0.0,
-      "reward": 0.22337154485285282,
-      "reward_std": 0.27424282068386674,
+      "reward": 0.19152415730059147,
+      "reward_std": 0.27376102562993765,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.15671529504470527,
+      "rewards/question_recreation_reward_func": 0.12736790860071778,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.06665624864399433,
+      "rewards/xmlcount_reward_func": 0.06415624869987369,
       "step": 54
     },
     {
-      "completion_length": 381.5357142857143,
+      "completion_length": 498.32142857142856,
       "epoch": 7.0,
-      "grad_norm": 16.75676727294922,
-      "kl": 0.00011531045960769657,
+      "grad_norm": 1.52250337600708,
+      "kl": 0.00023215323692836267,
       "learning_rate": 2.912198888849429e-08,
       "loss": 0.0,
-      "reward": 0.12595466205051967,
-      "reward_std": 0.10518132096954755,
+      "reward": 0.12091957139117378,
+      "reward_std": 0.2569341185901846,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.10281180537172727,
+      "rewards/question_recreation_reward_func": 0.1385624294302293,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.0231428574770689,
+      "rewards/xmlcount_reward_func": -0.017642855311610868,
       "step": 56
     },
     {
-      "completion_length": 259.21875,
+      "completion_length": 290.09375,
       "epoch": 7.258064516129032,
-      "grad_norm": 8.346614837646484,
-      "kl": 2.7651209791201836e-05,
+      "grad_norm": 1.6776108741760254,
+      "kl": 0.0008473875428762767,
       "learning_rate": 1.7755820045802144e-08,
-      "loss": -0.0,
-      "reward": 0.09485200280323625,
-      "reward_std": 0.1619195283856243,
+      "loss": 0.0,
+      "reward": 0.3240678207948804,
+      "reward_std": 0.37887709541246295,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.11832075333222747,
+      "rewards/question_recreation_reward_func": 0.15538030490279198,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": -0.023468749597668648,
+      "rewards/xmlcount_reward_func": 0.16868749679997563,
       "step": 58
     },
     {
-      "completion_length": 447.71875,
+      "completion_length": 422.78125,
       "epoch": 7.516129032258064,
-      "grad_norm": 3.6411349773406982,
-      "kl": 6.546302188326081e-05,
+      "grad_norm": 4.0938944816589355,
+      "kl": 7.57059717670927e-05,
       "learning_rate": 9.112501870194272e-09,
       "loss": 0.0,
-      "reward": 0.3702763707842678,
-      "reward_std": 0.3977031116373837,
+      "reward": 0.16282579209655523,
+      "reward_std": 0.3961476086406037,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.2630263774190098,
+      "rewards/question_recreation_reward_func": 0.12666953651933,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.10724999941885471,
+      "rewards/xmlcount_reward_func": 0.0361562481848523,
       "step": 60
     },
     {
       "epoch": 7.516129032258064,
       "step": 60,
       "total_flos": 0.0,
-      "train_loss": 5.784175674724186e-08,
-      "train_runtime": 2119.7857,
-      "train_samples_per_second": 0.453,
-      "train_steps_per_second": 0.028
+      "train_loss": 2.2126714176451402e-07,
+      "train_runtime": 2214.7036,
+      "train_samples_per_second": 0.433,
+      "train_steps_per_second": 0.027
     }
   ],
   "logging_steps": 2,