| { |
| "best_global_step": null, |
| "best_metric": null, |
| "best_model_checkpoint": null, |
| "epoch": 9.097560975609756, |
| "eval_steps": 500, |
| "global_step": 100, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "completion_length": 170.46875, |
| "epoch": 0.1951219512195122, |
| "grad_norm": 6.099827289581299, |
| "kl": 0.5973675912246108, |
| "learning_rate": 1.6666666666666665e-07, |
| "loss": 0.0006, |
| "reward": 2.4682722240686417, |
| "reward_std": 0.4695738647133112, |
| "rewards/concensus_correctness_reward_func": 0.0, |
| "rewards/consensus_reward_func": 0.0, |
| "rewards/cumulative_reward_2": 0.0, |
| "rewards/final_correctness_reward_func": 0.25, |
| "rewards/question_recreation_reward_func": 0.7388660460710526, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.3125, |
| "rewards/xmlcount_reward_func": 1.166906252503395, |
| "step": 2 |
| }, |
| { |
| "completion_length": 114.125, |
| "epoch": 0.3902439024390244, |
| "grad_norm": 5.554270267486572, |
| "kl": 1.5984905734658241, |
| "learning_rate": 5e-07, |
| "loss": 0.0016, |
| "reward": 2.225109323859215, |
| "reward_std": 0.6870723395841196, |
| "rewards/concensus_correctness_reward_func": 0.06012500077486038, |
| "rewards/consensus_reward_func": 0.0625, |
| "rewards/cumulative_reward_2": 0.0, |
| "rewards/final_correctness_reward_func": 0.0, |
| "rewards/question_recreation_reward_func": 0.5827342979609966, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.390625, |
| "rewards/xmlcount_reward_func": 1.1291249990463257, |
| "step": 4 |
| }, |
| { |
| "completion_length": 148.15625, |
| "epoch": 0.5853658536585366, |
| "grad_norm": 5.069469451904297, |
| "kl": 3.3319540731608868, |
| "learning_rate": 4.994757065594279e-07, |
| "loss": 0.0033, |
| "reward": 4.637000426650047, |
| "reward_std": 2.4823005218058825, |
| "rewards/concensus_correctness_reward_func": 1.4928749948740005, |
| "rewards/consensus_reward_func": 0.1875, |
| "rewards/cumulative_reward_2": 0.0, |
| "rewards/final_correctness_reward_func": 0.875, |
| "rewards/question_recreation_reward_func": 0.7499066442251205, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.328125, |
| "rewards/xmlcount_reward_func": 1.0035937614738941, |
| "step": 6 |
| }, |
| { |
| "completion_length": 159.59375, |
| "epoch": 0.7804878048780488, |
| "grad_norm": 22.011062622070312, |
| "kl": 5.900226164609194, |
| "learning_rate": 4.979050253066063e-07, |
| "loss": 0.0059, |
| "reward": 2.358607694506645, |
| "reward_std": 0.5496477753622457, |
| "rewards/concensus_correctness_reward_func": 0.06012500077486038, |
| "rewards/consensus_reward_func": 0.0625, |
| "rewards/cumulative_reward_2": 0.0, |
| "rewards/final_correctness_reward_func": 0.0, |
| "rewards/question_recreation_reward_func": 0.7734514437615871, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.296875, |
| "rewards/xmlcount_reward_func": 1.1656562685966492, |
| "step": 8 |
| }, |
| { |
| "completion_length": 175.625, |
| "epoch": 0.975609756097561, |
| "grad_norm": 4.487865447998047, |
| "kl": 0.8899677954614162, |
| "learning_rate": 4.952945442245597e-07, |
| "loss": 0.0009, |
| "reward": 2.114151880145073, |
| "reward_std": 0.5495768673717976, |
| "rewards/concensus_correctness_reward_func": 0.0598750002682209, |
| "rewards/consensus_reward_func": 0.0625, |
| "rewards/cumulative_reward_2": 0.0, |
| "rewards/final_correctness_reward_func": 0.0, |
| "rewards/question_recreation_reward_func": 0.5913393385708332, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.25, |
| "rewards/xmlcount_reward_func": 1.1504375040531158, |
| "step": 10 |
| }, |
| { |
| "completion_length": 137.7, |
| "epoch": 1.0975609756097562, |
| "grad_norm": 5.117649078369141, |
| "kl": 0.46071633845567705, |
| "learning_rate": 4.916552125781528e-07, |
| "loss": 0.0003, |
| "reward": 3.7879202365875244, |
| "reward_std": 1.7626623034477233, |
| "rewards/concensus_correctness_reward_func": 0.4884000062942505, |
| "rewards/consensus_reward_func": 0.3, |
| "rewards/cumulative_reward_2": 0.0, |
| "rewards/final_correctness_reward_func": 0.8, |
| "rewards/question_recreation_reward_func": 0.7498702496290207, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.3, |
| "rewards/xmlcount_reward_func": 1.1496500015258788, |
| "step": 12 |
| }, |
| { |
| "completion_length": 174.71875, |
| "epoch": 1.2926829268292683, |
| "grad_norm": 4.460564136505127, |
| "kl": 0.9902181550860405, |
| "learning_rate": 4.870022949890676e-07, |
| "loss": 0.001, |
| "reward": 2.211259737610817, |
| "reward_std": 0.41652211552718654, |
| "rewards/concensus_correctness_reward_func": 0.0, |
| "rewards/consensus_reward_func": 0.0, |
| "rewards/cumulative_reward_2": 0.0, |
| "rewards/final_correctness_reward_func": 0.0, |
| "rewards/question_recreation_reward_func": 0.7058847844600677, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.328125, |
| "rewards/xmlcount_reward_func": 1.1772499978542328, |
| "step": 14 |
| }, |
| { |
| "completion_length": 121.15625, |
| "epoch": 1.4878048780487805, |
| "grad_norm": 5.077511787414551, |
| "kl": 2.23406695201993, |
| "learning_rate": 4.81355307410676e-07, |
| "loss": 0.0022, |
| "reward": 4.244883641600609, |
| "reward_std": 2.1680759941227734, |
| "rewards/concensus_correctness_reward_func": 1.4302500039339066, |
| "rewards/consensus_reward_func": 0.1875, |
| "rewards/cumulative_reward_2": 0.0, |
| "rewards/final_correctness_reward_func": 0.25, |
| "rewards/question_recreation_reward_func": 0.9261960908770561, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.28125, |
| "rewards/xmlcount_reward_func": 1.1696875020861626, |
| "step": 16 |
| }, |
| { |
| "completion_length": 183.53125, |
| "epoch": 1.6829268292682928, |
| "grad_norm": 5.903010845184326, |
| "kl": 2.295085714198649, |
| "learning_rate": 4.747379352713488e-07, |
| "loss": 0.0023, |
| "reward": 2.392599403858185, |
| "reward_std": 0.9273251239210367, |
| "rewards/concensus_correctness_reward_func": 0.0, |
| "rewards/consensus_reward_func": 0.0, |
| "rewards/cumulative_reward_2": 0.0, |
| "rewards/final_correctness_reward_func": 0.5, |
| "rewards/question_recreation_reward_func": 0.792318120598793, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.1875, |
| "rewards/xmlcount_reward_func": 0.9127812534570694, |
| "step": 18 |
| }, |
| { |
| "completion_length": 142.8125, |
| "epoch": 1.8780487804878048, |
| "grad_norm": 1.9898862838745117, |
| "kl": 0.6429034024477005, |
| "learning_rate": 4.6717793412953776e-07, |
| "loss": 0.0006, |
| "reward": 2.9371677935123444, |
| "reward_std": 0.8608561623841524, |
| "rewards/concensus_correctness_reward_func": 0.12025000154972076, |
| "rewards/consensus_reward_func": 0.125, |
| "rewards/cumulative_reward_2": 0.0, |
| "rewards/final_correctness_reward_func": 0.5, |
| "rewards/question_recreation_reward_func": 0.6914178244769573, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.34375, |
| "rewards/xmlcount_reward_func": 1.1567500084638596, |
| "step": 20 |
| }, |
| { |
| "completion_length": 217.05, |
| "epoch": 2.0, |
| "grad_norm": 1.1733369827270508, |
| "kl": 4.470386900007725, |
| "learning_rate": 4.5870701325731773e-07, |
| "loss": 0.0028, |
| "reward": 2.0891710042953493, |
| "reward_std": 0.5484436511993408, |
| "rewards/concensus_correctness_reward_func": 0.0, |
| "rewards/consensus_reward_func": 0.0, |
| "rewards/cumulative_reward_2": 0.0, |
| "rewards/final_correctness_reward_func": 0.0, |
| "rewards/question_recreation_reward_func": 0.7873210072517395, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.25, |
| "rewards/xmlcount_reward_func": 1.0518500089645386, |
| "step": 22 |
| }, |
| { |
| "completion_length": 139.375, |
| "epoch": 2.1951219512195124, |
| "grad_norm": 4.537793159484863, |
| "kl": 0.816438952460885, |
| "learning_rate": 4.4936070264068016e-07, |
| "loss": 0.0008, |
| "reward": 2.3938701897859573, |
| "reward_std": 0.34673458884935826, |
| "rewards/concensus_correctness_reward_func": 0.0, |
| "rewards/consensus_reward_func": 0.0, |
| "rewards/cumulative_reward_2": 0.0, |
| "rewards/final_correctness_reward_func": 0.25, |
| "rewards/question_recreation_reward_func": 0.6716827414929867, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.296875, |
| "rewards/xmlcount_reward_func": 1.1753125041723251, |
| "step": 24 |
| }, |
| { |
| "completion_length": 245.0, |
| "epoch": 2.3902439024390243, |
| "grad_norm": 3.754828929901123, |
| "kl": 2.14161329343915, |
| "learning_rate": 4.391782039544238e-07, |
| "loss": 0.0021, |
| "reward": 4.017070218920708, |
| "reward_std": 2.1269510723650455, |
| "rewards/concensus_correctness_reward_func": 1.9351250007748604, |
| "rewards/consensus_reward_func": 0.0625, |
| "rewards/cumulative_reward_2": 0.0, |
| "rewards/final_correctness_reward_func": 0.25, |
| "rewards/question_recreation_reward_func": 0.5311014428734779, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.28125, |
| "rewards/xmlcount_reward_func": 0.9570937529206276, |
| "step": 26 |
| }, |
| { |
| "completion_length": 188.6875, |
| "epoch": 2.5853658536585367, |
| "grad_norm": 4.370049476623535, |
| "kl": 41.1987736299634, |
| "learning_rate": 4.282022261367073e-07, |
| "loss": 0.0412, |
| "reward": 3.2786895185709, |
| "reward_std": 2.1890580281615257, |
| "rewards/concensus_correctness_reward_func": 0.8053750023245811, |
| "rewards/consensus_reward_func": 0.1875, |
| "rewards/cumulative_reward_2": 0.0, |
| "rewards/final_correctness_reward_func": 0.1875, |
| "rewards/question_recreation_reward_func": 0.7307832725346088, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.265625, |
| "rewards/xmlcount_reward_func": 1.1019062548875809, |
| "step": 28 |
| }, |
| { |
| "completion_length": 201.25, |
| "epoch": 2.7804878048780486, |
| "grad_norm": 3.327321767807007, |
| "kl": 0.41819524113088846, |
| "learning_rate": 4.1647880625292027e-07, |
| "loss": 0.0004, |
| "reward": 2.4246954917907715, |
| "reward_std": 0.5871567726135254, |
| "rewards/concensus_correctness_reward_func": 0.0, |
| "rewards/consensus_reward_func": 0.0, |
| "rewards/cumulative_reward_2": 0.0, |
| "rewards/final_correctness_reward_func": 0.25, |
| "rewards/question_recreation_reward_func": 0.6943517699837685, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.34375, |
| "rewards/xmlcount_reward_func": 1.136593759059906, |
| "step": 30 |
| }, |
| { |
| "completion_length": 130.5625, |
| "epoch": 2.975609756097561, |
| "grad_norm": 4.2605109214782715, |
| "kl": 1.8532675383612514, |
| "learning_rate": 4.040571164002318e-07, |
| "loss": 0.0019, |
| "reward": 3.6226596236228943, |
| "reward_std": 2.205760184675455, |
| "rewards/concensus_correctness_reward_func": 0.8050000034272671, |
| "rewards/consensus_reward_func": 0.1875, |
| "rewards/cumulative_reward_2": 0.0, |
| "rewards/final_correctness_reward_func": 0.4375, |
| "rewards/question_recreation_reward_func": 0.7430970743298531, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.3125, |
| "rewards/xmlcount_reward_func": 1.1370624974370003, |
| "step": 32 |
| }, |
| { |
| "completion_length": 104.65, |
| "epoch": 3.097560975609756, |
| "grad_norm": 5.712991237640381, |
| "kl": 0.6544200539588928, |
| "learning_rate": 3.909892574627266e-07, |
| "loss": 0.0004, |
| "reward": 1.9994364500045776, |
| "reward_std": 0.34640331044793127, |
| "rewards/concensus_correctness_reward_func": 0.0, |
| "rewards/consensus_reward_func": 0.0, |
| "rewards/cumulative_reward_2": 0.0, |
| "rewards/final_correctness_reward_func": 0.0, |
| "rewards/question_recreation_reward_func": 0.6102864652872085, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.225, |
| "rewards/xmlcount_reward_func": 1.1641499996185303, |
| "step": 34 |
| }, |
| { |
| "completion_length": 164.625, |
| "epoch": 3.292682926829268, |
| "grad_norm": 3.5782923698425293, |
| "kl": 0.47456978261470795, |
| "learning_rate": 3.773300405821908e-07, |
| "loss": 0.0005, |
| "reward": 2.4073937088251114, |
| "reward_std": 0.8083425159566104, |
| "rewards/concensus_correctness_reward_func": 0.12012499943375587, |
| "rewards/consensus_reward_func": 0.125, |
| "rewards/cumulative_reward_2": 0.0, |
| "rewards/final_correctness_reward_func": 0.0, |
| "rewards/question_recreation_reward_func": 0.7384249614551663, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.265625, |
| "rewards/xmlcount_reward_func": 1.1582187488675117, |
| "step": 36 |
| }, |
| { |
| "completion_length": 145.4375, |
| "epoch": 3.4878048780487805, |
| "grad_norm": 3.6999077796936035, |
| "kl": 0.7185129821300507, |
| "learning_rate": 3.6313675726113475e-07, |
| "loss": 0.0007, |
| "reward": 3.2990617603063583, |
| "reward_std": 1.678582351654768, |
| "rewards/concensus_correctness_reward_func": 0.625, |
| "rewards/consensus_reward_func": 0.0, |
| "rewards/cumulative_reward_2": 0.0, |
| "rewards/final_correctness_reward_func": 0.5, |
| "rewards/question_recreation_reward_func": 0.6626242846250534, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.359375, |
| "rewards/xmlcount_reward_func": 1.1520625054836273, |
| "step": 38 |
| }, |
| { |
| "completion_length": 219.40625, |
| "epoch": 3.682926829268293, |
| "grad_norm": 3.1566965579986572, |
| "kl": 0.7571137994527817, |
| "learning_rate": 3.484689390623218e-07, |
| "loss": 0.0008, |
| "reward": 4.262388929724693, |
| "reward_std": 1.6960864844731987, |
| "rewards/concensus_correctness_reward_func": 1.875, |
| "rewards/consensus_reward_func": 0.0, |
| "rewards/cumulative_reward_2": 0.0, |
| "rewards/final_correctness_reward_func": 0.25, |
| "rewards/question_recreation_reward_func": 0.7855764981359243, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.296875, |
| "rewards/xmlcount_reward_func": 1.0549375042319298, |
| "step": 40 |
| }, |
| { |
| "completion_length": 124.125, |
| "epoch": 3.8780487804878048, |
| "grad_norm": 5.992835998535156, |
| "kl": 4.541106943972409, |
| "learning_rate": 3.3338810791270517e-07, |
| "loss": 0.0045, |
| "reward": 3.2296720147132874, |
| "reward_std": 1.5638214345090091, |
| "rewards/concensus_correctness_reward_func": 0.625, |
| "rewards/consensus_reward_func": 0.0, |
| "rewards/cumulative_reward_2": 0.0, |
| "rewards/final_correctness_reward_func": 0.25, |
| "rewards/question_recreation_reward_func": 0.7207032516598701, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.40625, |
| "rewards/xmlcount_reward_func": 1.2277187556028366, |
| "step": 42 |
| }, |
| { |
| "completion_length": 143.7, |
| "epoch": 4.0, |
| "grad_norm": 1.4582439661026, |
| "kl": 13.396405771374702, |
| "learning_rate": 3.179575180590857e-07, |
| "loss": 0.0084, |
| "reward": 5.374096298217774, |
| "reward_std": 3.2581148982048034, |
| "rewards/concensus_correctness_reward_func": 2.1923000037670137, |
| "rewards/consensus_reward_func": 0.2, |
| "rewards/cumulative_reward_2": 0.0, |
| "rewards/final_correctness_reward_func": 0.8, |
| "rewards/question_recreation_reward_func": 0.8091962337493896, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.25, |
| "rewards/xmlcount_reward_func": 1.122599995136261, |
| "step": 44 |
| }, |
| { |
| "completion_length": 160.03125, |
| "epoch": 4.195121951219512, |
| "grad_norm": 4.867384910583496, |
| "kl": 1.008437397889793, |
| "learning_rate": 3.022418907578188e-07, |
| "loss": 0.001, |
| "reward": 3.2849575728178024, |
| "reward_std": 1.9591746106743813, |
| "rewards/concensus_correctness_reward_func": 0.6851250007748604, |
| "rewards/consensus_reward_func": 0.0625, |
| "rewards/cumulative_reward_2": 0.0, |
| "rewards/final_correctness_reward_func": 0.25, |
| "rewards/question_recreation_reward_func": 0.746645025908947, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.40625, |
| "rewards/xmlcount_reward_func": 1.1344375163316727, |
| "step": 46 |
| }, |
| { |
| "completion_length": 168.625, |
| "epoch": 4.390243902439025, |
| "grad_norm": 5.173135280609131, |
| "kl": 0.954070495441556, |
| "learning_rate": 2.863071428113726e-07, |
| "loss": 0.001, |
| "reward": 3.638337269425392, |
| "reward_std": 1.8555740788578987, |
| "rewards/concensus_correctness_reward_func": 1.25, |
| "rewards/consensus_reward_func": 0.0, |
| "rewards/cumulative_reward_2": 0.0, |
| "rewards/final_correctness_reward_func": 0.25, |
| "rewards/question_recreation_reward_func": 0.6886498406529427, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.328125, |
| "rewards/xmlcount_reward_func": 1.1215624958276749, |
| "step": 48 |
| }, |
| { |
| "completion_length": 147.625, |
| "epoch": 4.585365853658536, |
| "grad_norm": 3.766300678253174, |
| "kl": 1.248833647929132, |
| "learning_rate": 2.7022011009035107e-07, |
| "loss": 0.0012, |
| "reward": 2.902125522494316, |
| "reward_std": 0.7541704792529345, |
| "rewards/concensus_correctness_reward_func": 0.12250000238418579, |
| "rewards/consensus_reward_func": 0.0625, |
| "rewards/cumulative_reward_2": 0.0, |
| "rewards/final_correctness_reward_func": 0.5, |
| "rewards/question_recreation_reward_func": 0.8029379919171333, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.265625, |
| "rewards/xmlcount_reward_func": 1.1485625207424164, |
| "step": 50 |
| }, |
| { |
| "completion_length": 221.5625, |
| "epoch": 4.780487804878049, |
| "grad_norm": 42.74195098876953, |
| "kl": 19.366400617174804, |
| "learning_rate": 2.540482672006254e-07, |
| "loss": 0.0194, |
| "reward": 2.4607008695602417, |
| "reward_std": 0.582477293908596, |
| "rewards/concensus_correctness_reward_func": 0.06012500077486038, |
| "rewards/consensus_reward_func": 0.0625, |
| "rewards/cumulative_reward_2": 0.0, |
| "rewards/final_correctness_reward_func": 0.25, |
| "rewards/question_recreation_reward_func": 0.7755759283900261, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.234375, |
| "rewards/xmlcount_reward_func": 1.0781250149011612, |
| "step": 52 |
| }, |
| { |
| "completion_length": 121.65625, |
| "epoch": 4.975609756097561, |
| "grad_norm": 4.044496059417725, |
| "kl": 1.2229662016034126, |
| "learning_rate": 2.37859444471388e-07, |
| "loss": 0.0012, |
| "reward": 3.0761840641498566, |
| "reward_std": 1.7169672884047031, |
| "rewards/concensus_correctness_reward_func": 0.6851250007748604, |
| "rewards/consensus_reward_func": 0.0625, |
| "rewards/cumulative_reward_2": 0.0, |
| "rewards/final_correctness_reward_func": 0.25, |
| "rewards/question_recreation_reward_func": 0.6576527766883373, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.296875, |
| "rewards/xmlcount_reward_func": 1.1240312531590462, |
| "step": 54 |
| }, |
| { |
| "completion_length": 190.15, |
| "epoch": 5.097560975609756, |
| "grad_norm": 2.5054287910461426, |
| "kl": 0.40689231753349303, |
| "learning_rate": 2.2172154345117894e-07, |
| "loss": 0.0003, |
| "reward": 2.3544807910919188, |
| "reward_std": 0.4992034614086151, |
| "rewards/concensus_correctness_reward_func": 0.0, |
| "rewards/consensus_reward_func": 0.0, |
| "rewards/cumulative_reward_2": 0.0, |
| "rewards/final_correctness_reward_func": 0.4, |
| "rewards/question_recreation_reward_func": 0.6489308118820191, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.175, |
| "rewards/xmlcount_reward_func": 1.130550003051758, |
| "step": 56 |
| }, |
| { |
| "completion_length": 167.46875, |
| "epoch": 5.2926829268292686, |
| "grad_norm": 4.939743518829346, |
| "kl": 1.0670354459434748, |
| "learning_rate": 2.0570225210519433e-07, |
| "loss": 0.0011, |
| "reward": 2.940119907259941, |
| "reward_std": 0.5183741468936205, |
| "rewards/concensus_correctness_reward_func": 0.06012500077486038, |
| "rewards/consensus_reward_func": 0.0625, |
| "rewards/cumulative_reward_2": 0.0, |
| "rewards/final_correctness_reward_func": 0.5, |
| "rewards/question_recreation_reward_func": 0.7578699514269829, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.375, |
| "rewards/xmlcount_reward_func": 1.1846249997615814, |
| "step": 58 |
| }, |
| { |
| "completion_length": 129.78125, |
| "epoch": 5.487804878048781, |
| "grad_norm": 14.262602806091309, |
| "kl": 6.911861918866634, |
| "learning_rate": 1.8986876090843664e-07, |
| "loss": 0.0069, |
| "reward": 2.512383133172989, |
| "reward_std": 0.6689571645110846, |
| "rewards/concensus_correctness_reward_func": 0.06012500077486038, |
| "rewards/consensus_reward_func": 0.0625, |
| "rewards/cumulative_reward_2": 0.0, |
| "rewards/final_correctness_reward_func": 0.25, |
| "rewards/question_recreation_reward_func": 0.7206018827855587, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.25, |
| "rewards/xmlcount_reward_func": 1.1691562682390213, |
| "step": 60 |
| }, |
| { |
| "completion_length": 153.84375, |
| "epoch": 5.682926829268292, |
| "grad_norm": 4.040970802307129, |
| "kl": 1.8997416272759438, |
| "learning_rate": 1.7428748102551234e-07, |
| "loss": 0.0019, |
| "reward": 4.094300732016563, |
| "reward_std": 3.2450541853904724, |
| "rewards/concensus_correctness_reward_func": 1.3101250007748604, |
| "rewards/consensus_reward_func": 0.0625, |
| "rewards/cumulative_reward_2": 0.0, |
| "rewards/final_correctness_reward_func": 0.5, |
| "rewards/question_recreation_reward_func": 0.7321133241057396, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.359375, |
| "rewards/xmlcount_reward_func": 1.130187500268221, |
| "step": 62 |
| }, |
| { |
| "completion_length": 149.0, |
| "epoch": 5.878048780487805, |
| "grad_norm": 4.407651424407959, |
| "kl": 1.1698362613096833, |
| "learning_rate": 1.5902376575912814e-07, |
| "loss": 0.0012, |
| "reward": 2.2420406192541122, |
| "reward_std": 0.5333225438371301, |
| "rewards/concensus_correctness_reward_func": 0.0598750002682209, |
| "rewards/consensus_reward_func": 0.0625, |
| "rewards/cumulative_reward_2": 0.0, |
| "rewards/final_correctness_reward_func": 0.0, |
| "rewards/question_recreation_reward_func": 0.792165607213974, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.21875, |
| "rewards/xmlcount_reward_func": 1.1087500154972076, |
| "step": 64 |
| }, |
| { |
| "completion_length": 112.15, |
| "epoch": 6.0, |
| "grad_norm": 3.1488583087921143, |
| "kl": 0.7777493536472321, |
| "learning_rate": 1.4414163643562753e-07, |
| "loss": 0.0005, |
| "reward": 2.53222017288208, |
| "reward_std": 0.6459437549114228, |
| "rewards/concensus_correctness_reward_func": 0.09580000042915345, |
| "rewards/consensus_reward_func": 0.1, |
| "rewards/cumulative_reward_2": 0.0, |
| "rewards/final_correctness_reward_func": 0.0, |
| "rewards/question_recreation_reward_func": 0.7860701441764831, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.35, |
| "rewards/xmlcount_reward_func": 1.2003499746322632, |
| "step": 66 |
| }, |
| { |
| "completion_length": 136.46875, |
| "epoch": 6.195121951219512, |
| "grad_norm": 5.8898539543151855, |
| "kl": 1.8731475602835417, |
| "learning_rate": 1.2970351387729872e-07, |
| "loss": 0.0019, |
| "reward": 2.372854396700859, |
| "reward_std": 0.46924145889352076, |
| "rewards/concensus_correctness_reward_func": 0.0, |
| "rewards/consensus_reward_func": 0.0, |
| "rewards/cumulative_reward_2": 0.0, |
| "rewards/final_correctness_reward_func": 0.25, |
| "rewards/question_recreation_reward_func": 0.6067919060587883, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.34375, |
| "rewards/xmlcount_reward_func": 1.1723125129938126, |
| "step": 68 |
| }, |
| { |
| "completion_length": 162.78125, |
| "epoch": 6.390243902439025, |
| "grad_norm": 6.582674026489258, |
| "kl": 1.30052063241601, |
| "learning_rate": 1.1576995658775404e-07, |
| "loss": 0.0013, |
| "reward": 4.535748735070229, |
| "reward_std": 1.4339275071397424, |
| "rewards/concensus_correctness_reward_func": 2.05774999409914, |
| "rewards/consensus_reward_func": 0.125, |
| "rewards/cumulative_reward_2": 0.0, |
| "rewards/final_correctness_reward_func": 0.25, |
| "rewards/question_recreation_reward_func": 0.6988736353814602, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.25, |
| "rewards/xmlcount_reward_func": 1.1541250050067902, |
| "step": 70 |
| }, |
| { |
| "completion_length": 108.125, |
| "epoch": 6.585365853658536, |
| "grad_norm": 6.37650728225708, |
| "kl": 0.5360646117478609, |
| "learning_rate": 1.0239940674851941e-07, |
| "loss": 0.0005, |
| "reward": 3.5589722096920013, |
| "reward_std": 1.8011039877310395, |
| "rewards/concensus_correctness_reward_func": 0.6851250007748604, |
| "rewards/consensus_reward_func": 0.0625, |
| "rewards/cumulative_reward_2": 0.0, |
| "rewards/final_correctness_reward_func": 0.5, |
| "rewards/question_recreation_reward_func": 0.7731909388676286, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.3125, |
| "rewards/xmlcount_reward_func": 1.2256562411785126, |
| "step": 72 |
| }, |
| { |
| "completion_length": 138.21875, |
| "epoch": 6.780487804878049, |
| "grad_norm": 4.734973430633545, |
| "kl": 3.8659601393155754, |
| "learning_rate": 8.964794509221507e-08, |
| "loss": 0.0039, |
| "reward": 2.692448616027832, |
| "reward_std": 0.35131036676466465, |
| "rewards/concensus_correctness_reward_func": 0.0, |
| "rewards/consensus_reward_func": 0.0, |
| "rewards/cumulative_reward_2": 0.0, |
| "rewards/final_correctness_reward_func": 0.25, |
| "rewards/question_recreation_reward_func": 0.8613236248493195, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.359375, |
| "rewards/xmlcount_reward_func": 1.221750020980835, |
| "step": 74 |
| }, |
| { |
| "completion_length": 151.0625, |
| "epoch": 6.975609756097561, |
| "grad_norm": 5.129486560821533, |
| "kl": 1.5537091912701726, |
| "learning_rate": 7.756905568047392e-08, |
| "loss": 0.0016, |
| "reward": 2.6046862453222275, |
| "reward_std": 0.7406165972352028, |
| "rewards/concensus_correctness_reward_func": 0.12250000238418579, |
| "rewards/consensus_reward_func": 0.0625, |
| "rewards/cumulative_reward_2": 0.0, |
| "rewards/final_correctness_reward_func": 0.25, |
| "rewards/question_recreation_reward_func": 0.6871863044798374, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.3125, |
| "rewards/xmlcount_reward_func": 1.1699999868869781, |
| "step": 76 |
| }, |
| { |
| "completion_length": 131.7, |
| "epoch": 7.097560975609756, |
| "grad_norm": 3.7079029083251953, |
| "kl": 1.5399453818798066, |
| "learning_rate": 6.621340157319996e-08, |
| "loss": 0.001, |
| "reward": 2.7136191368103026, |
| "reward_std": 0.9075609683990479, |
| "rewards/concensus_correctness_reward_func": 0.09620000123977661, |
| "rewards/consensus_reward_func": 0.1, |
| "rewards/cumulative_reward_2": 0.0, |
| "rewards/final_correctness_reward_func": 0.3, |
| "rewards/question_recreation_reward_func": 0.7861691594123841, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.25, |
| "rewards/xmlcount_reward_func": 1.18125, |
| "step": 78 |
| }, |
| { |
| "completion_length": 181.09375, |
| "epoch": 7.2926829268292686, |
| "grad_norm": 6.1535162925720215, |
| "kl": 0.573518211953342, |
| "learning_rate": 5.5628612330087724e-08, |
| "loss": 0.0006, |
| "reward": 3.8459194749593735, |
| "reward_std": 1.7336784086655825, |
| "rewards/concensus_correctness_reward_func": 1.25, |
| "rewards/consensus_reward_func": 0.0, |
| "rewards/cumulative_reward_2": 0.0, |
| "rewards/final_correctness_reward_func": 0.25, |
| "rewards/question_recreation_reward_func": 0.8142632357776165, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.34375, |
| "rewards/xmlcount_reward_func": 1.1879062503576279, |
| "step": 80 |
| }, |
| { |
| "completion_length": 152.875, |
| "epoch": 7.487804878048781, |
| "grad_norm": 6.165317535400391, |
| "kl": 2.319924622774124, |
| "learning_rate": 4.5859084235697235e-08, |
| "loss": 0.0023, |
| "reward": 2.0710987001657486, |
| "reward_std": 0.751331745646894, |
| "rewards/concensus_correctness_reward_func": 0.06012500077486038, |
| "rewards/consensus_reward_func": 0.0625, |
| "rewards/cumulative_reward_2": 0.0, |
| "rewards/final_correctness_reward_func": 0.0, |
| "rewards/question_recreation_reward_func": 0.55609873495996, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.265625, |
| "rewards/xmlcount_reward_func": 1.1267500072717667, |
| "step": 82 |
| }, |
| { |
| "completion_length": 174.9375, |
| "epoch": 7.682926829268292, |
| "grad_norm": 5.046776294708252, |
| "kl": 2.245447961613536, |
| "learning_rate": 3.6945794086007705e-08, |
| "loss": 0.0022, |
| "reward": 2.5292265713214874, |
| "reward_std": 1.143988709896803, |
| "rewards/concensus_correctness_reward_func": 0.24050000309944153, |
| "rewards/consensus_reward_func": 0.25, |
| "rewards/cumulative_reward_2": 0.0, |
| "rewards/final_correctness_reward_func": 0.0, |
| "rewards/question_recreation_reward_func": 0.6357265987899154, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.3125, |
| "rewards/xmlcount_reward_func": 1.0905000045895576, |
| "step": 84 |
| }, |
| { |
| "completion_length": 164.0, |
| "epoch": 7.878048780487805, |
| "grad_norm": 3.5354819297790527, |
| "kl": 0.5519012669101357, |
| "learning_rate": 2.892612731749414e-08, |
| "loss": 0.0006, |
| "reward": 3.282635271549225, |
| "reward_std": 1.7596494741737843, |
| "rewards/concensus_correctness_reward_func": 0.625, |
| "rewards/consensus_reward_func": 0.0, |
| "rewards/cumulative_reward_2": 0.0, |
| "rewards/final_correctness_reward_func": 0.375, |
| "rewards/question_recreation_reward_func": 0.807010255753994, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.328125, |
| "rewards/xmlcount_reward_func": 1.1475000083446503, |
| "step": 86 |
| }, |
| { |
| "completion_length": 108.4, |
| "epoch": 8.0, |
| "grad_norm": 2.0044972896575928, |
| "kl": 0.35757209062576295, |
| "learning_rate": 2.183372119961499e-08, |
| "loss": 0.0002, |
| "reward": 6.687426805496216, |
| "reward_std": 4.642478483915329, |
| "rewards/concensus_correctness_reward_func": 3.388399910926819, |
| "rewards/consensus_reward_func": 0.3, |
| "rewards/cumulative_reward_2": 0.0, |
| "rewards/final_correctness_reward_func": 0.8, |
| "rewards/question_recreation_reward_func": 0.6740266382694244, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.3, |
| "rewards/xmlcount_reward_func": 1.225, |
| "step": 88 |
| }, |
| { |
| "completion_length": 120.59375, |
| "epoch": 8.195121951219512, |
| "grad_norm": 3.868396520614624, |
| "kl": 2.700576603412628, |
| "learning_rate": 1.5698323748414122e-08, |
| "loss": 0.0027, |
| "reward": 3.760663390159607, |
| "reward_std": 2.0940339118242264, |
| "rewards/concensus_correctness_reward_func": 0.9279374964535236, |
| "rewards/consensus_reward_func": 0.25, |
| "rewards/cumulative_reward_2": 0.0, |
| "rewards/final_correctness_reward_func": 0.25, |
| "rewards/question_recreation_reward_func": 0.8017571456730366, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.328125, |
| "rewards/xmlcount_reward_func": 1.202843725681305, |
| "step": 90 |
| }, |
| { |
| "completion_length": 203.96875, |
| "epoch": 8.390243902439025, |
| "grad_norm": 4.034110069274902, |
| "kl": 3.7330283680930734, |
| "learning_rate": 1.054566895300324e-08, |
| "loss": 0.0037, |
| "reward": 3.24531951546669, |
| "reward_std": 1.75903601013124, |
| "rewards/concensus_correctness_reward_func": 0.8073750026524067, |
| "rewards/consensus_reward_func": 0.125, |
| "rewards/cumulative_reward_2": 0.0, |
| "rewards/final_correctness_reward_func": 0.25, |
| "rewards/question_recreation_reward_func": 0.6574131920933723, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.28125, |
| "rewards/xmlcount_reward_func": 1.1242812499403954, |
| "step": 92 |
| }, |
| { |
| "completion_length": 113.84375, |
| "epoch": 8.585365853658537, |
| "grad_norm": 3.601546287536621, |
| "kl": 2.1267421934753656, |
| "learning_rate": 6.397368838268496e-09, |
| "loss": 0.0021, |
| "reward": 2.7754287719726562, |
| "reward_std": 0.6228232402354479, |
| "rewards/concensus_correctness_reward_func": 0.05999999865889549, |
| "rewards/consensus_reward_func": 0.0625, |
| "rewards/cumulative_reward_2": 0.0, |
| "rewards/final_correctness_reward_func": 0.25, |
| "rewards/question_recreation_reward_func": 0.8581162914633751, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.359375, |
| "rewards/xmlcount_reward_func": 1.1854375004768372, |
| "step": 94 |
| }, |
| { |
| "completion_length": 198.03125, |
| "epoch": 8.78048780487805, |
| "grad_norm": 3.0875630378723145, |
| "kl": 0.6049554403871298, |
| "learning_rate": 3.2708228165273244e-09, |
| "loss": 0.0006, |
| "reward": 4.5220272690057755, |
| "reward_std": 1.8342011701315641, |
| "rewards/concensus_correctness_reward_func": 1.9952500015497208, |
| "rewards/consensus_reward_func": 0.125, |
| "rewards/cumulative_reward_2": 0.0, |
| "rewards/final_correctness_reward_func": 0.25, |
| "rewards/question_recreation_reward_func": 0.7617147043347359, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.28125, |
| "rewards/xmlcount_reward_func": 1.1088124960660934, |
| "step": 96 |
| }, |
| { |
| "completion_length": 147.875, |
| "epoch": 8.975609756097562, |
| "grad_norm": 6.384603977203369, |
| "kl": 2.2161214109510183, |
| "learning_rate": 1.1791447083465133e-09, |
| "loss": 0.0022, |
| "reward": 2.5484085381031036, |
| "reward_std": 0.6816056333482265, |
| "rewards/concensus_correctness_reward_func": 0.0, |
| "rewards/consensus_reward_func": 0.0, |
| "rewards/cumulative_reward_2": 0.0, |
| "rewards/final_correctness_reward_func": 0.5, |
| "rewards/question_recreation_reward_func": 0.5684084966778755, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.375, |
| "rewards/xmlcount_reward_func": 1.1050000041723251, |
| "step": 98 |
| }, |
| { |
| "completion_length": 105.9, |
| "epoch": 9.097560975609756, |
| "grad_norm": 3.4388906955718994, |
| "kl": 0.5531518459320068, |
| "learning_rate": 1.3110773862126667e-10, |
| "loss": 0.0003, |
| "reward": 2.561092567443848, |
| "reward_std": 0.5468424066901207, |
| "rewards/concensus_correctness_reward_func": 0.09610000252723694, |
| "rewards/consensus_reward_func": 0.1, |
| "rewards/cumulative_reward_2": 0.0, |
| "rewards/final_correctness_reward_func": 0.0, |
| "rewards/question_recreation_reward_func": 0.6962425768375397, |
| "rewards/soft_format_reward_func": 0.0, |
| "rewards/strict_format_reward_func": 0.45, |
| "rewards/xmlcount_reward_func": 1.21875, |
| "step": 100 |
| }, |
| { |
| "epoch": 9.097560975609756, |
| "step": 100, |
| "total_flos": 0.0, |
| "train_loss": 0.0029315173596842214, |
| "train_runtime": 554.1656, |
| "train_samples_per_second": 2.887, |
| "train_steps_per_second": 0.18 |
| } |
| ], |
| "logging_steps": 2, |
| "max_steps": 100, |
| "num_input_tokens_seen": 0, |
| "num_train_epochs": 10, |
| "save_steps": 100, |
| "stateful_callbacks": { |
| "TrainerControl": { |
| "args": { |
| "should_epoch_stop": false, |
| "should_evaluate": false, |
| "should_log": false, |
| "should_save": true, |
| "should_training_stop": true |
| }, |
| "attributes": {} |
| } |
| }, |
| "total_flos": 0.0, |
| "train_batch_size": 4, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
