End of training

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.07021208088845014,
-    "train_runtime": 2530.4162,
-    "train_samples": 29,
-    "train_samples_per_second": 0.016,
+    "train_loss": 0.010071155428886414,
+    "train_runtime": 2283.2649,
+    "train_samples": 83,
+    "train_samples_per_second": 0.018,
     "train_steps_per_second": 0.004
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7dc46cdccddbc6b0210462924189c2b879aa3bd06ccd36294a5f8002f2a47f19
+oid sha256:fd66b4b6378cc29432c13082199f0daf44c0dec032415e431f7e7c119a285750
 size 1976163472

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.07021208088845014,
-    "train_runtime": 2530.4162,
-    "train_samples": 29,
-    "train_samples_per_second": 0.016,
+    "train_loss": 0.010071155428886414,
+    "train_runtime": 2283.2649,
+    "train_samples": 83,
+    "train_samples_per_second": 0.018,
     "train_steps_per_second": 0.004
 }

trainer_state.json

@@ -2,7 +2,7 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.6896551724137931,
+  "epoch": 0.24096385542168675,
   "eval_steps": 500,
   "global_step": 10,
   "is_hyper_param_search": false,
@@ -16,21 +16,21 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 77.0,
-      "completions/max_terminated_length": 77.0,
-      "completions/mean_length": 39.75,
-      "completions/mean_terminated_length": 39.75,
-      "completions/min_length": 7.5,
-      "completions/min_terminated_length": 7.5,
-      "epoch": 0.13793103448275862,
+      "completions/max_length": 63.5,
+      "completions/max_terminated_length": 63.5,
+      "completions/mean_length": 35.0,
+      "completions/mean_terminated_length": 35.0,
+      "completions/min_length": 14.0,
+      "completions/min_terminated_length": 14.0,
+      "epoch": 0.04819277108433735,
       "frac_reward_zero_std": 0.0,
-      "grad_norm": 27.57093048095703,
+      "grad_norm": 17.314556121826172,
       "kl": 0.0,
       "learning_rate": 5e-07,
-      "loss": 0.1402,
-      "num_tokens": 1342.0,
-      "reward": 0.12268233392387629,
-      "reward_std": 0.02545860642567277,
+      "loss": 0.0107,
+      "num_tokens": 1304.0,
+      "reward": 0.12473818846046925,
+      "reward_std": 0.026818931568413973,
       "rewards/concensus_correctness_reward_func/mean": 0.0,
       "rewards/concensus_correctness_reward_func/std": 0.0,
       "rewards/consensus_reward_func/mean": 0.0,
@@ -39,8 +39,8 @@
       "rewards/cumulative_reward_2/std": 0.0,
       "rewards/final_correctness_reward_func/mean": 0.0,
       "rewards/final_correctness_reward_func/std": 0.0,
-      "rewards/question_recreation_reward_func/mean": 0.12268234137445688,
-      "rewards/question_recreation_reward_func/std": 0.029390745097771287,
+      "rewards/question_recreation_reward_func/mean": 0.12473818846046925,
+      "rewards/question_recreation_reward_func/std": 0.0307374969124794,
       "rewards/soft_format_reward_func/mean": 0.0,
       "rewards/soft_format_reward_func/std": 0.0,
       "rewards/strict_format_reward_func/mean": 0.0,
@@ -56,21 +56,21 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 125.0,
-      "completions/max_terminated_length": 125.0,
-      "completions/mean_length": 62.875,
-      "completions/mean_terminated_length": 62.875,
-      "completions/min_length": 13.5,
-      "completions/min_terminated_length": 13.5,
-      "epoch": 0.27586206896551724,
+      "completions/max_length": 72.5,
+      "completions/max_terminated_length": 72.5,
+      "completions/mean_length": 43.625,
+      "completions/mean_terminated_length": 43.625,
+      "completions/min_length": 24.0,
+      "completions/min_terminated_length": 24.0,
+      "epoch": 0.0963855421686747,
       "frac_reward_zero_std": 0.0,
-      "grad_norm": 36.416683197021484,
-      "kl": 0.006729700922733173,
+      "grad_norm": 43.791683197021484,
+      "kl": 0.0040386376567767,
       "learning_rate": 4.415111107797445e-07,
-      "loss": 0.1584,
-      "num_tokens": 2869.0,
-      "reward": 0.14873841404914856,
-      "reward_std": 0.018037823028862476,
+      "loss": -0.0489,
+      "num_tokens": 2677.0,
+      "reward": 0.0236467644572258,
+      "reward_std": 0.00882497476413846,
       "rewards/concensus_correctness_reward_func/mean": 0.0,
       "rewards/concensus_correctness_reward_func/std": 0.0,
       "rewards/consensus_reward_func/mean": 0.0,
@@ -79,8 +79,8 @@
       "rewards/cumulative_reward_2/std": 0.0,
       "rewards/final_correctness_reward_func/mean": 0.0,
       "rewards/final_correctness_reward_func/std": 0.0,
-      "rewards/question_recreation_reward_func/mean": 0.14873841404914856,
-      "rewards/question_recreation_reward_func/std": 0.05908002704381943,
+      "rewards/question_recreation_reward_func/mean": 0.023646765388548374,
+      "rewards/question_recreation_reward_func/std": 0.007400925969704986,
       "rewards/soft_format_reward_func/mean": 0.0,
       "rewards/soft_format_reward_func/std": 0.0,
       "rewards/strict_format_reward_func/mean": 0.0,
@@ -95,22 +95,22 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.125,
-      "completions/max_length": 142.0,
-      "completions/max_terminated_length": 31.5,
-      "completions/mean_length": 51.0,
-      "completions/mean_terminated_length": 22.791666984558105,
-      "completions/min_length": 15.5,
-      "completions/min_terminated_length": 15.5,
-      "epoch": 0.41379310344827586,
-      "frac_reward_zero_std": 0.25,
-      "grad_norm": 26.669572830200195,
-      "kl": 0.01210609171539545,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 77.0,
+      "completions/max_terminated_length": 77.0,
+      "completions/mean_length": 39.75,
+      "completions/mean_terminated_length": 39.75,
+      "completions/min_length": 16.5,
+      "completions/min_terminated_length": 16.5,
+      "epoch": 0.14457831325301204,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 39.4578857421875,
+      "kl": 0.0063580243731848896,
       "learning_rate": 2.934120444167326e-07,
-      "loss": -0.0109,
-      "num_tokens": 4301.0,
-      "reward": 0.07620356883853674,
-      "reward_std": 0.02216035732999444,
+      "loss": 0.0318,
+      "num_tokens": 4019.0,
+      "reward": 0.12428901437669992,
+      "reward_std": 0.030817807652056217,
       "rewards/concensus_correctness_reward_func/mean": 0.0,
       "rewards/concensus_correctness_reward_func/std": 0.0,
       "rewards/consensus_reward_func/mean": 0.0,
@@ -119,8 +119,8 @@
       "rewards/cumulative_reward_2/std": 0.0,
       "rewards/final_correctness_reward_func/mean": 0.0,
       "rewards/final_correctness_reward_func/std": 0.0,
-      "rewards/question_recreation_reward_func/mean": 0.07620356976985931,
-      "rewards/question_recreation_reward_func/std": 0.025473584420979023,
+      "rewards/question_recreation_reward_func/mean": 0.12428901344537735,
+      "rewards/question_recreation_reward_func/std": 0.030232174787670374,
       "rewards/soft_format_reward_func/mean": 0.0,
       "rewards/soft_format_reward_func/std": 0.0,
       "rewards/strict_format_reward_func/mean": 0.0,
@@ -136,21 +136,21 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 92.0,
-      "completions/max_terminated_length": 92.0,
-      "completions/mean_length": 39.25,
-      "completions/mean_terminated_length": 39.25,
-      "completions/min_length": 17.0,
-      "completions/min_terminated_length": 17.0,
-      "epoch": 0.5517241379310345,
+      "completions/max_length": 91.0,
+      "completions/max_terminated_length": 91.0,
+      "completions/mean_length": 51.875,
+      "completions/mean_terminated_length": 51.875,
+      "completions/min_length": 22.5,
+      "completions/min_terminated_length": 22.5,
+      "epoch": 0.1927710843373494,
       "frac_reward_zero_std": 0.0,
-      "grad_norm": 39.656028747558594,
-      "kl": 0.02664924575947225,
+      "grad_norm": 43.962554931640625,
+      "kl": 0.010772253677714616,
       "learning_rate": 1.2500000000000005e-07,
-      "loss": 0.0657,
-      "num_tokens": 5639.0,
-      "reward": 0.14269807375967503,
-      "reward_std": 0.015343487728387117,
+      "loss": 0.0536,
+      "num_tokens": 5458.0,
+      "reward": 0.10668943636119366,
+      "reward_std": 0.03776418976485729,
       "rewards/concensus_correctness_reward_func/mean": 0.0,
       "rewards/concensus_correctness_reward_func/std": 0.0,
       "rewards/consensus_reward_func/mean": 0.0,
@@ -159,8 +159,8 @@
       "rewards/cumulative_reward_2/std": 0.0,
       "rewards/final_correctness_reward_func/mean": 0.0,
       "rewards/final_correctness_reward_func/std": 0.0,
-      "rewards/question_recreation_reward_func/mean": 0.14269806630909443,
-      "rewards/question_recreation_reward_func/std": 0.0286758947186172,
+      "rewards/question_recreation_reward_func/mean": 0.10668943636119366,
+      "rewards/question_recreation_reward_func/std": 0.03313516452908516,
       "rewards/soft_format_reward_func/mean": 0.0,
       "rewards/soft_format_reward_func/std": 0.0,
       "rewards/strict_format_reward_func/mean": 0.0,
@@ -175,22 +175,22 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 58.0,
-      "completions/max_terminated_length": 58.0,
-      "completions/mean_length": 36.75,
-      "completions/mean_terminated_length": 36.75,
-      "completions/min_length": 22.0,
-      "completions/min_terminated_length": 22.0,
-      "epoch": 0.6896551724137931,
+      "completions/clipped_ratio": 0.125,
+      "completions/max_length": 154.0,
+      "completions/max_terminated_length": 94.0,
+      "completions/mean_length": 69.375,
+      "completions/mean_terminated_length": 44.54166603088379,
+      "completions/min_length": 14.5,
+      "completions/min_terminated_length": 14.5,
+      "epoch": 0.24096385542168675,
       "frac_reward_zero_std": 0.0,
-      "grad_norm": 35.242855072021484,
-      "kl": 0.01966581156011671,
+      "grad_norm": 17.73666763305664,
+      "kl": 0.008697476470842957,
       "learning_rate": 1.507684480352292e-08,
-      "loss": -0.0023,
-      "num_tokens": 6957.0,
-      "reward": 0.1422380842268467,
-      "reward_std": 0.021808857563883066,
+      "loss": 0.0032,
+      "num_tokens": 7037.0,
+      "reward": 0.10204933770000935,
+      "reward_std": 0.03418661467730999,
       "rewards/concensus_correctness_reward_func/mean": 0.0,
       "rewards/concensus_correctness_reward_func/std": 0.0,
       "rewards/consensus_reward_func/mean": 0.0,
@@ -199,8 +199,8 @@
       "rewards/cumulative_reward_2/std": 0.0,
       "rewards/final_correctness_reward_func/mean": 0.0,
       "rewards/final_correctness_reward_func/std": 0.0,
-      "rewards/question_recreation_reward_func/mean": 0.1422380842268467,
-      "rewards/question_recreation_reward_func/std": 0.03655920922756195,
+      "rewards/question_recreation_reward_func/mean": 0.10204933770000935,
+      "rewards/question_recreation_reward_func/std": 0.04046265175566077,
       "rewards/soft_format_reward_func/mean": 0.0,
       "rewards/soft_format_reward_func/std": 0.0,
       "rewards/strict_format_reward_func/mean": 0.0,
@@ -210,18 +210,18 @@
       "step": 10
     },
     {
-      "epoch": 0.6896551724137931,
+      "epoch": 0.24096385542168675,
       "step": 10,
       "total_flos": 0.0,
-      "train_loss": 0.07021208088845014,
-      "train_runtime": 2530.4162,
-      "train_samples_per_second": 0.016,
+      "train_loss": 0.010071155428886414,
+      "train_runtime": 2283.2649,
+      "train_samples_per_second": 0.018,
       "train_steps_per_second": 0.004
     }
   ],
   "logging_steps": 2,
   "max_steps": 10,
-  "num_input_tokens_seen": 6957,
+  "num_input_tokens_seen": 7037,
   "num_train_epochs": 1,
   "save_steps": 10,
   "stateful_callbacks": {