End of training

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": -0.00011151888082939157,
-    "train_runtime": 1866.5324,
-    "train_samples": 3,
-    "train_samples_per_second": 0.043,
-    "train_steps_per_second": 0.011
+    "train_loss": -0.003300715550648903,
+    "train_runtime": 932.3793,
+    "train_samples": 5,
+    "train_samples_per_second": 0.086,
+    "train_steps_per_second": 0.021
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:63073bffd65cd38e2813613d2c79a4a61e3a6b4d1f1c85be273e56e884cfd0c3
+oid sha256:cd74aabc43044b399b3fdbb155c26bd99656d58a8083aa93eee67aa286664299
 size 1976163472

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": -0.00011151888082939157,
-    "train_runtime": 1866.5324,
-    "train_samples": 3,
-    "train_samples_per_second": 0.043,
-    "train_steps_per_second": 0.011
+    "train_loss": -0.003300715550648903,
+    "train_runtime": 932.3793,
+    "train_samples": 5,
+    "train_samples_per_second": 0.086,
+    "train_steps_per_second": 0.021
 }

trainer_state.json

@@ -2,7 +2,7 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 19.666666666666668,
+  "epoch": 9.8,
   "eval_steps": 500,
   "global_step": 20,
   "is_hyper_param_search": false,
@@ -16,36 +16,36 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 113.0,
-      "completions/max_terminated_length": 113.0,
-      "completions/mean_length": 105.16666793823242,
-      "completions/mean_terminated_length": 105.16666793823242,
-      "completions/min_length": 89.5,
-      "completions/min_terminated_length": 89.5,
-      "epoch": 1.6666666666666665,
+      "completions/max_length": 21.0,
+      "completions/max_terminated_length": 21.0,
+      "completions/mean_length": 20.75,
+      "completions/mean_terminated_length": 20.75,
+      "completions/min_length": 20.5,
+      "completions/min_terminated_length": 20.5,
+      "epoch": 0.8,
       "grad_norm": 0.0,
       "kl": 0.0,
       "learning_rate": 5e-07,
       "loss": 0.0,
-      "num_tokens": 2858.0,
-      "reward": 8.479824781417847,
+      "num_tokens": 2214.0,
+      "reward": 1.273802012205124,
       "reward_std": 0.0,
-      "rewards/concensus_correctness_reward_func/mean": 2.9359999895095825,
-      "rewards/concensus_correctness_reward_func/std": 1.1304517984390259,
-      "rewards/consensus_reward_func/mean": 1.5,
-      "rewards/consensus_reward_func/std": 0.5773502588272095,
+      "rewards/concensus_correctness_reward_func/mean": 0.0,
+      "rewards/concensus_correctness_reward_func/std": 0.0,
+      "rewards/consensus_reward_func/mean": 0.0,
+      "rewards/consensus_reward_func/std": 0.0,
       "rewards/cumulative_reward_2/mean": 0.0,
       "rewards/cumulative_reward_2/std": 0.0,
-      "rewards/final_correctness_reward_func/mean": 2.0,
+      "rewards/final_correctness_reward_func/mean": 1.0,
       "rewards/final_correctness_reward_func/std": 0.0,
-      "rewards/question_recreation_reward_func/mean": 0.4813254624605179,
-      "rewards/question_recreation_reward_func/std": 0.4909558594226837,
+      "rewards/question_recreation_reward_func/mean": 0.02580203115940094,
+      "rewards/question_recreation_reward_func/std": 0.001379721456032712,
       "rewards/soft_format_reward_func/mean": 0.0,
       "rewards/soft_format_reward_func/std": 0.0,
-      "rewards/strict_format_reward_func/mean": 0.375,
-      "rewards/strict_format_reward_func/std": 0.14433756470680237,
-      "rewards/xmlcount_reward_func/mean": 1.1875,
-      "rewards/xmlcount_reward_func/std": 0.07216878235340118,
+      "rewards/strict_format_reward_func/mean": 0.0,
+      "rewards/strict_format_reward_func/std": 0.0,
+      "rewards/xmlcount_reward_func/mean": 0.24799999594688416,
+      "rewards/xmlcount_reward_func/std": 0.0,
       "step": 2
     },
     {
@@ -55,36 +55,36 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 113.0,
-      "completions/max_terminated_length": 113.0,
-      "completions/mean_length": 105.16666793823242,
-      "completions/mean_terminated_length": 105.16666793823242,
-      "completions/min_length": 89.5,
-      "completions/min_terminated_length": 89.5,
-      "epoch": 3.6666666666666665,
+      "completions/max_length": 22.0,
+      "completions/max_terminated_length": 22.0,
+      "completions/mean_length": 21.25,
+      "completions/mean_terminated_length": 21.25,
+      "completions/min_length": 20.5,
+      "completions/min_terminated_length": 20.5,
+      "epoch": 1.8,
       "grad_norm": 0.0,
       "kl": 0.0,
       "learning_rate": 4.864543104251586e-07,
       "loss": 0.0,
-      "num_tokens": 5716.0,
-      "reward": 8.479824781417847,
+      "num_tokens": 4432.0,
+      "reward": 1.2818379402160645,
       "reward_std": 0.0,
-      "rewards/concensus_correctness_reward_func/mean": 2.9359999895095825,
-      "rewards/concensus_correctness_reward_func/std": 1.1304517984390259,
-      "rewards/consensus_reward_func/mean": 1.5,
-      "rewards/consensus_reward_func/std": 0.5773502588272095,
+      "rewards/concensus_correctness_reward_func/mean": 0.0,
+      "rewards/concensus_correctness_reward_func/std": 0.0,
+      "rewards/consensus_reward_func/mean": 0.0,
+      "rewards/consensus_reward_func/std": 0.0,
       "rewards/cumulative_reward_2/mean": 0.0,
       "rewards/cumulative_reward_2/std": 0.0,
-      "rewards/final_correctness_reward_func/mean": 2.0,
-      "rewards/final_correctness_reward_func/std": 0.0,
-      "rewards/question_recreation_reward_func/mean": 0.4813254624605179,
-      "rewards/question_recreation_reward_func/std": 0.4909558594226837,
+      "rewards/final_correctness_reward_func/mean": 1.0,
+      "rewards/final_correctness_reward_func/std": 1.154700517654419,
+      "rewards/question_recreation_reward_func/mean": 0.03383800573647022,
+      "rewards/question_recreation_reward_func/std": 0.0016485399100929499,
       "rewards/soft_format_reward_func/mean": 0.0,
       "rewards/soft_format_reward_func/std": 0.0,
-      "rewards/strict_format_reward_func/mean": 0.375,
-      "rewards/strict_format_reward_func/std": 0.14433756470680237,
-      "rewards/xmlcount_reward_func/mean": 1.1875,
-      "rewards/xmlcount_reward_func/std": 0.07216878235340118,
+      "rewards/strict_format_reward_func/mean": 0.0,
+      "rewards/strict_format_reward_func/std": 0.0,
+      "rewards/xmlcount_reward_func/mean": 0.24799999594688416,
+      "rewards/xmlcount_reward_func/std": 0.0,
       "step": 4
     },
     {
@@ -94,36 +94,36 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 125.0,
-      "completions/max_terminated_length": 125.0,
-      "completions/mean_length": 112.83333206176758,
-      "completions/mean_terminated_length": 112.83333206176758,
-      "completions/min_length": 89.5,
-      "completions/min_terminated_length": 89.5,
-      "epoch": 5.666666666666667,
-      "grad_norm": 0.0,
+      "completions/max_length": 21.0,
+      "completions/max_terminated_length": 21.0,
+      "completions/mean_length": 21.0,
+      "completions/mean_terminated_length": 21.0,
+      "completions/min_length": 21.0,
+      "completions/min_terminated_length": 21.0,
+      "epoch": 2.8,
+      "grad_norm": 52.8972282409668,
       "kl": 0.0,
       "learning_rate": 4.472851273490984e-07,
-      "loss": 0.0,
-      "num_tokens": 8620.0,
-      "reward": 6.849369525909424,
-      "reward_std": 0.0,
-      "rewards/concensus_correctness_reward_func/mean": 1.9579999446868896,
-      "rewards/concensus_correctness_reward_func/std": 2.2609035968780518,
-      "rewards/consensus_reward_func/mean": 1.0,
-      "rewards/consensus_reward_func/std": 1.154700517654419,
+      "loss": -0.0038,
+      "num_tokens": 6651.0,
+      "reward": 0.8212482184171677,
+      "reward_std": 0.0007691325736232102,
+      "rewards/concensus_correctness_reward_func/mean": 0.0,
+      "rewards/concensus_correctness_reward_func/std": 0.0,
+      "rewards/consensus_reward_func/mean": 0.0,
+      "rewards/consensus_reward_func/std": 0.0,
       "rewards/cumulative_reward_2/mean": 0.0,
       "rewards/cumulative_reward_2/std": 0.0,
-      "rewards/final_correctness_reward_func/mean": 2.0,
-      "rewards/final_correctness_reward_func/std": 0.0,
-      "rewards/question_recreation_reward_func/mean": 0.5163697898387909,
-      "rewards/question_recreation_reward_func/std": 0.5474509298801422,
+      "rewards/final_correctness_reward_func/mean": 0.5,
+      "rewards/final_correctness_reward_func/std": 0.5773502588272095,
+      "rewards/question_recreation_reward_func/mean": 0.07324827183037996,
+      "rewards/question_recreation_reward_func/std": 0.001122168148867786,
       "rewards/soft_format_reward_func/mean": 0.0,
       "rewards/soft_format_reward_func/std": 0.0,
-      "rewards/strict_format_reward_func/mean": 0.25,
-      "rewards/strict_format_reward_func/std": 0.28867512941360474,
-      "rewards/xmlcount_reward_func/mean": 1.125,
-      "rewards/xmlcount_reward_func/std": 0.14433756470680237,
+      "rewards/strict_format_reward_func/mean": 0.0,
+      "rewards/strict_format_reward_func/std": 0.0,
+      "rewards/xmlcount_reward_func/mean": 0.24799999594688416,
+      "rewards/xmlcount_reward_func/std": 0.0,
       "step": 6
     },
     {
@@ -133,36 +133,36 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 123.0,
-      "completions/max_terminated_length": 123.0,
-      "completions/mean_length": 100.66666793823242,
-      "completions/mean_terminated_length": 100.66666793823242,
-      "completions/min_length": 89.5,
-      "completions/min_terminated_length": 89.5,
-      "epoch": 7.666666666666667,
-      "grad_norm": 0.0,
-      "kl": 0.0,
+      "completions/max_length": 21.0,
+      "completions/max_terminated_length": 21.0,
+      "completions/mean_length": 20.25,
+      "completions/mean_terminated_length": 20.25,
+      "completions/min_length": 18.0,
+      "completions/min_terminated_length": 18.0,
+      "epoch": 3.8,
+      "grad_norm": 3.309718522359617e-05,
+      "kl": 0.004937448339270789,
       "learning_rate": 3.867370395306068e-07,
-      "loss": 0.0,
-      "num_tokens": 11522.0,
-      "reward": 6.181347370147705,
-      "reward_std": 0.0,
-      "rewards/concensus_correctness_reward_func/mean": 1.459500014781952,
-      "rewards/concensus_correctness_reward_func/std": 1.6852854490280151,
-      "rewards/consensus_reward_func/mean": 1.0,
-      "rewards/consensus_reward_func/std": 1.154700517654419,
+      "loss": -0.0292,
+      "num_tokens": 8865.0,
+      "reward": 0.8165916055440903,
+      "reward_std": 0.0030181410256773233,
+      "rewards/concensus_correctness_reward_func/mean": 0.0,
+      "rewards/concensus_correctness_reward_func/std": 0.0,
+      "rewards/consensus_reward_func/mean": 0.0,
+      "rewards/consensus_reward_func/std": 0.0,
       "rewards/cumulative_reward_2/mean": 0.0,
       "rewards/cumulative_reward_2/std": 0.0,
-      "rewards/final_correctness_reward_func/mean": 2.0,
-      "rewards/final_correctness_reward_func/std": 0.0,
-      "rewards/question_recreation_reward_func/mean": 0.3468475788831711,
-      "rewards/question_recreation_reward_func/std": 0.34804321825504303,
+      "rewards/final_correctness_reward_func/mean": 0.5,
+      "rewards/final_correctness_reward_func/std": 0.5773502588272095,
+      "rewards/question_recreation_reward_func/mean": 0.06859164033085108,
+      "rewards/question_recreation_reward_func/std": 0.00623092451132834,
       "rewards/soft_format_reward_func/mean": 0.0,
       "rewards/soft_format_reward_func/std": 0.0,
-      "rewards/strict_format_reward_func/mean": 0.25,
-      "rewards/strict_format_reward_func/std": 0.28867512941360474,
-      "rewards/xmlcount_reward_func/mean": 1.125,
-      "rewards/xmlcount_reward_func/std": 0.14433756470680237,
+      "rewards/strict_format_reward_func/mean": 0.0,
+      "rewards/strict_format_reward_func/std": 0.0,
+      "rewards/xmlcount_reward_func/mean": 0.24799999594688416,
+      "rewards/xmlcount_reward_func/std": 0.0,
       "step": 8
     },
     {
@@ -172,36 +172,36 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 112.0,
-      "completions/max_terminated_length": 112.0,
-      "completions/mean_length": 97.0,
-      "completions/mean_terminated_length": 97.0,
-      "completions/min_length": 89.5,
-      "completions/min_terminated_length": 89.5,
-      "epoch": 9.666666666666666,
-      "grad_norm": 0.0,
-      "kl": 0.0,
+      "completions/max_length": 21.0,
+      "completions/max_terminated_length": 21.0,
+      "completions/mean_length": 20.75,
+      "completions/mean_terminated_length": 20.75,
+      "completions/min_length": 20.5,
+      "completions/min_terminated_length": 20.5,
+      "epoch": 4.8,
+      "grad_norm": 8.56333404186671e-09,
+      "kl": 9.624730929802539e-06,
       "learning_rate": 3.1137137178519977e-07,
       "loss": 0.0,
-      "num_tokens": 14378.0,
-      "reward": 7.34932804107666,
+      "num_tokens": 11079.0,
+      "reward": 1.3171695470809937,
       "reward_std": 0.0,
-      "rewards/concensus_correctness_reward_func/mean": 1.940500020980835,
-      "rewards/concensus_correctness_reward_func/std": 1.1298744678497314,
-      "rewards/consensus_reward_func/mean": 1.5,
-      "rewards/consensus_reward_func/std": 0.5773502588272095,
+      "rewards/concensus_correctness_reward_func/mean": 0.0,
+      "rewards/concensus_correctness_reward_func/std": 0.0,
+      "rewards/consensus_reward_func/mean": 0.0,
+      "rewards/consensus_reward_func/std": 0.0,
       "rewards/cumulative_reward_2/mean": 0.0,
       "rewards/cumulative_reward_2/std": 0.0,
-      "rewards/final_correctness_reward_func/mean": 2.0,
-      "rewards/final_correctness_reward_func/std": 0.0,
-      "rewards/question_recreation_reward_func/mean": 0.34632833302021027,
-      "rewards/question_recreation_reward_func/std": 0.3486427888274193,
+      "rewards/final_correctness_reward_func/mean": 1.0,
+      "rewards/final_correctness_reward_func/std": 1.154700517654419,
+      "rewards/question_recreation_reward_func/mean": 0.06916956789791584,
+      "rewards/question_recreation_reward_func/std": 0.00016566732665523887,
       "rewards/soft_format_reward_func/mean": 0.0,
       "rewards/soft_format_reward_func/std": 0.0,
-      "rewards/strict_format_reward_func/mean": 0.375,
-      "rewards/strict_format_reward_func/std": 0.14433756470680237,
-      "rewards/xmlcount_reward_func/mean": 1.1875,
-      "rewards/xmlcount_reward_func/std": 0.07216878235340118,
+      "rewards/strict_format_reward_func/mean": 0.0,
+      "rewards/strict_format_reward_func/std": 0.0,
+      "rewards/xmlcount_reward_func/mean": 0.24799999594688416,
+      "rewards/xmlcount_reward_func/std": 0.0,
       "step": 10
     },
     {
@@ -211,36 +211,36 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 129.0,
-      "completions/max_terminated_length": 129.0,
-      "completions/mean_length": 114.16666793823242,
-      "completions/mean_terminated_length": 114.16666793823242,
-      "completions/min_length": 89.5,
-      "completions/min_terminated_length": 89.5,
-      "epoch": 11.666666666666666,
-      "grad_norm": 0.0,
-      "kl": 0.0,
+      "completions/max_length": 22.0,
+      "completions/max_terminated_length": 22.0,
+      "completions/mean_length": 21.25,
+      "completions/mean_terminated_length": 21.25,
+      "completions/min_length": 20.5,
+      "completions/min_terminated_length": 20.5,
+      "epoch": 5.8,
+      "grad_norm": 2.63284931634189e-07,
+      "kl": 0.004934789435364628,
       "learning_rate": 2.2935516363191693e-07,
       "loss": 0.0,
-      "num_tokens": 17290.0,
-      "reward": 6.849369525909424,
+      "num_tokens": 13297.0,
+      "reward": 0.7760129868984222,
       "reward_std": 0.0,
-      "rewards/concensus_correctness_reward_func/mean": 1.9579999446868896,
-      "rewards/concensus_correctness_reward_func/std": 2.2609035968780518,
-      "rewards/consensus_reward_func/mean": 1.0,
-      "rewards/consensus_reward_func/std": 1.154700517654419,
+      "rewards/concensus_correctness_reward_func/mean": 0.0,
+      "rewards/concensus_correctness_reward_func/std": 0.0,
+      "rewards/consensus_reward_func/mean": 0.0,
+      "rewards/consensus_reward_func/std": 0.0,
       "rewards/cumulative_reward_2/mean": 0.0,
       "rewards/cumulative_reward_2/std": 0.0,
-      "rewards/final_correctness_reward_func/mean": 2.0,
-      "rewards/final_correctness_reward_func/std": 0.0,
-      "rewards/question_recreation_reward_func/mean": 0.5163697898387909,
-      "rewards/question_recreation_reward_func/std": 0.5474509298801422,
+      "rewards/final_correctness_reward_func/mean": 0.5,
+      "rewards/final_correctness_reward_func/std": 0.5773502588272095,
+      "rewards/question_recreation_reward_func/mean": 0.02801300771534443,
+      "rewards/question_recreation_reward_func/std": 0.0014285182114690542,
       "rewards/soft_format_reward_func/mean": 0.0,
       "rewards/soft_format_reward_func/std": 0.0,
-      "rewards/strict_format_reward_func/mean": 0.25,
-      "rewards/strict_format_reward_func/std": 0.28867512941360474,
-      "rewards/xmlcount_reward_func/mean": 1.125,
-      "rewards/xmlcount_reward_func/std": 0.14433756470680237,
+      "rewards/strict_format_reward_func/mean": 0.0,
+      "rewards/strict_format_reward_func/std": 0.0,
+      "rewards/xmlcount_reward_func/mean": 0.24799999594688416,
+      "rewards/xmlcount_reward_func/std": 0.0,
       "step": 12
     },
     {
@@ -250,35 +250,35 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 101.0,
-      "completions/max_terminated_length": 101.0,
-      "completions/mean_length": 85.66666412353516,
-      "completions/mean_terminated_length": 85.66666412353516,
-      "completions/min_length": 78.0,
-      "completions/min_terminated_length": 78.0,
-      "epoch": 13.666666666666666,
-      "grad_norm": 0.0,
-      "kl": 0.0,
+      "completions/max_length": 21.0,
+      "completions/max_terminated_length": 21.0,
+      "completions/mean_length": 20.75,
+      "completions/mean_terminated_length": 20.75,
+      "completions/min_length": 20.5,
+      "completions/min_terminated_length": 20.5,
+      "epoch": 6.8,
+      "grad_norm": 1.60744309596339e-06,
+      "kl": 0.004935034728207999,
       "learning_rate": 1.4957614383675767e-07,
       "loss": 0.0,
-      "num_tokens": 20054.0,
-      "reward": 7.936056613922119,
+      "num_tokens": 15515.0,
+      "reward": 1.3190129548311234,
       "reward_std": 0.0,
-      "rewards/concensus_correctness_reward_func/mean": 1.9240000247955322,
+      "rewards/concensus_correctness_reward_func/mean": 0.0,
       "rewards/concensus_correctness_reward_func/std": 0.0,
-      "rewards/consensus_reward_func/mean": 2.0,
+      "rewards/consensus_reward_func/mean": 0.0,
       "rewards/consensus_reward_func/std": 0.0,
       "rewards/cumulative_reward_2/mean": 0.0,
       "rewards/cumulative_reward_2/std": 0.0,
-      "rewards/final_correctness_reward_func/mean": 2.0,
+      "rewards/final_correctness_reward_func/mean": 1.0,
       "rewards/final_correctness_reward_func/std": 0.0,
-      "rewards/question_recreation_reward_func/mean": 0.2620568871498108,
-      "rewards/question_recreation_reward_func/std": 0.24471749365329742,
+      "rewards/question_recreation_reward_func/mean": 0.07101300172507763,
+      "rewards/question_recreation_reward_func/std": 0.0003500293714751024,
       "rewards/soft_format_reward_func/mean": 0.0,
       "rewards/soft_format_reward_func/std": 0.0,
-      "rewards/strict_format_reward_func/mean": 0.5,
+      "rewards/strict_format_reward_func/mean": 0.0,
       "rewards/strict_format_reward_func/std": 0.0,
-      "rewards/xmlcount_reward_func/mean": 1.25,
+      "rewards/xmlcount_reward_func/mean": 0.24799999594688416,
       "rewards/xmlcount_reward_func/std": 0.0,
       "step": 14
     },
@@ -289,36 +289,36 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 125.0,
-      "completions/max_terminated_length": 125.0,
-      "completions/mean_length": 108.83333206176758,
-      "completions/mean_terminated_length": 108.83333206176758,
-      "completions/min_length": 89.5,
-      "completions/min_terminated_length": 89.5,
-      "epoch": 15.666666666666666,
-      "grad_norm": 0.0,
-      "kl": 0.0,
+      "completions/max_length": 21.0,
+      "completions/max_terminated_length": 21.0,
+      "completions/mean_length": 20.75,
+      "completions/mean_terminated_length": 20.75,
+      "completions/min_length": 20.5,
+      "completions/min_terminated_length": 20.5,
+      "epoch": 7.8,
+      "grad_norm": 2.496720981071121e-07,
+      "kl": 9.651695065837629e-06,
       "learning_rate": 8.067960709356478e-08,
       "loss": 0.0,
-      "num_tokens": 22956.0,
-      "reward": 6.805034160614014,
+      "num_tokens": 17729.0,
+      "reward": 1.3170444816350937,
       "reward_std": 0.0,
-      "rewards/concensus_correctness_reward_func/mean": 1.9574999809265137,
-      "rewards/concensus_correctness_reward_func/std": 2.2603262662887573,
-      "rewards/consensus_reward_func/mean": 1.0,
-      "rewards/consensus_reward_func/std": 1.154700517654419,
+      "rewards/concensus_correctness_reward_func/mean": 0.0,
+      "rewards/concensus_correctness_reward_func/std": 0.0,
+      "rewards/consensus_reward_func/mean": 0.0,
+      "rewards/consensus_reward_func/std": 0.0,
       "rewards/cumulative_reward_2/mean": 0.0,
       "rewards/cumulative_reward_2/std": 0.0,
-      "rewards/final_correctness_reward_func/mean": 2.0,
+      "rewards/final_correctness_reward_func/mean": 1.0,
       "rewards/final_correctness_reward_func/std": 0.0,
-      "rewards/question_recreation_reward_func/mean": 0.47253431379795074,
-      "rewards/question_recreation_reward_func/std": 0.5011070221662521,
+      "rewards/question_recreation_reward_func/mean": 0.06904449872672558,
+      "rewards/question_recreation_reward_func/std": 2.124987804563716e-05,
       "rewards/soft_format_reward_func/mean": 0.0,
       "rewards/soft_format_reward_func/std": 0.0,
-      "rewards/strict_format_reward_func/mean": 0.25,
-      "rewards/strict_format_reward_func/std": 0.28867512941360474,
-      "rewards/xmlcount_reward_func/mean": 1.125,
-      "rewards/xmlcount_reward_func/std": 0.14433756470680237,
+      "rewards/strict_format_reward_func/mean": 0.0,
+      "rewards/strict_format_reward_func/std": 0.0,
+      "rewards/xmlcount_reward_func/mean": 0.24799999594688416,
+      "rewards/xmlcount_reward_func/std": 0.0,
       "step": 16
     },
     {
@@ -328,36 +328,36 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 125.0,
-      "completions/max_terminated_length": 125.0,
-      "completions/mean_length": 101.0,
-      "completions/mean_terminated_length": 101.0,
-      "completions/min_length": 77.5,
-      "completions/min_terminated_length": 77.5,
-      "epoch": 17.666666666666668,
-      "grad_norm": 5.54299783706665,
-      "kl": 0.0,
+      "completions/max_length": 20.5,
+      "completions/max_terminated_length": 20.5,
+      "completions/mean_length": 20.5,
+      "completions/mean_terminated_length": 20.5,
+      "completions/min_length": 20.5,
+      "completions/min_terminated_length": 20.5,
+      "epoch": 8.8,
+      "grad_norm": 4.384970608839467e-09,
+      "kl": 9.651695065837629e-06,
       "learning_rate": 3.013156219837776e-08,
-      "loss": -0.0011,
-      "num_tokens": 25826.0,
-      "reward": 6.22217059135437,
-      "reward_std": 0.0022791330702602863,
-      "rewards/concensus_correctness_reward_func/mean": 1.4599999785423279,
-      "rewards/concensus_correctness_reward_func/std": 1.6858627796173096,
-      "rewards/consensus_reward_func/mean": 1.0,
-      "rewards/consensus_reward_func/std": 1.154700517654419,
+      "loss": 0.0,
+      "num_tokens": 19943.0,
+      "reward": 1.3192957043647766,
+      "reward_std": 0.0,
+      "rewards/concensus_correctness_reward_func/mean": 0.0,
+      "rewards/concensus_correctness_reward_func/std": 0.0,
+      "rewards/consensus_reward_func/mean": 0.0,
+      "rewards/consensus_reward_func/std": 0.0,
       "rewards/cumulative_reward_2/mean": 0.0,
       "rewards/cumulative_reward_2/std": 0.0,
-      "rewards/final_correctness_reward_func/mean": 2.0,
-      "rewards/final_correctness_reward_func/std": 0.0,
-      "rewards/question_recreation_reward_func/mean": 0.3871704339981079,
-      "rewards/question_recreation_reward_func/std": 0.39475004374980927,
+      "rewards/final_correctness_reward_func/mean": 1.0,
+      "rewards/final_correctness_reward_func/std": 1.154700517654419,
+      "rewards/question_recreation_reward_func/mean": 0.07129578851163387,
+      "rewards/question_recreation_reward_func/std": 2.34974613704253e-05,
       "rewards/soft_format_reward_func/mean": 0.0,
       "rewards/soft_format_reward_func/std": 0.0,
-      "rewards/strict_format_reward_func/mean": 0.25,
-      "rewards/strict_format_reward_func/std": 0.28867512941360474,
-      "rewards/xmlcount_reward_func/mean": 1.125,
-      "rewards/xmlcount_reward_func/std": 0.14433756470680237,
+      "rewards/strict_format_reward_func/mean": 0.0,
+      "rewards/strict_format_reward_func/std": 0.0,
+      "rewards/xmlcount_reward_func/mean": 0.24799999594688416,
+      "rewards/xmlcount_reward_func/std": 0.0,
       "step": 18
     },
     {
@@ -367,52 +367,52 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 125.0,
-      "completions/max_terminated_length": 125.0,
-      "completions/mean_length": 105.16666412353516,
-      "completions/mean_terminated_length": 105.16666412353516,
-      "completions/min_length": 89.5,
-      "completions/min_terminated_length": 89.5,
-      "epoch": 19.666666666666668,
-      "grad_norm": 7.460154847649392e-06,
-      "kl": 9.199701889173184e-09,
+      "completions/max_length": 22.0,
+      "completions/max_terminated_length": 22.0,
+      "completions/mean_length": 21.25,
+      "completions/mean_terminated_length": 21.25,
+      "completions/min_length": 20.5,
+      "completions/min_terminated_length": 20.5,
+      "epoch": 9.8,
+      "grad_norm": 7.085214193125466e-09,
+      "kl": 0.00492547295011958,
       "learning_rate": 3.4096741493194193e-09,
       "loss": 0.0,
-      "num_tokens": 28731.0,
-      "reward": 6.225682735443115,
+      "num_tokens": 22161.0,
+      "reward": 1.2748364806175232,
       "reward_std": 0.0,
-      "rewards/concensus_correctness_reward_func/mean": 1.4599999785423279,
-      "rewards/concensus_correctness_reward_func/std": 1.6858627796173096,
-      "rewards/consensus_reward_func/mean": 1.0,
-      "rewards/consensus_reward_func/std": 1.154700517654419,
+      "rewards/concensus_correctness_reward_func/mean": 0.0,
+      "rewards/concensus_correctness_reward_func/std": 0.0,
+      "rewards/consensus_reward_func/mean": 0.0,
+      "rewards/consensus_reward_func/std": 0.0,
       "rewards/cumulative_reward_2/mean": 0.0,
       "rewards/cumulative_reward_2/std": 0.0,
-      "rewards/final_correctness_reward_func/mean": 2.0,
-      "rewards/final_correctness_reward_func/std": 0.0,
-      "rewards/question_recreation_reward_func/mean": 0.39068305492401123,
-      "rewards/question_recreation_reward_func/std": 0.3943871259689331,
+      "rewards/final_correctness_reward_func/mean": 1.0,
+      "rewards/final_correctness_reward_func/std": 1.154700517654419,
+      "rewards/question_recreation_reward_func/mean": 0.026836536824703217,
+      "rewards/question_recreation_reward_func/std": 7.004663348197937e-05,
       "rewards/soft_format_reward_func/mean": 0.0,
       "rewards/soft_format_reward_func/std": 0.0,
-      "rewards/strict_format_reward_func/mean": 0.25,
-      "rewards/strict_format_reward_func/std": 0.28867512941360474,
-      "rewards/xmlcount_reward_func/mean": 1.125,
-      "rewards/xmlcount_reward_func/std": 0.14433756470680237,
+      "rewards/strict_format_reward_func/mean": 0.0,
+      "rewards/strict_format_reward_func/std": 0.0,
+      "rewards/xmlcount_reward_func/mean": 0.24799999594688416,
+      "rewards/xmlcount_reward_func/std": 0.0,
       "step": 20
     },
     {
-      "epoch": 19.666666666666668,
+      "epoch": 9.8,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": -0.00011151888082939157,
-      "train_runtime": 1866.5324,
-      "train_samples_per_second": 0.043,
-      "train_steps_per_second": 0.011
+      "train_loss": -0.003300715550648903,
+      "train_runtime": 932.3793,
+      "train_samples_per_second": 0.086,
+      "train_steps_per_second": 0.021
     }
   ],
   "logging_steps": 2,
   "max_steps": 20,
-  "num_input_tokens_seen": 28731,
-  "num_train_epochs": 20,
+  "num_input_tokens_seen": 22161,
+  "num_train_epochs": 10,
   "save_steps": 25,
   "stateful_callbacks": {
     "TrainerControl": {