End of training

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.012940555065870284,
-    "train_runtime": 4717.7232,
-    "train_samples": 3,
-    "train_samples_per_second": 0.017,
-    "train_steps_per_second": 0.004
+    "train_loss": 0.037779451161623,
+    "train_runtime": 7169.9681,
+    "train_samples": 5,
+    "train_samples_per_second": 0.011,
+    "train_steps_per_second": 0.003
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:006bbd06767f2805a055debcec52ce7472120081e31468d9fd14e0ba06b082b5
+oid sha256:adecc22f303850912efbf20277ad42ab67f6089c6d70133581f073f6e2538487
 size 1976163472

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.012940555065870284,
-    "train_runtime": 4717.7232,
-    "train_samples": 3,
-    "train_samples_per_second": 0.017,
-    "train_steps_per_second": 0.004
+    "train_loss": 0.037779451161623,
+    "train_runtime": 7169.9681,
+    "train_samples": 5,
+    "train_samples_per_second": 0.011,
+    "train_steps_per_second": 0.003
 }

trainer_state.json

@@ -2,7 +2,7 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 19.666666666666668,
+  "epoch": 9.8,
   "eval_steps": 500,
   "global_step": 20,
   "is_hyper_param_search": false,
@@ -15,37 +15,37 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 152.0,
-      "completions/max_terminated_length": 152.0,
-      "completions/mean_length": 113.66666793823242,
-      "completions/mean_terminated_length": 113.66666793823242,
-      "completions/min_length": 69.5,
-      "completions/min_terminated_length": 69.5,
-      "epoch": 1.6666666666666665,
-      "grad_norm": 13.65064525604248,
+      "completions/clipped_ratio": 0.125,
+      "completions/max_length": 302.0,
+      "completions/max_terminated_length": 92.5,
+      "completions/mean_length": 123.375,
+      "completions/mean_terminated_length": 63.83333396911621,
+      "completions/min_length": 48.0,
+      "completions/min_terminated_length": 48.0,
+      "epoch": 0.8,
+      "grad_norm": 8.92754077911377,
       "kl": 0.0,
       "learning_rate": 5e-07,
-      "loss": 0.094,
-      "num_tokens": 3132.0,
-      "reward": 1.9971750974655151,
-      "reward_std": 1.0149143934249878,
-      "rewards/concensus_correctness_reward_func/mean": 0.24050000309944153,
-      "rewards/concensus_correctness_reward_func/std": 0.48100003600120544,
-      "rewards/consensus_reward_func/mean": 0.25,
-      "rewards/consensus_reward_func/std": 0.5,
+      "loss": -0.1218,
+      "num_tokens": 3214.0,
+      "reward": 0.3856615126132965,
+      "reward_std": 0.7816920205950737,
+      "rewards/concensus_correctness_reward_func/mean": 0.4375,
+      "rewards/concensus_correctness_reward_func/std": 0.875,
+      "rewards/consensus_reward_func/mean": 0.0,
+      "rewards/consensus_reward_func/std": 0.0,
       "rewards/cumulative_reward_2/mean": 0.0,
       "rewards/cumulative_reward_2/std": 0.0,
       "rewards/final_correctness_reward_func/mean": 0.0,
       "rewards/final_correctness_reward_func/std": 0.0,
-      "rewards/question_recreation_reward_func/mean": 0.1785501390695572,
-      "rewards/question_recreation_reward_func/std": 0.24844325333833694,
+      "rewards/question_recreation_reward_func/mean": 0.02203650400042534,
+      "rewards/question_recreation_reward_func/std": 0.023317964747548103,
       "rewards/soft_format_reward_func/mean": 0.0,
       "rewards/soft_format_reward_func/std": 0.0,
-      "rewards/strict_format_reward_func/mean": 0.25,
-      "rewards/strict_format_reward_func/std": 0.28867512941360474,
-      "rewards/xmlcount_reward_func/mean": 1.078125,
-      "rewards/xmlcount_reward_func/std": 0.3098391965031624,
+      "rewards/strict_format_reward_func/mean": 0.0,
+      "rewards/strict_format_reward_func/std": 0.0,
+      "rewards/xmlcount_reward_func/mean": -0.07387500256299973,
+      "rewards/xmlcount_reward_func/std": 0.269045926630497,
       "step": 2
     },
     {
@@ -55,36 +55,36 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 121.5,
-      "completions/max_terminated_length": 121.5,
-      "completions/mean_length": 93.16666793823242,
-      "completions/mean_terminated_length": 93.16666793823242,
-      "completions/min_length": 66.0,
-      "completions/min_terminated_length": 66.0,
-      "epoch": 3.6666666666666665,
-      "grad_norm": 10.07153606414795,
-      "kl": 0.003774605953367427,
+      "completions/max_length": 204.5,
+      "completions/max_terminated_length": 204.5,
+      "completions/mean_length": 108.875,
+      "completions/mean_terminated_length": 108.875,
+      "completions/min_length": 34.0,
+      "completions/min_terminated_length": 34.0,
+      "epoch": 1.8,
+      "grad_norm": 14.86384391784668,
+      "kl": 0.001416065962985158,
       "learning_rate": 4.864543104251586e-07,
-      "loss": 0.0403,
-      "num_tokens": 6059.0,
-      "reward": 2.220690667629242,
-      "reward_std": 0.9036273024976254,
-      "rewards/concensus_correctness_reward_func/mean": 0.24050000309944153,
-      "rewards/concensus_correctness_reward_func/std": 0.48100003600120544,
-      "rewards/consensus_reward_func/mean": 0.25,
-      "rewards/consensus_reward_func/std": 0.5,
+      "loss": 0.3279,
+      "num_tokens": 6661.0,
+      "reward": 0.022667515091598034,
+      "reward_std": 0.010929046286037192,
+      "rewards/concensus_correctness_reward_func/mean": 0.0,
+      "rewards/concensus_correctness_reward_func/std": 0.0,
+      "rewards/consensus_reward_func/mean": 0.0,
+      "rewards/consensus_reward_func/std": 0.0,
       "rewards/cumulative_reward_2/mean": 0.0,
       "rewards/cumulative_reward_2/std": 0.0,
       "rewards/final_correctness_reward_func/mean": 0.0,
       "rewards/final_correctness_reward_func/std": 0.0,
-      "rewards/question_recreation_reward_func/mean": 0.1989407055079937,
-      "rewards/question_recreation_reward_func/std": 0.28088878467679024,
+      "rewards/question_recreation_reward_func/mean": 0.015667515341192484,
+      "rewards/question_recreation_reward_func/std": 0.010519207920879126,
       "rewards/soft_format_reward_func/mean": 0.0,
       "rewards/soft_format_reward_func/std": 0.0,
-      "rewards/strict_format_reward_func/mean": 0.375,
-      "rewards/strict_format_reward_func/std": 0.25,
-      "rewards/xmlcount_reward_func/mean": 1.15625,
-      "rewards/xmlcount_reward_func/std": 0.1875,
+      "rewards/strict_format_reward_func/mean": 0.0,
+      "rewards/strict_format_reward_func/std": 0.0,
+      "rewards/xmlcount_reward_func/mean": 0.007000000216066837,
+      "rewards/xmlcount_reward_func/std": 0.01400000136345625,
       "step": 4
     },
     {
@@ -93,37 +93,37 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 159.0,
-      "completions/max_terminated_length": 159.0,
-      "completions/mean_length": 113.83333206176758,
-      "completions/mean_terminated_length": 113.83333206176758,
-      "completions/min_length": 85.0,
-      "completions/min_terminated_length": 85.0,
-      "epoch": 5.666666666666667,
-      "grad_norm": 18.319076538085938,
-      "kl": 0.02211279346374795,
+      "completions/clipped_ratio": 0.5,
+      "completions/max_length": 412.0,
+      "completions/max_terminated_length": 156.0,
+      "completions/mean_length": 316.125,
+      "completions/mean_terminated_length": 60.125,
+      "completions/min_length": 270.5,
+      "completions/min_terminated_length": 14.5,
+      "epoch": 2.8,
+      "grad_norm": 7.786389350891113,
+      "kl": 0.0026882924139499664,
       "learning_rate": 4.472851273490984e-07,
-      "loss": -0.1592,
-      "num_tokens": 9065.0,
-      "reward": 3.014554500579834,
-      "reward_std": 0.5706639671698213,
-      "rewards/concensus_correctness_reward_func/mean": 0.7215000092983246,
-      "rewards/concensus_correctness_reward_func/std": 1.0364110171794891,
-      "rewards/consensus_reward_func/mean": 0.75,
-      "rewards/consensus_reward_func/std": 1.0773502588272095,
+      "loss": 0.178,
+      "num_tokens": 10480.0,
+      "reward": 0.030308596324175596,
+      "reward_std": 0.0613415464758873,
+      "rewards/concensus_correctness_reward_func/mean": 0.0,
+      "rewards/concensus_correctness_reward_func/std": 0.0,
+      "rewards/consensus_reward_func/mean": 0.0,
+      "rewards/consensus_reward_func/std": 0.0,
       "rewards/cumulative_reward_2/mean": 0.0,
       "rewards/cumulative_reward_2/std": 0.0,
       "rewards/final_correctness_reward_func/mean": 0.0,
       "rewards/final_correctness_reward_func/std": 0.0,
-      "rewards/question_recreation_reward_func/mean": 0.07430431246757507,
-      "rewards/question_recreation_reward_func/std": 0.05623785126954317,
+      "rewards/question_recreation_reward_func/mean": 0.013933598063886166,
+      "rewards/question_recreation_reward_func/std": 0.014677805360406637,
       "rewards/soft_format_reward_func/mean": 0.0,
       "rewards/soft_format_reward_func/std": 0.0,
-      "rewards/strict_format_reward_func/mean": 0.3125,
-      "rewards/strict_format_reward_func/std": 0.125,
-      "rewards/xmlcount_reward_func/mean": 1.15625,
-      "rewards/xmlcount_reward_func/std": 0.0625,
+      "rewards/strict_format_reward_func/mean": 0.0,
+      "rewards/strict_format_reward_func/std": 0.0,
+      "rewards/xmlcount_reward_func/mean": 0.016375000588595867,
+      "rewards/xmlcount_reward_func/std": 0.051155371591448784,
       "step": 6
     },
     {
@@ -132,37 +132,37 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.16666666666666669,
-      "completions/max_length": 316.0,
-      "completions/max_terminated_length": 130.5,
-      "completions/mean_length": 180.8333282470703,
-      "completions/mean_terminated_length": 118.0,
-      "completions/min_length": 101.5,
-      "completions/min_terminated_length": 101.5,
-      "epoch": 7.666666666666667,
-      "grad_norm": 9.818954467773438,
-      "kl": 0.01149655319750309,
+      "completions/clipped_ratio": 0.5,
+      "completions/max_length": 447.5,
+      "completions/max_terminated_length": 191.5,
+      "completions/mean_length": 309.25,
+      "completions/mean_terminated_length": 53.25,
+      "completions/min_length": 258.0,
+      "completions/min_terminated_length": 2.0,
+      "epoch": 3.8,
+      "grad_norm": 5.402748107910156,
+      "kl": 0.005952609681116883,
       "learning_rate": 3.867370395306068e-07,
-      "loss": 0.0247,
-      "num_tokens": 12399.0,
-      "reward": 2.7228788137435913,
-      "reward_std": 1.667017936706543,
-      "rewards/concensus_correctness_reward_func/mean": 0.478999987244606,
-      "rewards/concensus_correctness_reward_func/std": 0.9579999446868896,
-      "rewards/consensus_reward_func/mean": 0.5,
-      "rewards/consensus_reward_func/std": 1.0,
+      "loss": -0.1153,
+      "num_tokens": 14444.0,
+      "reward": 0.01363831665366888,
+      "reward_std": 0.0067086233757436275,
+      "rewards/concensus_correctness_reward_func/mean": 0.0,
+      "rewards/concensus_correctness_reward_func/std": 0.0,
+      "rewards/consensus_reward_func/mean": 0.0,
+      "rewards/consensus_reward_func/std": 0.0,
       "rewards/cumulative_reward_2/mean": 0.0,
       "rewards/cumulative_reward_2/std": 0.0,
       "rewards/final_correctness_reward_func/mean": 0.0,
       "rewards/final_correctness_reward_func/std": 0.0,
-      "rewards/question_recreation_reward_func/mean": 0.19700367003679276,
-      "rewards/question_recreation_reward_func/std": 0.27262982353568077,
+      "rewards/question_recreation_reward_func/mean": 0.01363831665366888,
+      "rewards/question_recreation_reward_func/std": 0.005987111479043961,
       "rewards/soft_format_reward_func/mean": 0.0,
       "rewards/soft_format_reward_func/std": 0.0,
-      "rewards/strict_format_reward_func/mean": 0.375,
-      "rewards/strict_format_reward_func/std": 0.25,
-      "rewards/xmlcount_reward_func/mean": 1.171875,
-      "rewards/xmlcount_reward_func/std": 0.15625,
+      "rewards/strict_format_reward_func/mean": 0.0,
+      "rewards/strict_format_reward_func/std": 0.0,
+      "rewards/xmlcount_reward_func/mean": 0.0,
+      "rewards/xmlcount_reward_func/std": 0.0,
       "step": 8
     },
     {
@@ -172,36 +172,36 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 225.5,
-      "completions/max_terminated_length": 225.5,
-      "completions/mean_length": 149.5,
-      "completions/mean_terminated_length": 149.5,
-      "completions/min_length": 92.0,
-      "completions/min_terminated_length": 92.0,
-      "epoch": 9.666666666666666,
-      "grad_norm": 12.751887321472168,
-      "kl": 0.038790178950876,
+      "completions/max_length": 209.0,
+      "completions/max_terminated_length": 209.0,
+      "completions/mean_length": 181.625,
+      "completions/mean_terminated_length": 181.625,
+      "completions/min_length": 169.5,
+      "completions/min_terminated_length": 169.5,
+      "epoch": 4.8,
+      "grad_norm": 6.952495574951172,
+      "kl": 0.0149484759895131,
       "learning_rate": 3.1137137178519977e-07,
-      "loss": 0.1765,
-      "num_tokens": 15535.0,
-      "reward": 4.8462889194488525,
-      "reward_std": 1.6064382195472717,
-      "rewards/concensus_correctness_reward_func/mean": 1.4369999766349792,
-      "rewards/concensus_correctness_reward_func/std": 0.957999974489212,
-      "rewards/consensus_reward_func/mean": 1.5,
-      "rewards/consensus_reward_func/std": 1.0,
+      "loss": 0.0504,
+      "num_tokens": 17407.0,
+      "reward": 0.23214832320809364,
+      "reward_std": 0.37722403556108475,
+      "rewards/concensus_correctness_reward_func/mean": 0.0,
+      "rewards/concensus_correctness_reward_func/std": 0.0,
+      "rewards/consensus_reward_func/mean": 0.0,
+      "rewards/consensus_reward_func/std": 0.0,
       "rewards/cumulative_reward_2/mean": 0.0,
       "rewards/cumulative_reward_2/std": 0.0,
-      "rewards/final_correctness_reward_func/mean": 0.0,
-      "rewards/final_correctness_reward_func/std": 0.0,
-      "rewards/question_recreation_reward_func/mean": 0.3467889428138733,
-      "rewards/question_recreation_reward_func/std": 0.4016842842102051,
+      "rewards/final_correctness_reward_func/mean": 0.25,
+      "rewards/final_correctness_reward_func/std": 0.5,
+      "rewards/question_recreation_reward_func/mean": 0.015148311853408813,
+      "rewards/question_recreation_reward_func/std": 0.0038266002666205168,
       "rewards/soft_format_reward_func/mean": 0.0,
       "rewards/soft_format_reward_func/std": 0.0,
-      "rewards/strict_format_reward_func/mean": 0.3125,
-      "rewards/strict_format_reward_func/std": 0.26933756470680237,
-      "rewards/xmlcount_reward_func/mean": 1.25,
-      "rewards/xmlcount_reward_func/std": 0.0,
+      "rewards/strict_format_reward_func/mean": 0.0,
+      "rewards/strict_format_reward_func/std": 0.0,
+      "rewards/xmlcount_reward_func/mean": -0.03299999888986349,
+      "rewards/xmlcount_reward_func/std": 0.07268869318068027,
       "step": 10
     },
     {
@@ -210,37 +210,37 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 185.5,
-      "completions/max_terminated_length": 185.5,
-      "completions/mean_length": 126.0,
-      "completions/mean_terminated_length": 126.0,
-      "completions/min_length": 94.5,
-      "completions/min_terminated_length": 94.5,
-      "epoch": 11.666666666666666,
-      "grad_norm": 9.57380485534668,
-      "kl": 0.03469763998873532,
+      "completions/clipped_ratio": 0.125,
+      "completions/max_length": 301.0,
+      "completions/max_terminated_length": 158.0,
+      "completions/mean_length": 160.875,
+      "completions/mean_terminated_length": 114.16666412353516,
+      "completions/min_length": 89.0,
+      "completions/min_terminated_length": 89.0,
+      "epoch": 5.8,
+      "grad_norm": 16.89011573791504,
+      "kl": 0.017503770883195102,
       "learning_rate": 2.2935516363191693e-07,
-      "loss": -0.0734,
-      "num_tokens": 18587.0,
-      "reward": 4.602630615234375,
-      "reward_std": 1.6934871673583984,
-      "rewards/concensus_correctness_reward_func/mean": 1.443000078201294,
-      "rewards/concensus_correctness_reward_func/std": 0.9620000123977661,
-      "rewards/consensus_reward_func/mean": 1.5,
-      "rewards/consensus_reward_func/std": 1.0,
+      "loss": 0.1,
+      "num_tokens": 20938.0,
+      "reward": -0.1888856142759323,
+      "reward_std": 0.33080266416072845,
+      "rewards/concensus_correctness_reward_func/mean": 0.0,
+      "rewards/concensus_correctness_reward_func/std": 0.0,
+      "rewards/consensus_reward_func/mean": 0.0,
+      "rewards/consensus_reward_func/std": 0.0,
       "rewards/cumulative_reward_2/mean": 0.0,
       "rewards/cumulative_reward_2/std": 0.0,
       "rewards/final_correctness_reward_func/mean": 0.0,
       "rewards/final_correctness_reward_func/std": 0.0,
-      "rewards/question_recreation_reward_func/mean": 0.12838061526417732,
-      "rewards/question_recreation_reward_func/std": 0.12217769771814346,
+      "rewards/question_recreation_reward_func/mean": 0.015614384785294533,
+      "rewards/question_recreation_reward_func/std": 0.014846977777779102,
       "rewards/soft_format_reward_func/mean": 0.0,
       "rewards/soft_format_reward_func/std": 0.0,
-      "rewards/strict_format_reward_func/mean": 0.375,
-      "rewards/strict_format_reward_func/std": 0.25,
-      "rewards/xmlcount_reward_func/mean": 1.15625,
-      "rewards/xmlcount_reward_func/std": 0.1875,
+      "rewards/strict_format_reward_func/mean": 0.0,
+      "rewards/strict_format_reward_func/std": 0.0,
+      "rewards/xmlcount_reward_func/mean": -0.20449999906122684,
+      "rewards/xmlcount_reward_func/std": 0.4663179814815521,
       "step": 12
     },
     {
@@ -250,36 +250,36 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 152.0,
-      "completions/max_terminated_length": 152.0,
-      "completions/mean_length": 139.33333587646484,
-      "completions/mean_terminated_length": 139.33333587646484,
-      "completions/min_length": 122.0,
-      "completions/min_terminated_length": 122.0,
-      "epoch": 13.666666666666666,
-      "grad_norm": 9.227038383483887,
-      "kl": 0.04416784085333347,
+      "completions/max_length": 128.5,
+      "completions/max_terminated_length": 128.5,
+      "completions/mean_length": 87.5,
+      "completions/mean_terminated_length": 87.5,
+      "completions/min_length": 30.0,
+      "completions/min_terminated_length": 30.0,
+      "epoch": 6.8,
+      "grad_norm": 20.018957138061523,
+      "kl": 0.025806593243032694,
       "learning_rate": 1.4957614383675767e-07,
-      "loss": 0.0101,
-      "num_tokens": 21700.0,
-      "reward": 4.582450866699219,
-      "reward_std": 2.280969023704529,
-      "rewards/concensus_correctness_reward_func/mean": 1.1999999582767487,
-      "rewards/concensus_correctness_reward_func/std": 1.0342562794685364,
-      "rewards/consensus_reward_func/mean": 1.25,
-      "rewards/consensus_reward_func/std": 1.0773502588272095,
+      "loss": -0.1021,
+      "num_tokens": 23935.0,
+      "reward": 0.05173949897289276,
+      "reward_std": 0.0109914755448699,
+      "rewards/concensus_correctness_reward_func/mean": 0.0,
+      "rewards/concensus_correctness_reward_func/std": 0.0,
+      "rewards/consensus_reward_func/mean": 0.0,
+      "rewards/consensus_reward_func/std": 0.0,
       "rewards/cumulative_reward_2/mean": 0.0,
       "rewards/cumulative_reward_2/std": 0.0,
       "rewards/final_correctness_reward_func/mean": 0.0,
       "rewards/final_correctness_reward_func/std": 0.0,
-      "rewards/question_recreation_reward_func/mean": 0.5230759084224701,
-      "rewards/question_recreation_reward_func/std": 0.5501338243484497,
+      "rewards/question_recreation_reward_func/mean": 0.028114496264606714,
+      "rewards/question_recreation_reward_func/std": 0.025538412854075432,
       "rewards/soft_format_reward_func/mean": 0.0,
       "rewards/soft_format_reward_func/std": 0.0,
-      "rewards/strict_format_reward_func/mean": 0.375,
-      "rewards/strict_format_reward_func/std": 0.14433756470680237,
-      "rewards/xmlcount_reward_func/mean": 1.234375,
-      "rewards/xmlcount_reward_func/std": 0.03125,
+      "rewards/strict_format_reward_func/mean": 0.0,
+      "rewards/strict_format_reward_func/std": 0.0,
+      "rewards/xmlcount_reward_func/mean": 0.02362500037997961,
+      "rewards/xmlcount_reward_func/std": 0.030027911067008972,
       "step": 14
     },
     {
@@ -288,37 +288,37 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.16666666666666669,
-      "completions/max_length": 383.5,
-      "completions/max_terminated_length": 302.5,
-      "completions/mean_length": 249.16666412353516,
-      "completions/mean_terminated_length": 203.58333587646484,
-      "completions/min_length": 124.0,
-      "completions/min_terminated_length": 124.0,
-      "epoch": 15.666666666666666,
-      "grad_norm": 12.671793937683105,
-      "kl": 0.039287379826419055,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 225.5,
+      "completions/max_terminated_length": 225.5,
+      "completions/mean_length": 102.75,
+      "completions/mean_terminated_length": 102.75,
+      "completions/min_length": 47.0,
+      "completions/min_terminated_length": 47.0,
+      "epoch": 7.8,
+      "grad_norm": 18.695772171020508,
+      "kl": 0.02530490467324853,
       "learning_rate": 8.067960709356478e-08,
-      "loss": -0.0068,
-      "num_tokens": 25565.0,
-      "reward": 3.1929807662963867,
-      "reward_std": 0.9375366121530533,
-      "rewards/concensus_correctness_reward_func/mean": 0.7184999883174896,
-      "rewards/concensus_correctness_reward_func/std": 1.032946914434433,
-      "rewards/consensus_reward_func/mean": 0.75,
-      "rewards/consensus_reward_func/std": 1.0773502588272095,
+      "loss": -0.0895,
+      "num_tokens": 27073.0,
+      "reward": -0.059220071882009506,
+      "reward_std": 0.15402239561080933,
+      "rewards/concensus_correctness_reward_func/mean": 0.0,
+      "rewards/concensus_correctness_reward_func/std": 0.0,
+      "rewards/consensus_reward_func/mean": 0.0,
+      "rewards/consensus_reward_func/std": 0.0,
       "rewards/cumulative_reward_2/mean": 0.0,
       "rewards/cumulative_reward_2/std": 0.0,
       "rewards/final_correctness_reward_func/mean": 0.0,
       "rewards/final_correctness_reward_func/std": 0.0,
-      "rewards/question_recreation_reward_func/mean": 0.40010587871074677,
-      "rewards/question_recreation_reward_func/std": 0.45765654742717743,
+      "rewards/question_recreation_reward_func/mean": 0.01565493270754814,
+      "rewards/question_recreation_reward_func/std": 0.009137378772720695,
       "rewards/soft_format_reward_func/mean": 0.0,
       "rewards/soft_format_reward_func/std": 0.0,
-      "rewards/strict_format_reward_func/mean": 0.25,
-      "rewards/strict_format_reward_func/std": 0.28867512941360474,
-      "rewards/xmlcount_reward_func/mean": 1.074375033378601,
-      "rewards/xmlcount_reward_func/std": 0.3512499928474426,
+      "rewards/strict_format_reward_func/mean": 0.0,
+      "rewards/strict_format_reward_func/std": 0.0,
+      "rewards/xmlcount_reward_func/mean": -0.07487499713897705,
+      "rewards/xmlcount_reward_func/std": 0.15669719874858856,
       "step": 16
     },
     {
@@ -327,37 +327,37 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 350.5,
-      "completions/max_terminated_length": 350.5,
-      "completions/mean_length": 220.99999237060547,
-      "completions/mean_terminated_length": 220.99999237060547,
-      "completions/min_length": 98.5,
-      "completions/min_terminated_length": 98.5,
-      "epoch": 17.666666666666668,
-      "grad_norm": 7.500455856323242,
-      "kl": 0.01603590976446867,
+      "completions/clipped_ratio": 0.125,
+      "completions/max_length": 415.0,
+      "completions/max_terminated_length": 206.5,
+      "completions/mean_length": 241.25,
+      "completions/mean_terminated_length": 183.3333339691162,
+      "completions/min_length": 170.0,
+      "completions/min_terminated_length": 170.0,
+      "epoch": 8.8,
+      "grad_norm": 51.03651809692383,
+      "kl": 0.05000046588247642,
       "learning_rate": 3.013156219837776e-08,
-      "loss": 0.0981,
-      "num_tokens": 29204.0,
-      "reward": 5.051450490951538,
-      "reward_std": 1.452622190117836,
-      "rewards/concensus_correctness_reward_func/mean": 1.6819999814033508,
-      "rewards/concensus_correctness_reward_func/std": 0.48000001907348633,
-      "rewards/consensus_reward_func/mean": 1.75,
-      "rewards/consensus_reward_func/std": 0.5,
+      "loss": -0.0637,
+      "num_tokens": 30146.0,
+      "reward": -0.015393424779176712,
+      "reward_std": 0.0688084177672863,
+      "rewards/concensus_correctness_reward_func/mean": 0.0,
+      "rewards/concensus_correctness_reward_func/std": 0.0,
+      "rewards/consensus_reward_func/mean": 0.0,
+      "rewards/consensus_reward_func/std": 0.0,
       "rewards/cumulative_reward_2/mean": 0.0,
       "rewards/cumulative_reward_2/std": 0.0,
       "rewards/final_correctness_reward_func/mean": 0.0,
       "rewards/final_correctness_reward_func/std": 0.0,
-      "rewards/question_recreation_reward_func/mean": 0.2913255840539932,
-      "rewards/question_recreation_reward_func/std": 0.4711288511753082,
+      "rewards/question_recreation_reward_func/mean": 0.015356574673205614,
+      "rewards/question_recreation_reward_func/std": 0.013046635314822197,
       "rewards/soft_format_reward_func/mean": 0.0,
       "rewards/soft_format_reward_func/std": 0.0,
-      "rewards/strict_format_reward_func/mean": 0.3125,
-      "rewards/strict_format_reward_func/std": 0.26933756470680237,
-      "rewards/xmlcount_reward_func/mean": 1.015625,
-      "rewards/xmlcount_reward_func/std": 0.2991959750652313,
+      "rewards/strict_format_reward_func/mean": 0.0,
+      "rewards/strict_format_reward_func/std": 0.0,
+      "rewards/xmlcount_reward_func/mean": -0.03074999898672104,
+      "rewards/xmlcount_reward_func/std": 0.11776353418827057,
       "step": 18
     },
     {
@@ -366,53 +366,53 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 191.0,
-      "completions/max_terminated_length": 191.0,
-      "completions/mean_length": 154.83333587646484,
-      "completions/mean_terminated_length": 154.83333587646484,
-      "completions/min_length": 128.0,
-      "completions/min_terminated_length": 128.0,
-      "epoch": 19.666666666666668,
-      "grad_norm": 5.4542059898376465,
-      "kl": 0.0275694252923131,
+      "completions/clipped_ratio": 0.125,
+      "completions/max_length": 265.5,
+      "completions/max_terminated_length": 56.5,
+      "completions/mean_length": 97.625,
+      "completions/mean_terminated_length": 41.66666793823242,
+      "completions/min_length": 34.0,
+      "completions/min_terminated_length": 34.0,
+      "epoch": 9.8,
+      "grad_norm": 14.597654342651367,
+      "kl": 0.02688464312814176,
       "learning_rate": 3.4096741493194193e-09,
-      "loss": -0.0749,
-      "num_tokens": 32380.0,
-      "reward": 5.182299613952637,
-      "reward_std": 0.9995275139808655,
-      "rewards/concensus_correctness_reward_func/mean": 1.6819999814033508,
-      "rewards/concensus_correctness_reward_func/std": 0.47999998927116394,
-      "rewards/consensus_reward_func/mean": 1.75,
-      "rewards/consensus_reward_func/std": 0.5,
+      "loss": 0.214,
+      "num_tokens": 33969.0,
+      "reward": -0.003884643316268921,
+      "reward_std": 0.04420278873294592,
+      "rewards/concensus_correctness_reward_func/mean": 0.0,
+      "rewards/concensus_correctness_reward_func/std": 0.0,
+      "rewards/consensus_reward_func/mean": 0.0,
+      "rewards/consensus_reward_func/std": 0.0,
       "rewards/cumulative_reward_2/mean": 0.0,
       "rewards/cumulative_reward_2/std": 0.0,
       "rewards/final_correctness_reward_func/mean": 0.0,
       "rewards/final_correctness_reward_func/std": 0.0,
-      "rewards/question_recreation_reward_func/mean": 0.2502993941307068,
-      "rewards/question_recreation_reward_func/std": 0.3478831350803375,
+      "rewards/question_recreation_reward_func/mean": 0.020615354413166642,
+      "rewards/question_recreation_reward_func/std": 0.019722969736903906,
       "rewards/soft_format_reward_func/mean": 0.0,
       "rewards/soft_format_reward_func/std": 0.0,
-      "rewards/strict_format_reward_func/mean": 0.3125,
-      "rewards/strict_format_reward_func/std": 0.26933756470680237,
-      "rewards/xmlcount_reward_func/mean": 1.1875,
-      "rewards/xmlcount_reward_func/std": 0.0883883461356163,
+      "rewards/strict_format_reward_func/mean": 0.0,
+      "rewards/strict_format_reward_func/std": 0.0,
+      "rewards/xmlcount_reward_func/mean": -0.02449999935925007,
+      "rewards/xmlcount_reward_func/std": 0.04899999871850014,
       "step": 20
     },
     {
-      "epoch": 19.666666666666668,
+      "epoch": 9.8,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 0.012940555065870284,
-      "train_runtime": 4717.7232,
-      "train_samples_per_second": 0.017,
-      "train_steps_per_second": 0.004
+      "train_loss": 0.037779451161623,
+      "train_runtime": 7169.9681,
+      "train_samples_per_second": 0.011,
+      "train_steps_per_second": 0.003
     }
   ],
   "logging_steps": 2,
   "max_steps": 20,
-  "num_input_tokens_seen": 32380,
-  "num_train_epochs": 20,
+  "num_input_tokens_seen": 33969,
+  "num_train_epochs": 10,
   "save_steps": 25,
   "stateful_callbacks": {
     "TrainerControl": {