End of training

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.06928728232160211,
-    "train_runtime": 1855.3992,
+    "train_loss": 0.089614911028184,
+    "train_runtime": 1740.8949,
     "train_samples": 5,
-    "train_samples_per_second": 0.086,
+    "train_samples_per_second": 0.092,
     "train_steps_per_second": 0.011
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:95bb083789701670cbe94151fb0f00fd1f922d7f1c1a6738a7e0f2e783ce15bf
+oid sha256:25d1edacd0fbd8d8c1bfbae613a6179e41755b475c67376e997c751e09272a24
 size 1976163472

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.06928728232160211,
-    "train_runtime": 1855.3992,
+    "train_loss": 0.089614911028184,
+    "train_runtime": 1740.8949,
     "train_samples": 5,
-    "train_samples_per_second": 0.086,
+    "train_samples_per_second": 0.092,
     "train_steps_per_second": 0.011
 }

trainer_state.json

@@ -17,29 +17,29 @@
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.5,
       "completions/max_length": 96.0,
-      "completions/max_terminated_length": 53.0,
-      "completions/mean_length": 71.0,
-      "completions/mean_terminated_length": 46.16666603088379,
-      "completions/min_length": 36.5,
-      "completions/min_terminated_length": 36.5,
+      "completions/max_terminated_length": 45.0,
+      "completions/mean_length": 60.00000190734863,
+      "completions/mean_terminated_length": 25.5,
+      "completions/min_length": 4.5,
+      "completions/min_terminated_length": 4.5,
       "epoch": 1.8,
-      "grad_norm": 13.032360076904297,
-      "kl": -1.4287097838128204e-08,
+      "grad_norm": 14.51369857788086,
+      "kl": -1.2729463649829853e-08,
       "learning_rate": 5e-07,
-      "loss": 0.1108,
-      "num_tokens": 2713.0,
-      "reward": 0.12375914584845304,
-      "reward_std": 0.028197589330375195,
+      "loss": -0.023,
+      "num_tokens": 2627.0,
+      "reward": 0.22892774641513824,
+      "reward_std": 0.21130400896072388,
       "rewards/concensus_correctness_reward_func/mean": 0.0,
       "rewards/concensus_correctness_reward_func/std": 0.0,
       "rewards/consensus_reward_func/mean": 0.0,
       "rewards/consensus_reward_func/std": 0.0,
       "rewards/cumulative_reward_2/mean": 0.0,
       "rewards/cumulative_reward_2/std": 0.0,
-      "rewards/final_correctness_reward_func/mean": 0.0,
-      "rewards/final_correctness_reward_func/std": 0.0,
-      "rewards/question_recreation_reward_func/mean": 0.12375915423035622,
-      "rewards/question_recreation_reward_func/std": 0.03597170067951083,
+      "rewards/final_correctness_reward_func/mean": 0.125,
+      "rewards/final_correctness_reward_func/std": 0.3535533845424652,
+      "rewards/question_recreation_reward_func/mean": 0.10392776224762201,
+      "rewards/question_recreation_reward_func/std": 0.04285131534561515,
       "rewards/soft_format_reward_func/mean": 0.0,
       "rewards/soft_format_reward_func/std": 0.0,
       "rewards/strict_format_reward_func/mean": 0.0,
@@ -54,21 +54,21 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.7,
+      "completions/clipped_ratio": 0.9,
       "completions/max_length": 96.0,
-      "completions/max_terminated_length": 54.5,
-      "completions/mean_length": 79.20000076293945,
-      "completions/mean_terminated_length": 43.5,
-      "completions/min_length": 32.5,
-      "completions/min_terminated_length": 32.5,
+      "completions/max_terminated_length": 3.5,
+      "completions/mean_length": 87.0999984741211,
+      "completions/mean_terminated_length": 3.5,
+      "completions/min_length": 51.5,
+      "completions/min_terminated_length": 3.5,
       "epoch": 3.8,
-      "grad_norm": 10.995894432067871,
-      "kl": 0.0007601117586091277,
+      "grad_norm": 9.493478775024414,
+      "kl": 0.001022911081690836,
       "learning_rate": 4.864543104251586e-07,
-      "loss": -0.0212,
-      "num_tokens": 5490.0,
-      "reward": 0.027154644951224327,
-      "reward_std": 0.008858403074555099,
+      "loss": -0.0629,
+      "num_tokens": 5506.0,
+      "reward": 0.0016773659735918045,
+      "reward_std": 0.02505883341655135,
       "rewards/concensus_correctness_reward_func/mean": 0.0,
       "rewards/concensus_correctness_reward_func/std": 0.0,
       "rewards/consensus_reward_func/mean": 0.0,
@@ -77,14 +77,14 @@
       "rewards/cumulative_reward_2/std": 0.0,
       "rewards/final_correctness_reward_func/mean": 0.0,
       "rewards/final_correctness_reward_func/std": 0.0,
-      "rewards/question_recreation_reward_func/mean": 0.027154644951224327,
-      "rewards/question_recreation_reward_func/std": 0.01477902289479971,
+      "rewards/question_recreation_reward_func/mean": 0.015302364714443684,
+      "rewards/question_recreation_reward_func/std": 0.010023764800280333,
       "rewards/soft_format_reward_func/mean": 0.0,
       "rewards/soft_format_reward_func/std": 0.0,
       "rewards/strict_format_reward_func/mean": 0.0,
       "rewards/strict_format_reward_func/std": 0.0,
-      "rewards/xmlcount_reward_func/mean": 0.0,
-      "rewards/xmlcount_reward_func/std": 0.0,
+      "rewards/xmlcount_reward_func/mean": -0.013624999672174454,
+      "rewards/xmlcount_reward_func/std": 0.038537319749593735,
       "step": 4
     },
     {
@@ -95,35 +95,35 @@
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.5,
       "completions/max_length": 96.0,
-      "completions/max_terminated_length": 74.0,
-      "completions/mean_length": 74.29999923706055,
-      "completions/mean_terminated_length": 54.08333396911621,
-      "completions/min_length": 34.0,
-      "completions/min_terminated_length": 34.0,
+      "completions/max_terminated_length": 48.5,
+      "completions/mean_length": 62.70000076293945,
+      "completions/mean_terminated_length": 29.583333015441895,
+      "completions/min_length": 11.0,
+      "completions/min_terminated_length": 11.0,
       "epoch": 5.8,
-      "grad_norm": 15.450472831726074,
-      "kl": 0.005117912223795429,
+      "grad_norm": 13.253321647644043,
+      "kl": 0.007930153716188215,
       "learning_rate": 4.472851273490984e-07,
-      "loss": -0.0534,
-      "num_tokens": 8267.0,
-      "reward": 0.03235254064202309,
-      "reward_std": 0.010788492625579238,
+      "loss": 0.3243,
+      "num_tokens": 8106.0,
+      "reward": 0.2996043562889099,
+      "reward_std": 0.32131071388721466,
       "rewards/concensus_correctness_reward_func/mean": 0.0,
       "rewards/concensus_correctness_reward_func/std": 0.0,
       "rewards/consensus_reward_func/mean": 0.0,
       "rewards/consensus_reward_func/std": 0.0,
       "rewards/cumulative_reward_2/mean": 0.0,
       "rewards/cumulative_reward_2/std": 0.0,
-      "rewards/final_correctness_reward_func/mean": 0.0,
-      "rewards/final_correctness_reward_func/std": 0.0,
-      "rewards/question_recreation_reward_func/mean": 0.03235254157334566,
-      "rewards/question_recreation_reward_func/std": 0.014937533531337976,
+      "rewards/final_correctness_reward_func/mean": 0.25,
+      "rewards/final_correctness_reward_func/std": 0.7071067690849304,
+      "rewards/question_recreation_reward_func/mean": 0.021166879683732986,
+      "rewards/question_recreation_reward_func/std": 0.015909616835415363,
       "rewards/soft_format_reward_func/mean": 0.0,
       "rewards/soft_format_reward_func/std": 0.0,
       "rewards/strict_format_reward_func/mean": 0.0,
       "rewards/strict_format_reward_func/std": 0.0,
-      "rewards/xmlcount_reward_func/mean": 0.0,
-      "rewards/xmlcount_reward_func/std": 0.0,
+      "rewards/xmlcount_reward_func/mean": 0.02843749988824129,
+      "rewards/xmlcount_reward_func/std": 0.06866855919361115,
       "step": 6
     },
     {
@@ -132,37 +132,37 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.3,
+      "completions/clipped_ratio": 0.7,
       "completions/max_length": 96.0,
-      "completions/max_terminated_length": 88.0,
-      "completions/mean_length": 60.39999961853027,
-      "completions/mean_terminated_length": 47.125,
-      "completions/min_length": 18.5,
-      "completions/min_terminated_length": 18.5,
+      "completions/max_terminated_length": 60.0,
+      "completions/mean_length": 84.89999771118164,
+      "completions/mean_terminated_length": 51.0,
+      "completions/min_length": 42.0,
+      "completions/min_terminated_length": 42.0,
       "epoch": 7.8,
-      "grad_norm": 16.364418029785156,
-      "kl": 0.005086680190288462,
+      "grad_norm": 10.583436965942383,
+      "kl": 0.057811224742181366,
       "learning_rate": 3.8673703953060673e-07,
-      "loss": 0.1469,
-      "num_tokens": 10958.0,
-      "reward": 0.024977175518870354,
-      "reward_std": 0.005859495257027447,
+      "loss": 0.0703,
+      "num_tokens": 10879.0,
+      "reward": 0.27628672309219837,
+      "reward_std": 0.022481818683445454,
       "rewards/concensus_correctness_reward_func/mean": 0.0,
       "rewards/concensus_correctness_reward_func/std": 0.0,
       "rewards/consensus_reward_func/mean": 0.0,
       "rewards/consensus_reward_func/std": 0.0,
       "rewards/cumulative_reward_2/mean": 0.0,
       "rewards/cumulative_reward_2/std": 0.0,
-      "rewards/final_correctness_reward_func/mean": 0.0,
-      "rewards/final_correctness_reward_func/std": 0.0,
-      "rewards/question_recreation_reward_func/mean": 0.02497717458754778,
-      "rewards/question_recreation_reward_func/std": 0.010923161637037992,
+      "rewards/final_correctness_reward_func/mean": 0.25,
+      "rewards/final_correctness_reward_func/std": 0.4629100561141968,
+      "rewards/question_recreation_reward_func/mean": 0.01847423054277897,
+      "rewards/question_recreation_reward_func/std": 0.012820018921047449,
       "rewards/soft_format_reward_func/mean": 0.0,
       "rewards/soft_format_reward_func/std": 0.0,
       "rewards/strict_format_reward_func/mean": 0.0,
       "rewards/strict_format_reward_func/std": 0.0,
-      "rewards/xmlcount_reward_func/mean": 0.0,
-      "rewards/xmlcount_reward_func/std": 0.0,
+      "rewards/xmlcount_reward_func/mean": 0.0078125,
+      "rewards/xmlcount_reward_func/std": 0.022097086533904076,
       "step": 8
     },
     {
@@ -171,31 +171,31 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.39999999999999997,
+      "completions/clipped_ratio": 0.5,
       "completions/max_length": 96.0,
-      "completions/max_terminated_length": 80.0,
-      "completions/mean_length": 76.9000015258789,
-      "completions/mean_terminated_length": 62.75,
-      "completions/min_length": 36.0,
-      "completions/min_terminated_length": 36.0,
+      "completions/max_terminated_length": 47.5,
+      "completions/mean_length": 71.70000076293945,
+      "completions/mean_terminated_length": 34.5,
+      "completions/min_length": 13.0,
+      "completions/min_terminated_length": 13.0,
       "epoch": 9.8,
-      "grad_norm": 13.94202709197998,
-      "kl": 0.012420143248164095,
+      "grad_norm": 14.598203659057617,
+      "kl": 0.00804947045253357,
       "learning_rate": 3.1137137178519977e-07,
-      "loss": 0.1608,
-      "num_tokens": 13634.0,
-      "reward": 0.03278735093772411,
-      "reward_std": 0.01219230075366795,
+      "loss": 0.1708,
+      "num_tokens": 13520.0,
+      "reward": 0.2736130654811859,
+      "reward_std": 0.3600814640522003,
       "rewards/concensus_correctness_reward_func/mean": 0.0,
       "rewards/concensus_correctness_reward_func/std": 0.0,
       "rewards/consensus_reward_func/mean": 0.0,
       "rewards/consensus_reward_func/std": 0.0,
       "rewards/cumulative_reward_2/mean": 0.0,
       "rewards/cumulative_reward_2/std": 0.0,
-      "rewards/final_correctness_reward_func/mean": 0.0,
-      "rewards/final_correctness_reward_func/std": 0.0,
-      "rewards/question_recreation_reward_func/mean": 0.03278735093772411,
-      "rewards/question_recreation_reward_func/std": 0.01526944525539875,
+      "rewards/final_correctness_reward_func/mean": 0.25,
+      "rewards/final_correctness_reward_func/std": 0.7071067690849304,
+      "rewards/question_recreation_reward_func/mean": 0.02361306920647621,
+      "rewards/question_recreation_reward_func/std": 0.015552400611341,
       "rewards/soft_format_reward_func/mean": 0.0,
       "rewards/soft_format_reward_func/std": 0.0,
       "rewards/strict_format_reward_func/mean": 0.0,
@@ -210,31 +210,31 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.6000000000000001,
+      "completions/clipped_ratio": 0.5,
       "completions/max_length": 96.0,
-      "completions/max_terminated_length": 45.5,
-      "completions/mean_length": 78.70000076293945,
-      "completions/mean_terminated_length": 38.5,
-      "completions/min_length": 32.0,
-      "completions/min_terminated_length": 32.0,
+      "completions/max_terminated_length": 31.0,
+      "completions/mean_length": 57.60000038146973,
+      "completions/mean_terminated_length": 22.0,
+      "completions/min_length": 16.0,
+      "completions/min_terminated_length": 16.0,
       "epoch": 11.8,
-      "grad_norm": 12.199621200561523,
-      "kl": 0.008919582382077351,
+      "grad_norm": 23.344341278076172,
+      "kl": 0.13446828180894954,
       "learning_rate": 2.2935516363191693e-07,
-      "loss": 0.1325,
-      "num_tokens": 16400.0,
-      "reward": 0.11377206444740295,
-      "reward_std": 0.035611885599792004,
+      "loss": 0.0719,
+      "num_tokens": 15932.0,
+      "reward": 0.4726976007223129,
+      "reward_std": 0.19187587685883045,
       "rewards/concensus_correctness_reward_func/mean": 0.0,
       "rewards/concensus_correctness_reward_func/std": 0.0,
       "rewards/consensus_reward_func/mean": 0.0,
       "rewards/consensus_reward_func/std": 0.0,
       "rewards/cumulative_reward_2/mean": 0.0,
       "rewards/cumulative_reward_2/std": 0.0,
-      "rewards/final_correctness_reward_func/mean": 0.0,
-      "rewards/final_correctness_reward_func/std": 0.0,
-      "rewards/question_recreation_reward_func/mean": 0.11377206444740295,
-      "rewards/question_recreation_reward_func/std": 0.04100861307233572,
+      "rewards/final_correctness_reward_func/mean": 0.375,
+      "rewards/final_correctness_reward_func/std": 0.816463440656662,
+      "rewards/question_recreation_reward_func/mean": 0.09769760258495808,
+      "rewards/question_recreation_reward_func/std": 0.06193104526028037,
       "rewards/soft_format_reward_func/mean": 0.0,
       "rewards/soft_format_reward_func/std": 0.0,
       "rewards/strict_format_reward_func/mean": 0.0,
@@ -251,29 +251,29 @@
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.5,
       "completions/max_length": 96.0,
-      "completions/max_terminated_length": 57.0,
-      "completions/mean_length": 67.89999771118164,
-      "completions/mean_terminated_length": 35.91666603088379,
-      "completions/min_length": 20.0,
-      "completions/min_terminated_length": 20.0,
+      "completions/max_terminated_length": 25.0,
+      "completions/mean_length": 56.39999771118164,
+      "completions/mean_terminated_length": 11.25,
+      "completions/min_length": 2.0,
+      "completions/min_terminated_length": 2.0,
       "epoch": 13.8,
-      "grad_norm": 12.575211524963379,
-      "kl": 0.01954469917109236,
+      "grad_norm": 17.271100997924805,
+      "kl": 0.08348440888221376,
       "learning_rate": 1.4957614383675767e-07,
-      "loss": -0.012,
-      "num_tokens": 18911.0,
-      "reward": 0.03240520507097244,
-      "reward_std": 0.017977031879127026,
+      "loss": 0.1353,
+      "num_tokens": 18389.0,
+      "reward": 0.39730143547058105,
+      "reward_std": 0.18547067046165466,
       "rewards/concensus_correctness_reward_func/mean": 0.0,
       "rewards/concensus_correctness_reward_func/std": 0.0,
       "rewards/consensus_reward_func/mean": 0.0,
       "rewards/consensus_reward_func/std": 0.0,
       "rewards/cumulative_reward_2/mean": 0.0,
       "rewards/cumulative_reward_2/std": 0.0,
-      "rewards/final_correctness_reward_func/mean": 0.0,
-      "rewards/final_correctness_reward_func/std": 0.0,
-      "rewards/question_recreation_reward_func/mean": 0.03240520413964987,
-      "rewards/question_recreation_reward_func/std": 0.024350897409021854,
+      "rewards/final_correctness_reward_func/mean": 0.375,
+      "rewards/final_correctness_reward_func/std": 0.816463440656662,
+      "rewards/question_recreation_reward_func/mean": 0.022301463410258293,
+      "rewards/question_recreation_reward_func/std": 0.016719398088753223,
       "rewards/soft_format_reward_func/mean": 0.0,
       "rewards/soft_format_reward_func/std": 0.0,
       "rewards/strict_format_reward_func/mean": 0.0,
@@ -288,37 +288,37 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.19999999999999996,
+      "completions/clipped_ratio": 0.3,
       "completions/max_length": 96.0,
-      "completions/max_terminated_length": 74.0,
-      "completions/mean_length": 54.79999923706055,
-      "completions/mean_terminated_length": 44.5,
-      "completions/min_length": 21.5,
-      "completions/min_terminated_length": 21.5,
+      "completions/max_terminated_length": 72.5,
+      "completions/mean_length": 52.39999961853027,
+      "completions/mean_terminated_length": 36.29166603088379,
+      "completions/min_length": 13.0,
+      "completions/min_terminated_length": 13.0,
       "epoch": 15.8,
-      "grad_norm": 14.50256633758545,
-      "kl": 0.011643254954833537,
+      "grad_norm": 17.390409469604492,
+      "kl": 0.1617226421367377,
       "learning_rate": 8.067960709356475e-08,
-      "loss": -0.0041,
-      "num_tokens": 21540.0,
-      "reward": 0.12224335223436356,
-      "reward_std": 0.029093343764543533,
+      "loss": 0.0343,
+      "num_tokens": 20644.0,
+      "reward": 0.6499089896678925,
+      "reward_std": 0.1050245389342308,
       "rewards/concensus_correctness_reward_func/mean": 0.0,
       "rewards/concensus_correctness_reward_func/std": 0.0,
       "rewards/consensus_reward_func/mean": 0.0,
       "rewards/consensus_reward_func/std": 0.0,
       "rewards/cumulative_reward_2/mean": 0.0,
       "rewards/cumulative_reward_2/std": 0.0,
-      "rewards/final_correctness_reward_func/mean": 0.0,
-      "rewards/final_correctness_reward_func/std": 0.0,
-      "rewards/question_recreation_reward_func/mean": 0.12224335223436356,
-      "rewards/question_recreation_reward_func/std": 0.03663408011198044,
+      "rewards/final_correctness_reward_func/mean": 0.5,
+      "rewards/final_correctness_reward_func/std": 0.9258201122283936,
+      "rewards/question_recreation_reward_func/mean": 0.08803398394957185,
+      "rewards/question_recreation_reward_func/std": 0.0536980046890676,
       "rewards/soft_format_reward_func/mean": 0.0,
       "rewards/soft_format_reward_func/std": 0.0,
       "rewards/strict_format_reward_func/mean": 0.0,
       "rewards/strict_format_reward_func/std": 0.0,
-      "rewards/xmlcount_reward_func/mean": 0.0,
-      "rewards/xmlcount_reward_func/std": 0.0,
+      "rewards/xmlcount_reward_func/mean": 0.0618749987334013,
+      "rewards/xmlcount_reward_func/std": 0.17500893026590347,
       "step": 16
     },
     {
@@ -327,37 +327,37 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.19999999999999996,
+      "completions/clipped_ratio": 0.39999999999999997,
       "completions/max_length": 96.0,
-      "completions/max_terminated_length": 86.5,
-      "completions/mean_length": 68.69999885559082,
-      "completions/mean_terminated_length": 61.875,
-      "completions/min_length": 36.5,
-      "completions/min_terminated_length": 36.5,
+      "completions/max_terminated_length": 58.5,
+      "completions/mean_length": 62.39999961853027,
+      "completions/mean_terminated_length": 40.875,
+      "completions/min_length": 23.5,
+      "completions/min_terminated_length": 23.5,
       "epoch": 17.8,
-      "grad_norm": 12.17991828918457,
-      "kl": 0.013874782802304253,
+      "grad_norm": 13.08644962310791,
+      "kl": 0.08081291774578858,
       "learning_rate": 3.013156219837776e-08,
-      "loss": 0.0575,
-      "num_tokens": 24202.0,
-      "reward": 0.1218071561306715,
-      "reward_std": 0.023342850618064404,
+      "loss": -0.0031,
+      "num_tokens": 23161.0,
+      "reward": 0.35994406789541245,
+      "reward_std": 0.03296606941148639,
       "rewards/concensus_correctness_reward_func/mean": 0.0,
       "rewards/concensus_correctness_reward_func/std": 0.0,
       "rewards/consensus_reward_func/mean": 0.0,
       "rewards/consensus_reward_func/std": 0.0,
       "rewards/cumulative_reward_2/mean": 0.0,
       "rewards/cumulative_reward_2/std": 0.0,
-      "rewards/final_correctness_reward_func/mean": 0.0,
-      "rewards/final_correctness_reward_func/std": 0.0,
-      "rewards/question_recreation_reward_func/mean": 0.1218071486800909,
-      "rewards/question_recreation_reward_func/std": 0.03349189879372716,
+      "rewards/final_correctness_reward_func/mean": 0.25,
+      "rewards/final_correctness_reward_func/std": 0.4629100561141968,
+      "rewards/question_recreation_reward_func/mean": 0.11738158855587244,
+      "rewards/question_recreation_reward_func/std": 0.029859440866857767,
       "rewards/soft_format_reward_func/mean": 0.0,
       "rewards/soft_format_reward_func/std": 0.0,
       "rewards/strict_format_reward_func/mean": 0.0,
       "rewards/strict_format_reward_func/std": 0.0,
-      "rewards/xmlcount_reward_func/mean": 0.0,
-      "rewards/xmlcount_reward_func/std": 0.0,
+      "rewards/xmlcount_reward_func/mean": -0.007437500171363354,
+      "rewards/xmlcount_reward_func/std": 0.021036427468061447,
       "step": 18
     },
     {
@@ -366,21 +366,21 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.6,
+      "completions/clipped_ratio": 0.3,
       "completions/max_length": 96.0,
-      "completions/max_terminated_length": 60.0,
-      "completions/mean_length": 74.0,
-      "completions/mean_terminated_length": 41.0,
-      "completions/min_length": 22.0,
-      "completions/min_terminated_length": 22.0,
+      "completions/max_terminated_length": 70.0,
+      "completions/mean_length": 56.0,
+      "completions/mean_terminated_length": 39.33333396911621,
+      "completions/min_length": 8.5,
+      "completions/min_terminated_length": 8.5,
       "epoch": 19.8,
-      "grad_norm": 11.600945472717285,
-      "kl": 0.010468967841006815,
+      "grad_norm": 15.382369995117188,
+      "kl": 0.02692569710779935,
       "learning_rate": 3.4096741493194193e-09,
-      "loss": 0.1749,
-      "num_tokens": 26879.0,
-      "reward": 0.15840027295053005,
-      "reward_std": 0.19570728274993598,
+      "loss": 0.1782,
+      "num_tokens": 25599.0,
+      "reward": 0.16119842790067196,
+      "reward_std": 0.1692181215621531,
       "rewards/concensus_correctness_reward_func/mean": 0.0,
       "rewards/concensus_correctness_reward_func/std": 0.0,
       "rewards/consensus_reward_func/mean": 0.0,
@@ -389,29 +389,29 @@
       "rewards/cumulative_reward_2/std": 0.0,
       "rewards/final_correctness_reward_func/mean": 0.125,
       "rewards/final_correctness_reward_func/std": 0.3535533845424652,
-      "rewards/question_recreation_reward_func/mean": 0.023462770506739616,
-      "rewards/question_recreation_reward_func/std": 0.008887386415153742,
+      "rewards/question_recreation_reward_func/mean": 0.028385925106704235,
+      "rewards/question_recreation_reward_func/std": 0.014382507652044296,
       "rewards/soft_format_reward_func/mean": 0.0,
       "rewards/soft_format_reward_func/std": 0.0,
       "rewards/strict_format_reward_func/mean": 0.0,
       "rewards/strict_format_reward_func/std": 0.0,
-      "rewards/xmlcount_reward_func/mean": 0.009937499649822712,
-      "rewards/xmlcount_reward_func/std": 0.028107494115829468,
+      "rewards/xmlcount_reward_func/mean": 0.0078125,
+      "rewards/xmlcount_reward_func/std": 0.022097086533904076,
       "step": 20
     },
     {
       "epoch": 19.8,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 0.06928728232160211,
-      "train_runtime": 1855.3992,
-      "train_samples_per_second": 0.086,
+      "train_loss": 0.089614911028184,
+      "train_runtime": 1740.8949,
+      "train_samples_per_second": 0.092,
       "train_steps_per_second": 0.011
     }
   ],
   "logging_steps": 2,
   "max_steps": 20,
-  "num_input_tokens_seen": 26879,
+  "num_input_tokens_seen": 25599,
   "num_train_epochs": 20,
   "save_steps": 25,
   "stateful_callbacks": {