End of training

README.md

@@ -30,7 +30,7 @@ print(output["generated_text"])
 
 ## Training procedure
 
-[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/corobov-mitya-individual/huggingface/runs/an7qp0tx) 
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/corobov-mitya-individual/huggingface/runs/zcdsijaj) 
 
 
 This model was trained with GRPO, a method introduced in [DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models](https://huggingface.co/papers/2402.03300).

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.037779451161623,
-    "train_runtime": 7169.9681,
-    "train_samples": 5,
-    "train_samples_per_second": 0.011,
-    "train_steps_per_second": 0.003
+    "train_loss": -0.03877150900661945,
+    "train_runtime": 4996.2526,
+    "train_samples": 3,
+    "train_samples_per_second": 0.016,
+    "train_steps_per_second": 0.004
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:adecc22f303850912efbf20277ad42ab67f6089c6d70133581f073f6e2538487
+oid sha256:cdb70c227c8abf2dab49c2c098a88d98e158211501b71354a6d982fdb85fc70c
 size 1976163472

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.037779451161623,
-    "train_runtime": 7169.9681,
-    "train_samples": 5,
-    "train_samples_per_second": 0.011,
-    "train_steps_per_second": 0.003
+    "train_loss": -0.03877150900661945,
+    "train_runtime": 4996.2526,
+    "train_samples": 3,
+    "train_samples_per_second": 0.016,
+    "train_steps_per_second": 0.004
 }

trainer_state.json

@@ -2,7 +2,7 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 9.8,
+  "epoch": 19.666666666666668,
   "eval_steps": 500,
   "global_step": 20,
   "is_hyper_param_search": false,
@@ -15,37 +15,37 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.125,
-      "completions/max_length": 302.0,
-      "completions/max_terminated_length": 92.5,
-      "completions/mean_length": 123.375,
-      "completions/mean_terminated_length": 63.83333396911621,
-      "completions/min_length": 48.0,
-      "completions/min_terminated_length": 48.0,
-      "epoch": 0.8,
-      "grad_norm": 8.92754077911377,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 353.5,
+      "completions/max_terminated_length": 353.5,
+      "completions/mean_length": 172.83333587646484,
+      "completions/mean_terminated_length": 172.83333587646484,
+      "completions/min_length": 56.5,
+      "completions/min_terminated_length": 56.5,
+      "epoch": 1.6666666666666665,
+      "grad_norm": 24.607608795166016,
       "kl": 0.0,
       "learning_rate": 5e-07,
-      "loss": -0.1218,
-      "num_tokens": 3214.0,
-      "reward": 0.3856615126132965,
-      "reward_std": 0.7816920205950737,
-      "rewards/concensus_correctness_reward_func/mean": 0.4375,
-      "rewards/concensus_correctness_reward_func/std": 0.875,
-      "rewards/consensus_reward_func/mean": 0.0,
-      "rewards/consensus_reward_func/std": 0.0,
+      "loss": -0.161,
+      "num_tokens": 3192.0,
+      "reward": 0.7391411811113358,
+      "reward_std": 0.8254430899396539,
+      "rewards/concensus_correctness_reward_func/mean": 0.23999999463558197,
+      "rewards/concensus_correctness_reward_func/std": 0.47999998927116394,
+      "rewards/consensus_reward_func/mean": 0.25,
+      "rewards/consensus_reward_func/std": 0.5,
       "rewards/cumulative_reward_2/mean": 0.0,
       "rewards/cumulative_reward_2/std": 0.0,
       "rewards/final_correctness_reward_func/mean": 0.0,
       "rewards/final_correctness_reward_func/std": 0.0,
-      "rewards/question_recreation_reward_func/mean": 0.02203650400042534,
-      "rewards/question_recreation_reward_func/std": 0.023317964747548103,
+      "rewards/question_recreation_reward_func/mean": 0.08939116820693016,
+      "rewards/question_recreation_reward_func/std": 0.09541699755936861,
       "rewards/soft_format_reward_func/mean": 0.0,
       "rewards/soft_format_reward_func/std": 0.0,
       "rewards/strict_format_reward_func/mean": 0.0,
       "rewards/strict_format_reward_func/std": 0.0,
-      "rewards/xmlcount_reward_func/mean": -0.07387500256299973,
-      "rewards/xmlcount_reward_func/std": 0.269045926630497,
+      "rewards/xmlcount_reward_func/mean": 0.15974999964237213,
+      "rewards/xmlcount_reward_func/std": 0.2241138368844986,
       "step": 2
     },
     {
@@ -54,37 +54,37 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 204.5,
-      "completions/max_terminated_length": 204.5,
-      "completions/mean_length": 108.875,
-      "completions/mean_terminated_length": 108.875,
-      "completions/min_length": 34.0,
-      "completions/min_terminated_length": 34.0,
-      "epoch": 1.8,
-      "grad_norm": 14.86384391784668,
-      "kl": 0.001416065962985158,
+      "completions/clipped_ratio": 0.16666666666666669,
+      "completions/max_length": 346.0,
+      "completions/max_terminated_length": 189.5,
+      "completions/mean_length": 198.8333282470703,
+      "completions/mean_terminated_length": 141.5,
+      "completions/min_length": 77.0,
+      "completions/min_terminated_length": 77.0,
+      "epoch": 3.6666666666666665,
+      "grad_norm": 8.281935691833496,
+      "kl": 0.0013531837284972426,
       "learning_rate": 4.864543104251586e-07,
-      "loss": 0.3279,
-      "num_tokens": 6661.0,
-      "reward": 0.022667515091598034,
-      "reward_std": 0.010929046286037192,
-      "rewards/concensus_correctness_reward_func/mean": 0.0,
-      "rewards/concensus_correctness_reward_func/std": 0.0,
-      "rewards/consensus_reward_func/mean": 0.0,
-      "rewards/consensus_reward_func/std": 0.0,
+      "loss": 0.0243,
+      "num_tokens": 6675.0,
+      "reward": 0.7484749890863895,
+      "reward_std": 0.9315725984051824,
+      "rewards/concensus_correctness_reward_func/mean": 0.23999999463558197,
+      "rewards/concensus_correctness_reward_func/std": 0.47999998927116394,
+      "rewards/consensus_reward_func/mean": 0.25,
+      "rewards/consensus_reward_func/std": 0.5,
       "rewards/cumulative_reward_2/mean": 0.0,
       "rewards/cumulative_reward_2/std": 0.0,
       "rewards/final_correctness_reward_func/mean": 0.0,
       "rewards/final_correctness_reward_func/std": 0.0,
-      "rewards/question_recreation_reward_func/mean": 0.015667515341192484,
-      "rewards/question_recreation_reward_func/std": 0.010519207920879126,
+      "rewards/question_recreation_reward_func/mean": 0.040849958546459675,
+      "rewards/question_recreation_reward_func/std": 0.02256392315030098,
       "rewards/soft_format_reward_func/mean": 0.0,
       "rewards/soft_format_reward_func/std": 0.0,
       "rewards/strict_format_reward_func/mean": 0.0,
       "rewards/strict_format_reward_func/std": 0.0,
-      "rewards/xmlcount_reward_func/mean": 0.007000000216066837,
-      "rewards/xmlcount_reward_func/std": 0.01400000136345625,
+      "rewards/xmlcount_reward_func/mean": 0.21762500703334808,
+      "rewards/xmlcount_reward_func/std": 0.28903544694185257,
       "step": 4
     },
     {
@@ -93,21 +93,21 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.5,
-      "completions/max_length": 412.0,
-      "completions/max_terminated_length": 156.0,
-      "completions/mean_length": 316.125,
-      "completions/mean_terminated_length": 60.125,
-      "completions/min_length": 270.5,
-      "completions/min_terminated_length": 14.5,
-      "epoch": 2.8,
-      "grad_norm": 7.786389350891113,
-      "kl": 0.0026882924139499664,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 230.5,
+      "completions/max_terminated_length": 230.5,
+      "completions/mean_length": 104.0,
+      "completions/mean_terminated_length": 104.0,
+      "completions/min_length": 13.5,
+      "completions/min_terminated_length": 13.5,
+      "epoch": 5.666666666666667,
+      "grad_norm": 19.817241668701172,
+      "kl": 0.005833291725139134,
       "learning_rate": 4.472851273490984e-07,
-      "loss": 0.178,
-      "num_tokens": 10480.0,
-      "reward": 0.030308596324175596,
-      "reward_std": 0.0613415464758873,
+      "loss": 0.1028,
+      "num_tokens": 9569.0,
+      "reward": 0.15289875864982605,
+      "reward_std": 0.13462494127452374,
       "rewards/concensus_correctness_reward_func/mean": 0.0,
       "rewards/concensus_correctness_reward_func/std": 0.0,
       "rewards/consensus_reward_func/mean": 0.0,
@@ -116,14 +116,14 @@
       "rewards/cumulative_reward_2/std": 0.0,
       "rewards/final_correctness_reward_func/mean": 0.0,
       "rewards/final_correctness_reward_func/std": 0.0,
-      "rewards/question_recreation_reward_func/mean": 0.013933598063886166,
-      "rewards/question_recreation_reward_func/std": 0.014677805360406637,
+      "rewards/question_recreation_reward_func/mean": 0.052898744121193886,
+      "rewards/question_recreation_reward_func/std": 0.050960212014615536,
       "rewards/soft_format_reward_func/mean": 0.0,
       "rewards/soft_format_reward_func/std": 0.0,
       "rewards/strict_format_reward_func/mean": 0.0,
       "rewards/strict_format_reward_func/std": 0.0,
-      "rewards/xmlcount_reward_func/mean": 0.016375000588595867,
-      "rewards/xmlcount_reward_func/std": 0.051155371591448784,
+      "rewards/xmlcount_reward_func/mean": 0.10000000149011612,
+      "rewards/xmlcount_reward_func/std": 0.16105124354362488,
       "step": 6
     },
     {
@@ -132,21 +132,21 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.5,
-      "completions/max_length": 447.5,
-      "completions/max_terminated_length": 191.5,
-      "completions/mean_length": 309.25,
-      "completions/mean_terminated_length": 53.25,
-      "completions/min_length": 258.0,
-      "completions/min_terminated_length": 2.0,
-      "epoch": 3.8,
-      "grad_norm": 5.402748107910156,
-      "kl": 0.005952609681116883,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 214.5,
+      "completions/max_terminated_length": 214.5,
+      "completions/mean_length": 101.66666412353516,
+      "completions/mean_terminated_length": 101.66666412353516,
+      "completions/min_length": 24.0,
+      "completions/min_terminated_length": 24.0,
+      "epoch": 7.666666666666667,
+      "grad_norm": 16.099365234375,
+      "kl": 0.006465533399023116,
       "learning_rate": 3.867370395306068e-07,
-      "loss": -0.1153,
-      "num_tokens": 14444.0,
-      "reward": 0.01363831665366888,
-      "reward_std": 0.0067086233757436275,
+      "loss": -0.2668,
+      "num_tokens": 12403.0,
+      "reward": 0.16425441950559616,
+      "reward_std": 0.13405008241534233,
       "rewards/concensus_correctness_reward_func/mean": 0.0,
       "rewards/concensus_correctness_reward_func/std": 0.0,
       "rewards/consensus_reward_func/mean": 0.0,
@@ -155,14 +155,14 @@
       "rewards/cumulative_reward_2/std": 0.0,
       "rewards/final_correctness_reward_func/mean": 0.0,
       "rewards/final_correctness_reward_func/std": 0.0,
-      "rewards/question_recreation_reward_func/mean": 0.01363831665366888,
-      "rewards/question_recreation_reward_func/std": 0.005987111479043961,
+      "rewards/question_recreation_reward_func/mean": 0.12725441344082355,
+      "rewards/question_recreation_reward_func/std": 0.17341968975961208,
       "rewards/soft_format_reward_func/mean": 0.0,
       "rewards/soft_format_reward_func/std": 0.0,
       "rewards/strict_format_reward_func/mean": 0.0,
       "rewards/strict_format_reward_func/std": 0.0,
-      "rewards/xmlcount_reward_func/mean": 0.0,
-      "rewards/xmlcount_reward_func/std": 0.0,
+      "rewards/xmlcount_reward_func/mean": 0.03700000001117587,
+      "rewards/xmlcount_reward_func/std": 0.047584391199052334,
       "step": 8
     },
     {
@@ -172,36 +172,36 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 209.0,
-      "completions/max_terminated_length": 209.0,
-      "completions/mean_length": 181.625,
-      "completions/mean_terminated_length": 181.625,
-      "completions/min_length": 169.5,
-      "completions/min_terminated_length": 169.5,
-      "epoch": 4.8,
-      "grad_norm": 6.952495574951172,
-      "kl": 0.0149484759895131,
+      "completions/max_length": 181.0,
+      "completions/max_terminated_length": 181.0,
+      "completions/mean_length": 126.33333206176758,
+      "completions/mean_terminated_length": 126.33333206176758,
+      "completions/min_length": 77.5,
+      "completions/min_terminated_length": 77.5,
+      "epoch": 9.666666666666666,
+      "grad_norm": 12.087945938110352,
+      "kl": 0.010334744409192353,
       "learning_rate": 3.1137137178519977e-07,
-      "loss": 0.0504,
-      "num_tokens": 17407.0,
-      "reward": 0.23214832320809364,
-      "reward_std": 0.37722403556108475,
+      "loss": 0.0072,
+      "num_tokens": 15403.0,
+      "reward": 0.12861808016896248,
+      "reward_std": 0.07254930585622787,
       "rewards/concensus_correctness_reward_func/mean": 0.0,
       "rewards/concensus_correctness_reward_func/std": 0.0,
       "rewards/consensus_reward_func/mean": 0.0,
       "rewards/consensus_reward_func/std": 0.0,
       "rewards/cumulative_reward_2/mean": 0.0,
       "rewards/cumulative_reward_2/std": 0.0,
-      "rewards/final_correctness_reward_func/mean": 0.25,
-      "rewards/final_correctness_reward_func/std": 0.5,
-      "rewards/question_recreation_reward_func/mean": 0.015148311853408813,
-      "rewards/question_recreation_reward_func/std": 0.0038266002666205168,
+      "rewards/final_correctness_reward_func/mean": 0.0,
+      "rewards/final_correctness_reward_func/std": 0.0,
+      "rewards/question_recreation_reward_func/mean": 0.05049308016896248,
+      "rewards/question_recreation_reward_func/std": 0.03305052034556866,
       "rewards/soft_format_reward_func/mean": 0.0,
       "rewards/soft_format_reward_func/std": 0.0,
       "rewards/strict_format_reward_func/mean": 0.0,
       "rewards/strict_format_reward_func/std": 0.0,
-      "rewards/xmlcount_reward_func/mean": -0.03299999888986349,
-      "rewards/xmlcount_reward_func/std": 0.07268869318068027,
+      "rewards/xmlcount_reward_func/mean": 0.078125,
+      "rewards/xmlcount_reward_func/std": 0.1196383461356163,
       "step": 10
     },
     {
@@ -210,21 +210,21 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.125,
-      "completions/max_length": 301.0,
-      "completions/max_terminated_length": 158.0,
-      "completions/mean_length": 160.875,
-      "completions/mean_terminated_length": 114.16666412353516,
-      "completions/min_length": 89.0,
-      "completions/min_terminated_length": 89.0,
-      "epoch": 5.8,
-      "grad_norm": 16.89011573791504,
-      "kl": 0.017503770883195102,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 208.0,
+      "completions/max_terminated_length": 208.0,
+      "completions/mean_length": 105.5,
+      "completions/mean_terminated_length": 105.5,
+      "completions/min_length": 32.0,
+      "completions/min_terminated_length": 32.0,
+      "epoch": 11.666666666666666,
+      "grad_norm": 25.62042808532715,
+      "kl": 0.014036847351235338,
       "learning_rate": 2.2935516363191693e-07,
-      "loss": 0.1,
-      "num_tokens": 20938.0,
-      "reward": -0.1888856142759323,
-      "reward_std": 0.33080266416072845,
+      "loss": -0.1005,
+      "num_tokens": 18146.0,
+      "reward": 0.07443425804376602,
+      "reward_std": 0.025631051044911146,
       "rewards/concensus_correctness_reward_func/mean": 0.0,
       "rewards/concensus_correctness_reward_func/std": 0.0,
       "rewards/consensus_reward_func/mean": 0.0,
@@ -233,14 +233,14 @@
       "rewards/cumulative_reward_2/std": 0.0,
       "rewards/final_correctness_reward_func/mean": 0.0,
       "rewards/final_correctness_reward_func/std": 0.0,
-      "rewards/question_recreation_reward_func/mean": 0.015614384785294533,
-      "rewards/question_recreation_reward_func/std": 0.014846977777779102,
+      "rewards/question_recreation_reward_func/mean": 0.02743426151573658,
+      "rewards/question_recreation_reward_func/std": 0.011736967135220766,
       "rewards/soft_format_reward_func/mean": 0.0,
       "rewards/soft_format_reward_func/std": 0.0,
       "rewards/strict_format_reward_func/mean": 0.0,
       "rewards/strict_format_reward_func/std": 0.0,
-      "rewards/xmlcount_reward_func/mean": -0.20449999906122684,
-      "rewards/xmlcount_reward_func/std": 0.4663179814815521,
+      "rewards/xmlcount_reward_func/mean": 0.04700000025331974,
+      "rewards/xmlcount_reward_func/std": 0.06758439540863037,
       "step": 12
     },
     {
@@ -250,36 +250,36 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 128.5,
-      "completions/max_terminated_length": 128.5,
-      "completions/mean_length": 87.5,
-      "completions/mean_terminated_length": 87.5,
-      "completions/min_length": 30.0,
-      "completions/min_terminated_length": 30.0,
-      "epoch": 6.8,
-      "grad_norm": 20.018957138061523,
-      "kl": 0.025806593243032694,
+      "completions/max_length": 115.0,
+      "completions/max_terminated_length": 115.0,
+      "completions/mean_length": 93.5,
+      "completions/mean_terminated_length": 93.5,
+      "completions/min_length": 69.5,
+      "completions/min_terminated_length": 69.5,
+      "epoch": 13.666666666666666,
+      "grad_norm": 9.609339714050293,
+      "kl": 0.02532817842438817,
       "learning_rate": 1.4957614383675767e-07,
-      "loss": -0.1021,
-      "num_tokens": 23935.0,
-      "reward": 0.05173949897289276,
-      "reward_std": 0.0109914755448699,
-      "rewards/concensus_correctness_reward_func/mean": 0.0,
-      "rewards/concensus_correctness_reward_func/std": 0.0,
-      "rewards/consensus_reward_func/mean": 0.0,
-      "rewards/consensus_reward_func/std": 0.0,
+      "loss": -0.0654,
+      "num_tokens": 21002.0,
+      "reward": 0.8926658928394318,
+      "reward_std": 0.7070434279739857,
+      "rewards/concensus_correctness_reward_func/mean": 0.24050000309944153,
+      "rewards/concensus_correctness_reward_func/std": 0.48100000619888306,
+      "rewards/consensus_reward_func/mean": 0.25,
+      "rewards/consensus_reward_func/std": 0.5,
       "rewards/cumulative_reward_2/mean": 0.0,
       "rewards/cumulative_reward_2/std": 0.0,
       "rewards/final_correctness_reward_func/mean": 0.0,
       "rewards/final_correctness_reward_func/std": 0.0,
-      "rewards/question_recreation_reward_func/mean": 0.028114496264606714,
-      "rewards/question_recreation_reward_func/std": 0.025538412854075432,
+      "rewards/question_recreation_reward_func/mean": 0.10116582456976175,
+      "rewards/question_recreation_reward_func/std": 0.11067926976829767,
       "rewards/soft_format_reward_func/mean": 0.0,
       "rewards/soft_format_reward_func/std": 0.0,
       "rewards/strict_format_reward_func/mean": 0.0,
       "rewards/strict_format_reward_func/std": 0.0,
-      "rewards/xmlcount_reward_func/mean": 0.02362500037997961,
-      "rewards/xmlcount_reward_func/std": 0.030027911067008972,
+      "rewards/xmlcount_reward_func/mean": 0.3009999990463257,
+      "rewards/xmlcount_reward_func/std": 0.2542915344238281,
       "step": 14
     },
     {
@@ -288,21 +288,21 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 225.5,
-      "completions/max_terminated_length": 225.5,
-      "completions/mean_length": 102.75,
-      "completions/mean_terminated_length": 102.75,
-      "completions/min_length": 47.0,
-      "completions/min_terminated_length": 47.0,
-      "epoch": 7.8,
-      "grad_norm": 18.695772171020508,
-      "kl": 0.02530490467324853,
+      "completions/clipped_ratio": 0.16666666666666669,
+      "completions/max_length": 378.0,
+      "completions/max_terminated_length": 206.0,
+      "completions/mean_length": 197.33333587646484,
+      "completions/mean_terminated_length": 134.91666412353516,
+      "completions/min_length": 78.5,
+      "completions/min_terminated_length": 78.5,
+      "epoch": 15.666666666666666,
+      "grad_norm": 16.777217864990234,
+      "kl": 0.016524533144547604,
       "learning_rate": 8.067960709356478e-08,
-      "loss": -0.0895,
-      "num_tokens": 27073.0,
-      "reward": -0.059220071882009506,
-      "reward_std": 0.15402239561080933,
+      "loss": 0.0725,
+      "num_tokens": 24589.0,
+      "reward": 0.14827939122915268,
+      "reward_std": 0.14140347205102444,
       "rewards/concensus_correctness_reward_func/mean": 0.0,
       "rewards/concensus_correctness_reward_func/std": 0.0,
       "rewards/consensus_reward_func/mean": 0.0,
@@ -311,14 +311,14 @@
       "rewards/cumulative_reward_2/std": 0.0,
       "rewards/final_correctness_reward_func/mean": 0.0,
       "rewards/final_correctness_reward_func/std": 0.0,
-      "rewards/question_recreation_reward_func/mean": 0.01565493270754814,
-      "rewards/question_recreation_reward_func/std": 0.009137378772720695,
+      "rewards/question_recreation_reward_func/mean": 0.13127939216792583,
+      "rewards/question_recreation_reward_func/std": 0.19441921077668667,
       "rewards/soft_format_reward_func/mean": 0.0,
       "rewards/soft_format_reward_func/std": 0.0,
       "rewards/strict_format_reward_func/mean": 0.0,
       "rewards/strict_format_reward_func/std": 0.0,
-      "rewards/xmlcount_reward_func/mean": -0.07487499713897705,
-      "rewards/xmlcount_reward_func/std": 0.15669719874858856,
+      "rewards/xmlcount_reward_func/mean": 0.017000000923871994,
+      "rewards/xmlcount_reward_func/std": 0.08594898879528046,
       "step": 16
     },
     {
@@ -327,37 +327,37 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.125,
-      "completions/max_length": 415.0,
-      "completions/max_terminated_length": 206.5,
-      "completions/mean_length": 241.25,
-      "completions/mean_terminated_length": 183.3333339691162,
-      "completions/min_length": 170.0,
-      "completions/min_terminated_length": 170.0,
-      "epoch": 8.8,
-      "grad_norm": 51.03651809692383,
-      "kl": 0.05000046588247642,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 245.0,
+      "completions/max_terminated_length": 245.0,
+      "completions/mean_length": 169.33333587646484,
+      "completions/mean_terminated_length": 169.33333587646484,
+      "completions/min_length": 98.5,
+      "completions/min_terminated_length": 98.5,
+      "epoch": 17.666666666666668,
+      "grad_norm": 12.084244728088379,
+      "kl": 0.013836602825904265,
       "learning_rate": 3.013156219837776e-08,
-      "loss": -0.0637,
-      "num_tokens": 30146.0,
-      "reward": -0.015393424779176712,
-      "reward_std": 0.0688084177672863,
-      "rewards/concensus_correctness_reward_func/mean": 0.0,
-      "rewards/concensus_correctness_reward_func/std": 0.0,
-      "rewards/consensus_reward_func/mean": 0.0,
-      "rewards/consensus_reward_func/std": 0.0,
+      "loss": 0.1236,
+      "num_tokens": 27785.0,
+      "reward": 0.7921578735113144,
+      "reward_std": 1.0020692646503448,
+      "rewards/concensus_correctness_reward_func/mean": 0.24050000309944153,
+      "rewards/concensus_correctness_reward_func/std": 0.48100000619888306,
+      "rewards/consensus_reward_func/mean": 0.25,
+      "rewards/consensus_reward_func/std": 0.5,
       "rewards/cumulative_reward_2/mean": 0.0,
       "rewards/cumulative_reward_2/std": 0.0,
       "rewards/final_correctness_reward_func/mean": 0.0,
       "rewards/final_correctness_reward_func/std": 0.0,
-      "rewards/question_recreation_reward_func/mean": 0.015356574673205614,
-      "rewards/question_recreation_reward_func/std": 0.013046635314822197,
+      "rewards/question_recreation_reward_func/mean": 0.12290793936699629,
+      "rewards/question_recreation_reward_func/std": 0.19958087475970387,
       "rewards/soft_format_reward_func/mean": 0.0,
       "rewards/soft_format_reward_func/std": 0.0,
       "rewards/strict_format_reward_func/mean": 0.0,
       "rewards/strict_format_reward_func/std": 0.0,
-      "rewards/xmlcount_reward_func/mean": -0.03074999898672104,
-      "rewards/xmlcount_reward_func/std": 0.11776353418827057,
+      "rewards/xmlcount_reward_func/mean": 0.17875000089406967,
+      "rewards/xmlcount_reward_func/std": 0.28059088438749313,
       "step": 18
     },
     {
@@ -366,21 +366,21 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.125,
-      "completions/max_length": 265.5,
-      "completions/max_terminated_length": 56.5,
-      "completions/mean_length": 97.625,
-      "completions/mean_terminated_length": 41.66666793823242,
-      "completions/min_length": 34.0,
-      "completions/min_terminated_length": 34.0,
-      "epoch": 9.8,
-      "grad_norm": 14.597654342651367,
-      "kl": 0.02688464312814176,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 341.0,
+      "completions/max_terminated_length": 341.0,
+      "completions/mean_length": 187.5,
+      "completions/mean_terminated_length": 187.5,
+      "completions/min_length": 96.0,
+      "completions/min_terminated_length": 96.0,
+      "epoch": 19.666666666666668,
+      "grad_norm": 8.484414100646973,
+      "kl": 0.008221860975027084,
       "learning_rate": 3.4096741493194193e-09,
-      "loss": 0.214,
-      "num_tokens": 33969.0,
-      "reward": -0.003884643316268921,
-      "reward_std": 0.04420278873294592,
+      "loss": -0.1244,
+      "num_tokens": 31386.0,
+      "reward": 0.07486644759774208,
+      "reward_std": 0.07879441790282726,
       "rewards/concensus_correctness_reward_func/mean": 0.0,
       "rewards/concensus_correctness_reward_func/std": 0.0,
       "rewards/consensus_reward_func/mean": 0.0,
@@ -389,30 +389,30 @@
       "rewards/cumulative_reward_2/std": 0.0,
       "rewards/final_correctness_reward_func/mean": 0.0,
       "rewards/final_correctness_reward_func/std": 0.0,
-      "rewards/question_recreation_reward_func/mean": 0.020615354413166642,
-      "rewards/question_recreation_reward_func/std": 0.019722969736903906,
+      "rewards/question_recreation_reward_func/mean": 0.0378664480522275,
+      "rewards/question_recreation_reward_func/std": 0.04724898235872388,
       "rewards/soft_format_reward_func/mean": 0.0,
       "rewards/soft_format_reward_func/std": 0.0,
       "rewards/strict_format_reward_func/mean": 0.0,
       "rewards/strict_format_reward_func/std": 0.0,
-      "rewards/xmlcount_reward_func/mean": -0.02449999935925007,
-      "rewards/xmlcount_reward_func/std": 0.04899999871850014,
+      "rewards/xmlcount_reward_func/mean": 0.03700000047683716,
+      "rewards/xmlcount_reward_func/std": 0.07400000095367432,
       "step": 20
     },
     {
-      "epoch": 9.8,
+      "epoch": 19.666666666666668,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 0.037779451161623,
-      "train_runtime": 7169.9681,
-      "train_samples_per_second": 0.011,
-      "train_steps_per_second": 0.003
+      "train_loss": -0.03877150900661945,
+      "train_runtime": 4996.2526,
+      "train_samples_per_second": 0.016,
+      "train_steps_per_second": 0.004
     }
   ],
   "logging_steps": 2,
   "max_steps": 20,
-  "num_input_tokens_seen": 33969,
-  "num_train_epochs": 10,
+  "num_input_tokens_seen": 31386,
+  "num_train_epochs": 20,
   "save_steps": 25,
   "stateful_callbacks": {
     "TrainerControl": {

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ce333ccfd0eebed288b7f3195f0087f5d0abcd5c388d8f41ec2f2268933c2e63
+oid sha256:ff8519a297979b81c6f6328bec1149b7a9ae9dac22c186355657d01cde9d0fa5
 size 6929