End of training

README.md

@@ -40,7 +40,7 @@ This model was trained with GRPO, a method introduced in [DeepSeekMath: Pushing
 
 - TRL: 0.15.2
 - Transformers: 4.48.2
-- Pytorch: 2.5.1+cu121
+- Pytorch: 2.5.1
 - Datasets: 3.6.0
 - Tokenizers: 0.21.1
 

adapter_config.json

@@ -24,12 +24,12 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
+    "o_proj",
+    "k_proj",
+    "down_proj",
     "gate_proj",
     "up_proj",
     "q_proj",
-    "o_proj",
-    "down_proj",
-    "k_proj",
     "v_proj"
   ],
   "task_type": "CAUSAL_LM",

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:96970c5caa74a034e0f75503d062ccf3c79aa1e4bb1da783621dc643a7ef9a3d
+oid sha256:cc45f3686b0cba067c51fcf726eebe499a2e6560ef66389a088444fda4850481
 size 35237104

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 8.942109707277268e-05,
-    "train_runtime": 4099.4474,
-    "train_samples": 262,
-    "train_samples_per_second": 6.245,
-    "train_steps_per_second": 0.049
+    "train_loss": 5.2903332107234745e-06,
+    "train_runtime": 1863.6299,
+    "train_samples": 84,
+    "train_samples_per_second": 6.868,
+    "train_steps_per_second": 0.054
 }

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 8.942109707277268e-05,
-    "train_runtime": 4099.4474,
-    "train_samples": 262,
-    "train_samples_per_second": 6.245,
-    "train_steps_per_second": 0.049
+    "train_loss": 5.2903332107234745e-06,
+    "train_runtime": 1863.6299,
+    "train_samples": 84,
+    "train_samples_per_second": 6.868,
+    "train_steps_per_second": 0.054
 }

trainer_state.json

@@ -1,787 +1,407 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 11.793893129770993,
+  "epoch": 16.761904761904763,
   "eval_steps": 500,
-  "global_step": 200,
+  "global_step": 100,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 228.6078125,
-      "epoch": 0.3053435114503817,
-      "grad_norm": 1.6697295904159546,
-      "kl": 0.08263390087522567,
-      "learning_rate": 2.5e-07,
-      "loss": 0.0001,
-      "reward": 2.8884417802095412,
-      "reward_std": 3.0484682977199555,
-      "rewards/concensus_correctness_reward_func": 1.5492031248286366,
-      "rewards/consensus_reward_func": 0.2,
-      "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.21875,
-      "rewards/question_recreation_reward_func": 0.2991667349822819,
-      "rewards/soft_format_reward_func": 0.00078125,
-      "rewards/strict_format_reward_func": 0.00078125,
-      "rewards/xmlcount_reward_func": 0.6197593748569489,
-      "step": 5
-    },
-    {
-      "completion_length": 225.8765625,
-      "epoch": 0.6106870229007634,
-      "grad_norm": 1.4577786922454834,
-      "kl": 0.08271063729189336,
+      "completion_length": 214.846875,
+      "epoch": 0.9523809523809523,
+      "grad_norm": 4.286362171173096,
+      "kl": 0.007153257649042644,
       "learning_rate": 5e-07,
-      "loss": 0.0001,
-      "reward": 2.161433610320091,
-      "reward_std": 2.5517727360129356,
-      "rewards/concensus_correctness_reward_func": 0.857334372587502,
-      "rewards/consensus_reward_func": 0.203125,
-      "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.196875,
-      "rewards/question_recreation_reward_func": 0.2814491997472942,
-      "rewards/soft_format_reward_func": 0.00234375,
-      "rewards/strict_format_reward_func": 0.0015625,
-      "rewards/xmlcount_reward_func": 0.6187437549233437,
-      "step": 10
-    },
-    {
-      "completion_length": 223.1671875,
-      "epoch": 0.916030534351145,
-      "grad_norm": 40.90731430053711,
-      "kl": 0.08677452662959695,
-      "learning_rate": 4.991461232516674e-07,
-      "loss": 0.0001,
-      "reward": 2.2503652930259705,
-      "reward_std": 2.3072509482502936,
-      "rewards/concensus_correctness_reward_func": 0.9038093611598015,
-      "rewards/consensus_reward_func": 0.209375,
-      "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.18125,
-      "rewards/question_recreation_reward_func": 0.3275418411940336,
+      "loss": 0.0,
+      "reward": 0.9013831291347743,
+      "reward_std": 0.9495122246444225,
+      "rewards/concensus_correctness_reward_func": 0.07926875045523048,
+      "rewards/consensus_reward_func": 0.05,
+      "rewards/cumulative_reward_2": 0.0,
+      "rewards/final_correctness_reward_func": 0.165625,
+      "rewards/question_recreation_reward_func": 0.2006971864029765,
       "rewards/soft_format_reward_func": 0.00078125,
-      "rewards/strict_format_reward_func": 0.0015625,
-      "rewards/xmlcount_reward_func": 0.6260453149676323,
-      "step": 15
+      "rewards/strict_format_reward_func": 0.00234375,
+      "rewards/xmlcount_reward_func": 0.40266719218343494,
+      "step": 5
     },
     {
-      "completion_length": 225.08928571428572,
-      "epoch": 1.183206106870229,
-      "grad_norm": 2.2080225944519043,
-      "kl": 0.09310175127216748,
+      "completion_length": 217.63970588235293,
+      "epoch": 1.7619047619047619,
+      "grad_norm": 11.65625,
+      "kl": 0.003827589317498838,
       "learning_rate": 4.965903258506806e-07,
-      "loss": 0.0001,
-      "reward": 2.451156176839556,
-      "reward_std": 2.8527250630514964,
-      "rewards/concensus_correctness_reward_func": 1.0764642748449529,
-      "rewards/consensus_reward_func": 0.26071428571428573,
-      "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.22857142857142856,
-      "rewards/question_recreation_reward_func": 0.2879704715950148,
-      "rewards/soft_format_reward_func": 0.0008928571428571428,
-      "rewards/strict_format_reward_func": 0.0008928571428571428,
-      "rewards/xmlcount_reward_func": 0.5956499985286168,
-      "step": 20
-    },
-    {
-      "completion_length": 227.5796875,
-      "epoch": 1.4885496183206106,
-      "grad_norm": 1.866131067276001,
-      "kl": 0.08186168665997684,
-      "learning_rate": 4.923500664848326e-07,
-      "loss": 0.0001,
-      "reward": 2.17223744392395,
-      "reward_std": 2.3266281485557556,
-      "rewards/concensus_correctness_reward_func": 0.822290619648993,
-      "rewards/consensus_reward_func": 0.2,
-      "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.1625,
-      "rewards/question_recreation_reward_func": 0.36023432165384295,
-      "rewards/soft_format_reward_func": 0.0015625,
-      "rewards/strict_format_reward_func": 0.00078125,
-      "rewards/xmlcount_reward_func": 0.6248687498271466,
-      "step": 25
+      "loss": 0.0,
+      "reward": 0.9087017599274131,
+      "reward_std": 0.937035866519984,
+      "rewards/concensus_correctness_reward_func": 0.11116911733851713,
+      "rewards/consensus_reward_func": 0.04044117647058824,
+      "rewards/cumulative_reward_2": 0.0,
+      "rewards/final_correctness_reward_func": 0.1801470588235294,
+      "rewards/question_recreation_reward_func": 0.21326609646134517,
+      "rewards/soft_format_reward_func": 0.0009191176470588235,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.36275919000892076,
+      "step": 10
     },
     {
-      "completion_length": 227.615625,
-      "epoch": 1.7938931297709924,
-      "grad_norm": 1.6272974014282227,
-      "kl": 0.08450978258624672,
+      "completion_length": 215.58455882352942,
+      "epoch": 2.571428571428571,
+      "grad_norm": 4.602188587188721,
+      "kl": 0.007200981949127334,
       "learning_rate": 4.864543104251586e-07,
-      "loss": 0.0001,
-      "reward": 2.169202183187008,
-      "reward_std": 2.4988188624382017,
-      "rewards/concensus_correctness_reward_func": 0.8738718747161329,
-      "rewards/consensus_reward_func": 0.196875,
-      "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.215625,
-      "rewards/question_recreation_reward_func": 0.2872584268450737,
-      "rewards/soft_format_reward_func": 0.00078125,
-      "rewards/strict_format_reward_func": 0.00234375,
-      "rewards/xmlcount_reward_func": 0.5924468792974948,
-      "step": 30
-    },
-    {
-      "completion_length": 221.56607142857143,
-      "epoch": 2.0610687022900764,
-      "grad_norm": 1.8112502098083496,
-      "kl": 0.09125068480415004,
-      "learning_rate": 4.789433316637643e-07,
-      "loss": 0.0001,
-      "reward": 2.663744555200849,
-      "reward_std": 3.0470116555690767,
-      "rewards/concensus_correctness_reward_func": 1.3009928594742501,
-      "rewards/consensus_reward_func": 0.25357142857142856,
-      "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.175,
-      "rewards/question_recreation_reward_func": 0.28507488646677565,
-      "rewards/soft_format_reward_func": 0.0017857142857142857,
-      "rewards/strict_format_reward_func": 0.0017857142857142857,
-      "rewards/xmlcount_reward_func": 0.6455339295523507,
-      "step": 35
+      "loss": 0.0,
+      "reward": 0.8416915027534261,
+      "reward_std": 0.8690077452098622,
+      "rewards/concensus_correctness_reward_func": 0.060602940838126576,
+      "rewards/consensus_reward_func": 0.04044117647058824,
+      "rewards/cumulative_reward_2": 0.0,
+      "rewards/final_correctness_reward_func": 0.15073529411764705,
+      "rewards/question_recreation_reward_func": 0.22436061611070351,
+      "rewards/soft_format_reward_func": 0.0,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.3655514717102051,
+      "step": 15
     },
     {
-      "completion_length": 229.5390625,
-      "epoch": 2.366412213740458,
-      "grad_norm": 1.5192793607711792,
-      "kl": 0.08173536635003983,
+      "completion_length": 217.41176470588235,
+      "epoch": 3.380952380952381,
+      "grad_norm": 3.5898094177246094,
+      "kl": 0.006835748431245413,
       "learning_rate": 4.698684378016222e-07,
-      "loss": 0.0001,
-      "reward": 1.9662634998559951,
-      "reward_std": 2.307476855814457,
-      "rewards/concensus_correctness_reward_func": 0.6928218763321639,
-      "rewards/consensus_reward_func": 0.2,
-      "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.1625,
-      "rewards/question_recreation_reward_func": 0.313546329177916,
-      "rewards/soft_format_reward_func": 0.00078125,
-      "rewards/strict_format_reward_func": 0.00078125,
-      "rewards/xmlcount_reward_func": 0.5958328068256378,
-      "step": 40
-    },
-    {
-      "completion_length": 226.2484375,
-      "epoch": 2.67175572519084,
-      "grad_norm": 1.7399648427963257,
-      "kl": 0.0866468169959262,
-      "learning_rate": 4.592916195656321e-07,
-      "loss": 0.0001,
-      "reward": 2.2245844706892965,
-      "reward_std": 2.348295548558235,
-      "rewards/concensus_correctness_reward_func": 0.8688468646258116,
-      "rewards/consensus_reward_func": 0.21875,
-      "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.20625,
-      "rewards/question_recreation_reward_func": 0.3035891577601433,
-      "rewards/soft_format_reward_func": 0.0015625,
-      "rewards/strict_format_reward_func": 0.0015625,
-      "rewards/xmlcount_reward_func": 0.6240234360098839,
-      "step": 45
-    },
-    {
-      "completion_length": 229.1890625,
-      "epoch": 2.9770992366412212,
-      "grad_norm": 1.76873779296875,
-      "kl": 0.09108880357816815,
-      "learning_rate": 4.472851273490984e-07,
-      "loss": 0.0001,
-      "reward": 2.0511497393250466,
-      "reward_std": 1.911637831479311,
-      "rewards/concensus_correctness_reward_func": 0.6462500024586916,
-      "rewards/consensus_reward_func": 0.284375,
-      "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.196875,
-      "rewards/question_recreation_reward_func": 0.2953185113146901,
-      "rewards/soft_format_reward_func": 0.00078125,
+      "loss": 0.0,
+      "reward": 0.8475531142424134,
+      "reward_std": 0.9209246758152457,
+      "rewards/concensus_correctness_reward_func": 0.09411764736561214,
+      "rewards/consensus_reward_func": 0.025735294117647058,
+      "rewards/cumulative_reward_2": 0.0,
+      "rewards/final_correctness_reward_func": 0.16544117647058823,
+      "rewards/question_recreation_reward_func": 0.21184356072369745,
+      "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.6275500006973743,
-      "step": 50
+      "rewards/xmlcount_reward_func": 0.3504154397284283,
+      "step": 20
     },
     {
-      "completion_length": 230.36964285714285,
-      "epoch": 3.2442748091603053,
-      "grad_norm": 13.211128234863281,
-      "kl": 0.09072041969214167,
-      "learning_rate": 4.339309776682829e-07,
-      "loss": 0.0001,
-      "reward": 2.326539208207812,
-      "reward_std": 2.8010854831763674,
-      "rewards/concensus_correctness_reward_func": 1.093885723395007,
-      "rewards/consensus_reward_func": 0.19285714285714287,
-      "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.14642857142857144,
-      "rewards/question_recreation_reward_func": 0.2836945739175592,
-      "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0017857142857142857,
-      "rewards/xmlcount_reward_func": 0.6078875030790056,
-      "step": 55
+      "completion_length": 217.75551470588235,
+      "epoch": 4.190476190476191,
+      "grad_norm": 4.1127400398254395,
+      "kl": 0.004183680955868433,
+      "learning_rate": 4.472851273490984e-07,
+      "loss": 0.0,
+      "reward": 0.8926854492986903,
+      "reward_std": 0.9771755057222703,
+      "rewards/concensus_correctness_reward_func": 0.09619853119639789,
+      "rewards/consensus_reward_func": 0.0661764705882353,
+      "rewards/cumulative_reward_2": 0.0,
+      "rewards/final_correctness_reward_func": 0.14705882352941177,
+      "rewards/question_recreation_reward_func": 0.19953655747368054,
+      "rewards/soft_format_reward_func": 0.001838235294117647,
+      "rewards/strict_format_reward_func": 0.0009191176470588235,
+      "rewards/xmlcount_reward_func": 0.3809577248552266,
+      "step": 25
     },
     {
-      "completion_length": 222.096875,
-      "epoch": 3.549618320610687,
-      "grad_norm": 1.6315250396728516,
-      "kl": 0.08041337295435369,
+      "completion_length": 218.11580882352942,
+      "epoch": 5.0,
+      "grad_norm": 1.4711809158325195,
+      "kl": 0.004230352046707755,
       "learning_rate": 4.193203929064353e-07,
-      "loss": 0.0001,
-      "reward": 2.38142469227314,
-      "reward_std": 2.643636444211006,
-      "rewards/concensus_correctness_reward_func": 1.0123843706212938,
-      "rewards/consensus_reward_func": 0.196875,
-      "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.228125,
-      "rewards/question_recreation_reward_func": 0.3176996737718582,
-      "rewards/soft_format_reward_func": 0.0015625,
-      "rewards/strict_format_reward_func": 0.00078125,
-      "rewards/xmlcount_reward_func": 0.623996875435114,
-      "step": 60
-    },
-    {
-      "completion_length": 227.6765625,
-      "epoch": 3.854961832061069,
-      "grad_norm": 1.5087581872940063,
-      "kl": 0.09321223869919777,
-      "learning_rate": 4.0355317817241697e-07,
-      "loss": 0.0001,
-      "reward": 2.3100090995430946,
-      "reward_std": 2.631418401747942,
-      "rewards/concensus_correctness_reward_func": 0.9473375119268894,
-      "rewards/consensus_reward_func": 0.2375,
-      "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.20625,
-      "rewards/question_recreation_reward_func": 0.29184813760221007,
-      "rewards/soft_format_reward_func": 0.003125,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.623948446661234,
-      "step": 65
+      "loss": 0.0,
+      "reward": 0.9753129236838397,
+      "reward_std": 0.9230635635116521,
+      "rewards/concensus_correctness_reward_func": 0.09226102951694937,
+      "rewards/consensus_reward_func": 0.04411764705882353,
+      "rewards/cumulative_reward_2": 0.0,
+      "rewards/final_correctness_reward_func": 0.17647058823529413,
+      "rewards/question_recreation_reward_func": 0.23682946558384335,
+      "rewards/soft_format_reward_func": 0.0,
+      "rewards/strict_format_reward_func": 0.0009191176470588235,
+      "rewards/xmlcount_reward_func": 0.4247150727931191,
+      "step": 30
     },
     {
-      "completion_length": 226.1017857142857,
-      "epoch": 4.122137404580153,
-      "grad_norm": 1.5481903553009033,
-      "kl": 0.09283565463764328,
+      "completion_length": 219.0828125,
+      "epoch": 5.9523809523809526,
+      "grad_norm": 9.926897048950195,
+      "kl": 0.00585453501844313,
       "learning_rate": 3.867370395306068e-07,
-      "loss": 0.0001,
-      "reward": 2.0852925266538347,
-      "reward_std": 2.410001914841788,
-      "rewards/concensus_correctness_reward_func": 0.7340392986578601,
-      "rewards/consensus_reward_func": 0.21785714285714286,
+      "loss": 0.0,
+      "reward": 1.0385151661932468,
+      "reward_std": 1.107630380988121,
+      "rewards/concensus_correctness_reward_func": 0.19590625064447523,
+      "rewards/consensus_reward_func": 0.059375,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.2,
-      "rewards/question_recreation_reward_func": 0.3157889977097511,
+      "rewards/final_correctness_reward_func": 0.16875,
+      "rewards/question_recreation_reward_func": 0.23067768132314087,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.6176071413925716,
-      "step": 70
-    },
-    {
-      "completion_length": 224.4625,
-      "epoch": 4.427480916030534,
-      "grad_norm": 1.6221083402633667,
-      "kl": 0.09271430894732476,
-      "learning_rate": 3.689868482592684e-07,
-      "loss": 0.0001,
-      "reward": 2.5825726106762885,
-      "reward_std": 2.6193617291748525,
-      "rewards/concensus_correctness_reward_func": 1.1844156241044401,
-      "rewards/consensus_reward_func": 0.196875,
-      "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.228125,
-      "rewards/question_recreation_reward_func": 0.3128054209053516,
-      "rewards/soft_format_reward_func": 0.00078125,
       "rewards/strict_format_reward_func": 0.0015625,
-      "rewards/xmlcount_reward_func": 0.6580078117549419,
-      "step": 75
+      "rewards/xmlcount_reward_func": 0.382243749499321,
+      "step": 35
     },
     {
-      "completion_length": 227.3359375,
-      "epoch": 4.732824427480916,
-      "grad_norm": 1.699670672416687,
-      "kl": 0.09714379052165896,
+      "completion_length": 217.82169117647058,
+      "epoch": 6.761904761904762,
+      "grad_norm": 4.228250026702881,
+      "kl": 0.004935655852451044,
       "learning_rate": 3.5042385616324236e-07,
-      "loss": 0.0001,
-      "reward": 2.3627020329236985,
-      "reward_std": 2.499714456498623,
-      "rewards/concensus_correctness_reward_func": 1.0211906284559518,
-      "rewards/consensus_reward_func": 0.225,
-      "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.209375,
-      "rewards/question_recreation_reward_func": 0.2893488988280296,
+      "loss": 0.0,
+      "reward": 0.971110563067829,
+      "reward_std": 0.9750440252177855,
+      "rewards/concensus_correctness_reward_func": 0.1082463244743207,
+      "rewards/consensus_reward_func": 0.04779411764705882,
+      "rewards/cumulative_reward_2": 0.0,
+      "rewards/final_correctness_reward_func": 0.18382352941176472,
+      "rewards/question_recreation_reward_func": 0.2439561676891411,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.00234375,
-      "rewards/xmlcount_reward_func": 0.6154437460005283,
-      "step": 80
-    },
-    {
-      "completion_length": 227.6017857142857,
-      "epoch": 5.0,
-      "grad_norm": 0.8229286074638367,
-      "kl": 0.08785766730351108,
-      "learning_rate": 3.3117486730117087e-07,
-      "loss": 0.0001,
-      "reward": 2.0304165261132376,
-      "reward_std": 2.2844339413302284,
-      "rewards/concensus_correctness_reward_func": 0.7656464321272713,
-      "rewards/consensus_reward_func": 0.2,
-      "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.16071428571428573,
-      "rewards/question_recreation_reward_func": 0.2893593538020338,
-      "rewards/soft_format_reward_func": 0.0008928571428571428,
-      "rewards/strict_format_reward_func": 0.0008928571428571428,
-      "rewards/xmlcount_reward_func": 0.6129107126167842,
-      "step": 85
+      "rewards/strict_format_reward_func": 0.0009191176470588235,
+      "rewards/xmlcount_reward_func": 0.38637132504407096,
+      "step": 40
     },
     {
-      "completion_length": 224.35625,
-      "epoch": 5.305343511450381,
-      "grad_norm": 1.8476825952529907,
-      "kl": 0.09851492333691567,
+      "completion_length": 216.6672794117647,
+      "epoch": 7.571428571428571,
+      "grad_norm": 11.885100364685059,
+      "kl": 0.007807457162176862,
       "learning_rate": 3.1137137178519977e-07,
-      "loss": 0.0001,
-      "reward": 2.0146584630012514,
-      "reward_std": 2.040700928866863,
-      "rewards/concensus_correctness_reward_func": 0.6759656336158514,
-      "rewards/consensus_reward_func": 0.228125,
-      "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.178125,
-      "rewards/question_recreation_reward_func": 0.2884428523480892,
-      "rewards/soft_format_reward_func": 0.00234375,
-      "rewards/strict_format_reward_func": 0.00234375,
-      "rewards/xmlcount_reward_func": 0.6393125005066395,
-      "step": 90
-    },
-    {
-      "completion_length": 228.8078125,
-      "epoch": 5.6106870229007635,
-      "grad_norm": 2.1402406692504883,
-      "kl": 0.09463290963321924,
-      "learning_rate": 2.911486475701835e-07,
-      "loss": 0.0001,
-      "reward": 3.082321289181709,
-      "reward_std": 3.3287234142422677,
-      "rewards/concensus_correctness_reward_func": 1.6685624956153333,
-      "rewards/consensus_reward_func": 0.25625,
-      "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.190625,
-      "rewards/question_recreation_reward_func": 0.3137759966775775,
-      "rewards/soft_format_reward_func": 0.0015625,
-      "rewards/strict_format_reward_func": 0.0015625,
-      "rewards/xmlcount_reward_func": 0.6499828085303306,
-      "step": 95
+      "loss": 0.0,
+      "reward": 0.9136327408692416,
+      "reward_std": 0.9482370290686103,
+      "rewards/concensus_correctness_reward_func": 0.09872794173219625,
+      "rewards/consensus_reward_func": 0.05514705882352941,
+      "rewards/cumulative_reward_2": 0.0,
+      "rewards/final_correctness_reward_func": 0.15808823529411764,
+      "rewards/question_recreation_reward_func": 0.2349783167900408,
+      "rewards/soft_format_reward_func": 0.001838235294117647,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.36485293463749047,
+      "step": 45
     },
     {
-      "completion_length": 227.2109375,
-      "epoch": 5.916030534351145,
-      "grad_norm": 1.4899626970291138,
-      "kl": 0.08579296409152448,
+      "completion_length": 215.69301470588235,
+      "epoch": 8.380952380952381,
+      "grad_norm": 8.040736198425293,
+      "kl": 0.030681811523048535,
       "learning_rate": 2.706448363680831e-07,
-      "loss": 0.0001,
-      "reward": 2.692727318406105,
-      "reward_std": 2.8540393710136414,
-      "rewards/concensus_correctness_reward_func": 1.338937508687377,
-      "rewards/consensus_reward_func": 0.246875,
-      "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.18125,
-      "rewards/question_recreation_reward_func": 0.28955701310187576,
-      "rewards/soft_format_reward_func": 0.00078125,
-      "rewards/strict_format_reward_func": 0.00078125,
-      "rewards/xmlcount_reward_func": 0.6345453165471554,
-      "step": 100
-    },
-    {
-      "completion_length": 225.48392857142858,
-      "epoch": 6.183206106870229,
-      "grad_norm": 1.5983153581619263,
-      "kl": 0.08785093038209847,
-      "learning_rate": 2.5e-07,
-      "loss": 0.0001,
-      "reward": 2.0271904604775566,
-      "reward_std": 2.129315710919244,
-      "rewards/concensus_correctness_reward_func": 0.6535428636840411,
-      "rewards/consensus_reward_func": 0.275,
-      "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.12142857142857143,
-      "rewards/question_recreation_reward_func": 0.300606526860169,
-      "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0017857142857142857,
-      "rewards/xmlcount_reward_func": 0.6748267871992929,
-      "step": 105
+      "loss": 0.0,
+      "reward": 0.9279013907208162,
+      "reward_std": 0.9109513680724537,
+      "rewards/concensus_correctness_reward_func": 0.08287132454707342,
+      "rewards/consensus_reward_func": 0.05514705882352941,
+      "rewards/cumulative_reward_2": 0.0,
+      "rewards/final_correctness_reward_func": 0.17647058823529413,
+      "rewards/question_recreation_reward_func": 0.2283407300710678,
+      "rewards/soft_format_reward_func": 0.0009191176470588235,
+      "rewards/strict_format_reward_func": 0.0009191176470588235,
+      "rewards/xmlcount_reward_func": 0.3832334569271873,
+      "step": 50
     },
     {
-      "completion_length": 224.978125,
-      "epoch": 6.488549618320611,
-      "grad_norm": 2.0014476776123047,
-      "kl": 0.10433612796477973,
+      "completion_length": 219.5183823529412,
+      "epoch": 9.19047619047619,
+      "grad_norm": 6.5597825050354,
+      "kl": 0.004213525095324525,
       "learning_rate": 2.2935516363191693e-07,
-      "loss": 0.0001,
-      "reward": 2.448861801624298,
-      "reward_std": 2.524143137037754,
-      "rewards/concensus_correctness_reward_func": 0.9977124966681004,
-      "rewards/consensus_reward_func": 0.2625,
+      "loss": 0.0,
+      "reward": 0.8882785693687552,
+      "reward_std": 0.9178478770396289,
+      "rewards/concensus_correctness_reward_func": 0.07434191127472065,
+      "rewards/consensus_reward_func": 0.04411764705882353,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.28643678678199647,
-      "rewards/soft_format_reward_func": 0.00078125,
-      "rewards/strict_format_reward_func": 0.00234375,
-      "rewards/xmlcount_reward_func": 0.649087505787611,
-      "step": 110
-    },
-    {
-      "completion_length": 223.5515625,
-      "epoch": 6.793893129770993,
-      "grad_norm": 1.6419188976287842,
-      "kl": 0.09386491114273668,
-      "learning_rate": 2.0885135242981647e-07,
-      "loss": 0.0001,
-      "reward": 2.819219100475311,
-      "reward_std": 3.3134369418025016,
-      "rewards/concensus_correctness_reward_func": 1.461875000037253,
-      "rewards/consensus_reward_func": 0.246875,
-      "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.203125,
-      "rewards/question_recreation_reward_func": 0.27337694596499207,
+      "rewards/final_correctness_reward_func": 0.1875,
+      "rewards/question_recreation_reward_func": 0.22048261701403296,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.00078125,
-      "rewards/xmlcount_reward_func": 0.6331859365105629,
-      "step": 115
+      "rewards/strict_format_reward_func": 0.0009191176470588235,
+      "rewards/xmlcount_reward_func": 0.3609172793872216,
+      "step": 55
     },
     {
-      "completion_length": 228.37857142857143,
-      "epoch": 7.061068702290076,
-      "grad_norm": 1.7330297231674194,
-      "kl": 0.09407830823745046,
+      "completion_length": 216.66176470588235,
+      "epoch": 10.0,
+      "grad_norm": 1.862962007522583,
+      "kl": 0.002967422687695088,
       "learning_rate": 1.886286282148002e-07,
-      "loss": 0.0001,
-      "reward": 2.1238247888428825,
-      "reward_std": 2.135510918072292,
-      "rewards/concensus_correctness_reward_func": 0.8201964225087847,
-      "rewards/consensus_reward_func": 0.19285714285714287,
-      "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.17142857142857143,
-      "rewards/question_recreation_reward_func": 0.312046221111502,
-      "rewards/soft_format_reward_func": 0.0026785714285714286,
-      "rewards/strict_format_reward_func": 0.0008928571428571428,
-      "rewards/xmlcount_reward_func": 0.6237249961921147,
-      "step": 120
-    },
-    {
-      "completion_length": 225.4453125,
-      "epoch": 7.366412213740458,
-      "grad_norm": 1.3467189073562622,
-      "kl": 0.09670781339518726,
-      "learning_rate": 1.6882513269882913e-07,
-      "loss": 0.0001,
-      "reward": 2.9024840116500856,
-      "reward_std": 3.2552048370242117,
-      "rewards/concensus_correctness_reward_func": 1.4498093830421568,
-      "rewards/consensus_reward_func": 0.2375,
-      "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.228125,
-      "rewards/question_recreation_reward_func": 0.3152043020352721,
+      "loss": 0.0,
+      "reward": 0.8225221322739825,
+      "reward_std": 0.8722858481547412,
+      "rewards/concensus_correctness_reward_func": 0.07955514683442957,
+      "rewards/consensus_reward_func": 0.04779411764705882,
+      "rewards/cumulative_reward_2": 0.0,
+      "rewards/final_correctness_reward_func": 0.15073529411764705,
+      "rewards/question_recreation_reward_func": 0.18901477447327444,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0015625,
-      "rewards/xmlcount_reward_func": 0.6702828057110309,
-      "step": 125
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.3554227948188782,
+      "step": 60
     },
     {
-      "completion_length": 225.0,
-      "epoch": 7.67175572519084,
-      "grad_norm": 2.1679375171661377,
-      "kl": 0.08589964711572975,
+      "completion_length": 215.9703125,
+      "epoch": 10.952380952380953,
+      "grad_norm": 4.381515026092529,
+      "kl": 0.004608349371119402,
       "learning_rate": 1.4957614383675767e-07,
-      "loss": 0.0001,
-      "reward": 1.8035920888185502,
-      "reward_std": 1.8396185874938964,
-      "rewards/concensus_correctness_reward_func": 0.47076874738559127,
-      "rewards/consensus_reward_func": 0.215625,
-      "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.1625,
-      "rewards/question_recreation_reward_func": 0.30079679638147355,
-      "rewards/soft_format_reward_func": 0.0015625,
+      "loss": 0.0,
+      "reward": 0.9207330733537674,
+      "reward_std": 0.8800692193210125,
+      "rewards/concensus_correctness_reward_func": 0.07503749821335078,
+      "rewards/consensus_reward_func": 0.03125,
+      "rewards/cumulative_reward_2": 0.0,
+      "rewards/final_correctness_reward_func": 0.159375,
+      "rewards/question_recreation_reward_func": 0.247809642739594,
+      "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0015625,
-      "rewards/xmlcount_reward_func": 0.6507765665650368,
-      "step": 130
-    },
-    {
-      "completion_length": 229.65625,
-      "epoch": 7.977099236641221,
-      "grad_norm": 2.253136396408081,
-      "kl": 0.11240638778544962,
-      "learning_rate": 1.310131517407316e-07,
-      "loss": 0.0001,
-      "reward": 2.460201847553253,
-      "reward_std": 2.5180085256695746,
-      "rewards/concensus_correctness_reward_func": 1.081900003645569,
-      "rewards/consensus_reward_func": 0.246875,
-      "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.1875,
-      "rewards/question_recreation_reward_func": 0.29531903751194477,
-      "rewards/soft_format_reward_func": 0.00078125,
-      "rewards/strict_format_reward_func": 0.00078125,
-      "rewards/xmlcount_reward_func": 0.6470453068614006,
-      "step": 135
+      "rewards/xmlcount_reward_func": 0.40569843612611295,
+      "step": 65
     },
     {
-      "completion_length": 225.0625,
-      "epoch": 8.244274809160306,
-      "grad_norm": 1.5449919700622559,
-      "kl": 0.08950371891260148,
+      "completion_length": 217.2058823529412,
+      "epoch": 11.761904761904763,
+      "grad_norm": 3.9310142993927,
+      "kl": 0.0031400311820428163,
       "learning_rate": 1.1326296046939333e-07,
-      "loss": 0.0001,
-      "reward": 2.3891380735806056,
-      "reward_std": 2.811609878710338,
-      "rewards/concensus_correctness_reward_func": 1.032021415233612,
-      "rewards/consensus_reward_func": 0.22142857142857142,
-      "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.21071428571428572,
-      "rewards/question_recreation_reward_func": 0.2915398589202336,
+      "loss": 0.0,
+      "reward": 0.9569352628553615,
+      "reward_std": 0.9295545737533009,
+      "rewards/concensus_correctness_reward_func": 0.08112867603845456,
+      "rewards/consensus_reward_func": 0.04779411764705882,
+      "rewards/cumulative_reward_2": 0.0,
+      "rewards/final_correctness_reward_func": 0.15073529411764705,
+      "rewards/question_recreation_reward_func": 0.25045547919238315,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.6334339363234384,
-      "step": 140
-    },
-    {
-      "completion_length": 224.471875,
-      "epoch": 8.549618320610687,
-      "grad_norm": 1.8451206684112549,
-      "kl": 0.09755032181274145,
-      "learning_rate": 9.644682182758304e-08,
-      "loss": 0.0001,
-      "reward": 2.3509212240576742,
-      "reward_std": 2.5712964862585066,
-      "rewards/concensus_correctness_reward_func": 0.9543406262993812,
-      "rewards/consensus_reward_func": 0.25,
-      "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.209375,
-      "rewards/question_recreation_reward_func": 0.29743685238063333,
-      "rewards/soft_format_reward_func": 0.00078125,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.638987497985363,
-      "step": 145
+      "rewards/strict_format_reward_func": 0.0009191176470588235,
+      "rewards/xmlcount_reward_func": 0.42590257974670215,
+      "step": 70
     },
     {
-      "completion_length": 229.9328125,
-      "epoch": 8.854961832061068,
-      "grad_norm": 1.6817524433135986,
-      "kl": 0.09376556426286697,
+      "completion_length": 217.875,
+      "epoch": 12.571428571428571,
+      "grad_norm": 3.102419376373291,
+      "kl": 0.0023113159010844197,
       "learning_rate": 8.067960709356478e-08,
-      "loss": 0.0001,
-      "reward": 2.199650877714157,
-      "reward_std": 2.234014268964529,
-      "rewards/concensus_correctness_reward_func": 0.8738531303592026,
-      "rewards/consensus_reward_func": 0.215625,
-      "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.16875,
-      "rewards/question_recreation_reward_func": 0.3147571422159672,
-      "rewards/soft_format_reward_func": 0.00234375,
-      "rewards/strict_format_reward_func": 0.0015625,
-      "rewards/xmlcount_reward_func": 0.6227593772113323,
-      "step": 150
-    },
-    {
-      "completion_length": 227.23392857142858,
-      "epoch": 9.122137404580153,
-      "grad_norm": 1.5826033353805542,
-      "kl": 0.10102412780480725,
-      "learning_rate": 6.60690223317171e-08,
-      "loss": 0.0001,
-      "reward": 2.717519346305302,
-      "reward_std": 2.7952321265425,
-      "rewards/concensus_correctness_reward_func": 1.244885718183858,
-      "rewards/consensus_reward_func": 0.25,
-      "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.24642857142857144,
-      "rewards/question_recreation_reward_func": 0.30728720192398346,
-      "rewards/soft_format_reward_func": 0.0,
+      "loss": 0.0,
+      "reward": 0.8187810673433191,
+      "reward_std": 0.879577760310734,
+      "rewards/concensus_correctness_reward_func": 0.0920624991550165,
+      "rewards/consensus_reward_func": 0.025735294117647058,
+      "rewards/cumulative_reward_2": 0.0,
+      "rewards/final_correctness_reward_func": 0.13602941176470587,
+      "rewards/question_recreation_reward_func": 0.20141525446053812,
+      "rewards/soft_format_reward_func": 0.0009191176470588235,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.6689178551946368,
-      "step": 155
+      "rewards/xmlcount_reward_func": 0.3626194854869562,
+      "step": 75
     },
     {
-      "completion_length": 230.4765625,
-      "epoch": 9.427480916030534,
-      "grad_norm": 1.7758827209472656,
-      "kl": 0.09566058879718184,
+      "completion_length": 221.18014705882354,
+      "epoch": 13.380952380952381,
+      "grad_norm": 4.012791633605957,
+      "kl": 0.005384887541260789,
       "learning_rate": 5.271487265090163e-08,
-      "loss": 0.0001,
-      "reward": 2.054090151935816,
-      "reward_std": 2.34877013489604,
-      "rewards/concensus_correctness_reward_func": 0.8258124882355332,
-      "rewards/consensus_reward_func": 0.25625,
-      "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.08125,
-      "rewards/question_recreation_reward_func": 0.2795291979797184,
-      "rewards/soft_format_reward_func": 0.00234375,
-      "rewards/strict_format_reward_func": 0.0015625,
-      "rewards/xmlcount_reward_func": 0.6073421865701676,
-      "step": 160
-    },
-    {
-      "completion_length": 229.909375,
-      "epoch": 9.732824427480915,
-      "grad_norm": 1.4019196033477783,
-      "kl": 0.10127251031808555,
-      "learning_rate": 4.0708380434367864e-08,
-      "loss": 0.0001,
-      "reward": 2.4125003829598426,
-      "reward_std": 2.4047275736927984,
-      "rewards/concensus_correctness_reward_func": 1.0241687575355172,
-      "rewards/consensus_reward_func": 0.265625,
-      "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.16875,
-      "rewards/question_recreation_reward_func": 0.31353163830935954,
-      "rewards/soft_format_reward_func": 0.0015625,
-      "rewards/strict_format_reward_func": 0.0015625,
-      "rewards/xmlcount_reward_func": 0.6372999995946884,
-      "step": 165
+      "loss": 0.0,
+      "reward": 0.8298134128837025,
+      "reward_std": 0.8251312436426387,
+      "rewards/concensus_correctness_reward_func": 0.07310661739286255,
+      "rewards/consensus_reward_func": 0.022058823529411766,
+      "rewards/cumulative_reward_2": 0.0,
+      "rewards/final_correctness_reward_func": 0.1801470588235294,
+      "rewards/question_recreation_reward_func": 0.2030891538323725,
+      "rewards/soft_format_reward_func": 0.0009191176470588235,
+      "rewards/strict_format_reward_func": 0.0009191176470588235,
+      "rewards/xmlcount_reward_func": 0.3495735298184788,
+      "step": 80
     },
     {
-      "completion_length": 227.7017857142857,
-      "epoch": 10.0,
-      "grad_norm": 0.6785824298858643,
-      "kl": 0.08975713018860136,
+      "completion_length": 219.46691176470588,
+      "epoch": 14.19047619047619,
+      "grad_norm": 2.952268123626709,
+      "kl": 0.0029621824026381705,
       "learning_rate": 3.013156219837776e-08,
-      "loss": 0.0001,
-      "reward": 1.9554564390863691,
-      "reward_std": 2.0454944218908038,
-      "rewards/concensus_correctness_reward_func": 0.5938928590289184,
-      "rewards/consensus_reward_func": 0.22857142857142856,
-      "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.27023680912596837,
-      "rewards/soft_format_reward_func": 0.0008928571428571428,
-      "rewards/strict_format_reward_func": 0.0008928571428571428,
-      "rewards/xmlcount_reward_func": 0.6109696456364223,
-      "step": 170
-    },
-    {
-      "completion_length": 229.1828125,
-      "epoch": 10.305343511450381,
-      "grad_norm": 1.5233851671218872,
-      "kl": 0.08713910647202283,
-      "learning_rate": 2.1056668336235623e-08,
-      "loss": 0.0001,
-      "reward": 1.773926168680191,
-      "reward_std": 1.8827015846967696,
-      "rewards/concensus_correctness_reward_func": 0.5077062356285751,
-      "rewards/consensus_reward_func": 0.20625,
-      "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.175,
-      "rewards/question_recreation_reward_func": 0.2668996132910252,
-      "rewards/soft_format_reward_func": 0.0015625,
-      "rewards/strict_format_reward_func": 0.0015625,
-      "rewards/xmlcount_reward_func": 0.6149453103542328,
-      "step": 175
-    },
-    {
-      "completion_length": 224.91875,
-      "epoch": 10.610687022900763,
-      "grad_norm": 1.5735443830490112,
-      "kl": 0.09181699948385358,
-      "learning_rate": 1.3545689574841341e-08,
-      "loss": 0.0001,
-      "reward": 2.5688821971416473,
-      "reward_std": 3.0575607106089593,
-      "rewards/concensus_correctness_reward_func": 1.2135156329721213,
-      "rewards/consensus_reward_func": 0.209375,
-      "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.215625,
-      "rewards/question_recreation_reward_func": 0.3163775021210313,
+      "loss": 0.0,
+      "reward": 0.8352440113530439,
+      "reward_std": 0.8148685544729233,
+      "rewards/concensus_correctness_reward_func": 0.08091911651632365,
+      "rewards/consensus_reward_func": 0.04044117647058824,
+      "rewards/cumulative_reward_2": 0.0,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.21936532570158734,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.6139890596270561,
-      "step": 180
+      "rewards/strict_format_reward_func": 0.001838235294117647,
+      "rewards/xmlcount_reward_func": 0.3676801420309964,
+      "step": 85
     },
     {
-      "completion_length": 227.946875,
-      "epoch": 10.916030534351146,
-      "grad_norm": 1.6556141376495361,
-      "kl": 0.11013491982594133,
-      "learning_rate": 7.649933515167406e-09,
-      "loss": 0.0001,
-      "reward": 2.8750985130667686,
-      "reward_std": 3.1339681535959243,
-      "rewards/concensus_correctness_reward_func": 1.4591718828538434,
-      "rewards/consensus_reward_func": 0.265625,
-      "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.215625,
-      "rewards/question_recreation_reward_func": 0.30609070267528293,
-      "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.6285859420895576,
-      "step": 185
+      "completion_length": 219.0514705882353,
+      "epoch": 15.0,
+      "grad_norm": 2.2386655807495117,
+      "kl": 0.003243645081323955,
+      "learning_rate": 1.3545689574841341e-08,
+      "loss": 0.0,
+      "reward": 0.865555027828497,
+      "reward_std": 0.847034204093849,
+      "rewards/concensus_correctness_reward_func": 0.06844852908569224,
+      "rewards/consensus_reward_func": 0.04411764705882353,
+      "rewards/cumulative_reward_2": 0.0,
+      "rewards/final_correctness_reward_func": 0.14705882352941177,
+      "rewards/question_recreation_reward_func": 0.2121359138366054,
+      "rewards/soft_format_reward_func": 0.0009191176470588235,
+      "rewards/strict_format_reward_func": 0.0009191176470588235,
+      "rewards/xmlcount_reward_func": 0.3919558823108673,
+      "step": 90
     },
     {
-      "completion_length": 222.18392857142857,
-      "epoch": 11.183206106870228,
-      "grad_norm": 1.699262022972107,
-      "kl": 0.1069457121193409,
+      "completion_length": 221.35,
+      "epoch": 15.952380952380953,
+      "grad_norm": 5.4549055099487305,
+      "kl": 0.005640207437681966,
       "learning_rate": 3.4096741493194193e-09,
-      "loss": 0.0001,
-      "reward": 2.5891014899526326,
-      "reward_std": 2.7870528187070573,
-      "rewards/concensus_correctness_reward_func": 1.15280357918569,
-      "rewards/consensus_reward_func": 0.24285714285714285,
-      "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.24642857142857144,
-      "rewards/question_recreation_reward_func": 0.28257290316479544,
-      "rewards/soft_format_reward_func": 0.0008928571428571428,
-      "rewards/strict_format_reward_func": 0.0008928571428571428,
-      "rewards/xmlcount_reward_func": 0.662653568812779,
-      "step": 190
-    },
-    {
-      "completion_length": 227.6078125,
-      "epoch": 11.488549618320612,
-      "grad_norm": 2.025723695755005,
-      "kl": 0.09346012743189931,
-      "learning_rate": 8.538767483325383e-10,
-      "loss": 0.0001,
-      "reward": 2.5617597877979277,
-      "reward_std": 2.8330461889505387,
-      "rewards/concensus_correctness_reward_func": 1.2098406325094402,
-      "rewards/consensus_reward_func": 0.178125,
+      "loss": 0.0,
+      "reward": 0.977014084905386,
+      "reward_std": 1.0592330724000931,
+      "rewards/concensus_correctness_reward_func": 0.14016250055283308,
+      "rewards/consensus_reward_func": 0.05,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.246875,
-      "rewards/question_recreation_reward_func": 0.2995644178241491,
+      "rewards/final_correctness_reward_func": 0.18125,
+      "rewards/question_recreation_reward_func": 0.22033127043396233,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.00078125,
-      "rewards/xmlcount_reward_func": 0.6265734456479549,
-      "step": 195
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.38527031214907764,
+      "step": 95
     },
     {
-      "completion_length": 226.3453125,
-      "epoch": 11.793893129770993,
-      "grad_norm": 1.561265230178833,
-      "kl": 0.09566104216501117,
+      "completion_length": 218.4246323529412,
+      "epoch": 16.761904761904763,
+      "grad_norm": 4.802547931671143,
+      "kl": 0.003235554914948914,
       "learning_rate": 0.0,
-      "loss": 0.0001,
-      "reward": 2.1840140745043755,
-      "reward_std": 2.114570437371731,
-      "rewards/concensus_correctness_reward_func": 0.9061656036414206,
-      "rewards/consensus_reward_func": 0.228125,
-      "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.115625,
-      "rewards/question_recreation_reward_func": 0.3060203332453966,
+      "loss": 0.0,
+      "reward": 0.8579355530879077,
+      "reward_std": 0.9595372615491643,
+      "rewards/concensus_correctness_reward_func": 0.10365073623902657,
+      "rewards/consensus_reward_func": 0.025735294117647058,
+      "rewards/cumulative_reward_2": 0.0,
+      "rewards/final_correctness_reward_func": 0.13970588235294118,
+      "rewards/question_recreation_reward_func": 0.21492820759029949,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0015625,
-      "rewards/xmlcount_reward_func": 0.626515619456768,
-      "step": 200
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.3739154419916518,
+      "step": 100
     },
     {
-      "epoch": 11.793893129770993,
-      "step": 200,
+      "epoch": 16.761904761904763,
+      "step": 100,
       "total_flos": 0.0,
-      "train_loss": 8.942109707277268e-05,
-      "train_runtime": 4099.4474,
-      "train_samples_per_second": 6.245,
-      "train_steps_per_second": 0.049
+      "train_loss": 5.2903332107234745e-06,
+      "train_runtime": 1863.6299,
+      "train_samples_per_second": 6.868,
+      "train_steps_per_second": 0.054
     }
   ],
   "logging_steps": 5,
-  "max_steps": 200,
+  "max_steps": 100,
   "num_input_tokens_seen": 0,
-  "num_train_epochs": 13,
+  "num_train_epochs": 20,
   "save_steps": 50,
   "stateful_callbacks": {
     "TrainerControl": {

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:05ed80a22ff9c829edb7a4c8fd9cc4182f39df7c3e1c90d15f1ebf2fb8d1778c
+oid sha256:f61e7cd63625c52f24373a35fc8911f78375028dddc9c9ee533d76f160cb57ec
 size 5944