End of training

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 3.173273307766067e-05,
-    "train_runtime": 12794.8149,
+    "train_loss": 3.209044134564465e-05,
+    "train_runtime": 6335.5907,
     "train_samples": 17,
-    "train_samples_per_second": 0.013,
-    "train_steps_per_second": 0.001
+    "train_samples_per_second": 0.025,
+    "train_steps_per_second": 0.002
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:79bf70126adca640ef61e9f89147c3ceff0cc782f4c8b7ccded893f89ea8a8f3
+oid sha256:e248b0edca59cda743d58d2854ba76d32ca2e1befa1f575a24058c02405f8a1f
 size 1976163472

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 3.173273307766067e-05,
-    "train_runtime": 12794.8149,
+    "train_loss": 3.209044134564465e-05,
+    "train_runtime": 6335.5907,
     "train_samples": 17,
-    "train_samples_per_second": 0.013,
-    "train_steps_per_second": 0.001
+    "train_samples_per_second": 0.025,
+    "train_steps_per_second": 0.002
 }

trainer_state.json

@@ -10,108 +10,108 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 143.40625,
+      "completion_length": 165.75,
       "epoch": 0.9411764705882353,
-      "grad_norm": 4.284185409545898,
+      "grad_norm": 13.425328254699707,
       "kl": 0.0,
       "learning_rate": 5e-07,
-      "loss": 0.0,
-      "reward": 6.341443166136742,
-      "reward_std": 0.7442100643820595,
-      "rewards/concensus_correctness_reward_func": 2.2283749617636204,
-      "rewards/consensus_reward_func": 1.875,
+      "loss": -0.0,
+      "reward": 6.195518314838409,
+      "reward_std": 0.15492907539010048,
+      "rewards/concensus_correctness_reward_func": 2.160749964416027,
+      "rewards/consensus_reward_func": 2.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.375,
-      "rewards/question_recreation_reward_func": 0.9221307889092714,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.8891747251618654,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.046875,
-      "rewards/xmlcount_reward_func": 0.8940625116229057,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 1.0049687623977661,
       "step": 2
     },
     {
-      "completion_length": 109.5,
+      "completion_length": 99.66666666666667,
       "epoch": 1.4705882352941178,
-      "grad_norm": 12.020967483520508,
-      "kl": 0.026952934147427894,
+      "grad_norm": 8.04550838470459,
+      "kl": 0.033101409363249935,
       "learning_rate": 4.415111107797445e-07,
       "loss": 0.0,
-      "reward": 6.3079690138498945,
-      "reward_std": 0.1152670236511363,
-      "rewards/concensus_correctness_reward_func": 2.2435555193159313,
-      "rewards/consensus_reward_func": 2.0,
+      "reward": 5.772522926330566,
+      "reward_std": 0.42779586070941555,
+      "rewards/concensus_correctness_reward_func": 2.0278888609674244,
+      "rewards/consensus_reward_func": 1.8888888888888888,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.4444444444444444,
-      "rewards/question_recreation_reward_func": 0.7794692266422013,
+      "rewards/final_correctness_reward_func": 0.2222222222222222,
+      "rewards/question_recreation_reward_func": 0.7571342205111351,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.027777777777777776,
-      "rewards/xmlcount_reward_func": 0.8127222259839376,
+      "rewards/xmlcount_reward_func": 0.8486111296547784,
       "step": 4
     },
     {
-      "completion_length": 123.61111111111111,
+      "completion_length": 121.61111111111111,
       "epoch": 2.0,
-      "grad_norm": 1.6072788184828823e-06,
-      "kl": 0.05806939345267084,
+      "grad_norm": 5.110976086797336e-09,
+      "kl": 0.05847398496957289,
       "learning_rate": 2.934120444167326e-07,
       "loss": 0.0,
-      "reward": 6.237047221925524,
-      "reward_std": 0.2826267232497533,
-      "rewards/concensus_correctness_reward_func": 2.242888847986857,
+      "reward": 6.203311602274577,
+      "reward_std": 0.2927806369132466,
+      "rewards/concensus_correctness_reward_func": 2.1322221755981445,
       "rewards/consensus_reward_func": 2.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.1111111111111111,
-      "rewards/question_recreation_reward_func": 0.8891028325694302,
+      "rewards/question_recreation_reward_func": 0.887922905644195,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.027777777777777776,
-      "rewards/xmlcount_reward_func": 0.9661666684680514,
+      "rewards/strict_format_reward_func": 0.05555555555555555,
+      "rewards/xmlcount_reward_func": 1.0164999961853027,
       "step": 6
     },
     {
-      "completion_length": 169.75,
+      "completion_length": 103.1875,
       "epoch": 2.9411764705882355,
-      "grad_norm": 6.909427165985107,
-      "kl": 0.06250839692074805,
+      "grad_norm": 7.975579738616943,
+      "kl": 0.07568144043170832,
       "learning_rate": 1.2500000000000005e-07,
       "loss": 0.0001,
-      "reward": 6.214004263281822,
-      "reward_std": 0.4483142468961887,
-      "rewards/concensus_correctness_reward_func": 2.287937458604574,
-      "rewards/consensus_reward_func": 1.9375,
+      "reward": 6.343166649341583,
+      "reward_std": 0.17894219877780415,
+      "rewards/concensus_correctness_reward_func": 2.1609999611973763,
+      "rewards/consensus_reward_func": 2.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.8714732573425863,
+      "rewards/final_correctness_reward_func": 0.1875,
+      "rewards/question_recreation_reward_func": 0.8761980095005129,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.03125,
-      "rewards/xmlcount_reward_func": 0.9608437344431877,
+      "rewards/strict_format_reward_func": 0.109375,
+      "rewards/xmlcount_reward_func": 1.0090937577188015,
       "step": 8
     },
     {
-      "completion_length": 112.33333333333333,
+      "completion_length": 100.55555555555556,
       "epoch": 3.4705882352941178,
-      "grad_norm": 11.673796653747559,
-      "kl": 0.0752449690674742,
+      "grad_norm": 23.184375762939453,
+      "kl": 0.05748121626675129,
       "learning_rate": 1.507684480352292e-08,
       "loss": 0.0,
-      "reward": 6.055319547653198,
-      "reward_std": 0.37261243185235393,
-      "rewards/concensus_correctness_reward_func": 2.2428888612323337,
+      "reward": 6.09701861275567,
+      "reward_std": 0.1832191693958723,
+      "rewards/concensus_correctness_reward_func": 2.1322221888436212,
       "rewards/consensus_reward_func": 2.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.2222222222222222,
-      "rewards/question_recreation_reward_func": 0.7519309072134396,
+      "rewards/question_recreation_reward_func": 0.7799075724453561,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.8382777770360311,
+      "rewards/strict_format_reward_func": 0.08333333333333333,
+      "rewards/xmlcount_reward_func": 0.8793333371480306,
       "step": 10
     },
     {
       "epoch": 3.4705882352941178,
       "step": 10,
       "total_flos": 0.0,
-      "train_loss": 3.173273307766067e-05,
-      "train_runtime": 12794.8149,
-      "train_samples_per_second": 0.013,
-      "train_steps_per_second": 0.001
+      "train_loss": 3.209044134564465e-05,
+      "train_runtime": 6335.5907,
+      "train_samples_per_second": 0.025,
+      "train_steps_per_second": 0.002
     }
   ],
   "logging_steps": 2,