End of training

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 3.209044134564465e-05,
-    "train_runtime": 6335.5907,
+    "train_loss": 2.334083110326901e-05,
+    "train_runtime": 17913.3066,
     "train_samples": 17,
-    "train_samples_per_second": 0.025,
-    "train_steps_per_second": 0.002
+    "train_samples_per_second": 0.009,
+    "train_steps_per_second": 0.001
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e248b0edca59cda743d58d2854ba76d32ca2e1befa1f575a24058c02405f8a1f
+oid sha256:15d8162ff2f5da1973710d93f2b47430e7ce8999fe19ac9f1612de6271afd4b5
 size 1976163472

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 3.209044134564465e-05,
-    "train_runtime": 6335.5907,
+    "train_loss": 2.334083110326901e-05,
+    "train_runtime": 17913.3066,
     "train_samples": 17,
-    "train_samples_per_second": 0.025,
-    "train_steps_per_second": 0.002
+    "train_samples_per_second": 0.009,
+    "train_steps_per_second": 0.001
 }

trainer_state.json

@@ -10,108 +10,108 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 165.75,
+      "completion_length": 131.78125,
       "epoch": 0.9411764705882353,
-      "grad_norm": 13.425328254699707,
+      "grad_norm": 10.730029106140137,
       "kl": 0.0,
       "learning_rate": 5e-07,
       "loss": -0.0,
-      "reward": 6.195518314838409,
-      "reward_std": 0.15492907539010048,
-      "rewards/concensus_correctness_reward_func": 2.160749964416027,
-      "rewards/consensus_reward_func": 2.0,
+      "reward": 6.342288956046104,
+      "reward_std": 0.6563283740542829,
+      "rewards/concensus_correctness_reward_func": 2.102062474936247,
+      "rewards/consensus_reward_func": 1.9375,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.8891747251618654,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.9382578537333757,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": 1.0049687623977661,
+      "rewards/strict_format_reward_func": 0.15625,
+      "rewards/xmlcount_reward_func": 0.9582187570631504,
       "step": 2
     },
     {
-      "completion_length": 99.66666666666667,
+      "completion_length": 93.83333333333333,
       "epoch": 1.4705882352941178,
-      "grad_norm": 8.04550838470459,
-      "kl": 0.033101409363249935,
+      "grad_norm": 7.171228408813477,
+      "kl": 0.017023057715656858,
       "learning_rate": 4.415111107797445e-07,
       "loss": 0.0,
-      "reward": 5.772522926330566,
-      "reward_std": 0.42779586070941555,
-      "rewards/concensus_correctness_reward_func": 2.0278888609674244,
+      "reward": 6.55574173397488,
+      "reward_std": 0.43832636404679054,
+      "rewards/concensus_correctness_reward_func": 2.472111072805193,
       "rewards/consensus_reward_func": 1.8888888888888888,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.2222222222222222,
-      "rewards/question_recreation_reward_func": 0.7571342205111351,
+      "rewards/final_correctness_reward_func": 0.4444444444444444,
+      "rewards/question_recreation_reward_func": 0.7796862405310903,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.027777777777777776,
-      "rewards/xmlcount_reward_func": 0.8486111296547784,
+      "rewards/strict_format_reward_func": 0.1111111111111111,
+      "rewards/xmlcount_reward_func": 0.8594999843173556,
       "step": 4
     },
     {
-      "completion_length": 121.61111111111111,
+      "completion_length": 122.61111111111111,
       "epoch": 2.0,
-      "grad_norm": 5.110976086797336e-09,
-      "kl": 0.05847398496957289,
+      "grad_norm": 3.105441237494233e-08,
+      "kl": 0.02888714972262581,
       "learning_rate": 2.934120444167326e-07,
       "loss": 0.0,
-      "reward": 6.203311602274577,
-      "reward_std": 0.2927806369132466,
-      "rewards/concensus_correctness_reward_func": 2.1322221755981445,
+      "reward": 6.286137633853489,
+      "reward_std": 0.16779031413089898,
+      "rewards/concensus_correctness_reward_func": 2.134666668044196,
       "rewards/consensus_reward_func": 2.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.1111111111111111,
-      "rewards/question_recreation_reward_func": 0.887922905644195,
+      "rewards/question_recreation_reward_func": 0.8890821365122166,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.05555555555555555,
-      "rewards/xmlcount_reward_func": 1.0164999961853027,
+      "rewards/strict_format_reward_func": 0.1111111111111111,
+      "rewards/xmlcount_reward_func": 1.0401666694217258,
       "step": 6
     },
     {
-      "completion_length": 103.1875,
+      "completion_length": 121.0625,
       "epoch": 2.9411764705882355,
-      "grad_norm": 7.975579738616943,
-      "kl": 0.07568144043170832,
+      "grad_norm": 5.204808235168457,
+      "kl": 0.036964431335945847,
       "learning_rate": 1.2500000000000005e-07,
-      "loss": 0.0001,
-      "reward": 6.343166649341583,
-      "reward_std": 0.17894219877780415,
-      "rewards/concensus_correctness_reward_func": 2.1609999611973763,
+      "loss": 0.0,
+      "reward": 6.445510894060135,
+      "reward_std": 0.12171002507966477,
+      "rewards/concensus_correctness_reward_func": 2.28712497651577,
       "rewards/consensus_reward_func": 2.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.1875,
-      "rewards/question_recreation_reward_func": 0.8761980095005129,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.8760735102987383,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.109375,
-      "rewards/xmlcount_reward_func": 1.0090937577188015,
+      "rewards/xmlcount_reward_func": 0.9229374881833792,
       "step": 8
     },
     {
-      "completion_length": 100.55555555555556,
+      "completion_length": 135.33333333333334,
       "epoch": 3.4705882352941178,
-      "grad_norm": 23.184375762939453,
-      "kl": 0.05748121626675129,
+      "grad_norm": 13.752861976623535,
+      "kl": 0.08299477315611309,
       "learning_rate": 1.507684480352292e-08,
       "loss": 0.0,
-      "reward": 6.09701861275567,
-      "reward_std": 0.1832191693958723,
-      "rewards/concensus_correctness_reward_func": 2.1322221888436212,
+      "reward": 6.4182407591078015,
+      "reward_std": 0.18086219827334085,
+      "rewards/concensus_correctness_reward_func": 2.3553333150015936,
       "rewards/consensus_reward_func": 2.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.2222222222222222,
       "rewards/question_recreation_reward_func": 0.7799075724453561,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.08333333333333333,
-      "rewards/xmlcount_reward_func": 0.8793333371480306,
+      "rewards/strict_format_reward_func": 0.1111111111111111,
+      "rewards/xmlcount_reward_func": 0.9496666457917955,
       "step": 10
     },
     {
       "epoch": 3.4705882352941178,
       "step": 10,
       "total_flos": 0.0,
-      "train_loss": 3.209044134564465e-05,
-      "train_runtime": 6335.5907,
-      "train_samples_per_second": 0.025,
-      "train_steps_per_second": 0.002
+      "train_loss": 2.334083110326901e-05,
+      "train_runtime": 17913.3066,
+      "train_samples_per_second": 0.009,
+      "train_steps_per_second": 0.001
     }
   ],
   "logging_steps": 2,