End of training

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 1.11847843754731e-05,
-    "train_runtime": 205.7818,
+    "train_loss": 8.644163608551026e-06,
+    "train_runtime": 197.5707,
     "train_samples": 28,
-    "train_samples_per_second": 1.555,
-    "train_steps_per_second": 0.097
+    "train_samples_per_second": 1.62,
+    "train_steps_per_second": 0.101
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5678e4abcb885cc641c94af09b75a15fc281352efe58b4eee2261f43b4bdc4e1
+oid sha256:d528a7d528f3747a359348604a12a7c491f1765fae4bb4229aa80d5269938ec8
 size 1976163472

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 1.11847843754731e-05,
-    "train_runtime": 205.7818,
+    "train_loss": 8.644163608551026e-06,
+    "train_runtime": 197.5707,
     "train_samples": 28,
-    "train_samples_per_second": 1.555,
-    "train_steps_per_second": 0.097
+    "train_samples_per_second": 1.62,
+    "train_steps_per_second": 0.101
 }

trainer_state.json

@@ -10,203 +10,203 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 370.0625,
+      "completion_length": 351.71875,
       "epoch": 0.5714285714285714,
-      "grad_norm": 5.587164402008057,
+      "grad_norm": 6.811317443847656,
       "kl": 0.0,
       "learning_rate": 5e-07,
-      "loss": -0.0,
-      "reward": 1.8009525425732136,
-      "reward_std": 0.8436315204016864,
-      "rewards/concensus_correctness_reward_func": 0.29987499490380287,
-      "rewards/consensus_reward_func": 0.375,
+      "loss": 0.0,
+      "reward": 1.785135745536536,
+      "reward_std": 0.7207677089609206,
+      "rewards/concensus_correctness_reward_func": 0.21193749643862247,
+      "rewards/consensus_reward_func": 0.4375,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.4244213062338531,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.5477294897427782,
       "rewards/soft_format_reward_func": 0.015625,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.6235312521457672,
+      "rewards/xmlcount_reward_func": 0.5723437489941716,
       "step": 2
     },
     {
-      "completion_length": 241.79166666666666,
+      "completion_length": 300.7083333333333,
       "epoch": 1.0,
-      "grad_norm": 5.268057823181152,
-      "kl": 0.002790778259319874,
+      "grad_norm": 4.019150733947754,
+      "kl": 0.0015880058393425618,
       "learning_rate": 4.864543104251586e-07,
       "loss": 0.0,
-      "reward": 2.383975918094317,
-      "reward_std": 0.9525923697898785,
-      "rewards/concensus_correctness_reward_func": 0.5427499910195669,
-      "rewards/consensus_reward_func": 0.5,
+      "reward": 2.815759986639023,
+      "reward_std": 1.311702584226926,
+      "rewards/concensus_correctness_reward_func": 0.6080833251277605,
+      "rewards/consensus_reward_func": 0.75,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.16666666666666666,
-      "rewards/question_recreation_reward_func": 0.4531425765405099,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.5803016399343809,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.7214166720708212,
+      "rewards/strict_format_reward_func": 0.020833333333333332,
+      "rewards/xmlcount_reward_func": 0.8565416683753332,
       "step": 4
     },
     {
-      "completion_length": 352.6875,
+      "completion_length": 267.96875,
       "epoch": 1.5714285714285714,
-      "grad_norm": 7.042733192443848,
-      "kl": 0.00463899013993796,
+      "grad_norm": 6.299780368804932,
+      "kl": 0.0036450710395001806,
       "learning_rate": 4.472851273490984e-07,
       "loss": 0.0,
-      "reward": 2.075700271874666,
-      "reward_std": 1.4178721252828836,
-      "rewards/concensus_correctness_reward_func": 0.6191249918192625,
-      "rewards/consensus_reward_func": 0.625,
+      "reward": 2.7576604560017586,
+      "reward_std": 1.438764474238269,
+      "rewards/concensus_correctness_reward_func": 0.6950624939054251,
+      "rewards/consensus_reward_func": 0.75,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.1875,
-      "rewards/question_recreation_reward_func": 0.4460752762388438,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.5075979293324053,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.19799998681992292,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.7893750090152025,
       "step": 6
     },
     {
-      "completion_length": 342.125,
+      "completion_length": 263.75,
       "epoch": 2.0,
-      "grad_norm": 3.0797219276428223,
-      "kl": 0.006928815477294847,
+      "grad_norm": 4.339041709899902,
+      "kl": 0.005562390239598851,
       "learning_rate": 3.867370395306068e-07,
       "loss": 0.0,
-      "reward": 2.9164270808299384,
-      "reward_std": 1.4713270713885624,
-      "rewards/concensus_correctness_reward_func": 0.8288333266973495,
-      "rewards/consensus_reward_func": 0.75,
+      "reward": 2.794683963060379,
+      "reward_std": 1.0713556228826444,
+      "rewards/concensus_correctness_reward_func": 0.5786666596929232,
+      "rewards/consensus_reward_func": 1.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.08333333333333333,
-      "rewards/question_recreation_reward_func": 0.570885420932124,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.5789339219530424,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.6833750059207281,
+      "rewards/strict_format_reward_func": 0.020833333333333332,
+      "rewards/xmlcount_reward_func": 0.6162500108281771,
       "step": 8
     },
     {
-      "completion_length": 296.9375,
+      "completion_length": 219.1875,
       "epoch": 2.571428571428571,
-      "grad_norm": 7.378518104553223,
-      "kl": 0.012051919649820775,
+      "grad_norm": 7.3506574630737305,
+      "kl": 0.03309335559606552,
       "learning_rate": 3.1137137178519977e-07,
       "loss": 0.0,
-      "reward": 3.2202155999839306,
-      "reward_std": 1.7594602214521728,
-      "rewards/concensus_correctness_reward_func": 1.6434374917298555,
-      "rewards/consensus_reward_func": 0.5,
+      "reward": 3.0435366928577423,
+      "reward_std": 1.0485685534076765,
+      "rewards/concensus_correctness_reward_func": 0.6486249938607216,
+      "rewards/consensus_reward_func": 1.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.33384072268381715,
+      "rewards/question_recreation_reward_func": 0.48016168363392353,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": 0.7273125052452087,
+      "rewards/xmlcount_reward_func": 0.899125000461936,
       "step": 10
     },
     {
-      "completion_length": 271.875,
+      "completion_length": 318.7916666666667,
       "epoch": 3.0,
-      "grad_norm": 5.178144931793213,
-      "kl": 0.012011676677502692,
+      "grad_norm": 3.507310390472412,
+      "kl": 0.007355922580851863,
       "learning_rate": 2.2935516363191693e-07,
       "loss": 0.0,
-      "reward": 2.871542831261953,
-      "reward_std": 1.1954153617843986,
-      "rewards/concensus_correctness_reward_func": 0.6278333266576132,
-      "rewards/consensus_reward_func": 0.5833333333333334,
+      "reward": 3.699404043455919,
+      "reward_std": 1.2840088397885363,
+      "rewards/concensus_correctness_reward_func": 1.0116666505734127,
+      "rewards/consensus_reward_func": 1.4166666666666667,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.16666666666666666,
-      "rewards/question_recreation_reward_func": 0.6830428292353948,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.6990290557344755,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.8106666579842567,
+      "rewards/strict_format_reward_func": 0.020833333333333332,
+      "rewards/xmlcount_reward_func": 0.5512083147962888,
       "step": 12
     },
     {
-      "completion_length": 318.4375,
+      "completion_length": 298.1875,
       "epoch": 3.571428571428571,
-      "grad_norm": 6.178627014160156,
-      "kl": 0.014399387815501541,
+      "grad_norm": 6.966953754425049,
+      "kl": 0.013355173650779761,
       "learning_rate": 1.4957614383675767e-07,
       "loss": 0.0,
-      "reward": 2.97923981025815,
-      "reward_std": 2.4084168089320883,
-      "rewards/concensus_correctness_reward_func": 1.0642499960958958,
-      "rewards/consensus_reward_func": 0.5625,
+      "reward": 2.5207905806601048,
+      "reward_std": 1.5945063475519419,
+      "rewards/concensus_correctness_reward_func": 0.508437491953373,
+      "rewards/consensus_reward_func": 0.75,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.5395523381885141,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.5373843046836555,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.6879374999552965,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.7093437574803829,
       "step": 14
     },
     {
-      "completion_length": 426.7083333333333,
+      "completion_length": 243.0,
       "epoch": 4.0,
-      "grad_norm": 4.621153354644775,
-      "kl": 0.03207185295953726,
+      "grad_norm": 5.53330659866333,
+      "kl": 0.015268806600943208,
       "learning_rate": 8.067960709356478e-08,
       "loss": 0.0,
-      "reward": 2.2798982113599777,
-      "reward_std": 1.6485638072093327,
-      "rewards/concensus_correctness_reward_func": 0.6247499883174896,
-      "rewards/consensus_reward_func": 0.6666666666666666,
+      "reward": 1.8712952236334484,
+      "reward_std": 1.110026717806856,
+      "rewards/concensus_correctness_reward_func": 0.250750000278155,
+      "rewards/consensus_reward_func": 0.4166666666666667,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.16666666666666666,
-      "rewards/question_recreation_reward_func": 0.43839817494153976,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.4589619180187583,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.38341666013002396,
+      "rewards/xmlcount_reward_func": 0.7449166675408682,
       "step": 16
     },
     {
-      "completion_length": 262.8125,
+      "completion_length": 381.1875,
       "epoch": 4.571428571428571,
-      "grad_norm": 6.285000801086426,
-      "kl": 0.03336340782698244,
+      "grad_norm": 6.215991020202637,
+      "kl": 0.00821731197356712,
       "learning_rate": 3.013156219837776e-08,
       "loss": 0.0,
-      "reward": 2.810444377362728,
-      "reward_std": 1.7215215421747416,
-      "rewards/concensus_correctness_reward_func": 0.7313749901950359,
-      "rewards/consensus_reward_func": 0.8125,
+      "reward": 2.828702801838517,
+      "reward_std": 1.5806347676552832,
+      "rewards/concensus_correctness_reward_func": 0.5827499907463789,
+      "rewards/consensus_reward_func": 1.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.5584443034604192,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.47042151528876275,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": 0.5675000064074993,
+      "rewards/xmlcount_reward_func": 0.7599062472581863,
       "step": 18
     },
     {
-      "completion_length": 396.9583333333333,
+      "completion_length": 334.0416666666667,
       "epoch": 5.0,
-      "grad_norm": 5.309892177581787,
-      "kl": 0.009607602056348696,
+      "grad_norm": 4.492783069610596,
+      "kl": 0.00856334783990557,
       "learning_rate": 3.4096741493194193e-09,
       "loss": 0.0,
-      "reward": 1.503704349199931,
-      "reward_std": 1.614993463580807,
-      "rewards/concensus_correctness_reward_func": 0.2646666653454304,
-      "rewards/consensus_reward_func": 0.5,
+      "reward": 2.5465123057365417,
+      "reward_std": 1.2983978067835171,
+      "rewards/concensus_correctness_reward_func": 0.6519999926288923,
+      "rewards/consensus_reward_func": 0.8333333333333334,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.6006626834472021,
+      "rewards/question_recreation_reward_func": 0.46867894288152456,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.020833333333333332,
-      "rewards/xmlcount_reward_func": 0.11754167079925537,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.5925000111262003,
       "step": 20
     },
     {
       "epoch": 5.0,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 1.11847843754731e-05,
-      "train_runtime": 205.7818,
-      "train_samples_per_second": 1.555,
-      "train_steps_per_second": 0.097
+      "train_loss": 8.644163608551026e-06,
+      "train_runtime": 197.5707,
+      "train_samples_per_second": 1.62,
+      "train_steps_per_second": 0.101
     }
   ],
   "logging_steps": 2,