End of training

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.5893889988888986,
-    "train_runtime": 128.9396,
+    "train_loss": 112738.47402167018,
+    "train_runtime": 120.4527,
     "train_samples": 28,
-    "train_samples_per_second": 2.482,
-    "train_steps_per_second": 0.155
+    "train_samples_per_second": 2.657,
+    "train_steps_per_second": 0.166
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:cff8eaef64bc4e53ac33daa4f7372427207618a96fdd1c99b4b4fead0485235e
+oid sha256:e4dd321b86c4618e847033c86f24449ab4ef0986295c656b5bdac3223e0f6f3c
 size 1976163472

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.5893889988888986,
-    "train_runtime": 128.9396,
+    "train_loss": 112738.47402167018,
+    "train_runtime": 120.4527,
     "train_samples": 28,
-    "train_samples_per_second": 2.482,
-    "train_steps_per_second": 0.155
+    "train_samples_per_second": 2.657,
+    "train_steps_per_second": 0.166
 }

trainer_state.json

@@ -10,203 +10,203 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 299.53125,
+      "completion_length": 249.40625,
       "epoch": 0.5714285714285714,
-      "grad_norm": 42.628971099853516,
+      "grad_norm": 78.55313110351562,
       "kl": 0.0,
       "learning_rate": 5e-07,
       "loss": -0.0,
-      "reward": 3.9247607700526714,
-      "reward_std": 2.2996415526286,
-      "rewards/concensus_correctness_reward_func": 1.444374993443489,
-      "rewards/consensus_reward_func": 0.9375,
+      "reward": 3.0591967664659023,
+      "reward_std": 1.667841853806749,
+      "rewards/concensus_correctness_reward_func": 0.6743124946951866,
+      "rewards/consensus_reward_func": 0.6875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.7389170434325933,
-      "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.09375,
-      "rewards/xmlcount_reward_func": 0.6477187490090728,
+      "rewards/final_correctness_reward_func": 0.1875,
+      "rewards/question_recreation_reward_func": 0.6536654952215031,
+      "rewards/soft_format_reward_func": 0.015625,
+      "rewards/strict_format_reward_func": 0.078125,
+      "rewards/xmlcount_reward_func": 0.7624687403440475,
       "step": 2
     },
     {
-      "completion_length": 151.04166666666666,
+      "completion_length": 149.375,
       "epoch": 1.0,
-      "grad_norm": 34.50354766845703,
-      "kl": 0.6318123211773733,
+      "grad_norm": 86.88078308105469,
+      "kl": 0.05806352787961563,
       "learning_rate": 4.864543104251586e-07,
-      "loss": 0.0005,
-      "reward": 5.500972350438436,
-      "reward_std": 1.0254100343833368,
-      "rewards/concensus_correctness_reward_func": 1.94283331433932,
-      "rewards/consensus_reward_func": 1.3333333333333333,
+      "loss": 0.0,
+      "reward": 5.5724403858184814,
+      "reward_std": 1.5296151217771694,
+      "rewards/concensus_correctness_reward_func": 1.7816666613022487,
+      "rewards/consensus_reward_func": 1.5,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.3333333333333333,
-      "rewards/question_recreation_reward_func": 0.7553057310481867,
+      "rewards/question_recreation_reward_func": 0.7538986553748449,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.14583333333333334,
-      "rewards/xmlcount_reward_func": 0.9903333385785421,
+      "rewards/strict_format_reward_func": 0.16666666666666666,
+      "rewards/xmlcount_reward_func": 1.0368749996026356,
       "step": 4
     },
     {
-      "completion_length": 153.59375,
+      "completion_length": 152.34375,
       "epoch": 1.5714285714285714,
-      "grad_norm": 255134.84375,
-      "kl": 5747.716522115748,
+      "grad_norm": 34.086280822753906,
+      "kl": 0.6559465810423717,
       "learning_rate": 4.472851273490984e-07,
-      "loss": 5.7477,
-      "reward": 5.574208900332451,
-      "reward_std": 0.8293443049769849,
-      "rewards/concensus_correctness_reward_func": 1.73862499371171,
-      "rewards/consensus_reward_func": 1.3125,
+      "loss": 0.0007,
+      "reward": 6.155833105556667,
+      "reward_std": 0.6093435278016841,
+      "rewards/concensus_correctness_reward_func": 2.099375005811453,
+      "rewards/consensus_reward_func": 1.5625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.375,
-      "rewards/question_recreation_reward_func": 0.8341151364147663,
+      "rewards/final_correctness_reward_func": 0.3125,
+      "rewards/question_recreation_reward_func": 0.7866768958047032,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.1875,
-      "rewards/xmlcount_reward_func": 1.1264687441289425,
+      "rewards/strict_format_reward_func": 0.3125,
+      "rewards/xmlcount_reward_func": 1.0822812505066395,
       "step": 6
     },
     {
-      "completion_length": 186.16666666666666,
+      "completion_length": 175.41666666666666,
       "epoch": 2.0,
-      "grad_norm": 589.8479614257812,
-      "kl": 45.244061558973044,
+      "grad_norm": 102.05463409423828,
+      "kl": 1.1159729743376374,
       "learning_rate": 3.867370395306068e-07,
-      "loss": 0.0339,
-      "reward": 4.733554303646088,
-      "reward_std": 1.148453081647555,
-      "rewards/concensus_correctness_reward_func": 1.2845833202203114,
-      "rewards/consensus_reward_func": 1.25,
+      "loss": 0.0008,
+      "reward": 5.1590457409620285,
+      "reward_std": 1.2149010785312082,
+      "rewards/concensus_correctness_reward_func": 1.4724166442950566,
+      "rewards/consensus_reward_func": 1.4166666666666667,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.3333333333333333,
-      "rewards/question_recreation_reward_func": 0.757595919072628,
+      "rewards/final_correctness_reward_func": 0.5,
+      "rewards/question_recreation_reward_func": 0.7165040796001753,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.14583333333333334,
-      "rewards/xmlcount_reward_func": 0.9622083355983099,
+      "rewards/strict_format_reward_func": 0.0625,
+      "rewards/xmlcount_reward_func": 0.9909583280483881,
       "step": 8
     },
     {
-      "completion_length": 162.4375,
+      "completion_length": 181.1875,
       "epoch": 2.571428571428571,
-      "grad_norm": 487.8678283691406,
-      "kl": 15.251416414394043,
+      "grad_norm": 1401648.875,
+      "kl": 4485897.915470626,
       "learning_rate": 3.1137137178519977e-07,
-      "loss": 0.0153,
-      "reward": 5.47632609307766,
-      "reward_std": 1.0686873188242316,
-      "rewards/concensus_correctness_reward_func": 1.6484999917447567,
-      "rewards/consensus_reward_func": 1.4375,
+      "loss": 4485.897,
+      "reward": 5.614339977502823,
+      "reward_std": 1.5171476181130856,
+      "rewards/concensus_correctness_reward_func": 1.811499997973442,
+      "rewards/consensus_reward_func": 1.375,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.842013506218791,
+      "rewards/final_correctness_reward_func": 0.4375,
+      "rewards/question_recreation_reward_func": 0.7781211100518703,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.171875,
-      "rewards/xmlcount_reward_func": 1.126437496393919,
+      "rewards/strict_format_reward_func": 0.1875,
+      "rewards/xmlcount_reward_func": 1.0247187428176403,
       "step": 10
     },
     {
-      "completion_length": 188.79166666666666,
+      "completion_length": 150.79166666666666,
       "epoch": 3.0,
-      "grad_norm": 56.841827392578125,
-      "kl": 0.6789928351839384,
+      "grad_norm": 34.14075469970703,
+      "kl": 219711.29470605837,
       "learning_rate": 2.2935516363191693e-07,
-      "loss": 0.0005,
-      "reward": 5.649905860424042,
-      "reward_std": 1.2127385487159092,
-      "rewards/concensus_correctness_reward_func": 1.7670833220084508,
-      "rewards/consensus_reward_func": 1.6666666666666667,
+      "loss": 164.7835,
+      "reward": 6.4597495794296265,
+      "reward_std": 0.2802316110270719,
+      "rewards/concensus_correctness_reward_func": 2.0669999917348227,
+      "rewards/consensus_reward_func": 1.8333333333333333,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.7885308315356573,
+      "rewards/final_correctness_reward_func": 0.3333333333333333,
+      "rewards/question_recreation_reward_func": 0.8656661808490753,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.125,
-      "rewards/xmlcount_reward_func": 1.0526250104109447,
+      "rewards/strict_format_reward_func": 0.1875,
+      "rewards/xmlcount_reward_func": 1.1729166706403096,
       "step": 12
     },
     {
-      "completion_length": 174.4375,
+      "completion_length": 187.09375,
       "epoch": 3.571428571428571,
-      "grad_norm": 45.8433837890625,
-      "kl": 1.8573934370651841,
+      "grad_norm": 469.4254150390625,
+      "kl": 209.55569416983053,
       "learning_rate": 1.4957614383675767e-07,
-      "loss": 0.0019,
-      "reward": 6.0423071682453156,
-      "reward_std": 1.3475016155280173,
-      "rewards/concensus_correctness_reward_func": 2.326937511563301,
-      "rewards/consensus_reward_func": 1.375,
+      "loss": 0.2096,
+      "reward": 6.7032526433467865,
+      "reward_std": 1.293150786194019,
+      "rewards/concensus_correctness_reward_func": 2.4413749873638153,
+      "rewards/consensus_reward_func": 1.625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.4375,
-      "rewards/question_recreation_reward_func": 0.7390572149306536,
+      "rewards/final_correctness_reward_func": 0.375,
+      "rewards/question_recreation_reward_func": 0.8526588641107082,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.109375,
-      "rewards/xmlcount_reward_func": 1.0544375032186508,
+      "rewards/strict_format_reward_func": 0.234375,
+      "rewards/xmlcount_reward_func": 1.174843743443489,
       "step": 14
     },
     {
-      "completion_length": 191.625,
+      "completion_length": 156.54166666666666,
       "epoch": 4.0,
-      "grad_norm": 137.88986206054688,
-      "kl": 0.6375924314682683,
+      "grad_norm": 19.79854965209961,
+      "kl": 128.49675853115818,
       "learning_rate": 8.067960709356478e-08,
-      "loss": 0.0005,
-      "reward": 5.098466714223226,
-      "reward_std": 1.0510680011163156,
-      "rewards/concensus_correctness_reward_func": 1.558916653196017,
-      "rewards/consensus_reward_func": 1.4166666666666667,
+      "loss": 0.0964,
+      "reward": 5.564403712749481,
+      "reward_std": 1.683505776959161,
+      "rewards/concensus_correctness_reward_func": 1.6036666532357533,
+      "rewards/consensus_reward_func": 1.3333333333333333,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.16666666666666666,
-      "rewards/question_recreation_reward_func": 0.8831332474946976,
+      "rewards/final_correctness_reward_func": 0.3333333333333333,
+      "rewards/question_recreation_reward_func": 0.8824037462472916,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.125,
-      "rewards/xmlcount_reward_func": 0.9480833411216736,
+      "rewards/strict_format_reward_func": 0.2916666666666667,
+      "rewards/xmlcount_reward_func": 1.119999997317791,
       "step": 16
     },
     {
-      "completion_length": 175.8125,
+      "completion_length": 157.4375,
       "epoch": 4.571428571428571,
-      "grad_norm": 59.4065055847168,
-      "kl": 2.3382355459034443,
+      "grad_norm": 53211725824.0,
+      "kl": 1122733816.4475722,
       "learning_rate": 3.013156219837776e-08,
-      "loss": 0.0023,
-      "reward": 5.580400392413139,
-      "reward_std": 1.3598273791285465,
-      "rewards/concensus_correctness_reward_func": 1.6806874871253967,
+      "loss": 1122733.75,
+      "reward": 5.74696709215641,
+      "reward_std": 1.4755408964119852,
+      "rewards/concensus_correctness_reward_func": 1.7621249966323376,
       "rewards/consensus_reward_func": 1.625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.7823691540397704,
+      "rewards/final_correctness_reward_func": 0.4375,
+      "rewards/question_recreation_reward_func": 0.6688420784194022,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.171875,
-      "rewards/xmlcount_reward_func": 1.070468757301569,
+      "rewards/strict_format_reward_func": 0.203125,
+      "rewards/xmlcount_reward_func": 1.0503750005736947,
       "step": 18
     },
     {
-      "completion_length": 158.45833333333334,
+      "completion_length": 160.08333333333334,
       "epoch": 5.0,
-      "grad_norm": 4546.7431640625,
-      "kl": 121.77881032818307,
+      "grad_norm": 22.75950050354004,
+      "kl": 3.0091245751827955,
       "learning_rate": 3.4096741493194193e-09,
-      "loss": 0.0913,
-      "reward": 5.049804945786794,
-      "reward_std": 0.7113708908048769,
-      "rewards/concensus_correctness_reward_func": 1.4803333183129628,
-      "rewards/consensus_reward_func": 1.1666666666666667,
+      "loss": 0.0023,
+      "reward": 5.480962634086609,
+      "reward_std": 1.3035912602208555,
+      "rewards/concensus_correctness_reward_func": 1.6363333264986675,
+      "rewards/consensus_reward_func": 1.3333333333333333,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.3333333333333333,
-      "rewards/question_recreation_reward_func": 0.7588882595300674,
+      "rewards/question_recreation_reward_func": 0.8412126104036967,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.20833333333333334,
-      "rewards/xmlcount_reward_func": 1.1022500023245811,
+      "rewards/strict_format_reward_func": 0.22916666666666666,
+      "rewards/xmlcount_reward_func": 1.107583334048589,
       "step": 20
     },
     {
       "epoch": 5.0,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 0.5893889988888986,
-      "train_runtime": 128.9396,
-      "train_samples_per_second": 2.482,
-      "train_steps_per_second": 0.155
+      "train_loss": 112738.47402167018,
+      "train_runtime": 120.4527,
+      "train_samples_per_second": 2.657,
+      "train_steps_per_second": 0.166
     }
   ],
   "logging_steps": 2,