End of training

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 5.40037734391517,
-    "train_runtime": 148.422,
+    "train_loss": 0.9518747762893327,
+    "train_runtime": 158.2361,
     "train_samples": 160,
-    "train_samples_per_second": 2.156,
-    "train_steps_per_second": 0.135
+    "train_samples_per_second": 2.022,
+    "train_steps_per_second": 0.126
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5da4ac043691ad25f4ed610ae099638001caa64994c061cf70e9183485fc6974
+oid sha256:c3b5fb63ca9a9bac682c342cf1544ab05ac413ff2e5c9551d2a2ed6ed9a69184
 size 1976163472

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 5.40037734391517,
-    "train_runtime": 148.422,
+    "train_loss": 0.9518747762893327,
+    "train_runtime": 158.2361,
     "train_samples": 160,
-    "train_samples_per_second": 2.156,
-    "train_steps_per_second": 0.135
+    "train_samples_per_second": 2.022,
+    "train_steps_per_second": 0.126
 }

trainer_state.json

@@ -10,190 +10,190 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 214.15625,
+      "completion_length": 252.5625,
       "epoch": 0.1,
-      "grad_norm": 22.216941833496094,
+      "grad_norm": 22.417692184448242,
       "kl": 0.0,
       "learning_rate": 5e-07,
-      "loss": 0.0,
-      "reward": 3.9006508318707347,
-      "reward_std": 0.5246393299676129,
-      "rewards/concensus_correctness_reward_func": 1.1009999923408031,
-      "rewards/consensus_reward_func": 1.0625,
+      "loss": -0.0,
+      "reward": 3.3942950665950775,
+      "reward_std": 0.989583509406657,
+      "rewards/concensus_correctness_reward_func": 0.8876875024288893,
+      "rewards/consensus_reward_func": 1.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.6793695546220988,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.6617951137013733,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.21875,
-      "rewards/xmlcount_reward_func": 0.7765312492847443,
+      "rewards/xmlcount_reward_func": 0.6260625068098307,
       "step": 2
     },
     {
-      "completion_length": 153.5625,
+      "completion_length": 150.03125,
       "epoch": 0.2,
-      "grad_norm": 20.44854164123535,
-      "kl": 0.1254931440926157,
+      "grad_norm": 24.588348388671875,
+      "kl": 0.34198364353505895,
       "learning_rate": 4.864543104251586e-07,
-      "loss": 0.0001,
-      "reward": 6.288067072629929,
-      "reward_std": 0.5811158895376138,
-      "rewards/concensus_correctness_reward_func": 1.6979374811053276,
-      "rewards/consensus_reward_func": 1.875,
+      "loss": 0.0003,
+      "reward": 6.036934971809387,
+      "reward_std": 0.4371946807950735,
+      "rewards/concensus_correctness_reward_func": 1.6122499853372574,
+      "rewards/consensus_reward_func": 1.75,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.9120046440511942,
+      "rewards/question_recreation_reward_func": 0.9453724790364504,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.34375,
-      "rewards/xmlcount_reward_func": 1.2093750014901161,
+      "rewards/xmlcount_reward_func": 1.1355624999850988,
       "step": 4
     },
     {
-      "completion_length": 194.0625,
+      "completion_length": 200.875,
       "epoch": 0.3,
-      "grad_norm": 27783.4765625,
-      "kl": 919.1792740374804,
+      "grad_norm": 20.9364070892334,
+      "kl": 2.615051311207935,
       "learning_rate": 4.472851273490984e-07,
-      "loss": 0.9192,
-      "reward": 6.010144874453545,
-      "reward_std": 0.7817130146140698,
-      "rewards/concensus_correctness_reward_func": 1.6994999833405018,
-      "rewards/consensus_reward_func": 1.9375,
+      "loss": 0.0026,
+      "reward": 6.104766398668289,
+      "reward_std": 0.3827528031542897,
+      "rewards/concensus_correctness_reward_func": 1.6582500077784061,
+      "rewards/consensus_reward_func": 1.875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.9526761472225189,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.9630476571619511,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.359375,
-      "rewards/xmlcount_reward_func": 0.9985937625169754,
+      "rewards/strict_format_reward_func": 0.328125,
+      "rewards/xmlcount_reward_func": 1.1553437449038029,
       "step": 6
     },
     {
-      "completion_length": 142.375,
+      "completion_length": 135.53125,
       "epoch": 0.4,
-      "grad_norm": 831.9281616210938,
-      "kl": 31.678775895619765,
+      "grad_norm": 1757.8309326171875,
+      "kl": 9380.595343698515,
       "learning_rate": 3.867370395306068e-07,
-      "loss": 0.0317,
-      "reward": 6.461392790079117,
-      "reward_std": 0.3590539181313943,
-      "rewards/concensus_correctness_reward_func": 1.9264999954029918,
-      "rewards/consensus_reward_func": 1.875,
+      "loss": 9.3806,
+      "reward": 5.714629948139191,
+      "reward_std": 0.9692007284902502,
+      "rewards/concensus_correctness_reward_func": 1.603812504094094,
+      "rewards/consensus_reward_func": 1.5625,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.1875,
-      "rewards/question_recreation_reward_func": 0.8699865005910397,
+      "rewards/question_recreation_reward_func": 0.897161303088069,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.390625,
-      "rewards/xmlcount_reward_func": 1.2117812484502792,
+      "rewards/strict_format_reward_func": 0.34375,
+      "rewards/xmlcount_reward_func": 1.11990624666214,
       "step": 8
     },
     {
-      "completion_length": 131.09375,
+      "completion_length": 127.125,
       "epoch": 0.5,
-      "grad_norm": 51.59836959838867,
-      "kl": 9.099941534281243,
+      "grad_norm": 24.506160736083984,
+      "kl": 0.7436663164990023,
       "learning_rate": 3.1137137178519977e-07,
-      "loss": 0.0091,
-      "reward": 7.209178298711777,
-      "reward_std": 0.16811024746857584,
-      "rewards/concensus_correctness_reward_func": 2.2494999915361404,
+      "loss": 0.0007,
+      "reward": 7.066307455301285,
+      "reward_std": 0.3984663321170956,
+      "rewards/concensus_correctness_reward_func": 2.132374994456768,
       "rewards/consensus_reward_func": 2.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.375,
-      "rewards/question_recreation_reward_func": 0.9613657891750336,
+      "rewards/final_correctness_reward_func": 0.3125,
+      "rewards/question_recreation_reward_func": 0.9456512089818716,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.40625,
-      "rewards/xmlcount_reward_func": 1.217062495648861,
+      "rewards/strict_format_reward_func": 0.453125,
+      "rewards/xmlcount_reward_func": 1.22265625,
       "step": 10
     },
     {
-      "completion_length": 129.625,
+      "completion_length": 134.96875,
       "epoch": 0.6,
-      "grad_norm": 20.847944259643555,
-      "kl": 0.8772291259083431,
+      "grad_norm": 27.838016510009766,
+      "kl": 4.309598408872262,
       "learning_rate": 2.2935516363191693e-07,
-      "loss": 0.0009,
-      "reward": 6.097430318593979,
-      "reward_std": 0.2864232168503804,
-      "rewards/concensus_correctness_reward_func": 1.7592499777674675,
-      "rewards/consensus_reward_func": 1.8125,
+      "loss": 0.0043,
+      "reward": 6.028566122055054,
+      "reward_std": 0.3237042485270649,
+      "rewards/concensus_correctness_reward_func": 1.651874989271164,
+      "rewards/consensus_reward_func": 1.6875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.8994615543633699,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.8966911472380161,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.40625,
-      "rewards/xmlcount_reward_func": 1.2199687510728836,
+      "rewards/strict_format_reward_func": 0.4375,
+      "rewards/xmlcount_reward_func": 1.2300000041723251,
       "step": 12
     },
     {
-      "completion_length": 136.625,
+      "completion_length": 144.21875,
       "epoch": 0.7,
-      "grad_norm": 1207.919189453125,
-      "kl": 40.688665235298686,
+      "grad_norm": 90.1445541381836,
+      "kl": 6.900334812235087,
       "learning_rate": 1.4957614383675767e-07,
-      "loss": 0.0407,
-      "reward": 6.123583257198334,
-      "reward_std": 0.6078165378421545,
-      "rewards/concensus_correctness_reward_func": 1.7557500042021275,
-      "rewards/consensus_reward_func": 1.75,
+      "loss": 0.0069,
+      "reward": 6.494320347905159,
+      "reward_std": 0.5424829662442789,
+      "rewards/concensus_correctness_reward_func": 2.0426250100135803,
+      "rewards/consensus_reward_func": 1.6875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.9663332924246788,
+      "rewards/final_correctness_reward_func": 0.3125,
+      "rewards/question_recreation_reward_func": 0.9569765664637089,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.375,
-      "rewards/xmlcount_reward_func": 1.2140000015497208,
+      "rewards/strict_format_reward_func": 0.34375,
+      "rewards/xmlcount_reward_func": 1.1509687453508377,
       "step": 14
     },
     {
-      "completion_length": 137.3125,
+      "completion_length": 139.9375,
       "epoch": 0.8,
-      "grad_norm": 40.619361877441406,
-      "kl": 5.575095858424902,
+      "grad_norm": 65.96639251708984,
+      "kl": 2.3929815529845655,
       "learning_rate": 8.067960709356478e-08,
-      "loss": 0.0056,
-      "reward": 6.810699820518494,
-      "reward_std": 0.2520275285933167,
-      "rewards/concensus_correctness_reward_func": 2.0633749961853027,
-      "rewards/consensus_reward_func": 2.0,
+      "loss": 0.0024,
+      "reward": 6.913275599479675,
+      "reward_std": 0.47360870630291174,
+      "rewards/concensus_correctness_reward_func": 2.1199999935925007,
+      "rewards/consensus_reward_func": 1.9375,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.1875,
-      "rewards/question_recreation_reward_func": 0.9416372999548912,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.9604942761361599,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.40625,
-      "rewards/xmlcount_reward_func": 1.211937502026558,
+      "rewards/strict_format_reward_func": 0.4375,
+      "rewards/xmlcount_reward_func": 1.2077812515199184,
       "step": 16
     },
     {
-      "completion_length": 122.5,
+      "completion_length": 129.09375,
       "epoch": 0.9,
-      "grad_norm": 37.19350051879883,
-      "kl": 52978.122300678864,
+      "grad_norm": 22.48250389099121,
+      "kl": 119.92963805189356,
       "learning_rate": 3.013156219837776e-08,
-      "loss": 52.9781,
-      "reward": 6.109934449195862,
-      "reward_std": 0.58212883642409,
-      "rewards/concensus_correctness_reward_func": 1.7430625036358833,
-      "rewards/consensus_reward_func": 1.5625,
+      "loss": 0.1199,
+      "reward": 5.9071336686611176,
+      "reward_std": 0.34385218027455267,
+      "rewards/concensus_correctness_reward_func": 1.5897500030696392,
+      "rewards/consensus_reward_func": 1.6875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.3125,
-      "rewards/question_recreation_reward_func": 0.9283719323575497,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.9265399239957333,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.359375,
-      "rewards/xmlcount_reward_func": 1.2041249945759773,
+      "rewards/strict_format_reward_func": 0.40625,
+      "rewards/xmlcount_reward_func": 1.1720937490463257,
       "step": 18
     },
     {
-      "completion_length": 148.28125,
+      "completion_length": 129.65625,
       "epoch": 1.0,
-      "grad_norm": 721.8731079101562,
-      "kl": 18.435496295103803,
+      "grad_norm": 36.973785400390625,
+      "kl": 0.9043536563403904,
       "learning_rate": 3.4096741493194193e-09,
-      "loss": 0.0184,
-      "reward": 6.203249305486679,
-      "reward_std": 0.460769604993402,
-      "rewards/concensus_correctness_reward_func": 1.706125009804964,
-      "rewards/consensus_reward_func": 1.75,
+      "loss": 0.0009,
+      "reward": 6.220566600561142,
+      "reward_std": 0.4913365952670574,
+      "rewards/concensus_correctness_reward_func": 1.7718124948441982,
+      "rewards/consensus_reward_func": 1.8125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.9697804637253284,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.9214103538542986,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.4375,
       "rewards/xmlcount_reward_func": 1.21484375,
@@ -203,10 +203,10 @@
       "epoch": 1.0,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 5.40037734391517,
-      "train_runtime": 148.422,
-      "train_samples_per_second": 2.156,
-      "train_steps_per_second": 0.135
+      "train_loss": 0.9518747762893327,
+      "train_runtime": 158.2361,
+      "train_samples_per_second": 2.022,
+      "train_steps_per_second": 0.126
     }
   ],
   "logging_steps": 2,