End of training

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 13.891589600990846,
-    "train_runtime": 140.3079,
-    "train_samples": 28,
-    "train_samples_per_second": 2.281,
-    "train_steps_per_second": 0.143
+    "train_loss": 2.6187665071120137e-06,
+    "train_runtime": 768.4129,
+    "train_samples": 79,
+    "train_samples_per_second": 0.416,
+    "train_steps_per_second": 0.026
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1fbe076f2c55240717df0543d294813d4e77c226871e6b46026485697a7199ea
+oid sha256:70f9995bf9835fd712718ec7892061c160930203863df53eaf8d500573f5a39e
 size 1976163472

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 13.891589600990846,
-    "train_runtime": 140.3079,
-    "train_samples": 28,
-    "train_samples_per_second": 2.281,
-    "train_steps_per_second": 0.143
+    "train_loss": 2.6187665071120137e-06,
+    "train_runtime": 768.4129,
+    "train_samples": 79,
+    "train_samples_per_second": 0.416,
+    "train_steps_per_second": 0.026
 }

trainer_state.json

@@ -2,7 +2,7 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 5.0,
+  "epoch": 2.0,
   "eval_steps": 500,
   "global_step": 20,
   "is_hyper_param_search": false,
@@ -10,209 +10,209 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 261.625,
-      "epoch": 0.5714285714285714,
-      "grad_norm": 18.2692813873291,
+      "completion_length": 271.4375,
+      "epoch": 0.20253164556962025,
+      "grad_norm": 8.816926956176758,
       "kl": 0.0,
       "learning_rate": 5e-07,
       "loss": 0.0,
-      "reward": 3.2587675862014294,
-      "reward_std": 0.5479843649081886,
-      "rewards/concensus_correctness_reward_func": 0.6271249894052744,
-      "rewards/consensus_reward_func": 0.9375,
+      "reward": 0.36623164458433166,
+      "reward_std": 0.7491341227432713,
+      "rewards/concensus_correctness_reward_func": 0.05999999865889549,
+      "rewards/consensus_reward_func": 0.0625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.8197362683713436,
+      "rewards/final_correctness_reward_func": 0.1875,
+      "rewards/question_recreation_reward_func": 0.18841915280791,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.8119062604382634,
+      "rewards/xmlcount_reward_func": -0.13218749314546585,
       "step": 2
     },
     {
-      "completion_length": 152.95833333333334,
-      "epoch": 1.0,
-      "grad_norm": 12.842881202697754,
-      "kl": 0.09050525856825213,
+      "completion_length": 361.4375,
+      "epoch": 0.4050632911392405,
+      "grad_norm": 17.59312629699707,
+      "kl": 0.0013711865340155782,
       "learning_rate": 4.864543104251586e-07,
-      "loss": 0.0001,
-      "reward": 5.476259032885234,
-      "reward_std": 0.9015039427710386,
-      "rewards/concensus_correctness_reward_func": 1.4560000076889992,
-      "rewards/consensus_reward_func": 1.6666666666666667,
+      "loss": 0.0,
+      "reward": 0.4331077783717774,
+      "reward_std": 0.5598573155584745,
+      "rewards/concensus_correctness_reward_func": 0.02775000035762787,
+      "rewards/consensus_reward_func": 0.125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.3333333333333333,
-      "rewards/question_recreation_reward_func": 0.9221756458282471,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.17860777111491188,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 1.0980833570162456,
+      "rewards/xmlcount_reward_func": -0.14825000939890742,
       "step": 4
     },
     {
-      "completion_length": 137.0625,
-      "epoch": 1.5714285714285714,
-      "grad_norm": 40.57078552246094,
-      "kl": 2.395180522580631,
+      "completion_length": 323.65625,
+      "epoch": 0.6075949367088608,
+      "grad_norm": 122.53370666503906,
+      "kl": 0.0090612619969761,
       "learning_rate": 4.472851273490984e-07,
-      "loss": 0.0024,
-      "reward": 5.467783257365227,
-      "reward_std": 0.4834165150823537,
-      "rewards/concensus_correctness_reward_func": 1.5619999952614307,
-      "rewards/consensus_reward_func": 1.6875,
+      "loss": 0.0,
+      "reward": 1.1600094995519612,
+      "reward_std": 1.234167092028656,
+      "rewards/concensus_correctness_reward_func": 0.625,
+      "rewards/consensus_reward_func": 0.0625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.7703457288444042,
-      "rewards/soft_format_reward_func": 0.0,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.20494700092240237,
+      "rewards/soft_format_reward_func": 0.015625,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 1.1979374960064888,
+      "rewards/xmlcount_reward_func": 0.18943750020116568,
       "step": 6
     },
     {
-      "completion_length": 150.5,
-      "epoch": 2.0,
-      "grad_norm": 15.625757217407227,
-      "kl": 11.704404253512621,
+      "completion_length": 338.34375,
+      "epoch": 0.810126582278481,
+      "grad_norm": 6.451416969299316,
+      "kl": 0.0015921097838145215,
       "learning_rate": 3.867370395306068e-07,
-      "loss": 0.0088,
-      "reward": 4.858903328577678,
-      "reward_std": 0.58683955820743,
-      "rewards/concensus_correctness_reward_func": 1.1974999929467838,
-      "rewards/consensus_reward_func": 1.6666666666666667,
+      "loss": 0.0,
+      "reward": 0.5769440656295046,
+      "reward_std": 0.24592963242321275,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0625,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.8203616303702196,
+      "rewards/question_recreation_reward_func": 0.27731904422398657,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 1.1743750274181366,
+      "rewards/xmlcount_reward_func": 0.237125008367002,
       "step": 8
     },
     {
-      "completion_length": 170.3125,
-      "epoch": 2.571428571428571,
-      "grad_norm": 18749.9765625,
-      "kl": 426.2471541694831,
+      "completion_length": 396.3666666666667,
+      "epoch": 1.0,
+      "grad_norm": 19.4079532623291,
+      "kl": 0.0008428459094526867,
       "learning_rate": 3.1137137178519977e-07,
-      "loss": 0.4262,
-      "reward": 5.340568527579308,
-      "reward_std": 0.521991155212163,
-      "rewards/concensus_correctness_reward_func": 1.579624991863966,
-      "rewards/consensus_reward_func": 1.5625,
+      "loss": 0.0,
+      "reward": 1.2873985360066096,
+      "reward_std": 1.6557406775653363,
+      "rewards/concensus_correctness_reward_func": 0.7303999980290731,
+      "rewards/consensus_reward_func": 0.13333333333333333,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.7725684833712876,
-      "rewards/soft_format_reward_func": 0.0,
+      "rewards/final_correctness_reward_func": 0.13333333333333333,
+      "rewards/question_recreation_reward_func": 0.19319855545957884,
+      "rewards/soft_format_reward_func": 0.016666666666666666,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 1.1758750081062317,
+      "rewards/xmlcount_reward_func": 0.08046667128801346,
       "step": 10
     },
     {
-      "completion_length": 149.16666666666666,
-      "epoch": 3.0,
-      "grad_norm": 13.14773941040039,
-      "kl": 0.6229217213888963,
+      "completion_length": 325.9375,
+      "epoch": 1.2025316455696202,
+      "grad_norm": 16.229761123657227,
+      "kl": 0.0029048858195892535,
       "learning_rate": 2.2935516363191693e-07,
-      "loss": 0.0005,
-      "reward": 5.155540744463603,
-      "reward_std": 0.5695952884852886,
-      "rewards/concensus_correctness_reward_func": 1.4084999958674114,
-      "rewards/consensus_reward_func": 1.6666666666666667,
+      "loss": 0.0,
+      "reward": 0.5114847992081195,
+      "reward_std": 0.6120182508020662,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.9319572498401006,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.22107853856869042,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 1.1484166781107585,
+      "rewards/strict_format_reward_func": 0.03125,
+      "rewards/xmlcount_reward_func": 0.1966562531888485,
       "step": 12
     },
     {
-      "completion_length": 164.40625,
-      "epoch": 3.571428571428571,
-      "grad_norm": 701.1793823242188,
-      "kl": 26.294148512999527,
+      "completion_length": 306.65625,
+      "epoch": 1.4050632911392404,
+      "grad_norm": 11.222420692443848,
+      "kl": 0.002048927537543932,
       "learning_rate": 1.4957614383675767e-07,
-      "loss": 0.0263,
-      "reward": 5.567795515060425,
-      "reward_std": 0.31949186109704897,
-      "rewards/concensus_correctness_reward_func": 1.6592499986290932,
-      "rewards/consensus_reward_func": 1.75,
+      "loss": 0.0,
+      "reward": 0.6656969569157809,
+      "reward_std": 0.7207401255145669,
+      "rewards/concensus_correctness_reward_func": 0.059562500566244125,
+      "rewards/consensus_reward_func": 0.125,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.8718266356736422,
+      "rewards/question_recreation_reward_func": 0.223384442680981,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 1.1617187708616257,
+      "rewards/xmlcount_reward_func": 0.132750004529953,
       "step": 14
     },
     {
-      "completion_length": 162.16666666666666,
-      "epoch": 4.0,
-      "grad_norm": 8.945233345031738,
-      "kl": 1.7373863802446674,
+      "completion_length": 301.46875,
+      "epoch": 1.6075949367088609,
+      "grad_norm": 38.977447509765625,
+      "kl": 0.0022216661363927415,
       "learning_rate": 8.067960709356478e-08,
-      "loss": 0.0013,
-      "reward": 5.225785593191783,
-      "reward_std": 0.7394425706006587,
-      "rewards/concensus_correctness_reward_func": 1.3721666547159355,
-      "rewards/consensus_reward_func": 1.6666666666666667,
+      "loss": 0.0,
+      "reward": 0.7955324916401878,
+      "reward_std": 0.6592622805619612,
+      "rewards/concensus_correctness_reward_func": 0.1197499968111515,
+      "rewards/consensus_reward_func": 0.125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.08333333333333333,
-      "rewards/question_recreation_reward_func": 0.8910355667273203,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.31178249258664437,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.020833333333333332,
-      "rewards/xmlcount_reward_func": 1.1917500098546345,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.11400000448338687,
       "step": 16
     },
     {
-      "completion_length": 173.65625,
-      "epoch": 4.571428571428571,
-      "grad_norm": 15.458992958068848,
-      "kl": 138449.00829719938,
+      "completion_length": 374.09375,
+      "epoch": 1.810126582278481,
+      "grad_norm": 40.63166809082031,
+      "kl": 0.001135692305979319,
       "learning_rate": 3.013156219837776e-08,
-      "loss": 138.449,
-      "reward": 5.486883968114853,
-      "reward_std": 0.41487747368228156,
-      "rewards/concensus_correctness_reward_func": 1.5810624957084656,
-      "rewards/consensus_reward_func": 1.8125,
+      "loss": 0.0,
+      "reward": 0.38595891930162907,
+      "reward_std": 0.38352038772427477,
+      "rewards/concensus_correctness_reward_func": 0.05999999865889549,
+      "rewards/consensus_reward_func": 0.0625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.8087901342660189,
-      "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": 1.143906258046627,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.12880266876891255,
+      "rewards/soft_format_reward_func": 0.015625,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.11903125047683716,
       "step": 18
     },
     {
-      "completion_length": 183.375,
-      "epoch": 5.0,
-      "grad_norm": 19.41164779663086,
-      "kl": 1.7835005586966872,
+      "completion_length": 401.93333333333334,
+      "epoch": 2.0,
+      "grad_norm": 68.24220275878906,
+      "kl": 0.00539769782529523,
       "learning_rate": 3.4096741493194193e-09,
-      "loss": 0.0013,
-      "reward": 5.1484537323315935,
-      "reward_std": 0.49525849769512814,
-      "rewards/concensus_correctness_reward_func": 1.3932499910394351,
-      "rewards/consensus_reward_func": 1.5833333333333333,
+      "loss": 0.0,
+      "reward": 0.7710167534649373,
+      "reward_std": 0.5045542840845882,
+      "rewards/concensus_correctness_reward_func": 0.13080000082651774,
+      "rewards/consensus_reward_func": 0.06666666666666667,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.16666666666666666,
-      "rewards/question_recreation_reward_func": 0.8702036167184511,
+      "rewards/final_correctness_reward_func": 0.13333333333333333,
+      "rewards/question_recreation_reward_func": 0.22905009041229885,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 1.1350000103314717,
+      "rewards/xmlcount_reward_func": 0.21116666992505392,
       "step": 20
     },
     {
-      "epoch": 5.0,
+      "epoch": 2.0,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 13.891589600990846,
-      "train_runtime": 140.3079,
-      "train_samples_per_second": 2.281,
-      "train_steps_per_second": 0.143
+      "train_loss": 2.6187665071120137e-06,
+      "train_runtime": 768.4129,
+      "train_samples_per_second": 0.416,
+      "train_steps_per_second": 0.026
     }
   ],
   "logging_steps": 2,
   "max_steps": 20,
   "num_input_tokens_seen": 0,
-  "num_train_epochs": 7,
+  "num_train_epochs": 3,
   "save_steps": 25,
   "stateful_callbacks": {
     "TrainerControl": {

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:60094aa5d4e497ee4cf699df5996409f341ccf6f12f5a74c360e77094041b9b6
+oid sha256:91a1c0fd8fef83819c429ca79990ccf5fccad40f019854baa11d2af64e601086
 size 6008