End of training

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 8.280637219718967,
-    "train_runtime": 115.5142,
-    "train_samples": 22,
-    "train_samples_per_second": 2.078,
-    "train_steps_per_second": 0.13
+    "train_loss": 1.4178454875946046e-06,
+    "train_runtime": 1361.509,
+    "train_samples": 56,
+    "train_samples_per_second": 0.176,
+    "train_steps_per_second": 0.011
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:914a24979bdd48a90c532198175ce0d9bcabf25c953cd250f07acdc89a71d1ad
+oid sha256:c39eb5263ed42e3cbed3ced90f7d14865d695d96feca4ecbd0ae1d01ae769dc5
 size 1976163472

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 8.280637219718967,
-    "train_runtime": 115.5142,
-    "train_samples": 22,
-    "train_samples_per_second": 2.078,
-    "train_steps_per_second": 0.13
+    "train_loss": 1.4178454875946046e-06,
+    "train_runtime": 1361.509,
+    "train_samples": 56,
+    "train_samples_per_second": 0.176,
+    "train_steps_per_second": 0.011
 }

trainer_state.json

@@ -2,7 +2,7 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 5.0,
+  "epoch": 2.142857142857143,
   "eval_steps": 500,
   "global_step": 15,
   "is_hyper_param_search": false,
@@ -10,164 +10,164 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 263.53125,
-      "epoch": 0.7272727272727273,
-      "grad_norm": 28.811277389526367,
+      "completion_length": 313.9375,
+      "epoch": 0.2857142857142857,
+      "grad_norm": 6.98803186416626,
       "kl": 0.0,
       "learning_rate": 5e-07,
       "loss": -0.0,
-      "reward": 3.30694979429245,
-      "reward_std": 0.7047338464763016,
-      "rewards/concensus_correctness_reward_func": 0.6910000029020011,
-      "rewards/consensus_reward_func": 0.875,
+      "reward": 0.8072698791511357,
+      "reward_std": 1.1071819385979325,
+      "rewards/concensus_correctness_reward_func": 0.05974999815225601,
+      "rewards/consensus_reward_func": 0.0625,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.740949796512723,
+      "rewards/question_recreation_reward_func": 0.36967611219733953,
       "rewards/soft_format_reward_func": 0.015625,
-      "rewards/strict_format_reward_func": 0.171875,
-      "rewards/xmlcount_reward_func": 0.6874999981373549,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.1747187536675483,
       "step": 2
     },
     {
-      "completion_length": 201.21428571428572,
-      "epoch": 1.3636363636363638,
-      "grad_norm": 42.29725646972656,
-      "kl": 7.259678252839616,
+      "completion_length": 456.4375,
+      "epoch": 0.5714285714285714,
+      "grad_norm": 6.565876483917236,
+      "kl": 0.0009796264093893114,
       "learning_rate": 4.752422169756047e-07,
-      "loss": 0.0064,
-      "reward": 5.871921556336539,
-      "reward_std": 0.3950730026034372,
-      "rewards/concensus_correctness_reward_func": 1.5897142589092255,
-      "rewards/consensus_reward_func": 1.5714285714285714,
+      "loss": 0.0,
+      "reward": 0.8925973402801901,
+      "reward_std": 1.1312729835044593,
+      "rewards/concensus_correctness_reward_func": 0.24249999970197678,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.42857142857142855,
-      "rewards/question_recreation_reward_func": 0.8278144470282963,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.3265973336820025,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.23214285714285715,
-      "rewards/xmlcount_reward_func": 1.2222499932561601,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.26100000459700823,
       "step": 4
     },
     {
-      "completion_length": 163.78571428571428,
-      "epoch": 2.0,
-      "grad_norm": 230056.28125,
-      "kl": 5528.924554995926,
+      "completion_length": 319.53125,
+      "epoch": 0.8571428571428571,
+      "grad_norm": 6.887750625610352,
+      "kl": 0.0011162323780808947,
       "learning_rate": 4.058724504646834e-07,
-      "loss": 4.8378,
-      "reward": 5.534039872033255,
-      "reward_std": 0.9441594547991242,
-      "rewards/concensus_correctness_reward_func": 1.4844999983906746,
-      "rewards/consensus_reward_func": 1.5714285714285714,
+      "loss": 0.0,
+      "reward": 0.9045767441857606,
+      "reward_std": 0.864304929447826,
+      "rewards/concensus_correctness_reward_func": 0.17912499606609344,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.21428571428571427,
-      "rewards/question_recreation_reward_func": 0.8267541421311242,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.3269830149365589,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.26785714285714285,
-      "rewards/xmlcount_reward_func": 1.1692142827170235,
+      "rewards/strict_format_reward_func": 0.03125,
+      "rewards/xmlcount_reward_func": 0.3672187486663461,
       "step": 6
     },
     {
-      "completion_length": 165.375,
-      "epoch": 2.7272727272727275,
-      "grad_norm": 1824.9788818359375,
-      "kl": 46.490739446366206,
+      "completion_length": 398.875,
+      "epoch": 1.1428571428571428,
+      "grad_norm": 12.83385181427002,
+      "kl": 0.0018791476104524918,
       "learning_rate": 3.056302334890786e-07,
-      "loss": 0.0465,
-      "reward": 6.231653705239296,
-      "reward_std": 0.2912933654151857,
-      "rewards/concensus_correctness_reward_func": 1.6437499858438969,
-      "rewards/consensus_reward_func": 1.8125,
+      "loss": 0.0,
+      "reward": 0.8395995200844482,
+      "reward_std": 0.8317519237753004,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.375,
-      "rewards/question_recreation_reward_func": 0.8316849917173386,
+      "rewards/final_correctness_reward_func": 0.1875,
+      "rewards/question_recreation_reward_func": 0.25088077047257684,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.34375,
-      "rewards/xmlcount_reward_func": 1.224968746304512,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.3387187476619147,
       "step": 8
     },
     {
-      "completion_length": 165.42857142857142,
-      "epoch": 3.3636363636363638,
-      "grad_norm": 115.09622955322266,
-      "kl": 137.9989093542099,
+      "completion_length": 417.625,
+      "epoch": 1.4285714285714286,
+      "grad_norm": 17.527530670166016,
+      "kl": 0.0017757427813194226,
       "learning_rate": 1.9436976651092142e-07,
-      "loss": 0.1207,
-      "reward": 5.702515159334455,
-      "reward_std": 0.8088822598968234,
-      "rewards/concensus_correctness_reward_func": 1.6274999880364962,
-      "rewards/consensus_reward_func": 1.5714285714285714,
+      "loss": 0.0,
+      "reward": 0.5782972080633044,
+      "reward_std": 0.6126703205518425,
+      "rewards/concensus_correctness_reward_func": 0.0625,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.14285714285714285,
-      "rewards/question_recreation_reward_func": 0.9152294525078365,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.37370346777606755,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.26785714285714285,
-      "rewards/xmlcount_reward_func": 1.177642856325422,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.14209374599158764,
       "step": 10
     },
     {
-      "completion_length": 189.57142857142858,
-      "epoch": 4.0,
-      "grad_norm": 63.57939147949219,
-      "kl": 2.590619974769652,
+      "completion_length": 350.6875,
+      "epoch": 1.7142857142857144,
+      "grad_norm": 15.639988899230957,
+      "kl": 0.0017420653257431695,
       "learning_rate": 9.412754953531663e-08,
-      "loss": 0.0023,
-      "reward": 5.409352123737335,
-      "reward_std": 1.450669475864353,
-      "rewards/concensus_correctness_reward_func": 1.6674999818205833,
-      "rewards/consensus_reward_func": 1.2857142857142858,
+      "loss": 0.0,
+      "reward": 0.7548560223076493,
+      "reward_std": 0.8821847003418952,
+      "rewards/concensus_correctness_reward_func": 0.1223750002682209,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.2857142857142857,
-      "rewards/question_recreation_reward_func": 0.7612092856849942,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.35623101284727454,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.23214285714285715,
-      "rewards/xmlcount_reward_func": 1.1770714223384857,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.21375000104308128,
       "step": 12
     },
     {
-      "completion_length": 175.0625,
-      "epoch": 4.7272727272727275,
-      "grad_norm": 740719.1875,
-      "kl": 57090.395359053975,
+      "completion_length": 346.1875,
+      "epoch": 2.0,
+      "grad_norm": 6.904570579528809,
+      "kl": 0.002683696340682218,
       "learning_rate": 2.475778302439524e-08,
-      "loss": 57.0904,
-      "reward": 5.76124407351017,
-      "reward_std": 0.8618016238033306,
-      "rewards/concensus_correctness_reward_func": 1.6354999784380198,
-      "rewards/consensus_reward_func": 1.625,
+      "loss": 0.0,
+      "reward": 1.1041226710949559,
+      "reward_std": 0.709871930317604,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.8227753229439259,
-      "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.265625,
-      "rewards/xmlcount_reward_func": 1.1623437516391277,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.5565914369944949,
+      "rewards/soft_format_reward_func": 0.015625,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.4069062480702996,
       "step": 14
     },
     {
-      "completion_length": 143.0,
-      "epoch": 5.0,
-      "kl": 1.8872459878524144,
-      "reward": 4.5298177401224775,
-      "reward_std": 1.0606419742107391,
-      "rewards/concensus_correctness_reward_func": 0.636333333949248,
-      "rewards/consensus_reward_func": 1.5,
+      "completion_length": 367.0,
+      "epoch": 2.142857142857143,
+      "kl": 0.0015956754941726103,
+      "reward": 0.680220011388883,
+      "reward_std": 0.5986478577833623,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.3333333333333333,
-      "rewards/question_recreation_reward_func": 0.7836510688066483,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.3217825188767165,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.2916666666666667,
-      "rewards/xmlcount_reward_func": 0.9848333348830541,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.3584375027567148,
       "step": 15,
       "total_flos": 0.0,
-      "train_loss": 8.280637219718967,
-      "train_runtime": 115.5142,
-      "train_samples_per_second": 2.078,
-      "train_steps_per_second": 0.13
+      "train_loss": 1.4178454875946046e-06,
+      "train_runtime": 1361.509,
+      "train_samples_per_second": 0.176,
+      "train_steps_per_second": 0.011
     }
   ],
   "logging_steps": 2,
   "max_steps": 15,
   "num_input_tokens_seen": 0,
-  "num_train_epochs": 8,
+  "num_train_epochs": 3,
   "save_steps": 25,
   "stateful_callbacks": {
     "TrainerControl": {

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f870bf1e4292ac79e86b80ab79d9b736e5c265fc639ace85935f591e01d092d3
+oid sha256:59de0e4d9e3b9026cd75fb7f0e815759370050dc6664302afbe2d1f3efaa0b5d
 size 5944