End of training

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.16747612971521447,
-    "train_runtime": 161.8685,
+    "train_loss": 5.484697436541319,
+    "train_runtime": 165.0886,
     "train_samples": 160,
-    "train_samples_per_second": 1.977,
-    "train_steps_per_second": 0.124
+    "train_samples_per_second": 1.938,
+    "train_steps_per_second": 0.121
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a7c66b7cb1dac894e7eb2ded1fd4f6e18c0cbba37cb59cbea30b8b19da12aef8
+oid sha256:67baf9a3f96678870a522cba1d16884f31dd828b30385434b768f96145af2182
 size 1976163472

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.16747612971521447,
-    "train_runtime": 161.8685,
+    "train_loss": 5.484697436541319,
+    "train_runtime": 165.0886,
     "train_samples": 160,
-    "train_samples_per_second": 1.977,
-    "train_steps_per_second": 0.124
+    "train_samples_per_second": 1.938,
+    "train_steps_per_second": 0.121
 }

trainer_state.json

@@ -10,203 +10,203 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 278.3125,
+      "completion_length": 297.5,
       "epoch": 0.1,
-      "grad_norm": 30.306766510009766,
+      "grad_norm": 26.61813735961914,
       "kl": 0.0,
       "learning_rate": 5e-07,
       "loss": -0.0,
-      "reward": 3.417320553679019,
-      "reward_std": 0.7929421009321231,
-      "rewards/concensus_correctness_reward_func": 0.9435624927282333,
-      "rewards/consensus_reward_func": 1.0,
+      "reward": 3.743449727073312,
+      "reward_std": 0.4033267687627813,
+      "rewards/concensus_correctness_reward_func": 1.133312501013279,
+      "rewards/consensus_reward_func": 1.0625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.5869455883512273,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.5725435714703053,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.15625,
-      "rewards/xmlcount_reward_func": 0.7305625043809414,
+      "rewards/strict_format_reward_func": 0.203125,
+      "rewards/xmlcount_reward_func": 0.6469687456265092,
       "step": 2
     },
     {
-      "completion_length": 190.875,
+      "completion_length": 152.78125,
       "epoch": 0.2,
-      "grad_norm": 32.36166000366211,
-      "kl": 0.3710639944765717,
+      "grad_norm": 3002670.75,
+      "kl": 54187.647869876004,
       "learning_rate": 4.864543104251586e-07,
-      "loss": 0.0004,
-      "reward": 5.694687262177467,
-      "reward_std": 0.8986590428976342,
-      "rewards/concensus_correctness_reward_func": 1.6944999918341637,
-      "rewards/consensus_reward_func": 1.6875,
+      "loss": 54.1876,
+      "reward": 6.1946185529232025,
+      "reward_std": 0.6147320005693473,
+      "rewards/concensus_correctness_reward_func": 1.8531874902546406,
+      "rewards/consensus_reward_func": 1.75,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.8439998179674149,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.8713061073794961,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.296875,
-      "rewards/xmlcount_reward_func": 1.0468124970793724,
+      "rewards/strict_format_reward_func": 0.34375,
+      "rewards/xmlcount_reward_func": 1.1263750046491623,
       "step": 4
     },
     {
-      "completion_length": 145.875,
+      "completion_length": 186.25,
       "epoch": 0.3,
-      "grad_norm": 24.805095672607422,
-      "kl": 0.581909274449572,
+      "grad_norm": 37.145965576171875,
+      "kl": 8.248837063321844,
       "learning_rate": 4.472851273490984e-07,
-      "loss": 0.0006,
-      "reward": 6.487181305885315,
-      "reward_std": 0.17793172437814064,
-      "rewards/concensus_correctness_reward_func": 1.8696249760687351,
-      "rewards/consensus_reward_func": 1.875,
+      "loss": 0.0082,
+      "reward": 5.679559916257858,
+      "reward_std": 0.32521654991433024,
+      "rewards/concensus_correctness_reward_func": 1.4357500039041042,
+      "rewards/consensus_reward_func": 1.75,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.1875,
-      "rewards/question_recreation_reward_func": 0.9601500444114208,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.9139662310481071,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.390625,
-      "rewards/xmlcount_reward_func": 1.204281248152256,
+      "rewards/strict_format_reward_func": 0.328125,
+      "rewards/xmlcount_reward_func": 1.126718744635582,
       "step": 6
     },
     {
-      "completion_length": 175.25,
+      "completion_length": 174.0,
       "epoch": 0.4,
-      "grad_norm": 534.1702270507812,
-      "kl": 76.98514324938878,
+      "grad_norm": 613.678955078125,
+      "kl": 41.35943519131979,
       "learning_rate": 3.867370395306068e-07,
-      "loss": 0.077,
-      "reward": 5.755250185728073,
-      "reward_std": 0.7580820816219784,
-      "rewards/concensus_correctness_reward_func": 1.7667500004172325,
-      "rewards/consensus_reward_func": 1.6875,
+      "loss": 0.0414,
+      "reward": 5.720165818929672,
+      "reward_std": 0.9133461741730571,
+      "rewards/concensus_correctness_reward_func": 1.6651874957606196,
+      "rewards/consensus_reward_func": 1.625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.1875,
-      "rewards/question_recreation_reward_func": 0.8275314536876976,
+      "rewards/final_correctness_reward_func": 0.3125,
+      "rewards/question_recreation_reward_func": 0.8708219872787595,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.3125,
-      "rewards/xmlcount_reward_func": 0.9734687507152557,
+      "rewards/strict_format_reward_func": 0.21875,
+      "rewards/xmlcount_reward_func": 1.0279062557965517,
       "step": 8
     },
     {
-      "completion_length": 151.0,
+      "completion_length": 138.75,
       "epoch": 0.5,
-      "grad_norm": 49.06590270996094,
-      "kl": 21.88503791950643,
+      "grad_norm": 68.1861801147461,
+      "kl": 2.098579771351069,
       "learning_rate": 3.1137137178519977e-07,
-      "loss": 0.0219,
-      "reward": 6.543248265981674,
-      "reward_std": 0.5147177229519002,
-      "rewards/concensus_correctness_reward_func": 2.1673749685287476,
-      "rewards/consensus_reward_func": 2.0,
+      "loss": 0.0021,
+      "reward": 6.643429189920425,
+      "reward_std": 0.7670735517167486,
+      "rewards/concensus_correctness_reward_func": 2.149812500923872,
+      "rewards/consensus_reward_func": 1.875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.9027169682085514,
+      "rewards/final_correctness_reward_func": 0.375,
+      "rewards/question_recreation_reward_func": 0.9242104254662991,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.3125,
-      "rewards/xmlcount_reward_func": 1.0356562435626984,
+      "rewards/strict_format_reward_func": 0.265625,
+      "rewards/xmlcount_reward_func": 1.0537812523543835,
       "step": 10
     },
     {
-      "completion_length": 160.5625,
+      "completion_length": 130.1875,
       "epoch": 0.6,
-      "grad_norm": 95.7334976196289,
-      "kl": 77.77790280617774,
+      "grad_norm": 351.7011413574219,
+      "kl": 38.816855053883046,
       "learning_rate": 2.2935516363191693e-07,
-      "loss": 0.0778,
-      "reward": 5.421116538345814,
-      "reward_std": 0.6716473484411836,
-      "rewards/concensus_correctness_reward_func": 1.555874977260828,
-      "rewards/consensus_reward_func": 1.625,
+      "loss": 0.0388,
+      "reward": 6.589949309825897,
+      "reward_std": 0.9503946190234274,
+      "rewards/concensus_correctness_reward_func": 2.2669374756515026,
+      "rewards/consensus_reward_func": 1.75,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.7852728068828583,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.8636056985706091,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.359375,
-      "rewards/xmlcount_reward_func": 1.0955937504768372,
+      "rewards/strict_format_reward_func": 0.390625,
+      "rewards/xmlcount_reward_func": 1.19378125295043,
       "step": 12
     },
     {
-      "completion_length": 151.5625,
+      "completion_length": 165.09375,
       "epoch": 0.7,
-      "grad_norm": 16727.4921875,
-      "kl": 804.6062668529339,
+      "grad_norm": 12886.0927734375,
+      "kl": 339.3810063265264,
       "learning_rate": 1.4957614383675767e-07,
-      "loss": 0.8046,
-      "reward": 5.7431196719408035,
-      "reward_std": 0.7721475283615291,
-      "rewards/concensus_correctness_reward_func": 1.689125008881092,
-      "rewards/consensus_reward_func": 1.8125,
+      "loss": 0.3394,
+      "reward": 5.94477491080761,
+      "reward_std": 0.8131171045824885,
+      "rewards/concensus_correctness_reward_func": 1.7751874960958958,
+      "rewards/consensus_reward_func": 1.875,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.912525961175561,
+      "rewards/question_recreation_reward_func": 0.9484624117612839,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.296875,
-      "rewards/xmlcount_reward_func": 1.032093746587634,
+      "rewards/strict_format_reward_func": 0.28125,
+      "rewards/xmlcount_reward_func": 1.0648749992251396,
       "step": 14
     },
     {
-      "completion_length": 121.28125,
+      "completion_length": 131.53125,
       "epoch": 0.8,
-      "grad_norm": 98.13787841796875,
-      "kl": 679.9979079507466,
+      "grad_norm": 13532.1064453125,
+      "kl": 205.99427369842306,
       "learning_rate": 8.067960709356478e-08,
-      "loss": 0.68,
-      "reward": 5.9227196872234344,
-      "reward_std": 0.9738721310568508,
-      "rewards/concensus_correctness_reward_func": 1.6648124866187572,
-      "rewards/consensus_reward_func": 1.75,
+      "loss": 0.206,
+      "reward": 6.167793929576874,
+      "reward_std": 0.833345370425377,
+      "rewards/concensus_correctness_reward_func": 1.7546249814331532,
+      "rewards/consensus_reward_func": 1.8125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.850438398309052,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.9484501853585243,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.375,
-      "rewards/xmlcount_reward_func": 1.157468743622303,
+      "rewards/strict_format_reward_func": 0.40625,
+      "rewards/xmlcount_reward_func": 1.1834687516093254,
       "step": 16
     },
     {
-      "completion_length": 146.4375,
+      "completion_length": 147.90625,
       "epoch": 0.9,
-      "grad_norm": 202.61131286621094,
-      "kl": 8.440069527830929,
+      "grad_norm": 37.940574645996094,
+      "kl": 8.128756958059967,
       "learning_rate": 3.013156219837776e-08,
-      "loss": 0.0084,
-      "reward": 5.58927384018898,
-      "reward_std": 1.127526328433305,
-      "rewards/concensus_correctness_reward_func": 1.5550624951720238,
+      "loss": 0.0081,
+      "reward": 6.150354638695717,
+      "reward_std": 1.2210767110082088,
+      "rewards/concensus_correctness_reward_func": 2.165000006556511,
       "rewards/consensus_reward_func": 1.6875,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.8999613896012306,
+      "rewards/question_recreation_reward_func": 0.9025107845664024,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.296875,
-      "rewards/xmlcount_reward_func": 1.0248750001192093,
+      "rewards/strict_format_reward_func": 0.1875,
+      "rewards/xmlcount_reward_func": 1.0828437507152557,
       "step": 18
     },
     {
-      "completion_length": 125.71875,
+      "completion_length": 189.4375,
       "epoch": 1.0,
-      "grad_norm": 44.48989486694336,
-      "kl": 4.118530319072306,
+      "grad_norm": 325.0932922363281,
+      "kl": 15.322000436484814,
       "learning_rate": 3.4096741493194193e-09,
-      "loss": 0.0041,
-      "reward": 5.763610690832138,
-      "reward_std": 1.142304808134213,
-      "rewards/concensus_correctness_reward_func": 1.784624993801117,
-      "rewards/consensus_reward_func": 1.625,
+      "loss": 0.0153,
+      "reward": 6.026204079389572,
+      "reward_std": 0.7251240487676114,
+      "rewards/concensus_correctness_reward_func": 1.8362499922513962,
+      "rewards/consensus_reward_func": 1.875,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.8503607120364904,
+      "rewards/question_recreation_reward_func": 0.8849227903410792,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.265625,
-      "rewards/xmlcount_reward_func": 1.1129999905824661,
+      "rewards/strict_format_reward_func": 0.234375,
+      "rewards/xmlcount_reward_func": 1.0706562548875809,
       "step": 20
     },
     {
       "epoch": 1.0,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 0.16747612971521447,
-      "train_runtime": 161.8685,
-      "train_samples_per_second": 1.977,
-      "train_steps_per_second": 0.124
+      "train_loss": 5.484697436541319,
+      "train_runtime": 165.0886,
+      "train_samples_per_second": 1.938,
+      "train_steps_per_second": 0.121
     }
   ],
   "logging_steps": 2,