End of training

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.0009786995709873737,
-    "train_runtime": 134.3867,
+    "train_loss": 0.0008016080327251984,
+    "train_runtime": 111.0152,
     "train_samples": 160,
-    "train_samples_per_second": 2.381,
-    "train_steps_per_second": 0.149
+    "train_samples_per_second": 2.882,
+    "train_steps_per_second": 0.18
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fa00768b87236685c7a6e42f728690869017f15647cc4be1fc21596a35249765
+oid sha256:2ea7b046133f7d49f5ec226ae302e78674f56453eaaea0137e2af8877081f8b4
 size 1976163472

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.0009786995709873737,
-    "train_runtime": 134.3867,
+    "train_loss": 0.0008016080327251984,
+    "train_runtime": 111.0152,
     "train_samples": 160,
-    "train_samples_per_second": 2.381,
-    "train_steps_per_second": 0.149
+    "train_samples_per_second": 2.882,
+    "train_steps_per_second": 0.18
 }

trainer_state.json

@@ -10,203 +10,203 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 323.3125,
+      "completion_length": 281.25,
       "epoch": 0.1,
-      "grad_norm": 8.017651557922363,
+      "grad_norm": 11.220077514648438,
       "kl": 0.0,
       "learning_rate": 5e-07,
-      "loss": 0.0,
-      "reward": 2.563552338629961,
-      "reward_std": 2.1488499622792006,
-      "rewards/concensus_correctness_reward_func": 0.9308750033378601,
-      "rewards/consensus_reward_func": 0.4375,
+      "loss": -0.0,
+      "reward": 2.3628913946449757,
+      "reward_std": 1.7425345129013294,
+      "rewards/concensus_correctness_reward_func": 0.6074374988675117,
+      "rewards/consensus_reward_func": 0.625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.5906460355035961,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.5488913534209132,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.5420312769711018,
+      "rewards/xmlcount_reward_func": 0.5815625246614218,
       "step": 2
     },
     {
-      "completion_length": 210.5625,
+      "completion_length": 214.71875,
       "epoch": 0.2,
-      "grad_norm": 7.595134258270264,
-      "kl": 0.00439721306975116,
+      "grad_norm": 14.205344200134277,
+      "kl": 0.009285321953939274,
       "learning_rate": 4.864543104251586e-07,
       "loss": 0.0,
-      "reward": 3.520405486226082,
-      "reward_std": 1.2246719358954579,
-      "rewards/concensus_correctness_reward_func": 0.7968124955659732,
-      "rewards/consensus_reward_func": 1.0625,
+      "reward": 3.392438106238842,
+      "reward_std": 1.2985186432488263,
+      "rewards/concensus_correctness_reward_func": 0.7482500001788139,
+      "rewards/consensus_reward_func": 0.875,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.6318742204457521,
+      "rewards/question_recreation_reward_func": 0.683906783349812,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.966718778014183,
+      "rewards/xmlcount_reward_func": 1.0227812826633453,
       "step": 4
     },
     {
-      "completion_length": 264.84375,
+      "completion_length": 200.4375,
       "epoch": 0.3,
-      "grad_norm": 8.46875286102295,
-      "kl": 0.028284174797590822,
+      "grad_norm": 20.50322723388672,
+      "kl": 0.538932702911552,
       "learning_rate": 4.472851273490984e-07,
-      "loss": 0.0,
-      "reward": 3.280557908117771,
-      "reward_std": 1.4112022835761309,
-      "rewards/concensus_correctness_reward_func": 0.7302500009536743,
-      "rewards/consensus_reward_func": 0.875,
+      "loss": 0.0005,
+      "reward": 3.473274417221546,
+      "reward_std": 1.2387595605905517,
+      "rewards/concensus_correctness_reward_func": 0.803250003606081,
+      "rewards/consensus_reward_func": 0.8125,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.6834016898646951,
+      "rewards/question_recreation_reward_func": 0.728399284183979,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.9294062629342079,
+      "rewards/xmlcount_reward_func": 1.0666250251233578,
       "step": 6
     },
     {
-      "completion_length": 181.65625,
+      "completion_length": 162.21875,
       "epoch": 0.4,
-      "grad_norm": 11.66727352142334,
-      "kl": 0.09929609816754237,
+      "grad_norm": 12.368329048156738,
+      "kl": 0.08565096225356683,
       "learning_rate": 3.867370395306068e-07,
       "loss": 0.0001,
-      "reward": 3.6070429645478725,
-      "reward_std": 1.5279794879024848,
-      "rewards/concensus_correctness_reward_func": 0.885687499307096,
-      "rewards/consensus_reward_func": 0.9375,
+      "reward": 3.893532305955887,
+      "reward_std": 1.9143780157901347,
+      "rewards/concensus_correctness_reward_func": 0.9625624967738986,
+      "rewards/consensus_reward_func": 1.1875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.5651679411821533,
-      "rewards/soft_format_reward_func": 0.015625,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.5282509727403522,
+      "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.9530625063925982,
+      "rewards/xmlcount_reward_func": 1.0902187786996365,
       "step": 8
     },
     {
-      "completion_length": 241.0,
+      "completion_length": 139.71875,
       "epoch": 0.5,
-      "grad_norm": 13.426807403564453,
-      "kl": 0.8362448264670093,
+      "grad_norm": 13.570387840270996,
+      "kl": 1.3568879710510373,
       "learning_rate": 3.1137137178519977e-07,
-      "loss": 0.0008,
-      "reward": 3.7212521210312843,
-      "reward_std": 1.639379691099748,
-      "rewards/concensus_correctness_reward_func": 0.87049999833107,
-      "rewards/consensus_reward_func": 0.8125,
+      "loss": 0.0014,
+      "reward": 4.62184239923954,
+      "reward_std": 1.8512163756677182,
+      "rewards/concensus_correctness_reward_func": 1.282937504351139,
+      "rewards/consensus_reward_func": 1.125,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.5,
-      "rewards/question_recreation_reward_func": 0.6762520521879196,
+      "rewards/question_recreation_reward_func": 0.6110609890893102,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.8620000295341015,
+      "rewards/xmlcount_reward_func": 1.1028437800705433,
       "step": 10
     },
     {
-      "completion_length": 197.65625,
+      "completion_length": 179.78125,
       "epoch": 0.6,
-      "grad_norm": 8.333313941955566,
-      "kl": 0.804443325905595,
+      "grad_norm": 13.817785263061523,
+      "kl": 1.6189894516137429,
       "learning_rate": 2.2935516363191693e-07,
-      "loss": 0.0008,
-      "reward": 3.636394202709198,
-      "reward_std": 1.5618274274165742,
-      "rewards/concensus_correctness_reward_func": 0.9395624920725822,
-      "rewards/consensus_reward_func": 1.0,
+      "loss": 0.0016,
+      "reward": 3.3854666724801064,
+      "reward_std": 0.7775582046888303,
+      "rewards/concensus_correctness_reward_func": 0.90625,
+      "rewards/consensus_reward_func": 0.75,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.6416129586286843,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.6140603653620929,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.9302187636494637,
+      "rewards/xmlcount_reward_func": 1.052656276151538,
       "step": 12
     },
     {
-      "completion_length": 262.78125,
+      "completion_length": 192.0625,
       "epoch": 0.7,
-      "grad_norm": 9.710261344909668,
-      "kl": 0.609934190637432,
+      "grad_norm": 11.607613563537598,
+      "kl": 0.9145036803674884,
       "learning_rate": 1.4957614383675767e-07,
-      "loss": 0.0006,
-      "reward": 3.5298764258623123,
-      "reward_std": 1.5647379670408554,
-      "rewards/concensus_correctness_reward_func": 0.9007500000298023,
-      "rewards/consensus_reward_func": 1.0625,
+      "loss": 0.0009,
+      "reward": 4.008361168205738,
+      "reward_std": 1.162059747366584,
+      "rewards/concensus_correctness_reward_func": 1.1055000089108944,
+      "rewards/consensus_reward_func": 1.1875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.7030013324692845,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.6631735619157553,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.8636250160634518,
+      "rewards/xmlcount_reward_func": 0.9896875210106373,
       "step": 14
     },
     {
-      "completion_length": 262.71875,
+      "completion_length": 138.4375,
       "epoch": 0.8,
-      "grad_norm": 9.240312576293945,
-      "kl": 1.1370018686284311,
+      "grad_norm": 14.20344066619873,
+      "kl": 1.9467103342758492,
       "learning_rate": 8.067960709356478e-08,
-      "loss": 0.0011,
-      "reward": 4.615200288593769,
-      "reward_std": 1.5115430569276214,
-      "rewards/concensus_correctness_reward_func": 1.7112500220537186,
-      "rewards/consensus_reward_func": 1.0,
+      "loss": 0.0019,
+      "reward": 3.873469777405262,
+      "reward_std": 1.7628455930534983,
+      "rewards/concensus_correctness_reward_func": 0.934249997138977,
+      "rewards/consensus_reward_func": 1.125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.7081689362530597,
-      "rewards/soft_format_reward_func": 0.015625,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.7548758981283754,
+      "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.9301562681794167,
+      "rewards/xmlcount_reward_func": 1.0593437775969505,
       "step": 16
     },
     {
-      "completion_length": 228.15625,
+      "completion_length": 224.1875,
       "epoch": 0.9,
-      "grad_norm": 30.264890670776367,
-      "kl": 4.979973241686821,
+      "grad_norm": 13.969220161437988,
+      "kl": 0.8128210354479961,
       "learning_rate": 3.013156219837776e-08,
-      "loss": 0.005,
-      "reward": 3.6262373626232147,
-      "reward_std": 1.359476147841633,
-      "rewards/concensus_correctness_reward_func": 0.9268124997615814,
-      "rewards/consensus_reward_func": 1.0,
+      "loss": 0.0008,
+      "reward": 3.397010952234268,
+      "reward_std": 1.0946342277748045,
+      "rewards/concensus_correctness_reward_func": 0.7531875018030405,
+      "rewards/consensus_reward_func": 0.9375,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.665112272836268,
+      "rewards/question_recreation_reward_func": 0.6452609216794372,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.9718125052750111,
+      "rewards/xmlcount_reward_func": 0.9985625185072422,
       "step": 18
     },
     {
-      "completion_length": 167.875,
+      "completion_length": 143.4375,
       "epoch": 1.0,
-      "grad_norm": 11.473233222961426,
-      "kl": 1.2813922382192686,
+      "grad_norm": 97.46627807617188,
+      "kl": 0.7393010172527283,
       "learning_rate": 3.4096741493194193e-09,
-      "loss": 0.0013,
-      "reward": 3.8251934871077538,
-      "reward_std": 1.499690835364163,
-      "rewards/concensus_correctness_reward_func": 0.9646874964237213,
-      "rewards/consensus_reward_func": 1.0,
+      "loss": 0.0007,
+      "reward": 4.482209220528603,
+      "reward_std": 0.7270907972415444,
+      "rewards/concensus_correctness_reward_func": 1.3186875022947788,
+      "rewards/consensus_reward_func": 1.1875,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.6929746754467487,
+      "rewards/question_recreation_reward_func": 0.7328341100364923,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 1.0425312668085098,
+      "rewards/xmlcount_reward_func": 1.1181875206530094,
       "step": 20
     },
     {
       "epoch": 1.0,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 0.0009786995709873737,
-      "train_runtime": 134.3867,
-      "train_samples_per_second": 2.381,
-      "train_steps_per_second": 0.149
+      "train_loss": 0.0008016080327251984,
+      "train_runtime": 111.0152,
+      "train_samples_per_second": 2.882,
+      "train_steps_per_second": 0.18
     }
   ],
   "logging_steps": 2,