End of training

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.3775766265403945,
-    "train_runtime": 158.6634,
+    "train_loss": 0.02531889981837594,
+    "train_runtime": 157.2181,
     "train_samples": 160,
-    "train_samples_per_second": 2.017,
-    "train_steps_per_second": 0.126
+    "train_samples_per_second": 2.035,
+    "train_steps_per_second": 0.127
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:dde680a4c5d84a93456736a868d99007d94ab9c2822230fd62950b52aa5f1036
+oid sha256:dee56cfec3e0bc592f9fe6a78661c6206c87d31f1f93d04f69781bd641270767
 size 1976163472

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.3775766265403945,
-    "train_runtime": 158.6634,
+    "train_loss": 0.02531889981837594,
+    "train_runtime": 157.2181,
     "train_samples": 160,
-    "train_samples_per_second": 2.017,
-    "train_steps_per_second": 0.126
+    "train_samples_per_second": 2.035,
+    "train_steps_per_second": 0.127
 }

trainer_state.json

@@ -10,203 +10,203 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 248.375,
+      "completion_length": 304.0,
       "epoch": 0.1,
-      "grad_norm": 16.94546890258789,
+      "grad_norm": 21.056718826293945,
       "kl": 0.0,
       "learning_rate": 5e-07,
       "loss": 0.0,
-      "reward": 3.726687029004097,
-      "reward_std": 0.8719135313294828,
-      "rewards/concensus_correctness_reward_func": 0.9646250009536743,
-      "rewards/consensus_reward_func": 0.75,
+      "reward": 4.178548539988697,
+      "reward_std": 0.645618889015168,
+      "rewards/concensus_correctness_reward_func": 1.2848749905824661,
+      "rewards/consensus_reward_func": 1.1875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.3125,
-      "rewards/question_recreation_reward_func": 0.8028745176270604,
-      "rewards/soft_format_reward_func": 0.015625,
-      "rewards/strict_format_reward_func": 0.140625,
-      "rewards/xmlcount_reward_func": 0.74043751321733,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.621923585422337,
+      "rewards/soft_format_reward_func": 0.0,
+      "rewards/strict_format_reward_func": 0.125,
+      "rewards/xmlcount_reward_func": 0.7092500035651028,
       "step": 2
     },
     {
-      "completion_length": 168.15625,
+      "completion_length": 170.4375,
       "epoch": 0.2,
-      "grad_norm": 56.39082717895508,
-      "kl": 1.63070479844464,
+      "grad_norm": 25.17281150817871,
+      "kl": 0.05341240478446707,
       "learning_rate": 4.864543104251586e-07,
-      "loss": 0.0016,
-      "reward": 5.480429410934448,
-      "reward_std": 0.3091476298868656,
-      "rewards/concensus_correctness_reward_func": 1.4978749975562096,
-      "rewards/consensus_reward_func": 1.875,
+      "loss": 0.0001,
+      "reward": 5.996723979711533,
+      "reward_std": 0.8833582359948196,
+      "rewards/concensus_correctness_reward_func": 1.7934374883770943,
+      "rewards/consensus_reward_func": 1.8125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.77839822974056,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.8079114654101431,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.234375,
-      "rewards/xmlcount_reward_func": 1.0947812534868717,
+      "rewards/strict_format_reward_func": 0.265625,
+      "rewards/xmlcount_reward_func": 1.0672499984502792,
       "step": 4
     },
     {
-      "completion_length": 174.5,
+      "completion_length": 154.03125,
       "epoch": 0.3,
-      "grad_norm": 15645.306640625,
-      "kl": 579.2013410500076,
+      "grad_norm": 32.91071319580078,
+      "kl": 0.3560464845504612,
       "learning_rate": 4.472851273490984e-07,
-      "loss": 0.5792,
-      "reward": 6.264685034751892,
-      "reward_std": 0.4674139647104312,
-      "rewards/concensus_correctness_reward_func": 1.778187483549118,
-      "rewards/consensus_reward_func": 1.8125,
+      "loss": 0.0004,
+      "reward": 5.7404483407735825,
+      "reward_std": 0.492848154011881,
+      "rewards/concensus_correctness_reward_func": 1.5335624851286411,
+      "rewards/consensus_reward_func": 1.625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.3125,
-      "rewards/question_recreation_reward_func": 0.8925288319587708,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.7900107745081186,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.28125,
-      "rewards/xmlcount_reward_func": 1.1877187490463257,
+      "rewards/strict_format_reward_func": 0.34375,
+      "rewards/xmlcount_reward_func": 1.1981250010430813,
       "step": 6
     },
     {
-      "completion_length": 152.375,
+      "completion_length": 148.625,
       "epoch": 0.4,
-      "grad_norm": 58.33176040649414,
-      "kl": 40.23226916906424,
+      "grad_norm": 21.569852828979492,
+      "kl": 0.5424519316293299,
       "learning_rate": 3.867370395306068e-07,
-      "loss": 0.0402,
-      "reward": 6.479055359959602,
-      "reward_std": 1.0061776030124747,
-      "rewards/concensus_correctness_reward_func": 2.2228125068359077,
-      "rewards/consensus_reward_func": 1.75,
+      "loss": 0.0005,
+      "reward": 6.806638553738594,
+      "reward_std": 1.2124032883657492,
+      "rewards/concensus_correctness_reward_func": 2.5408750283531845,
+      "rewards/consensus_reward_func": 1.6875,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.3125,
-      "rewards/question_recreation_reward_func": 0.7656803373247385,
+      "rewards/question_recreation_reward_func": 0.8630447909235954,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.296875,
-      "rewards/xmlcount_reward_func": 1.1311875060200691,
+      "rewards/strict_format_reward_func": 0.28125,
+      "rewards/xmlcount_reward_func": 1.1214687526226044,
       "step": 8
     },
     {
-      "completion_length": 135.53125,
+      "completion_length": 127.875,
       "epoch": 0.5,
-      "grad_norm": 473.90350341796875,
-      "kl": 21.327185112517327,
+      "grad_norm": 62.4689826965332,
+      "kl": 156.11455886077601,
       "learning_rate": 3.1137137178519977e-07,
-      "loss": 0.0213,
-      "reward": 6.760695576667786,
-      "reward_std": 0.6887843045406044,
-      "rewards/concensus_correctness_reward_func": 2.2292499765753746,
-      "rewards/consensus_reward_func": 1.9375,
+      "loss": 0.1561,
+      "reward": 6.762893989682198,
+      "reward_std": 0.609527827007696,
+      "rewards/concensus_correctness_reward_func": 2.166999977082014,
+      "rewards/consensus_reward_func": 1.875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.8453517183661461,
+      "rewards/final_correctness_reward_func": 0.3125,
+      "rewards/question_recreation_reward_func": 0.8805815447121859,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.328125,
-      "rewards/xmlcount_reward_func": 1.1704687476158142,
+      "rewards/xmlcount_reward_func": 1.1996874958276749,
       "step": 10
     },
     {
-      "completion_length": 162.34375,
+      "completion_length": 150.15625,
       "epoch": 0.6,
-      "grad_norm": 38.34357833862305,
-      "kl": 0.8932573557831347,
+      "grad_norm": 27.662405014038086,
+      "kl": 1.8935456158360466,
       "learning_rate": 2.2935516363191693e-07,
-      "loss": 0.0009,
-      "reward": 5.6704335659742355,
-      "reward_std": 0.6159919730853289,
-      "rewards/concensus_correctness_reward_func": 1.6258124746382236,
-      "rewards/consensus_reward_func": 1.8125,
+      "loss": 0.0019,
+      "reward": 5.805030718445778,
+      "reward_std": 0.4186253456864506,
+      "rewards/concensus_correctness_reward_func": 1.7014999762177467,
+      "rewards/consensus_reward_func": 1.875,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.7967772949486971,
+      "rewards/question_recreation_reward_func": 0.718936949968338,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.34375,
-      "rewards/xmlcount_reward_func": 1.091593749821186,
+      "rewards/strict_format_reward_func": 0.390625,
+      "rewards/xmlcount_reward_func": 1.1189687550067902,
       "step": 12
     },
     {
-      "completion_length": 140.75,
+      "completion_length": 130.09375,
       "epoch": 0.7,
-      "grad_norm": 1027.1484375,
-      "kl": 984.0450744732516,
+      "grad_norm": 55.68687438964844,
+      "kl": 7.544132360257208,
       "learning_rate": 1.4957614383675767e-07,
-      "loss": 0.984,
-      "reward": 7.15090399980545,
-      "reward_std": 1.0265336243246566,
-      "rewards/concensus_correctness_reward_func": 2.6768125034868717,
-      "rewards/consensus_reward_func": 1.875,
+      "loss": 0.0075,
+      "reward": 5.769488573074341,
+      "reward_std": 0.5740991415805183,
+      "rewards/concensus_correctness_reward_func": 1.618437498807907,
+      "rewards/consensus_reward_func": 1.8125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.845685207284987,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.7887072912417352,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.328125,
-      "rewards/xmlcount_reward_func": 1.1752812527120113,
+      "rewards/strict_format_reward_func": 0.34375,
+      "rewards/xmlcount_reward_func": 1.1435937508940697,
       "step": 14
     },
     {
-      "completion_length": 140.75,
+      "completion_length": 137.875,
       "epoch": 0.8,
-      "grad_norm": 32.436222076416016,
-      "kl": 26.871364967664704,
+      "grad_norm": 78.64262390136719,
+      "kl": 8.407416695728898,
       "learning_rate": 8.067960709356478e-08,
-      "loss": 0.0269,
-      "reward": 6.505572438240051,
-      "reward_std": 0.6704019304211215,
-      "rewards/concensus_correctness_reward_func": 1.8783750012516975,
-      "rewards/consensus_reward_func": 1.875,
+      "loss": 0.0084,
+      "reward": 6.370813250541687,
+      "reward_std": 0.5956368539482355,
+      "rewards/concensus_correctness_reward_func": 2.050624992698431,
+      "rewards/consensus_reward_func": 1.8125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.3125,
-      "rewards/question_recreation_reward_func": 0.8701348614413291,
+      "rewards/final_correctness_reward_func": 0.1875,
+      "rewards/question_recreation_reward_func": 0.7730007863137871,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.375,
-      "rewards/xmlcount_reward_func": 1.1945624984800816,
+      "rewards/strict_format_reward_func": 0.359375,
+      "rewards/xmlcount_reward_func": 1.1878125071525574,
       "step": 16
     },
     {
-      "completion_length": 132.125,
+      "completion_length": 158.28125,
       "epoch": 0.9,
-      "grad_norm": 22.22176742553711,
-      "kl": 2.2486448741983622,
+      "grad_norm": 494.2891540527344,
+      "kl": 26.95943553140387,
       "learning_rate": 3.013156219837776e-08,
-      "loss": 0.0022,
-      "reward": 6.13468325138092,
-      "reward_std": 0.3558603721903637,
-      "rewards/concensus_correctness_reward_func": 1.6389999873936176,
-      "rewards/consensus_reward_func": 1.8125,
+      "loss": 0.027,
+      "reward": 6.122102856636047,
+      "reward_std": 1.537369220983237,
+      "rewards/concensus_correctness_reward_func": 2.282374992966652,
+      "rewards/consensus_reward_func": 1.625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.1875,
-      "rewards/question_recreation_reward_func": 0.9093082211911678,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.789227906614542,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.390625,
-      "rewards/xmlcount_reward_func": 1.1957499980926514,
+      "rewards/strict_format_reward_func": 0.25,
+      "rewards/xmlcount_reward_func": 1.0505000054836273,
       "step": 18
     },
     {
-      "completion_length": 155.0,
+      "completion_length": 136.90625,
       "epoch": 1.0,
-      "grad_norm": 18.39291763305664,
-      "kl": 2119.248297330283,
+      "grad_norm": 31.231962203979492,
+      "kl": 51.322200544178486,
       "learning_rate": 3.4096741493194193e-09,
-      "loss": 2.1192,
-      "reward": 5.990508034825325,
-      "reward_std": 0.5463871430838481,
-      "rewards/concensus_correctness_reward_func": 1.7003749758005142,
-      "rewards/consensus_reward_func": 1.8125,
+      "loss": 0.0513,
+      "reward": 6.534483835101128,
+      "reward_std": 0.5124675603583455,
+      "rewards/concensus_correctness_reward_func": 1.995937503874302,
+      "rewards/consensus_reward_func": 1.9375,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.1875,
-      "rewards/question_recreation_reward_func": 0.8488205298781395,
+      "rewards/question_recreation_reward_func": 0.8151400880888104,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.328125,
-      "rewards/xmlcount_reward_func": 1.1131874956190586,
+      "rewards/strict_format_reward_func": 0.390625,
+      "rewards/xmlcount_reward_func": 1.2077812552452087,
       "step": 20
     },
     {
       "epoch": 1.0,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 0.3775766265403945,
-      "train_runtime": 158.6634,
-      "train_samples_per_second": 2.017,
-      "train_steps_per_second": 0.126
+      "train_loss": 0.02531889981837594,
+      "train_runtime": 157.2181,
+      "train_samples_per_second": 2.035,
+      "train_steps_per_second": 0.127
     }
   ],
   "logging_steps": 2,