End of training

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.0032690246604033745,
-    "train_runtime": 95.0198,
+    "train_loss": 2.7494842162559507,
+    "train_runtime": 97.2024,
     "train_samples": 160,
-    "train_samples_per_second": 3.368,
-    "train_steps_per_second": 0.21
+    "train_samples_per_second": 3.292,
+    "train_steps_per_second": 0.206
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1fd9d38270e0b365bb467c7fc9213ac3c89bce1faea6949fc4d9ea2fe114f764
+oid sha256:87b2c3bee7da6bd5d24ab8c1c78dd86022da182ad9de919df3a1507980d3139d
 size 1976163472

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.0032690246604033745,
-    "train_runtime": 95.0198,
+    "train_loss": 2.7494842162559507,
+    "train_runtime": 97.2024,
     "train_samples": 160,
-    "train_samples_per_second": 3.368,
-    "train_steps_per_second": 0.21
+    "train_samples_per_second": 3.292,
+    "train_steps_per_second": 0.206
 }

trainer_state.json

@@ -10,203 +10,203 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 232.09375,
+      "completion_length": 284.0,
       "epoch": 0.1,
-      "grad_norm": 29.780792236328125,
+      "grad_norm": 28.643468856811523,
       "kl": 0.0,
       "learning_rate": 5e-07,
-      "loss": -0.0,
-      "reward": 2.748943938408047,
-      "reward_std": 0.9673663395515177,
-      "rewards/concensus_correctness_reward_func": 0.7265000082552433,
-      "rewards/consensus_reward_func": 0.75,
+      "loss": 0.0,
+      "reward": 2.988352857530117,
+      "reward_std": 1.0429220204532612,
+      "rewards/concensus_correctness_reward_func": 0.8278750032186508,
+      "rewards/consensus_reward_func": 0.875,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.5500689675100148,
+      "rewards/question_recreation_reward_func": 0.5378527842694893,
       "rewards/soft_format_reward_func": 0.015625,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.7067499980330467,
+      "rewards/xmlcount_reward_func": 0.7319999802857637,
       "step": 2
     },
     {
-      "completion_length": 176.03125,
+      "completion_length": 162.0625,
       "epoch": 0.2,
-      "grad_norm": 20.436641693115234,
-      "kl": 0.05092630523722619,
+      "grad_norm": 19.01485252380371,
+      "kl": 0.14813337312079966,
       "learning_rate": 4.864543104251586e-07,
       "loss": 0.0001,
-      "reward": 5.3132626712322235,
-      "reward_std": 0.5458125858276617,
-      "rewards/concensus_correctness_reward_func": 1.5475625060498714,
-      "rewards/consensus_reward_func": 1.8125,
+      "reward": 5.656299531459808,
+      "reward_std": 0.3762048778589815,
+      "rewards/concensus_correctness_reward_func": 1.671812493354082,
+      "rewards/consensus_reward_func": 1.875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.7860437128692865,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.8333931616507471,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 1.1671562120318413,
+      "rewards/xmlcount_reward_func": 1.213593702763319,
       "step": 4
     },
     {
-      "completion_length": 179.03125,
+      "completion_length": 166.21875,
       "epoch": 0.3,
-      "grad_norm": 45.27792739868164,
-      "kl": 2.5084967764560133,
+      "grad_norm": 26.311038970947266,
+      "kl": 0.42348220106214285,
       "learning_rate": 4.472851273490984e-07,
-      "loss": 0.0025,
-      "reward": 5.82921689748764,
-      "reward_std": 1.5218889913521707,
-      "rewards/concensus_correctness_reward_func": 2.16118748113513,
-      "rewards/consensus_reward_func": 1.5625,
+      "loss": 0.0004,
+      "reward": 5.496340714395046,
+      "reward_std": 1.3250514825631399,
+      "rewards/concensus_correctness_reward_func": 1.9405000023543835,
+      "rewards/consensus_reward_func": 1.4375,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.1875,
-      "rewards/question_recreation_reward_func": 0.8439668118953705,
+      "rewards/question_recreation_reward_func": 0.8062468525022268,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 1.0740624759346247,
+      "rewards/xmlcount_reward_func": 1.1245937086641788,
       "step": 6
     },
     {
-      "completion_length": 162.25,
+      "completion_length": 158.125,
       "epoch": 0.4,
-      "grad_norm": 27.008052825927734,
-      "kl": 0.8038429818116128,
+      "grad_norm": 26.313640594482422,
+      "kl": 1.6625678152777255,
       "learning_rate": 3.867370395306068e-07,
-      "loss": 0.0008,
-      "reward": 4.970751509070396,
-      "reward_std": 0.8328131509770174,
-      "rewards/concensus_correctness_reward_func": 1.351249999832362,
-      "rewards/consensus_reward_func": 1.5,
+      "loss": 0.0017,
+      "reward": 5.847894251346588,
+      "reward_std": 0.8010366430826252,
+      "rewards/concensus_correctness_reward_func": 1.8802499799057841,
+      "rewards/consensus_reward_func": 1.6875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.8635014370083809,
+      "rewards/final_correctness_reward_func": 0.375,
+      "rewards/question_recreation_reward_func": 0.7160815456882119,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 1.193499967455864,
+      "rewards/xmlcount_reward_func": 1.1890624575316906,
       "step": 8
     },
     {
-      "completion_length": 141.46875,
+      "completion_length": 148.3125,
       "epoch": 0.5,
-      "grad_norm": 35.949424743652344,
-      "kl": 3.9549555676057935,
+      "grad_norm": 45.98735046386719,
+      "kl": 27449.420265319757,
       "learning_rate": 3.1137137178519977e-07,
-      "loss": 0.004,
-      "reward": 5.719552978873253,
-      "reward_std": 0.8067643920658156,
-      "rewards/concensus_correctness_reward_func": 1.829874999821186,
-      "rewards/consensus_reward_func": 1.625,
+      "loss": 27.4494,
+      "reward": 5.816218912601471,
+      "reward_std": 0.6078874306840589,
+      "rewards/concensus_correctness_reward_func": 1.9053125008940697,
+      "rewards/consensus_reward_func": 1.8125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.8506779558956623,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.7735312916338444,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 1.163999978452921,
+      "rewards/xmlcount_reward_func": 1.1998749561607838,
       "step": 10
     },
     {
-      "completion_length": 161.53125,
+      "completion_length": 171.96875,
       "epoch": 0.6,
-      "grad_norm": 23.505184173583984,
-      "kl": 2.307737836614251,
+      "grad_norm": 25.97221565246582,
+      "kl": 3.4606125969439745,
       "learning_rate": 2.2935516363191693e-07,
-      "loss": 0.0023,
-      "reward": 4.948680937290192,
-      "reward_std": 0.8787264387356117,
-      "rewards/concensus_correctness_reward_func": 1.4945624954998493,
-      "rewards/consensus_reward_func": 1.625,
+      "loss": 0.0035,
+      "reward": 5.34786581993103,
+      "reward_std": 0.8366972071089549,
+      "rewards/concensus_correctness_reward_func": 1.571687500923872,
+      "rewards/consensus_reward_func": 1.75,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.6966183818876743,
-      "rewards/soft_format_reward_func": 0.015625,
+      "rewards/question_recreation_reward_func": 0.8823031485080719,
+      "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 1.1168749816715717,
+      "rewards/xmlcount_reward_func": 1.1438749693334103,
       "step": 12
     },
     {
-      "completion_length": 144.46875,
+      "completion_length": 143.25,
       "epoch": 0.7,
-      "grad_norm": 103.61343383789062,
-      "kl": 16.992877448908985,
+      "grad_norm": 28.309701919555664,
+      "kl": 3.211598624009639,
       "learning_rate": 1.4957614383675767e-07,
-      "loss": 0.017,
-      "reward": 5.655805706977844,
-      "reward_std": 0.9154406100860797,
-      "rewards/concensus_correctness_reward_func": 1.7131874971091747,
-      "rewards/consensus_reward_func": 1.75,
+      "loss": 0.0032,
+      "reward": 5.006202816963196,
+      "reward_std": 1.1830032421348733,
+      "rewards/concensus_correctness_reward_func": 1.508250005543232,
+      "rewards/consensus_reward_func": 1.5625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.1875,
-      "rewards/question_recreation_reward_func": 0.7997118066996336,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.7309526707977057,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 1.2054062262177467,
+      "rewards/xmlcount_reward_func": 1.204499937593937,
       "step": 14
     },
     {
-      "completion_length": 144.03125,
+      "completion_length": 150.71875,
       "epoch": 0.8,
-      "grad_norm": 26.544294357299805,
-      "kl": 2.246969759464264,
+      "grad_norm": 22.530658721923828,
+      "kl": 9.211048339493573,
       "learning_rate": 8.067960709356478e-08,
-      "loss": 0.0022,
-      "reward": 5.492044880986214,
-      "reward_std": 0.5297679766081274,
-      "rewards/concensus_correctness_reward_func": 1.634062498807907,
-      "rewards/consensus_reward_func": 1.875,
+      "loss": 0.0092,
+      "reward": 5.990000277757645,
+      "reward_std": 0.2861854724033037,
+      "rewards/concensus_correctness_reward_func": 1.8882499895989895,
+      "rewards/consensus_reward_func": 1.8125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.7621072232723236,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.8423438891768456,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 1.1583749651908875,
+      "rewards/xmlcount_reward_func": 1.196906205266714,
       "step": 16
     },
     {
-      "completion_length": 159.875,
+      "completion_length": 159.40625,
       "epoch": 0.9,
-      "grad_norm": 75.8289794921875,
-      "kl": 2.813395518809557,
+      "grad_norm": 489.46051025390625,
+      "kl": 18.507965696509928,
       "learning_rate": 3.013156219837776e-08,
-      "loss": 0.0028,
-      "reward": 5.568623270839453,
-      "reward_std": 1.7908655412174994,
-      "rewards/concensus_correctness_reward_func": 2.014750000089407,
-      "rewards/consensus_reward_func": 1.5625,
+      "loss": 0.0185,
+      "reward": 4.922803536057472,
+      "reward_std": 1.2908645485149464,
+      "rewards/concensus_correctness_reward_func": 1.4476875066757202,
+      "rewards/consensus_reward_func": 1.5,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.7789357244619168,
+      "rewards/question_recreation_reward_func": 0.7194908875972033,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 1.0874374751001596,
+      "rewards/xmlcount_reward_func": 1.1306249648332596,
       "step": 18
     },
     {
-      "completion_length": 150.4375,
+      "completion_length": 138.8125,
       "epoch": 1.0,
-      "grad_norm": 19.319944381713867,
-      "kl": 1.0169571689330041,
+      "grad_norm": 36.206031799316406,
+      "kl": 8.796861524460837,
       "learning_rate": 3.4096741493194193e-09,
-      "loss": 0.001,
-      "reward": 5.82940012216568,
-      "reward_std": 1.0650014284910867,
-      "rewards/concensus_correctness_reward_func": 1.7777500040829182,
-      "rewards/consensus_reward_func": 1.75,
+      "loss": 0.0088,
+      "reward": 5.56746606528759,
+      "reward_std": 0.6262589020479936,
+      "rewards/concensus_correctness_reward_func": 1.7263749949634075,
+      "rewards/consensus_reward_func": 1.6875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.8705874998122454,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.807747196406126,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 1.1810624562203884,
+      "rewards/xmlcount_reward_func": 1.2208436951041222,
       "step": 20
     },
     {
       "epoch": 1.0,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 0.0032690246604033745,
-      "train_runtime": 95.0198,
-      "train_samples_per_second": 3.368,
-      "train_steps_per_second": 0.21
+      "train_loss": 2.7494842162559507,
+      "train_runtime": 97.2024,
+      "train_samples_per_second": 3.292,
+      "train_steps_per_second": 0.206
     }
   ],
   "logging_steps": 2,