End of training

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.47205696956516474,
-    "train_runtime": 151.675,
+    "train_loss": 5.40037734391517,
+    "train_runtime": 148.422,
     "train_samples": 160,
-    "train_samples_per_second": 2.11,
-    "train_steps_per_second": 0.132
+    "train_samples_per_second": 2.156,
+    "train_steps_per_second": 0.135
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a2abb0f3ff4980e96c6547352a1d23348d10d72c29cdcdbd8743b93953cee7a0
+oid sha256:5da4ac043691ad25f4ed610ae099638001caa64994c061cf70e9183485fc6974
 size 1976163472

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.47205696956516474,
-    "train_runtime": 151.675,
+    "train_loss": 5.40037734391517,
+    "train_runtime": 148.422,
     "train_samples": 160,
-    "train_samples_per_second": 2.11,
-    "train_steps_per_second": 0.132
+    "train_samples_per_second": 2.156,
+    "train_steps_per_second": 0.135
 }

trainer_state.json

@@ -10,203 +10,203 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 219.25,
+      "completion_length": 214.15625,
       "epoch": 0.1,
-      "grad_norm": 36.475006103515625,
+      "grad_norm": 22.216941833496094,
       "kl": 0.0,
       "learning_rate": 5e-07,
-      "loss": -0.0,
-      "reward": 3.9063082225620747,
-      "reward_std": 0.8945223242044449,
-      "rewards/concensus_correctness_reward_func": 1.197062499821186,
-      "rewards/consensus_reward_func": 1.125,
+      "loss": 0.0,
+      "reward": 3.9006508318707347,
+      "reward_std": 0.5246393299676129,
+      "rewards/concensus_correctness_reward_func": 1.1009999923408031,
+      "rewards/consensus_reward_func": 1.0625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.634433209488634,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.6793695546220988,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.140625,
-      "rewards/xmlcount_reward_func": 0.6841874970123172,
+      "rewards/strict_format_reward_func": 0.21875,
+      "rewards/xmlcount_reward_func": 0.7765312492847443,
       "step": 2
     },
     {
-      "completion_length": 133.9375,
+      "completion_length": 153.5625,
       "epoch": 0.2,
-      "grad_norm": 98.08300018310547,
-      "kl": 0.3790229291189462,
+      "grad_norm": 20.44854164123535,
+      "kl": 0.1254931440926157,
       "learning_rate": 4.864543104251586e-07,
-      "loss": 0.0004,
-      "reward": 6.21312315762043,
-      "reward_std": 1.315331432968378,
-      "rewards/concensus_correctness_reward_func": 2.089437495917082,
-      "rewards/consensus_reward_func": 1.6875,
+      "loss": 0.0001,
+      "reward": 6.288067072629929,
+      "reward_std": 0.5811158895376138,
+      "rewards/concensus_correctness_reward_func": 1.6979374811053276,
+      "rewards/consensus_reward_func": 1.875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.9614044353365898,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.9120046440511942,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.25,
-      "rewards/xmlcount_reward_func": 1.0997812524437904,
+      "rewards/strict_format_reward_func": 0.34375,
+      "rewards/xmlcount_reward_func": 1.2093750014901161,
       "step": 4
     },
     {
-      "completion_length": 178.1875,
+      "completion_length": 194.0625,
       "epoch": 0.3,
-      "grad_norm": 36.2740592956543,
-      "kl": 0.4786758206319064,
+      "grad_norm": 27783.4765625,
+      "kl": 919.1792740374804,
       "learning_rate": 4.472851273490984e-07,
-      "loss": 0.0005,
-      "reward": 5.581935882568359,
-      "reward_std": 0.9109209656016901,
-      "rewards/concensus_correctness_reward_func": 1.5193749889731407,
-      "rewards/consensus_reward_func": 1.625,
+      "loss": 0.9192,
+      "reward": 6.010144874453545,
+      "reward_std": 0.7817130146140698,
+      "rewards/concensus_correctness_reward_func": 1.6994999833405018,
+      "rewards/consensus_reward_func": 1.9375,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.9968421719968319,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.9526761472225189,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.25,
-      "rewards/xmlcount_reward_func": 1.0657187476754189,
+      "rewards/strict_format_reward_func": 0.359375,
+      "rewards/xmlcount_reward_func": 0.9985937625169754,
       "step": 6
     },
     {
-      "completion_length": 151.875,
+      "completion_length": 142.375,
       "epoch": 0.4,
-      "grad_norm": 19.702165603637695,
-      "kl": 0.2741884889546782,
+      "grad_norm": 831.9281616210938,
+      "kl": 31.678775895619765,
       "learning_rate": 3.867370395306068e-07,
-      "loss": 0.0003,
-      "reward": 5.628999873995781,
-      "reward_std": 0.45297699933871627,
-      "rewards/concensus_correctness_reward_func": 1.480437506455928,
-      "rewards/consensus_reward_func": 1.625,
+      "loss": 0.0317,
+      "reward": 6.461392790079117,
+      "reward_std": 0.3590539181313943,
+      "rewards/concensus_correctness_reward_func": 1.9264999954029918,
+      "rewards/consensus_reward_func": 1.875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.9062810204923153,
+      "rewards/final_correctness_reward_func": 0.1875,
+      "rewards/question_recreation_reward_func": 0.8699865005910397,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.28125,
-      "rewards/xmlcount_reward_func": 1.0860312581062317,
+      "rewards/strict_format_reward_func": 0.390625,
+      "rewards/xmlcount_reward_func": 1.2117812484502792,
       "step": 8
     },
     {
-      "completion_length": 129.59375,
+      "completion_length": 131.09375,
       "epoch": 0.5,
-      "grad_norm": 30.488317489624023,
-      "kl": 0.2393563985824585,
+      "grad_norm": 51.59836959838867,
+      "kl": 9.099941534281243,
       "learning_rate": 3.1137137178519977e-07,
-      "loss": 0.0002,
-      "reward": 6.794623285531998,
-      "reward_std": 0.9424451283412054,
-      "rewards/concensus_correctness_reward_func": 2.033812489360571,
-      "rewards/consensus_reward_func": 1.8125,
+      "loss": 0.0091,
+      "reward": 7.209178298711777,
+      "reward_std": 0.16811024746857584,
+      "rewards/concensus_correctness_reward_func": 2.2494999915361404,
+      "rewards/consensus_reward_func": 2.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.4375,
-      "rewards/question_recreation_reward_func": 0.9692170210182667,
+      "rewards/final_correctness_reward_func": 0.375,
+      "rewards/question_recreation_reward_func": 0.9613657891750336,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.375,
-      "rewards/xmlcount_reward_func": 1.1665937528014183,
+      "rewards/strict_format_reward_func": 0.40625,
+      "rewards/xmlcount_reward_func": 1.217062495648861,
       "step": 10
     },
     {
-      "completion_length": 156.5625,
+      "completion_length": 129.625,
       "epoch": 0.6,
-      "grad_norm": 272.53741455078125,
-      "kl": 9.845261040958576,
+      "grad_norm": 20.847944259643555,
+      "kl": 0.8772291259083431,
       "learning_rate": 2.2935516363191693e-07,
-      "loss": 0.0098,
-      "reward": 5.63976863026619,
-      "reward_std": 0.7742573891300708,
-      "rewards/concensus_correctness_reward_func": 1.6123124696314335,
-      "rewards/consensus_reward_func": 1.5625,
+      "loss": 0.0009,
+      "reward": 6.097430318593979,
+      "reward_std": 0.2864232168503804,
+      "rewards/concensus_correctness_reward_func": 1.7592499777674675,
+      "rewards/consensus_reward_func": 1.8125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.8787062028422952,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.8994615543633699,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.375,
-      "rewards/xmlcount_reward_func": 1.1487499997019768,
+      "rewards/strict_format_reward_func": 0.40625,
+      "rewards/xmlcount_reward_func": 1.2199687510728836,
       "step": 12
     },
     {
-      "completion_length": 140.46875,
+      "completion_length": 136.625,
       "epoch": 0.7,
-      "grad_norm": 28.47071647644043,
-      "kl": 4612.2689645413775,
+      "grad_norm": 1207.919189453125,
+      "kl": 40.688665235298686,
       "learning_rate": 1.4957614383675767e-07,
-      "loss": 4.6123,
-      "reward": 6.150349587202072,
-      "reward_std": 0.696038618858438,
-      "rewards/concensus_correctness_reward_func": 1.8968749977648258,
+      "loss": 0.0407,
+      "reward": 6.123583257198334,
+      "reward_std": 0.6078165378421545,
+      "rewards/concensus_correctness_reward_func": 1.7557500042021275,
       "rewards/consensus_reward_func": 1.75,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.8873808388598263,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.9663332924246788,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.3125,
-      "rewards/xmlcount_reward_func": 1.0535937510430813,
+      "rewards/strict_format_reward_func": 0.375,
+      "rewards/xmlcount_reward_func": 1.2140000015497208,
       "step": 14
     },
     {
-      "completion_length": 135.8125,
+      "completion_length": 137.3125,
       "epoch": 0.8,
-      "grad_norm": 55.1845703125,
-      "kl": 1.8462436823174357,
+      "grad_norm": 40.619361877441406,
+      "kl": 5.575095858424902,
       "learning_rate": 8.067960709356478e-08,
-      "loss": 0.0018,
-      "reward": 6.7156277894973755,
-      "reward_std": 0.374933703802526,
-      "rewards/concensus_correctness_reward_func": 2.0350000001490116,
+      "loss": 0.0056,
+      "reward": 6.810699820518494,
+      "reward_std": 0.2520275285933167,
+      "rewards/concensus_correctness_reward_func": 2.0633749961853027,
       "rewards/consensus_reward_func": 2.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.3125,
-      "rewards/question_recreation_reward_func": 0.8633777701761574,
+      "rewards/final_correctness_reward_func": 0.1875,
+      "rewards/question_recreation_reward_func": 0.9416372999548912,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.3125,
-      "rewards/xmlcount_reward_func": 1.1922500059008598,
+      "rewards/strict_format_reward_func": 0.40625,
+      "rewards/xmlcount_reward_func": 1.211937502026558,
       "step": 16
     },
     {
-      "completion_length": 136.75,
+      "completion_length": 122.5,
       "epoch": 0.9,
-      "grad_norm": 580.36279296875,
-      "kl": 86.7045733507257,
+      "grad_norm": 37.19350051879883,
+      "kl": 52978.122300678864,
       "learning_rate": 3.013156219837776e-08,
-      "loss": 0.0867,
-      "reward": 5.618547663092613,
-      "reward_std": 0.6109558623866178,
-      "rewards/concensus_correctness_reward_func": 1.6356249898672104,
-      "rewards/consensus_reward_func": 1.6875,
+      "loss": 52.9781,
+      "reward": 6.109934449195862,
+      "reward_std": 0.58212883642409,
+      "rewards/concensus_correctness_reward_func": 1.7430625036358833,
+      "rewards/consensus_reward_func": 1.5625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.8768601454794407,
+      "rewards/final_correctness_reward_func": 0.3125,
+      "rewards/question_recreation_reward_func": 0.9283719323575497,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.3125,
-      "rewards/xmlcount_reward_func": 1.10606250166893,
+      "rewards/strict_format_reward_func": 0.359375,
+      "rewards/xmlcount_reward_func": 1.2041249945759773,
       "step": 18
     },
     {
-      "completion_length": 129.9375,
+      "completion_length": 148.28125,
       "epoch": 1.0,
-      "grad_norm": 34.112430572509766,
-      "kl": 8.528769343975,
+      "grad_norm": 721.8731079101562,
+      "kl": 18.435496295103803,
       "learning_rate": 3.4096741493194193e-09,
-      "loss": 0.0085,
-      "reward": 6.253787875175476,
-      "reward_std": 0.7373550900956616,
-      "rewards/concensus_correctness_reward_func": 1.8726249933242798,
+      "loss": 0.0184,
+      "reward": 6.203249305486679,
+      "reward_std": 0.460769604993402,
+      "rewards/concensus_correctness_reward_func": 1.706125009804964,
       "rewards/consensus_reward_func": 1.75,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.8808816038072109,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.9697804637253284,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.359375,
-      "rewards/xmlcount_reward_func": 1.140906248241663,
+      "rewards/strict_format_reward_func": 0.4375,
+      "rewards/xmlcount_reward_func": 1.21484375,
       "step": 20
     },
     {
       "epoch": 1.0,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 0.47205696956516474,
-      "train_runtime": 151.675,
-      "train_samples_per_second": 2.11,
-      "train_steps_per_second": 0.132
+      "train_loss": 5.40037734391517,
+      "train_runtime": 148.422,
+      "train_samples_per_second": 2.156,
+      "train_steps_per_second": 0.135
     }
   ],
   "logging_steps": 2,