End of training

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 9.117599711316871e-06,
-    "train_runtime": 360.6804,
+    "train_loss": 1.6838312149047852e-06,
+    "train_runtime": 318.2882,
     "train_samples": 160,
-    "train_samples_per_second": 0.887,
-    "train_steps_per_second": 0.055
+    "train_samples_per_second": 1.005,
+    "train_steps_per_second": 0.063
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d256fae7e77041b593bee4d9ac36959593b8cf3440ad21d9f55a7812557ce754
+oid sha256:8991881f2d6c28b94356372a3c54e0157e3d2dc517e5f8ac2da541cdd5834e7c
 size 1976163472

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 9.117599711316871e-06,
-    "train_runtime": 360.6804,
+    "train_loss": 1.6838312149047852e-06,
+    "train_runtime": 318.2882,
     "train_samples": 160,
-    "train_samples_per_second": 0.887,
-    "train_steps_per_second": 0.055
+    "train_samples_per_second": 1.005,
+    "train_steps_per_second": 0.063
 }

trainer_state.json

@@ -10,203 +10,203 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 364.1875,
+      "completion_length": 351.71875,
       "epoch": 0.1,
-      "grad_norm": 7.79181432723999,
+      "grad_norm": 8.513227462768555,
       "kl": 0.0,
       "learning_rate": 5e-07,
       "loss": -0.0,
-      "reward": 0.39489845104981214,
-      "reward_std": 0.9017761385766789,
-      "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.0,
+      "reward": 0.8320725136436522,
+      "reward_std": 0.8263479982997524,
+      "rewards/concensus_correctness_reward_func": 0.0755624994635582,
+      "rewards/consensus_reward_func": 0.1875,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.1875,
-      "rewards/question_recreation_reward_func": 0.36814845085609704,
-      "rewards/soft_format_reward_func": 0.0,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.37107251823181286,
+      "rewards/soft_format_reward_func": 0.015625,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": -0.16074999328702688,
+      "rewards/xmlcount_reward_func": 0.11981249786913395,
       "step": 2
     },
     {
-      "completion_length": 432.375,
+      "completion_length": 403.84375,
       "epoch": 0.2,
-      "grad_norm": 9.590618133544922,
-      "kl": 0.0014355634102685144,
+      "grad_norm": 4.751884460449219,
+      "kl": 0.0016504357699886896,
       "learning_rate": 4.864543104251586e-07,
       "loss": 0.0,
-      "reward": 0.7185110447462648,
-      "reward_std": 0.732723499299027,
-      "rewards/concensus_correctness_reward_func": 0.008187499828636646,
-      "rewards/consensus_reward_func": 0.0,
+      "reward": 0.8297341071302071,
+      "reward_std": 0.9467946260992903,
+      "rewards/concensus_correctness_reward_func": 0.10125000029802322,
+      "rewards/consensus_reward_func": 0.125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.3436360324267298,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.40404659567866474,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.30418750178068876,
+      "rewards/xmlcount_reward_func": 0.19943749252706766,
       "step": 4
     },
     {
-      "completion_length": 382.71875,
+      "completion_length": 401.75,
       "epoch": 0.3,
-      "grad_norm": 4.208951473236084,
-      "kl": 0.0012705847912002355,
+      "grad_norm": 3.9849419593811035,
+      "kl": 0.0013705312012461945,
       "learning_rate": 4.472851273490984e-07,
       "loss": 0.0,
-      "reward": 0.6679772045463324,
-      "reward_std": 0.8955608154865331,
-      "rewards/concensus_correctness_reward_func": 0.045499999076128006,
+      "reward": 0.7022149509284645,
+      "reward_std": 0.5484118369640782,
+      "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.3797584424028173,
+      "rewards/final_correctness_reward_func": 0.1875,
+      "rewards/question_recreation_reward_func": 0.4711212064139545,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.05521875782869756,
+      "rewards/xmlcount_reward_func": -0.018906254321336746,
       "step": 6
     },
     {
-      "completion_length": 457.59375,
+      "completion_length": 448.03125,
       "epoch": 0.4,
-      "grad_norm": 4.035867691040039,
-      "kl": 0.002204360036557773,
+      "grad_norm": 3.4099152088165283,
+      "kl": 0.0012933166872244328,
       "learning_rate": 3.867370395306068e-07,
       "loss": 0.0,
-      "reward": 0.5797329153865576,
-      "reward_std": 0.6650298394015408,
-      "rewards/concensus_correctness_reward_func": 0.024437500163912773,
-      "rewards/consensus_reward_func": 0.0,
+      "reward": 0.8546537263318896,
+      "reward_std": 0.7673704327316955,
+      "rewards/concensus_correctness_reward_func": 0.0064999996684491634,
+      "rewards/consensus_reward_func": 0.1875,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.4533579223207198,
+      "rewards/question_recreation_reward_func": 0.38755996932741255,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.101937499595806,
+      "rewards/xmlcount_reward_func": 0.2730937581509352,
       "step": 8
     },
     {
-      "completion_length": 542.875,
+      "completion_length": 408.53125,
       "epoch": 0.5,
-      "grad_norm": 4.634764671325684,
-      "kl": 0.005208585236687213,
+      "grad_norm": 4.682156085968018,
+      "kl": 0.0020962205962860025,
       "learning_rate": 3.1137137178519977e-07,
       "loss": 0.0,
-      "reward": 0.9039936233311892,
-      "reward_std": 1.0827069451333955,
-      "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.0,
+      "reward": 2.0500756152905524,
+      "reward_std": 1.4114575510611758,
+      "rewards/concensus_correctness_reward_func": 0.892937496304512,
+      "rewards/consensus_reward_func": 0.25,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.5436498466879129,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.339450626866892,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": 0.28221875708550215,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.3176874998025596,
       "step": 10
     },
     {
-      "completion_length": 345.46875,
+      "completion_length": 356.15625,
       "epoch": 0.6,
-      "grad_norm": 6.202835559844971,
-      "kl": 0.02693040803569602,
+      "grad_norm": 7.706838607788086,
+      "kl": 0.0017779090485419147,
       "learning_rate": 2.2935516363191693e-07,
       "loss": 0.0,
-      "reward": 0.618082148488611,
-      "reward_std": 0.7239898710104171,
-      "rewards/concensus_correctness_reward_func": 0.024687500670552254,
-      "rewards/consensus_reward_func": 0.125,
+      "reward": 0.7448974205181003,
+      "reward_std": 0.6400141692720354,
+      "rewards/concensus_correctness_reward_func": 0.031375001184642315,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.2905508743133396,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.3690224272431806,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.015625,
-      "rewards/xmlcount_reward_func": 0.16221874905750155,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.2195000002393499,
       "step": 12
     },
     {
-      "completion_length": 437.09375,
+      "completion_length": 444.375,
       "epoch": 0.7,
-      "grad_norm": 4.228725910186768,
-      "kl": 0.006790324652683921,
+      "grad_norm": 3.907611131668091,
+      "kl": 0.0013627830157929566,
       "learning_rate": 1.4957614383675767e-07,
       "loss": 0.0,
-      "reward": 0.5991453088354319,
-      "reward_std": 0.561388993752189,
-      "rewards/concensus_correctness_reward_func": 0.021687500178813934,
-      "rewards/consensus_reward_func": 0.0,
+      "reward": 2.271299608051777,
+      "reward_std": 1.8669061198015697,
+      "rewards/concensus_correctness_reward_func": 0.7388749998062849,
+      "rewards/consensus_reward_func": 0.4375,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.3984578078961931,
-      "rewards/soft_format_reward_func": 0.015625,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.6308308928273618,
+      "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.16337500140070915,
+      "rewards/xmlcount_reward_func": 0.21409375220537186,
       "step": 14
     },
     {
-      "completion_length": 378.78125,
+      "completion_length": 318.375,
       "epoch": 0.8,
-      "grad_norm": 7.585692882537842,
-      "kl": 0.010340058604924707,
+      "grad_norm": 4.985864162445068,
+      "kl": 0.002357518140343018,
       "learning_rate": 8.067960709356478e-08,
       "loss": 0.0,
-      "reward": 0.7918121283873916,
-      "reward_std": 0.7190128332586028,
-      "rewards/concensus_correctness_reward_func": 0.03837500046938658,
-      "rewards/consensus_reward_func": 0.0625,
+      "reward": 1.5525614071812015,
+      "reward_std": 0.9043538695550524,
+      "rewards/concensus_correctness_reward_func": 0.20749999582767487,
+      "rewards/consensus_reward_func": 0.3125,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.1875,
-      "rewards/question_recreation_reward_func": 0.38859339867485687,
+      "rewards/question_recreation_reward_func": 0.4891551311593503,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.11484375223517418,
+      "rewards/xmlcount_reward_func": 0.3559062508866191,
       "step": 16
     },
     {
-      "completion_length": 405.21875,
+      "completion_length": 335.875,
       "epoch": 0.9,
-      "grad_norm": 3.7810328006744385,
-      "kl": 0.005060435782070272,
+      "grad_norm": 10.401203155517578,
+      "kl": 0.0028135806787759066,
       "learning_rate": 3.013156219837776e-08,
       "loss": 0.0,
-      "reward": 0.6734253508038819,
-      "reward_std": 0.8879609380383044,
-      "rewards/concensus_correctness_reward_func": 0.045625001192092896,
+      "reward": 1.749845893587917,
+      "reward_std": 1.5136296135606244,
+      "rewards/concensus_correctness_reward_func": 0.6584999989718199,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.4548003734089434,
+      "rewards/final_correctness_reward_func": 0.1875,
+      "rewards/question_recreation_reward_func": 0.4342209289316088,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.04799999948590994,
+      "rewards/xmlcount_reward_func": 0.46962500223889947,
       "step": 18
     },
     {
-      "completion_length": 416.15625,
+      "completion_length": 429.78125,
       "epoch": 1.0,
-      "grad_norm": 5.279428005218506,
-      "kl": 0.03254748803738039,
+      "grad_norm": 4.305922508239746,
+      "kl": 0.0020146886017755605,
       "learning_rate": 3.4096741493194193e-09,
       "loss": 0.0,
-      "reward": 2.115490459022112,
-      "reward_std": 1.9778084652352845,
-      "rewards/concensus_correctness_reward_func": 0.902562496252358,
+      "reward": 1.6820365519961342,
+      "reward_std": 1.639612780767493,
+      "rewards/concensus_correctness_reward_func": 0.7378125041723251,
       "rewards/consensus_reward_func": 0.1875,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.1875,
-      "rewards/question_recreation_reward_func": 0.5558030122192577,
+      "rewards/question_recreation_reward_func": 0.42944281999371015,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.28212499618530273,
+      "rewards/xmlcount_reward_func": 0.13978125574067235,
       "step": 20
     },
     {
       "epoch": 1.0,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 9.117599711316871e-06,
-      "train_runtime": 360.6804,
-      "train_samples_per_second": 0.887,
-      "train_steps_per_second": 0.055
+      "train_loss": 1.6838312149047852e-06,
+      "train_runtime": 318.2882,
+      "train_samples_per_second": 1.005,
+      "train_steps_per_second": 0.063
     }
   ],
   "logging_steps": 2,