End of training

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 1.025590872814064e-05,
-    "train_runtime": 265.9955,
+    "train_loss": 3.325208206206298e-05,
+    "train_runtime": 266.3118,
     "train_samples": 160,
-    "train_samples_per_second": 1.203,
+    "train_samples_per_second": 1.202,
     "train_steps_per_second": 0.075
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:98d62767c591d63774b69b3ea37da24654de23b1c5b641f8777b929dd1d44438
+oid sha256:275e6390d7a74df0de1a024532816301b98c0eae94f06a0f4198aad149b8dbf6
 size 1976163472

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 1.025590872814064e-05,
-    "train_runtime": 265.9955,
+    "train_loss": 3.325208206206298e-05,
+    "train_runtime": 266.3118,
     "train_samples": 160,
-    "train_samples_per_second": 1.203,
+    "train_samples_per_second": 1.202,
     "train_steps_per_second": 0.075
 }

trainer_state.json

@@ -10,202 +10,202 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 446.5,
+      "completion_length": 350.03125,
       "epoch": 0.1,
-      "grad_norm": 7.707586288452148,
+      "grad_norm": 7.12115478515625,
       "kl": 0.0,
       "learning_rate": 5e-07,
       "loss": -0.0,
-      "reward": 1.853087481111288,
-      "reward_std": 1.2459120045241434,
-      "rewards/concensus_correctness_reward_func": 0.42012499272823334,
-      "rewards/consensus_reward_func": 0.4375,
+      "reward": 2.2255071950203273,
+      "reward_std": 1.224156864976976,
+      "rewards/concensus_correctness_reward_func": 0.5882499972358346,
+      "rewards/consensus_reward_func": 0.5,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.25,
-      "rewards/question_recreation_reward_func": 0.597775018773973,
+      "rewards/question_recreation_reward_func": 0.4241946841648314,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.14768752170493826,
+      "rewards/xmlcount_reward_func": 0.46306250244379044,
       "step": 2
     },
     {
-      "completion_length": 258.0,
+      "completion_length": 365.90625,
       "epoch": 0.2,
-      "grad_norm": 5.790719032287598,
-      "kl": 0.0017292632583121303,
+      "grad_norm": 6.482650279998779,
+      "kl": 0.0013846163419657387,
       "learning_rate": 4.864543104251586e-07,
       "loss": 0.0,
-      "reward": 2.8558200262486935,
-      "reward_std": 0.9629578137537464,
-      "rewards/concensus_correctness_reward_func": 0.553812500089407,
-      "rewards/consensus_reward_func": 0.6875,
+      "reward": 2.9605965055525303,
+      "reward_std": 1.8632249562069774,
+      "rewards/concensus_correctness_reward_func": 0.5551249980926514,
+      "rewards/consensus_reward_func": 0.625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.6252262452617288,
-      "rewards/soft_format_reward_func": 0.015625,
+      "rewards/final_correctness_reward_func": 0.3125,
+      "rewards/question_recreation_reward_func": 0.612190262414515,
+      "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.8486562445759773,
+      "rewards/xmlcount_reward_func": 0.8557812534272671,
       "step": 4
     },
     {
-      "completion_length": 324.5,
+      "completion_length": 364.09375,
       "epoch": 0.3,
-      "grad_norm": 4.269913196563721,
-      "kl": 0.0022732225770596415,
+      "grad_norm": 4.9196295738220215,
+      "kl": 0.0018765363929560408,
       "learning_rate": 4.472851273490984e-07,
       "loss": 0.0,
-      "reward": 3.787586487829685,
-      "reward_std": 1.7468879896914586,
-      "rewards/concensus_correctness_reward_func": 1.3090000171214342,
+      "reward": 2.944482371211052,
+      "reward_std": 1.9826700370758772,
+      "rewards/concensus_correctness_reward_func": 0.6771875005215406,
       "rewards/consensus_reward_func": 0.75,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.5,
-      "rewards/question_recreation_reward_func": 0.6204927363432944,
+      "rewards/final_correctness_reward_func": 0.1875,
+      "rewards/question_recreation_reward_func": 0.7243573628365993,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.6080937422811985,
+      "rewards/xmlcount_reward_func": 0.6054375264793634,
       "step": 6
     },
     {
-      "completion_length": 284.84375,
+      "completion_length": 350.59375,
       "epoch": 0.4,
-      "grad_norm": 4.578140735626221,
-      "kl": 0.0031328682307503186,
+      "grad_norm": 5.708310604095459,
+      "kl": 0.005335728461432154,
       "learning_rate": 3.867370395306068e-07,
       "loss": 0.0,
-      "reward": 3.9615301713347435,
-      "reward_std": 1.6491755824536085,
-      "rewards/concensus_correctness_reward_func": 1.1211875025182962,
-      "rewards/consensus_reward_func": 0.9375,
+      "reward": 2.213101528584957,
+      "reward_std": 1.5623105892445892,
+      "rewards/concensus_correctness_reward_func": 0.498562503606081,
+      "rewards/consensus_reward_func": 0.5625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.375,
-      "rewards/question_recreation_reward_func": 0.5936862211674452,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.5546640297397971,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.9341562688350677,
+      "rewards/xmlcount_reward_func": 0.4723750129342079,
       "step": 8
     },
     {
-      "completion_length": 258.65625,
+      "completion_length": 263.53125,
       "epoch": 0.5,
-      "grad_norm": 5.295426845550537,
-      "kl": 0.007704104245931376,
+      "grad_norm": 6.409860610961914,
+      "kl": 0.00669573646155186,
       "learning_rate": 3.1137137178519977e-07,
       "loss": 0.0,
-      "reward": 3.2043592408299446,
-      "reward_std": 1.6422009095549583,
-      "rewards/concensus_correctness_reward_func": 0.7212499976158142,
-      "rewards/consensus_reward_func": 0.875,
+      "reward": 3.0519943945109844,
+      "reward_std": 1.7202243172796443,
+      "rewards/concensus_correctness_reward_func": 0.7220625076442957,
+      "rewards/consensus_reward_func": 0.8125,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.573140453081578,
+      "rewards/question_recreation_reward_func": 0.5301193697378039,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.9099687561392784,
+      "rewards/xmlcount_reward_func": 0.862312488257885,
       "step": 10
     },
     {
-      "completion_length": 323.1875,
+      "completion_length": 242.3125,
       "epoch": 0.6,
-      "grad_norm": 5.510501861572266,
-      "kl": 0.011015730764484033,
+      "grad_norm": 6.210168361663818,
+      "kl": 0.009614975278964266,
       "learning_rate": 2.2935516363191693e-07,
       "loss": 0.0,
-      "reward": 3.006742224097252,
-      "reward_std": 1.1151306866668165,
-      "rewards/concensus_correctness_reward_func": 0.6842499990016222,
-      "rewards/consensus_reward_func": 0.875,
+      "reward": 2.844990584999323,
+      "reward_std": 1.331618865369819,
+      "rewards/concensus_correctness_reward_func": 0.6087499987334013,
+      "rewards/consensus_reward_func": 0.625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.6348984064534307,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.549146810430102,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.8125937543809414,
+      "rewards/xmlcount_reward_func": 0.9370937384665012,
       "step": 12
     },
     {
-      "completion_length": 265.03125,
+      "completion_length": 245.84375,
       "epoch": 0.7,
-      "grad_norm": 5.572478771209717,
-      "kl": 0.01679254014743492,
+      "grad_norm": 5.952077865600586,
+      "kl": 0.1754204363678582,
       "learning_rate": 1.4957614383675767e-07,
-      "loss": 0.0,
-      "reward": 2.8126327581703663,
-      "reward_std": 1.8483319870356354,
-      "rewards/concensus_correctness_reward_func": 0.5091250017285347,
-      "rewards/consensus_reward_func": 0.5,
+      "loss": 0.0002,
+      "reward": 3.334137696772814,
+      "reward_std": 0.8904841914772987,
+      "rewards/concensus_correctness_reward_func": 0.7768749948590994,
+      "rewards/consensus_reward_func": 0.8125,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.375,
-      "rewards/question_recreation_reward_func": 0.5358202885836363,
+      "rewards/question_recreation_reward_func": 0.5237626568414271,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.8926875195465982,
+      "rewards/xmlcount_reward_func": 0.8459999896585941,
       "step": 14
     },
     {
-      "completion_length": 350.125,
+      "completion_length": 305.25,
       "epoch": 0.8,
-      "grad_norm": 7.969921588897705,
-      "kl": 0.011116135690826923,
+      "grad_norm": 6.027353763580322,
+      "kl": 0.04947703628567979,
       "learning_rate": 8.067960709356478e-08,
       "loss": 0.0,
-      "reward": 2.8975531551986933,
-      "reward_std": 1.043988436809741,
-      "rewards/concensus_correctness_reward_func": 0.5460000038146973,
-      "rewards/consensus_reward_func": 0.625,
+      "reward": 3.23410002887249,
+      "reward_std": 1.3452857581432909,
+      "rewards/concensus_correctness_reward_func": 0.6507500056177378,
+      "rewards/consensus_reward_func": 0.75,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.1875,
-      "rewards/question_recreation_reward_func": 0.6895218668505549,
+      "rewards/final_correctness_reward_func": 0.375,
+      "rewards/question_recreation_reward_func": 0.6314438171684742,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.849531251937151,
+      "rewards/xmlcount_reward_func": 0.8269062582403421,
       "step": 16
     },
     {
-      "completion_length": 283.3125,
+      "completion_length": 221.375,
       "epoch": 0.9,
-      "grad_norm": 6.680671691894531,
-      "kl": 0.022523066741996445,
+      "grad_norm": 5.727414608001709,
+      "kl": 0.05646272320882417,
       "learning_rate": 3.013156219837776e-08,
-      "loss": 0.0,
-      "reward": 3.0637912526726723,
-      "reward_std": 1.36955469584791,
-      "rewards/concensus_correctness_reward_func": 0.6422499939799309,
-      "rewards/consensus_reward_func": 0.8125,
+      "loss": 0.0001,
+      "reward": 5.031358875334263,
+      "reward_std": 0.9862957114237361,
+      "rewards/concensus_correctness_reward_func": 2.143124995753169,
+      "rewards/consensus_reward_func": 1.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.1875,
-      "rewards/question_recreation_reward_func": 0.7074787458404899,
+      "rewards/question_recreation_reward_func": 0.6074526156298816,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.7140625081956387,
+      "rewards/xmlcount_reward_func": 1.093281265348196,
       "step": 18
     },
     {
-      "completion_length": 318.3125,
+      "completion_length": 357.3125,
       "epoch": 1.0,
-      "grad_norm": 5.309074401855469,
-      "kl": 0.021950500711682253,
+      "grad_norm": 6.372084140777588,
+      "kl": 0.024577953990956303,
       "learning_rate": 3.4096741493194193e-09,
       "loss": 0.0,
-      "reward": 2.391891084611416,
-      "reward_std": 1.2925091469660401,
-      "rewards/concensus_correctness_reward_func": 0.34806250780820847,
-      "rewards/consensus_reward_func": 0.4375,
+      "reward": 2.749328803271055,
+      "reward_std": 1.5625416764523834,
+      "rewards/concensus_correctness_reward_func": 0.49949999153614044,
+      "rewards/consensus_reward_func": 0.375,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.3125,
-      "rewards/question_recreation_reward_func": 0.6306723039597273,
+      "rewards/final_correctness_reward_func": 0.5,
+      "rewards/question_recreation_reward_func": 0.6372662745416164,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.6631562616676092,
+      "rewards/xmlcount_reward_func": 0.737562507390976,
       "step": 20
     },
     {
       "epoch": 1.0,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 1.025590872814064e-05,
-      "train_runtime": 265.9955,
-      "train_samples_per_second": 1.203,
+      "train_loss": 3.325208206206298e-05,
+      "train_runtime": 266.3118,
+      "train_samples_per_second": 1.202,
       "train_steps_per_second": 0.075
     }
   ],