End of training

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 5.764640900451923e-06,
-    "train_runtime": 312.0329,
+    "train_loss": 0.00023715728893876077,
+    "train_runtime": 370.9739,
     "train_samples": 160,
-    "train_samples_per_second": 1.026,
-    "train_steps_per_second": 0.064
+    "train_samples_per_second": 0.863,
+    "train_steps_per_second": 0.054
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ac95cc25855169f3851831ca28645d0c790faeab3158cc4861bf7053b9c49017
+oid sha256:405a168bb6a75899b7ea652a72f87e75ce806de97df61fa2c7c64a4f06e3f275
 size 1976163472

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 5.764640900451923e-06,
-    "train_runtime": 312.0329,
+    "train_loss": 0.00023715728893876077,
+    "train_runtime": 370.9739,
     "train_samples": 160,
-    "train_samples_per_second": 1.026,
-    "train_steps_per_second": 0.064
+    "train_samples_per_second": 0.863,
+    "train_steps_per_second": 0.054
 }

trainer_state.json

@@ -10,203 +10,203 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 334.6875,
+      "completion_length": 393.75,
       "epoch": 0.1,
-      "grad_norm": 5.855443477630615,
+      "grad_norm": 5.188680648803711,
       "kl": 0.0,
       "learning_rate": 5e-07,
       "loss": -0.0,
-      "reward": 1.1098423983203247,
-      "reward_std": 1.6228562939795665,
-      "rewards/concensus_correctness_reward_func": 0.625,
-      "rewards/consensus_reward_func": 0.0,
+      "reward": 0.9277644922258332,
+      "reward_std": 0.7210114789195359,
+      "rewards/concensus_correctness_reward_func": 0.11312500201165676,
+      "rewards/consensus_reward_func": 0.0625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.1875,
-      "rewards/question_recreation_reward_func": 0.35637367074377835,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.4264832512708381,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": -0.059031250420957804,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.24753125524148345,
       "step": 2
     },
     {
-      "completion_length": 377.875,
+      "completion_length": 430.84375,
       "epoch": 0.2,
-      "grad_norm": 5.300206661224365,
-      "kl": 0.0024700732283235993,
+      "grad_norm": 232.78201293945312,
+      "kl": 0.18494924689730396,
       "learning_rate": 4.864543104251586e-07,
-      "loss": 0.0,
-      "reward": 0.3390364852966741,
-      "reward_std": 0.7709111558851873,
+      "loss": 0.0002,
+      "reward": 0.7872139913961291,
+      "reward_std": 0.4320281593827531,
       "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0625,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.3044114876538515,
+      "rewards/question_recreation_reward_func": 0.37062026001513004,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": -0.02787499246187508,
+      "rewards/xmlcount_reward_func": 0.291593739297241,
       "step": 4
     },
     {
-      "completion_length": 385.21875,
+      "completion_length": 504.3125,
       "epoch": 0.3,
-      "grad_norm": 22.370466232299805,
-      "kl": 0.0020103070928598754,
+      "grad_norm": 3.7133069038391113,
+      "kl": 0.0013250585143396165,
       "learning_rate": 4.472851273490984e-07,
       "loss": 0.0,
-      "reward": 0.37886355444788933,
-      "reward_std": 0.4881445004721172,
-      "rewards/concensus_correctness_reward_func": 0.0,
+      "reward": 0.48609744632267393,
+      "reward_std": 0.43579606293860707,
+      "rewards/concensus_correctness_reward_func": 0.051062499172985554,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.22114479448646307,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.26303494660533033,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.09521875134669244,
+      "rewards/xmlcount_reward_func": 0.04699999815784395,
       "step": 6
     },
     {
-      "completion_length": 327.3125,
+      "completion_length": 456.40625,
       "epoch": 0.4,
-      "grad_norm": 15.930990219116211,
-      "kl": 0.0016392190518672578,
+      "grad_norm": 111.57942199707031,
+      "kl": 2.1505834279450937,
       "learning_rate": 3.867370395306068e-07,
-      "loss": 0.0,
-      "reward": 0.5796866102609783,
-      "reward_std": 0.4016084271715954,
+      "loss": 0.0022,
+      "reward": 0.07964608876500279,
+      "reward_std": 0.7304952847771347,
       "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.0625,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.2572803513612598,
+      "rewards/question_recreation_reward_func": 0.2753335883608088,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.25990624725818634,
+      "rewards/xmlcount_reward_func": -0.19568750681355596,
       "step": 8
     },
     {
-      "completion_length": 364.25,
+      "completion_length": 457.25,
       "epoch": 0.5,
-      "grad_norm": 7.68777322769165,
-      "kl": 0.005957612967904424,
+      "grad_norm": 3.990921974182129,
+      "kl": 0.003516843630677613,
       "learning_rate": 3.1137137178519977e-07,
       "loss": 0.0,
-      "reward": 0.5165964886546135,
-      "reward_std": 0.6143831994850188,
-      "rewards/concensus_correctness_reward_func": 0.0,
+      "reward": 0.691937566967681,
+      "reward_std": 0.5611258368480776,
+      "rewards/concensus_correctness_reward_func": 0.0078125,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.2783464976819232,
-      "rewards/soft_format_reward_func": 0.015625,
+      "rewards/final_correctness_reward_func": 0.1875,
+      "rewards/question_recreation_reward_func": 0.26021880889311433,
+      "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.1601250022649765,
+      "rewards/xmlcount_reward_func": 0.23640624433755875,
       "step": 10
     },
     {
-      "completion_length": 380.15625,
+      "completion_length": 512.0625,
       "epoch": 0.6,
-      "grad_norm": 6.398386001586914,
-      "kl": 0.003765162509807851,
+      "grad_norm": 4.030053615570068,
+      "kl": 0.0023621325381100178,
       "learning_rate": 2.2935516363191693e-07,
       "loss": 0.0,
-      "reward": 1.185325913131237,
-      "reward_std": 1.6851115431054495,
-      "rewards/concensus_correctness_reward_func": 0.6428124997764826,
-      "rewards/consensus_reward_func": 0.0625,
+      "reward": 0.5346925361081958,
+      "reward_std": 0.6594595573842525,
+      "rewards/concensus_correctness_reward_func": 0.034500000067055225,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.3609509505331516,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.34913003619294614,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.056562501937150955,
+      "rewards/xmlcount_reward_func": 0.15106250625103712,
       "step": 12
     },
     {
-      "completion_length": 387.6875,
+      "completion_length": 452.15625,
       "epoch": 0.7,
-      "grad_norm": 10.472870826721191,
-      "kl": 0.022627558621024946,
+      "grad_norm": 9.331066131591797,
+      "kl": 0.007394553460471798,
       "learning_rate": 1.4957614383675767e-07,
       "loss": 0.0,
-      "reward": 0.4109920918708667,
-      "reward_std": 0.3846119233639911,
-      "rewards/concensus_correctness_reward_func": 0.0,
+      "reward": 0.3333409542683512,
+      "reward_std": 0.8716766675934196,
+      "rewards/concensus_correctness_reward_func": 0.01837499998509884,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.2638358509866521,
+      "rewards/question_recreation_reward_func": 0.3838409634772688,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.14715624786913395,
+      "rewards/xmlcount_reward_func": -0.06887499429285526,
       "step": 14
     },
     {
-      "completion_length": 317.34375,
+      "completion_length": 476.6875,
       "epoch": 0.8,
-      "grad_norm": 14.636994361877441,
-      "kl": 0.0074590980730135925,
+      "grad_norm": 4.383597373962402,
+      "kl": 0.0026971642728312872,
       "learning_rate": 8.067960709356478e-08,
       "loss": 0.0,
-      "reward": 0.2813199677038938,
-      "reward_std": 0.749781655962579,
-      "rewards/concensus_correctness_reward_func": 0.027249999344348907,
+      "reward": 0.6619523297995329,
+      "reward_std": 0.8007291500689462,
+      "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.3051324697153177,
+      "rewards/question_recreation_reward_func": 0.5646398302633315,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": -0.05106249405071139,
+      "rewards/xmlcount_reward_func": 0.09731250209733844,
       "step": 16
     },
     {
-      "completion_length": 399.46875,
+      "completion_length": 491.90625,
       "epoch": 0.9,
-      "grad_norm": 13.077838897705078,
-      "kl": 0.002335159126232611,
+      "grad_norm": 4.677735328674316,
+      "kl": 0.0028275340446271002,
       "learning_rate": 3.013156219837776e-08,
       "loss": 0.0,
-      "reward": 1.5585488975048065,
-      "reward_std": 0.568805246221018,
-      "rewards/concensus_correctness_reward_func": 1.25,
+      "reward": 0.8493442741164472,
+      "reward_std": 0.5758356966034626,
+      "rewards/concensus_correctness_reward_func": 0.04468749836087227,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.2209863435709849,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.27390676047070883,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.02506249281577766,
+      "rewards/xmlcount_reward_func": 0.2807499999180436,
       "step": 18
     },
     {
-      "completion_length": 420.5,
+      "completion_length": 389.96875,
       "epoch": 1.0,
-      "grad_norm": 47.53742980957031,
-      "kl": 0.009703567069664132,
+      "grad_norm": 72.06743621826172,
+      "kl": 0.015022013009001967,
       "learning_rate": 3.4096741493194193e-09,
       "loss": 0.0,
-      "reward": 0.797133304644376,
-      "reward_std": 0.7819674629718065,
-      "rewards/concensus_correctness_reward_func": 0.0078125,
-      "rewards/consensus_reward_func": 0.125,
+      "reward": 1.3650949345901608,
+      "reward_std": 1.8583283452317119,
+      "rewards/concensus_correctness_reward_func": 0.6311249998398125,
+      "rewards/consensus_reward_func": 0.0625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.1875,
-      "rewards/question_recreation_reward_func": 0.3550707916729152,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.33409492881037295,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.12174999853596091,
+      "rewards/xmlcount_reward_func": 0.08737499744165689,
       "step": 20
     },
     {
       "epoch": 1.0,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 5.764640900451923e-06,
-      "train_runtime": 312.0329,
-      "train_samples_per_second": 1.026,
-      "train_steps_per_second": 0.064
+      "train_loss": 0.00023715728893876077,
+      "train_runtime": 370.9739,
+      "train_samples_per_second": 0.863,
+      "train_steps_per_second": 0.054
     }
   ],
   "logging_steps": 2,