End of training

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 8.345013884536457e-06,
-    "train_runtime": 1784.0509,
+    "train_loss": 2.411922059764038e-06,
+    "train_runtime": 2650.5489,
     "train_samples": 28,
-    "train_samples_per_second": 0.179,
-    "train_steps_per_second": 0.011
+    "train_samples_per_second": 0.121,
+    "train_steps_per_second": 0.008
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7c762afa85d8c75ab052303f54b6cac910e6366b44154d4dda3466595ae75ffa
+oid sha256:6b544624b52f86d57a378c827c5a4779fd10f86be94c2856b4b7261e4431e047
 size 1976163472

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 8.345013884536457e-06,
-    "train_runtime": 1784.0509,
+    "train_loss": 2.411922059764038e-06,
+    "train_runtime": 2650.5489,
     "train_samples": 28,
-    "train_samples_per_second": 0.179,
-    "train_steps_per_second": 0.011
+    "train_samples_per_second": 0.121,
+    "train_steps_per_second": 0.008
 }

trainer_state.json

@@ -10,203 +10,203 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 404.90625,
+      "completion_length": 442.65625,
       "epoch": 0.5714285714285714,
-      "grad_norm": 14.533000946044922,
+      "grad_norm": 7.137018203735352,
       "kl": 0.0,
       "learning_rate": 5e-07,
-      "loss": 0.0,
-      "reward": 0.43868499156087637,
-      "reward_std": 0.3679787404835224,
-      "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.0,
+      "loss": -0.0,
+      "reward": 0.5420733785722405,
+      "reward_std": 0.36219248750421684,
+      "rewards/concensus_correctness_reward_func": 0.017625000327825546,
+      "rewards/consensus_reward_func": 0.125,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.2568724912125617,
+      "rewards/question_recreation_reward_func": 0.24694837513379753,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.18181249871850014,
+      "rewards/xmlcount_reward_func": 0.15250000217929482,
       "step": 2
     },
     {
-      "completion_length": 326.6666666666667,
+      "completion_length": 360.5,
       "epoch": 1.0,
-      "grad_norm": 11.852654457092285,
-      "kl": 0.002071247967251111,
+      "grad_norm": 2.7042696475982666,
+      "kl": 0.0007317917788896011,
       "learning_rate": 4.864543104251586e-07,
       "loss": 0.0,
-      "reward": 0.1856961726055791,
-      "reward_std": 0.29518962391496945,
+      "reward": 0.2949130789687236,
+      "reward_std": 0.30900699730652076,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.16311283545413366,
+      "rewards/question_recreation_reward_func": 0.22091308049857616,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.022583337190250557,
+      "rewards/xmlcount_reward_func": 0.07400000215663265,
       "step": 4
     },
     {
-      "completion_length": 436.28125,
+      "completion_length": 377.09375,
       "epoch": 1.5714285714285714,
-      "grad_norm": 44.79896545410156,
-      "kl": 0.0019136109767714515,
+      "grad_norm": 140.6943359375,
+      "kl": 0.0025935436788131483,
       "learning_rate": 4.472851273490984e-07,
       "loss": 0.0,
-      "reward": 0.4418511223047972,
-      "reward_std": 0.25800913470447995,
-      "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.0,
+      "reward": 0.5349589511752129,
+      "reward_std": 0.4536192311497871,
+      "rewards/concensus_correctness_reward_func": 0.045312498696148396,
+      "rewards/consensus_reward_func": 0.0625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.2885073731886223,
-      "rewards/soft_format_reward_func": 0.015625,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.2757089544320479,
+      "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.13771874643862247,
+      "rewards/xmlcount_reward_func": 0.08893749746493995,
       "step": 6
     },
     {
-      "completion_length": 413.0833333333333,
+      "completion_length": 328.375,
       "epoch": 2.0,
-      "grad_norm": 3.477694511413574,
-      "kl": 0.0020777028742789603,
+      "grad_norm": 6.15055513381958,
+      "kl": 0.0009562060222378932,
       "learning_rate": 3.867370395306068e-07,
       "loss": 0.0,
-      "reward": 0.05461296625435352,
-      "reward_std": 0.43925169059851515,
+      "reward": 0.202367508240665,
+      "reward_std": 0.22718117238643268,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.2794879542974134,
+      "rewards/question_recreation_reward_func": 0.11857584250780444,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": -0.22487499254445234,
+      "rewards/xmlcount_reward_func": 0.08379166573286057,
       "step": 8
     },
     {
-      "completion_length": 335.8125,
+      "completion_length": 356.9375,
       "epoch": 2.571428571428571,
-      "grad_norm": 6.241124153137207,
-      "kl": 0.007286508047400275,
+      "grad_norm": 12.800816535949707,
+      "kl": 0.0064311328023904935,
       "learning_rate": 3.1137137178519977e-07,
       "loss": 0.0,
-      "reward": 0.3183134066639468,
-      "reward_std": 0.4347515512199607,
-      "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.0,
+      "reward": 0.5643500033766031,
+      "reward_std": 0.5321357457432896,
+      "rewards/concensus_correctness_reward_func": 0.008812500163912773,
+      "rewards/consensus_reward_func": 0.0625,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.270188408321701,
+      "rewards/question_recreation_reward_func": 0.2930999907257501,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": -0.014374999795109034,
+      "rewards/xmlcount_reward_func": 0.13743750134017318,
       "step": 10
     },
     {
-      "completion_length": 451.9583333333333,
+      "completion_length": 275.0,
       "epoch": 3.0,
-      "grad_norm": 76.113037109375,
-      "kl": 0.00921955972444266,
+      "grad_norm": 5.229515552520752,
+      "kl": 0.00183830058570796,
       "learning_rate": 2.2935516363191693e-07,
       "loss": 0.0,
-      "reward": 0.2820186714331309,
-      "reward_std": 0.3407151723901431,
+      "reward": 0.34687480283901095,
+      "reward_std": 0.4368026171190043,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.2511853330458204,
+      "rewards/final_correctness_reward_func": 0.08333333333333333,
+      "rewards/question_recreation_reward_func": 0.2073331281232337,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.03083333062628905,
+      "rewards/xmlcount_reward_func": 0.05620833610494932,
       "step": 12
     },
     {
-      "completion_length": 284.03125,
+      "completion_length": 441.40625,
       "epoch": 3.571428571428571,
-      "grad_norm": 7.384803295135498,
-      "kl": 0.00221475418220507,
+      "grad_norm": 58.959373474121094,
+      "kl": 0.002568057843745919,
       "learning_rate": 1.4957614383675767e-07,
       "loss": 0.0,
-      "reward": 0.42047514545265585,
-      "reward_std": 0.5029277658904903,
-      "rewards/concensus_correctness_reward_func": 0.022937499918043613,
-      "rewards/consensus_reward_func": 0.0625,
+      "reward": 0.28832691139541566,
+      "reward_std": 0.2864814767963253,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.2086626668460667,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.217514410847798,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.0013750039506703615,
+      "rewards/xmlcount_reward_func": 0.07081249915063381,
       "step": 14
     },
     {
-      "completion_length": 337.0833333333333,
+      "completion_length": 225.25,
       "epoch": 4.0,
-      "grad_norm": 7.833550453186035,
-      "kl": 0.026275320638281602,
+      "grad_norm": 4.326447010040283,
+      "kl": 0.004686735337600112,
       "learning_rate": 8.067960709356478e-08,
       "loss": 0.0,
-      "reward": 0.17484355298802257,
-      "reward_std": 0.4374485242490967,
+      "reward": 0.44169732338438433,
+      "reward_std": 0.26000941286717233,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.2331352272691826,
+      "rewards/question_recreation_reward_func": 0.2755723277417322,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": -0.05829165906955799,
+      "rewards/xmlcount_reward_func": 0.1661250051110983,
       "step": 16
     },
     {
-      "completion_length": 386.65625,
+      "completion_length": 416.03125,
       "epoch": 4.571428571428571,
-      "grad_norm": 11.165565490722656,
-      "kl": 0.002814602070429828,
+      "grad_norm": 5.953683853149414,
+      "kl": 0.0024949461294454522,
       "learning_rate": 3.013156219837776e-08,
       "loss": 0.0,
-      "reward": 0.5532054845243692,
-      "reward_std": 0.594974227657076,
-      "rewards/concensus_correctness_reward_func": 0.00925000011920929,
-      "rewards/consensus_reward_func": 0.0625,
+      "reward": 0.4505774106364697,
+      "reward_std": 0.3652912781544728,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.3261742278264137,
+      "rewards/question_recreation_reward_func": 0.36542115616612136,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.1552812505979091,
+      "rewards/xmlcount_reward_func": 0.08515624981373549,
       "step": 18
     },
     {
-      "completion_length": 460.625,
+      "completion_length": 376.75,
       "epoch": 5.0,
-      "grad_norm": 41.27642822265625,
-      "kl": 0.05248331223750332,
+      "grad_norm": 4.8638811111450195,
+      "kl": 0.005203787494489613,
       "learning_rate": 3.4096741493194193e-09,
       "loss": 0.0,
-      "reward": 0.34862043731845915,
-      "reward_std": 0.37990098469890654,
+      "reward": 0.6231528092175722,
+      "reward_std": 0.7030623334770402,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.16666666666666666,
-      "rewards/question_recreation_reward_func": 0.1707871137962987,
+      "rewards/final_correctness_reward_func": 0.08333333333333333,
+      "rewards/question_recreation_reward_func": 0.3369028040518363,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.011166666944821676,
+      "rewards/xmlcount_reward_func": 0.20291666686534882,
       "step": 20
     },
     {
       "epoch": 5.0,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 8.345013884536457e-06,
-      "train_runtime": 1784.0509,
-      "train_samples_per_second": 0.179,
-      "train_steps_per_second": 0.011
+      "train_loss": 2.411922059764038e-06,
+      "train_runtime": 2650.5489,
+      "train_samples_per_second": 0.121,
+      "train_steps_per_second": 0.008
     }
   ],
   "logging_steps": 2,