End of training

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 2.411922059764038e-06,
-    "train_runtime": 2650.5489,
-    "train_samples": 28,
-    "train_samples_per_second": 0.121,
+    "train_loss": 4.90974634885788e-06,
+    "train_runtime": 2432.4537,
+    "train_samples": 38,
+    "train_samples_per_second": 0.132,
     "train_steps_per_second": 0.008
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6b544624b52f86d57a378c827c5a4779fd10f86be94c2856b4b7261e4431e047
+oid sha256:ad4f6e8efe6a091c74dc67ad180db55a0c3d341a55d040a9ec47b695b3a1db13
 size 1976163472

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 2.411922059764038e-06,
-    "train_runtime": 2650.5489,
-    "train_samples": 28,
-    "train_samples_per_second": 0.121,
+    "train_loss": 4.90974634885788e-06,
+    "train_runtime": 2432.4537,
+    "train_samples": 38,
+    "train_samples_per_second": 0.132,
     "train_steps_per_second": 0.008
 }

trainer_state.json

@@ -2,7 +2,7 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 5.0,
+  "epoch": 4.0,
   "eval_steps": 500,
   "global_step": 20,
   "is_hyper_param_search": false,
@@ -10,209 +10,209 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 442.65625,
-      "epoch": 0.5714285714285714,
-      "grad_norm": 7.137018203735352,
+      "completion_length": 355.3125,
+      "epoch": 0.42105263157894735,
+      "grad_norm": 102.69873809814453,
       "kl": 0.0,
       "learning_rate": 5e-07,
       "loss": -0.0,
-      "reward": 0.5420733785722405,
-      "reward_std": 0.36219248750421684,
-      "rewards/concensus_correctness_reward_func": 0.017625000327825546,
-      "rewards/consensus_reward_func": 0.125,
+      "reward": 0.21049223048612475,
+      "reward_std": 0.883908929725294,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.24694837513379753,
+      "rewards/question_recreation_reward_func": 0.3366484845755622,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.15250000217929482,
+      "rewards/xmlcount_reward_func": -0.12615624256432056,
       "step": 2
     },
     {
-      "completion_length": 360.5,
-      "epoch": 1.0,
-      "grad_norm": 2.7042696475982666,
-      "kl": 0.0007317917788896011,
+      "completion_length": 282.46875,
+      "epoch": 0.8421052631578947,
+      "grad_norm": 11.320378303527832,
+      "kl": 0.0014440583036048338,
       "learning_rate": 4.864543104251586e-07,
       "loss": 0.0,
-      "reward": 0.2949130789687236,
-      "reward_std": 0.30900699730652076,
-      "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.0,
+      "reward": 0.9027491768356413,
+      "reward_std": 0.6032452237559482,
+      "rewards/concensus_correctness_reward_func": 0.059812501072883606,
+      "rewards/consensus_reward_func": 0.0625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.22091308049857616,
+      "rewards/final_correctness_reward_func": 0.1875,
+      "rewards/question_recreation_reward_func": 0.36512421316001564,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.07400000215663265,
+      "rewards/xmlcount_reward_func": 0.22781249205581844,
       "step": 4
     },
     {
-      "completion_length": 377.09375,
-      "epoch": 1.5714285714285714,
-      "grad_norm": 140.6943359375,
-      "kl": 0.0025935436788131483,
+      "completion_length": 455.60714285714283,
+      "epoch": 1.2105263157894737,
+      "grad_norm": 5.977652549743652,
+      "kl": 0.0017597835332188489,
       "learning_rate": 4.472851273490984e-07,
       "loss": 0.0,
-      "reward": 0.5349589511752129,
-      "reward_std": 0.4536192311497871,
-      "rewards/concensus_correctness_reward_func": 0.045312498696148396,
-      "rewards/consensus_reward_func": 0.0625,
+      "reward": 0.9831949581525156,
+      "reward_std": 0.6607641908007541,
+      "rewards/concensus_correctness_reward_func": 0.13250000136239187,
+      "rewards/consensus_reward_func": 0.07142857142857142,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.2757089544320479,
+      "rewards/final_correctness_reward_func": 0.14285714285714285,
+      "rewards/question_recreation_reward_func": 0.4053377916238138,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.08893749746493995,
+      "rewards/xmlcount_reward_func": 0.2310714295932225,
       "step": 6
     },
     {
-      "completion_length": 328.375,
-      "epoch": 2.0,
-      "grad_norm": 6.15055513381958,
-      "kl": 0.0009562060222378932,
+      "completion_length": 250.28125,
+      "epoch": 1.631578947368421,
+      "grad_norm": 25.159584045410156,
+      "kl": 0.00793385949509684,
       "learning_rate": 3.867370395306068e-07,
       "loss": 0.0,
-      "reward": 0.202367508240665,
-      "reward_std": 0.22718117238643268,
-      "rewards/concensus_correctness_reward_func": 0.0,
+      "reward": 0.5738056732807308,
+      "reward_std": 0.6877566411567386,
+      "rewards/concensus_correctness_reward_func": 0.05999999865889549,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.11857584250780444,
+      "rewards/question_recreation_reward_func": 0.2658369077835232,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.08379166573286057,
+      "rewards/xmlcount_reward_func": 0.24796875566244125,
       "step": 8
     },
     {
-      "completion_length": 356.9375,
-      "epoch": 2.571428571428571,
-      "grad_norm": 12.800816535949707,
-      "kl": 0.0064311328023904935,
+      "completion_length": 346.5357142857143,
+      "epoch": 2.0,
+      "grad_norm": 5.285414218902588,
+      "kl": 0.0035223962206925663,
       "learning_rate": 3.1137137178519977e-07,
       "loss": 0.0,
-      "reward": 0.5643500033766031,
-      "reward_std": 0.5321357457432896,
-      "rewards/concensus_correctness_reward_func": 0.008812500163912773,
-      "rewards/consensus_reward_func": 0.0625,
+      "reward": 0.7206419948488474,
+      "reward_std": 0.6630100615854774,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.2930999907257501,
+      "rewards/final_correctness_reward_func": 0.14285714285714285,
+      "rewards/question_recreation_reward_func": 0.3452491302575384,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.13743750134017318,
+      "rewards/xmlcount_reward_func": 0.23253571029220307,
       "step": 10
     },
     {
-      "completion_length": 275.0,
-      "epoch": 3.0,
-      "grad_norm": 5.229515552520752,
-      "kl": 0.00183830058570796,
+      "completion_length": 312.03125,
+      "epoch": 2.4210526315789473,
+      "grad_norm": 5.405126571655273,
+      "kl": 0.0024529736692784354,
       "learning_rate": 2.2935516363191693e-07,
       "loss": 0.0,
-      "reward": 0.34687480283901095,
-      "reward_std": 0.4368026171190043,
+      "reward": 0.7590260014403611,
+      "reward_std": 0.7920715521322563,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.08333333333333333,
-      "rewards/question_recreation_reward_func": 0.2073331281232337,
+      "rewards/final_correctness_reward_func": 0.3125,
+      "rewards/question_recreation_reward_func": 0.29855725495144725,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.05620833610494932,
+      "rewards/xmlcount_reward_func": 0.14796874905005097,
       "step": 12
     },
     {
-      "completion_length": 441.40625,
-      "epoch": 3.571428571428571,
-      "grad_norm": 58.959373474121094,
-      "kl": 0.002568057843745919,
+      "completion_length": 470.5625,
+      "epoch": 2.8421052631578947,
+      "grad_norm": 3.935940980911255,
+      "kl": 0.006017757339577656,
       "learning_rate": 1.4957614383675767e-07,
       "loss": 0.0,
-      "reward": 0.28832691139541566,
-      "reward_std": 0.2864814767963253,
-      "rewards/concensus_correctness_reward_func": 0.0,
+      "reward": 0.764203691855073,
+      "reward_std": 0.511967592057772,
+      "rewards/concensus_correctness_reward_func": 0.05999999865889549,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.217514410847798,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.3350786907831207,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.07081249915063381,
+      "rewards/xmlcount_reward_func": 0.3066250067204237,
       "step": 14
     },
     {
-      "completion_length": 225.25,
-      "epoch": 4.0,
-      "grad_norm": 4.326447010040283,
-      "kl": 0.004686735337600112,
+      "completion_length": 378.0,
+      "epoch": 3.2105263157894735,
+      "grad_norm": 22.409475326538086,
+      "kl": 0.005201808779799778,
       "learning_rate": 8.067960709356478e-08,
       "loss": 0.0,
-      "reward": 0.44169732338438433,
-      "reward_std": 0.26000941286717233,
-      "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.0,
+      "reward": 1.00338282798683,
+      "reward_std": 0.779172311399764,
+      "rewards/concensus_correctness_reward_func": 0.06835714408329555,
+      "rewards/consensus_reward_func": 0.07142857142857142,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.2755723277417322,
+      "rewards/question_recreation_reward_func": 0.44459709249037715,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.1661250051110983,
+      "rewards/xmlcount_reward_func": 0.41899999869721277,
       "step": 16
     },
     {
-      "completion_length": 416.03125,
-      "epoch": 4.571428571428571,
-      "grad_norm": 5.953683853149414,
-      "kl": 0.0024949461294454522,
+      "completion_length": 357.3125,
+      "epoch": 3.6315789473684212,
+      "grad_norm": 5.217537879943848,
+      "kl": 0.005285432001983281,
       "learning_rate": 3.013156219837776e-08,
       "loss": 0.0,
-      "reward": 0.4505774106364697,
-      "reward_std": 0.3652912781544728,
-      "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.0,
+      "reward": 1.409838660620153,
+      "reward_std": 0.8962057730532251,
+      "rewards/concensus_correctness_reward_func": 0.23587500303983688,
+      "rewards/consensus_reward_func": 0.125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.36542115616612136,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.4597761657787487,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.08515624981373549,
+      "rewards/xmlcount_reward_func": 0.33918749913573265,
       "step": 18
     },
     {
-      "completion_length": 376.75,
-      "epoch": 5.0,
-      "grad_norm": 4.8638811111450195,
-      "kl": 0.005203787494489613,
+      "completion_length": 447.57142857142856,
+      "epoch": 4.0,
+      "grad_norm": 6.929130554199219,
+      "kl": 0.019091870286501944,
       "learning_rate": 3.4096741493194193e-09,
       "loss": 0.0,
-      "reward": 0.6231528092175722,
-      "reward_std": 0.7030623334770402,
+      "reward": 0.5597098918099489,
+      "reward_std": 0.7218386048584112,
       "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.07142857142857142,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.08333333333333333,
-      "rewards/question_recreation_reward_func": 0.3369028040518363,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.34185273446408765,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.20291666686534882,
+      "rewards/xmlcount_reward_func": 0.1464285653616701,
       "step": 20
     },
     {
-      "epoch": 5.0,
+      "epoch": 4.0,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 2.411922059764038e-06,
-      "train_runtime": 2650.5489,
-      "train_samples_per_second": 0.121,
+      "train_loss": 4.90974634885788e-06,
+      "train_runtime": 2432.4537,
+      "train_samples_per_second": 0.132,
       "train_steps_per_second": 0.008
     }
   ],
   "logging_steps": 2,
   "max_steps": 20,
   "num_input_tokens_seen": 0,
-  "num_train_epochs": 7,
+  "num_train_epochs": 5,
   "save_steps": 25,
   "stateful_callbacks": {
     "TrainerControl": {