End of training

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.00015548239141480736,
-    "train_runtime": 835.2221,
-    "train_samples": 246,
-    "train_samples_per_second": 0.383,
-    "train_steps_per_second": 0.024
+    "train_loss": 4.895885365385766e-06,
+    "train_runtime": 325.8563,
+    "train_samples": 160,
+    "train_samples_per_second": 0.982,
+    "train_steps_per_second": 0.061
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e8087f9490598e1c202be69663271e1ca175eec1d88feb348e99170aa136dac2
+oid sha256:34a18f8d3f28d2a0f8164ef057ee62fc9006edd0c0fe6ef5de84a1c03edddc27
 size 1976163472

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.00015548239141480736,
-    "train_runtime": 835.2221,
-    "train_samples": 246,
-    "train_samples_per_second": 0.383,
-    "train_steps_per_second": 0.024
+    "train_loss": 4.895885365385766e-06,
+    "train_runtime": 325.8563,
+    "train_samples": 160,
+    "train_samples_per_second": 0.982,
+    "train_steps_per_second": 0.061
 }

trainer_state.json

@@ -2,7 +2,7 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.6504065040650406,
+  "epoch": 1.0,
   "eval_steps": 500,
   "global_step": 20,
   "is_hyper_param_search": false,
@@ -10,203 +10,203 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 312.90625,
-      "epoch": 0.06504065040650407,
-      "grad_norm": 111.86184692382812,
+      "completion_length": 357.1875,
+      "epoch": 0.1,
+      "grad_norm": 6.888583660125732,
       "kl": 0.0,
       "learning_rate": 4.965903258506806e-07,
       "loss": -0.0,
-      "reward": 0.38406348531134427,
-      "reward_std": 0.42611162207322195,
+      "reward": 0.25579002825543284,
+      "reward_std": 0.446488927322207,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.17321973131038249,
+      "rewards/question_recreation_reward_func": 0.22263377904891968,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.14834375027567148,
+      "rewards/xmlcount_reward_func": -0.029343747533857822,
       "step": 2
     },
     {
-      "completion_length": 265.625,
-      "epoch": 0.13008130081300814,
-      "grad_norm": 32.45527648925781,
-      "kl": 0.0017551507444295567,
+      "completion_length": 429.1875,
+      "epoch": 0.2,
+      "grad_norm": 14.042842864990234,
+      "kl": 0.0031932991187204607,
       "learning_rate": 4.698684378016222e-07,
       "loss": 0.0,
-      "reward": 0.198526976746507,
-      "reward_std": 0.19390703603858128,
+      "reward": 0.255347105441615,
+      "reward_std": 0.3866303440881893,
       "rewards/concensus_correctness_reward_func": 0.0,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.0782457260065712,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.24169085221365094,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.057781249983236194,
+      "rewards/xmlcount_reward_func": 0.01365625043399632,
       "step": 4
     },
     {
-      "completion_length": 354.75,
-      "epoch": 0.1951219512195122,
-      "grad_norm": 16.893728256225586,
-      "kl": 0.0011393952687512865,
+      "completion_length": 350.25,
+      "epoch": 0.3,
+      "grad_norm": 7.698403358459473,
+      "kl": 0.001231183203344699,
       "learning_rate": 4.193203929064353e-07,
       "loss": 0.0,
-      "reward": 0.21917021238186862,
-      "reward_std": 0.2813764516322408,
-      "rewards/concensus_correctness_reward_func": 0.0,
+      "reward": 0.2183529941830784,
+      "reward_std": 0.2752405374776572,
+      "rewards/concensus_correctness_reward_func": 0.010812499560415745,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.15120146072877105,
+      "rewards/question_recreation_reward_func": 0.19875924361986108,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.06796875211875886,
+      "rewards/xmlcount_reward_func": 0.0087812514975667,
       "step": 6
     },
     {
-      "completion_length": 260.5625,
-      "epoch": 0.2601626016260163,
-      "grad_norm": 71.35713958740234,
-      "kl": 0.011541976742591942,
+      "completion_length": 416.4375,
+      "epoch": 0.4,
+      "grad_norm": 203.54537963867188,
+      "kl": 0.01736884316414944,
       "learning_rate": 3.5042385616324236e-07,
       "loss": 0.0,
-      "reward": 0.3831669706851244,
-      "reward_std": 0.3858762901800219,
+      "reward": 0.31171806302154437,
+      "reward_std": 0.5207656847196631,
       "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.0625,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.18316696758847684,
+      "rewards/question_recreation_reward_func": 0.3278430561767891,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.1375000011175871,
+      "rewards/xmlcount_reward_func": -0.016124999936437234,
       "step": 8
     },
     {
-      "completion_length": 239.625,
-      "epoch": 0.3252032520325203,
-      "grad_norm": 144.3345947265625,
-      "kl": 1.334491158118908,
+      "completion_length": 325.9375,
+      "epoch": 0.5,
+      "grad_norm": 71.53971862792969,
+      "kl": 0.005782900771009736,
       "learning_rate": 2.706448363680831e-07,
-      "loss": 0.0013,
-      "reward": 1.163124865386635,
-      "reward_std": 1.3022385529820895,
-      "rewards/concensus_correctness_reward_func": 0.625,
+      "loss": 0.0,
+      "reward": 1.1208667854079977,
+      "reward_std": 1.3186431895255737,
+      "rewards/concensus_correctness_reward_func": 0.6328125,
       "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.3125,
-      "rewards/question_recreation_reward_func": 0.16240612702677026,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.29371054912917316,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.06321875005960464,
+      "rewards/xmlcount_reward_func": 0.0693437559530139,
       "step": 10
     },
     {
-      "completion_length": 302.625,
-      "epoch": 0.3902439024390244,
-      "grad_norm": 19.480737686157227,
-      "kl": 0.005554946790653048,
+      "completion_length": 409.1875,
+      "epoch": 0.6,
+      "grad_norm": 6.795338153839111,
+      "kl": 0.0016026653465814888,
       "learning_rate": 1.886286282148002e-07,
       "loss": 0.0,
-      "reward": 0.18286312883719802,
-      "reward_std": 0.09963599013281055,
-      "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.0,
+      "reward": 0.3760008474928327,
+      "reward_std": 0.8878942199517041,
+      "rewards/concensus_correctness_reward_func": 0.0286250002682209,
+      "rewards/consensus_reward_func": 0.125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.10270687309093773,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.3062821010244079,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.017656249459832907,
+      "rewards/xmlcount_reward_func": -0.2089062463492155,
       "step": 12
     },
     {
-      "completion_length": 353.40625,
-      "epoch": 0.45528455284552843,
-      "grad_norm": 163.0377197265625,
-      "kl": 0.0320172518495383,
+      "completion_length": 339.875,
+      "epoch": 0.7,
+      "grad_norm": 58.777374267578125,
+      "kl": 0.002584025995020056,
       "learning_rate": 1.1326296046939333e-07,
       "loss": 0.0,
-      "reward": 0.21497111866483465,
-      "reward_std": 0.20311534999905234,
-      "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.0,
+      "reward": 0.2751228383858688,
+      "reward_std": 0.6605659697379451,
+      "rewards/concensus_correctness_reward_func": 0.01425000000745058,
+      "rewards/consensus_reward_func": 0.0625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.13703362256637774,
+      "rewards/final_correctness_reward_func": 0.1875,
+      "rewards/question_recreation_reward_func": 0.1413415894494392,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.07793750055134296,
+      "rewards/xmlcount_reward_func": -0.1304687513038516,
       "step": 14
     },
     {
-      "completion_length": 241.8125,
-      "epoch": 0.5203252032520326,
-      "grad_norm": 108.89897155761719,
-      "kl": 0.06319558176619466,
+      "completion_length": 354.8125,
+      "epoch": 0.8,
+      "grad_norm": 6.398130893707275,
+      "kl": 0.0022334530076477677,
       "learning_rate": 5.271487265090163e-08,
-      "loss": 0.0001,
-      "reward": 0.857462587300688,
-      "reward_std": 1.0997640631758259,
-      "rewards/concensus_correctness_reward_func": 0.625,
-      "rewards/consensus_reward_func": 0.0,
+      "loss": 0.0,
+      "reward": 0.39246670436114073,
+      "reward_std": 0.3623120840638876,
+      "rewards/concensus_correctness_reward_func": -0.015812499448657036,
+      "rewards/consensus_reward_func": 0.0625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.13090006925631315,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.2991229514591396,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.0390625,
+      "rewards/xmlcount_reward_func": 0.04665624676272273,
       "step": 16
     },
     {
-      "completion_length": 313.59375,
-      "epoch": 0.5853658536585366,
-      "grad_norm": 101.26744079589844,
-      "kl": 0.09484852727109683,
+      "completion_length": 394.625,
+      "epoch": 0.9,
+      "grad_norm": 7.904482841491699,
+      "kl": 0.017078750890505034,
       "learning_rate": 1.3545689574841341e-08,
-      "loss": 0.0001,
-      "reward": 0.01994338713120669,
-      "reward_std": 0.5154625551658683,
-      "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.0625,
+      "loss": 0.0,
+      "reward": 0.4695802601054311,
+      "reward_std": 0.2639739840233233,
+      "rewards/concensus_correctness_reward_func": 0.025875000283122063,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.1413183781260159,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.27270525274798274,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": -0.18387499451637268,
+      "rewards/xmlcount_reward_func": 0.10850000148639083,
       "step": 18
     },
     {
-      "completion_length": 224.125,
-      "epoch": 0.6504065040650406,
-      "grad_norm": 65.41374206542969,
-      "kl": 0.010168664273805916,
+      "completion_length": 400.4375,
+      "epoch": 1.0,
+      "grad_norm": 20.407468795776367,
+      "kl": 0.002406862549833022,
       "learning_rate": 0.0,
-      "loss": 0.0,
-      "reward": 0.9727522840839811,
-      "reward_std": 1.1393494626126994,
-      "rewards/concensus_correctness_reward_func": 0.625,
-      "rewards/consensus_reward_func": 0.0,
+      "loss": -0.0,
+      "reward": 0.8111102178227156,
+      "reward_std": 1.3617777990330069,
+      "rewards/concensus_correctness_reward_func": 0.6328125,
+      "rewards/consensus_reward_func": 0.0625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.12134604249149561,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.20814148837234825,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.10140624968335032,
+      "rewards/xmlcount_reward_func": -0.09234375134110451,
       "step": 20
     },
     {
-      "epoch": 0.6504065040650406,
+      "epoch": 1.0,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 0.00015548239141480736,
-      "train_runtime": 835.2221,
-      "train_samples_per_second": 0.383,
-      "train_steps_per_second": 0.024
+      "train_loss": 4.895885365385766e-06,
+      "train_runtime": 325.8563,
+      "train_samples_per_second": 0.982,
+      "train_steps_per_second": 0.061
     }
   ],
   "logging_steps": 2,