End of training

README.md

@@ -40,7 +40,7 @@ This model was trained with GRPO, a method introduced in [DeepSeekMath: Pushing
 - TRL: 0.15.2
 - Transformers: 4.51.3
 - Pytorch: 2.5.1
-- Datasets: 3.5.0
+- Datasets: 3.5.1
 - Tokenizers: 0.21.1
 
 ## Citations

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 112738.47402167018,
-    "train_runtime": 120.4527,
-    "train_samples": 28,
-    "train_samples_per_second": 2.657,
-    "train_steps_per_second": 0.166
+    "train_loss": 4.051951691508293e-06,
+    "train_runtime": 400.9226,
+    "train_samples": 92,
+    "train_samples_per_second": 0.798,
+    "train_steps_per_second": 0.05
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e4dd321b86c4618e847033c86f24449ab4ef0986295c656b5bdac3223e0f6f3c
+oid sha256:7c6dec50e47d18e5cffc0d385cfe34a8ecf10a3e6e0d08c6e2de7291d75e2db2
 size 1976163472

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 112738.47402167018,
-    "train_runtime": 120.4527,
-    "train_samples": 28,
-    "train_samples_per_second": 2.657,
-    "train_steps_per_second": 0.166
+    "train_loss": 4.051951691508293e-06,
+    "train_runtime": 400.9226,
+    "train_samples": 92,
+    "train_samples_per_second": 0.798,
+    "train_steps_per_second": 0.05
 }

trainer_state.json

@@ -2,7 +2,7 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 5.0,
+  "epoch": 1.6956521739130435,
   "eval_steps": 500,
   "global_step": 20,
   "is_hyper_param_search": false,
@@ -10,209 +10,209 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 249.40625,
-      "epoch": 0.5714285714285714,
-      "grad_norm": 78.55313110351562,
+      "completion_length": 190.625,
+      "epoch": 0.17391304347826086,
+      "grad_norm": 15.500188827514648,
       "kl": 0.0,
       "learning_rate": 5e-07,
-      "loss": -0.0,
-      "reward": 3.0591967664659023,
-      "reward_std": 1.667841853806749,
-      "rewards/concensus_correctness_reward_func": 0.6743124946951866,
-      "rewards/consensus_reward_func": 0.6875,
+      "loss": 0.0,
+      "reward": 0.8078505160519853,
+      "reward_std": 0.7219250428606756,
+      "rewards/concensus_correctness_reward_func": 0.06012500077486038,
+      "rewards/consensus_reward_func": 0.25,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.1875,
-      "rewards/question_recreation_reward_func": 0.6536654952215031,
-      "rewards/soft_format_reward_func": 0.015625,
-      "rewards/strict_format_reward_func": 0.078125,
-      "rewards/xmlcount_reward_func": 0.7624687403440475,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.21953801915515214,
+      "rewards/soft_format_reward_func": 0.0,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.13756249891594052,
       "step": 2
     },
     {
-      "completion_length": 149.375,
-      "epoch": 1.0,
-      "grad_norm": 86.88078308105469,
-      "kl": 0.05806352787961563,
+      "completion_length": 287.65625,
+      "epoch": 0.34782608695652173,
+      "grad_norm": 66.68354797363281,
+      "kl": 0.0012720471095235553,
       "learning_rate": 4.864543104251586e-07,
       "loss": 0.0,
-      "reward": 5.5724403858184814,
-      "reward_std": 1.5296151217771694,
-      "rewards/concensus_correctness_reward_func": 1.7816666613022487,
-      "rewards/consensus_reward_func": 1.5,
+      "reward": 0.4001400780980475,
+      "reward_std": 0.5209844395576511,
+      "rewards/concensus_correctness_reward_func": 0.03981250151991844,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.3333333333333333,
-      "rewards/question_recreation_reward_func": 0.7538986553748449,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.21017132356064394,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.16666666666666666,
-      "rewards/xmlcount_reward_func": 1.0368749996026356,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.0876562490593642,
       "step": 4
     },
     {
-      "completion_length": 152.34375,
-      "epoch": 1.5714285714285714,
-      "grad_norm": 34.086280822753906,
-      "kl": 0.6559465810423717,
+      "completion_length": 327.84375,
+      "epoch": 0.5217391304347826,
+      "grad_norm": 89.11880493164062,
+      "kl": 0.001105570492654806,
       "learning_rate": 4.472851273490984e-07,
-      "loss": 0.0007,
-      "reward": 6.155833105556667,
-      "reward_std": 0.6093435278016841,
-      "rewards/concensus_correctness_reward_func": 2.099375005811453,
-      "rewards/consensus_reward_func": 1.5625,
+      "loss": 0.0,
+      "reward": 0.5919948052032851,
+      "reward_std": 0.6676725139259361,
+      "rewards/concensus_correctness_reward_func": 0.0078125,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.3125,
-      "rewards/question_recreation_reward_func": 0.7866768958047032,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.19855730346171185,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.3125,
-      "rewards/xmlcount_reward_func": 1.0822812505066395,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.13562499964609742,
       "step": 6
     },
     {
-      "completion_length": 175.41666666666666,
-      "epoch": 2.0,
-      "grad_norm": 102.05463409423828,
-      "kl": 1.1159729743376374,
+      "completion_length": 252.75,
+      "epoch": 0.6956521739130435,
+      "grad_norm": 49.65816116333008,
+      "kl": 0.0012518168077804148,
       "learning_rate": 3.867370395306068e-07,
-      "loss": 0.0008,
-      "reward": 5.1590457409620285,
-      "reward_std": 1.2149010785312082,
-      "rewards/concensus_correctness_reward_func": 1.4724166442950566,
-      "rewards/consensus_reward_func": 1.4166666666666667,
+      "loss": 0.0,
+      "reward": 0.4244745868491009,
+      "reward_std": 0.6876564208650962,
+      "rewards/concensus_correctness_reward_func": 0.05999999865889549,
+      "rewards/consensus_reward_func": 0.0625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.5,
-      "rewards/question_recreation_reward_func": 0.7165040796001753,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.18391207925742492,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.0625,
-      "rewards/xmlcount_reward_func": 0.9909583280483881,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.055562496185302734,
       "step": 8
     },
     {
-      "completion_length": 181.1875,
-      "epoch": 2.571428571428571,
-      "grad_norm": 1401648.875,
-      "kl": 4485897.915470626,
+      "completion_length": 304.875,
+      "epoch": 0.8695652173913043,
+      "grad_norm": 11.769674301147461,
+      "kl": 0.0011520595126057742,
       "learning_rate": 3.1137137178519977e-07,
-      "loss": 4485.897,
-      "reward": 5.614339977502823,
-      "reward_std": 1.5171476181130856,
-      "rewards/concensus_correctness_reward_func": 1.811499997973442,
-      "rewards/consensus_reward_func": 1.375,
+      "loss": 0.0,
+      "reward": 0.4081341839628294,
+      "reward_std": 0.5512065841176081,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.4375,
-      "rewards/question_recreation_reward_func": 0.7781211100518703,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.20441542949993163,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.1875,
-      "rewards/xmlcount_reward_func": 1.0247187428176403,
+      "rewards/strict_format_reward_func": 0.015625,
+      "rewards/xmlcount_reward_func": 0.1255937460809946,
       "step": 10
     },
     {
-      "completion_length": 150.79166666666666,
-      "epoch": 3.0,
-      "grad_norm": 34.14075469970703,
-      "kl": 219711.29470605837,
+      "completion_length": 362.1666666666667,
+      "epoch": 1.0,
+      "grad_norm": 19.223752975463867,
+      "kl": 0.0012859399915517618,
       "learning_rate": 2.2935516363191693e-07,
-      "loss": 164.7835,
-      "reward": 6.4597495794296265,
-      "reward_std": 0.2802316110270719,
-      "rewards/concensus_correctness_reward_func": 2.0669999917348227,
-      "rewards/consensus_reward_func": 1.8333333333333333,
+      "loss": 0.0,
+      "reward": 0.5378691589770218,
+      "reward_std": 0.5761304936531815,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.3333333333333333,
-      "rewards/question_recreation_reward_func": 0.8656661808490753,
+      "rewards/final_correctness_reward_func": 0.16666666666666666,
+      "rewards/question_recreation_reward_func": 0.2493691734659175,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.1875,
-      "rewards/xmlcount_reward_func": 1.1729166706403096,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.12183333374559879,
       "step": 12
     },
     {
-      "completion_length": 187.09375,
-      "epoch": 3.571428571428571,
-      "grad_norm": 469.4254150390625,
-      "kl": 209.55569416983053,
+      "completion_length": 302.6875,
+      "epoch": 1.1739130434782608,
+      "grad_norm": 55.770511627197266,
+      "kl": 0.026156973304750863,
       "learning_rate": 1.4957614383675767e-07,
-      "loss": 0.2096,
-      "reward": 6.7032526433467865,
-      "reward_std": 1.293150786194019,
-      "rewards/concensus_correctness_reward_func": 2.4413749873638153,
-      "rewards/consensus_reward_func": 1.625,
+      "loss": 0.0,
+      "reward": 0.6981925278087147,
+      "reward_std": 0.7400844875933217,
+      "rewards/concensus_correctness_reward_func": 0.06012500077486038,
+      "rewards/consensus_reward_func": 0.125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.375,
-      "rewards/question_recreation_reward_func": 0.8526588641107082,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.2547237664402928,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.234375,
-      "rewards/xmlcount_reward_func": 1.174843743443489,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.1333437452558428,
       "step": 14
     },
     {
-      "completion_length": 156.54166666666666,
-      "epoch": 4.0,
-      "grad_norm": 19.79854965209961,
-      "kl": 128.49675853115818,
+      "completion_length": 227.78125,
+      "epoch": 1.3478260869565217,
+      "grad_norm": 11.940630912780762,
+      "kl": 0.001112268855649745,
       "learning_rate": 8.067960709356478e-08,
-      "loss": 0.0964,
-      "reward": 5.564403712749481,
-      "reward_std": 1.683505776959161,
-      "rewards/concensus_correctness_reward_func": 1.6036666532357533,
-      "rewards/consensus_reward_func": 1.3333333333333333,
+      "loss": 0.0,
+      "reward": 0.48283951880875975,
+      "reward_std": 0.6924986405065283,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.125,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.3333333333333333,
-      "rewards/question_recreation_reward_func": 0.8824037462472916,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.2560895170317963,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.2916666666666667,
-      "rewards/xmlcount_reward_func": 1.119999997317791,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.10175000299932435,
       "step": 16
     },
     {
-      "completion_length": 157.4375,
-      "epoch": 4.571428571428571,
-      "grad_norm": 53211725824.0,
-      "kl": 1122733816.4475722,
+      "completion_length": 389.5,
+      "epoch": 1.5217391304347827,
+      "grad_norm": 98.38214874267578,
+      "kl": 0.0016772262242739089,
       "learning_rate": 3.013156219837776e-08,
-      "loss": 1122733.75,
-      "reward": 5.74696709215641,
-      "reward_std": 1.4755408964119852,
-      "rewards/concensus_correctness_reward_func": 1.7621249966323376,
-      "rewards/consensus_reward_func": 1.625,
+      "loss": 0.0,
+      "reward": 0.3492350783199072,
+      "reward_std": 0.34461650425328116,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.4375,
-      "rewards/question_recreation_reward_func": 0.6688420784194022,
+      "rewards/final_correctness_reward_func": 0.0,
+      "rewards/question_recreation_reward_func": 0.187610074528493,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.203125,
-      "rewards/xmlcount_reward_func": 1.0503750005736947,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.09912500088103116,
       "step": 18
     },
     {
-      "completion_length": 160.08333333333334,
-      "epoch": 5.0,
-      "grad_norm": 22.75950050354004,
-      "kl": 3.0091245751827955,
+      "completion_length": 396.21875,
+      "epoch": 1.6956521739130435,
+      "grad_norm": 50.862178802490234,
+      "kl": 0.00580468270345591,
       "learning_rate": 3.4096741493194193e-09,
-      "loss": 0.0023,
-      "reward": 5.480962634086609,
-      "reward_std": 1.3035912602208555,
-      "rewards/concensus_correctness_reward_func": 1.6363333264986675,
-      "rewards/consensus_reward_func": 1.3333333333333333,
+      "loss": 0.0,
+      "reward": 0.26614804682321846,
+      "reward_std": 0.4568546366026567,
+      "rewards/concensus_correctness_reward_func": 0.0,
+      "rewards/consensus_reward_func": 0.0,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.3333333333333333,
-      "rewards/question_recreation_reward_func": 0.8412126104036967,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.12105428869836032,
       "rewards/soft_format_reward_func": 0.0,
-      "rewards/strict_format_reward_func": 0.22916666666666666,
-      "rewards/xmlcount_reward_func": 1.107583334048589,
+      "rewards/strict_format_reward_func": 0.0,
+      "rewards/xmlcount_reward_func": 0.020093752536922693,
       "step": 20
     },
     {
-      "epoch": 5.0,
+      "epoch": 1.6956521739130435,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 112738.47402167018,
-      "train_runtime": 120.4527,
-      "train_samples_per_second": 2.657,
-      "train_steps_per_second": 0.166
+      "train_loss": 4.051951691508293e-06,
+      "train_runtime": 400.9226,
+      "train_samples_per_second": 0.798,
+      "train_steps_per_second": 0.05
     }
   ],
   "logging_steps": 2,
   "max_steps": 20,
   "num_input_tokens_seen": 0,
-  "num_train_epochs": 7,
+  "num_train_epochs": 2,
   "save_steps": 25,
   "stateful_callbacks": {
     "TrainerControl": {

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bf452952a00ff82f507e6aea2cde42beee8396b22d4f6c75ad745c17d2f42738
+oid sha256:d057628bd4f4876cd7196bb74d82ee0f7f969435e2307a78e9ccd7d3769f38a8
 size 5944