End of training

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": -0.030795110389590263,
-    "train_runtime": 10341.1372,
-    "train_samples": 88,
-    "train_samples_per_second": 0.004,
+    "train_loss": -0.007105642557144165,
+    "train_runtime": 13988.242,
+    "train_samples": 104,
+    "train_samples_per_second": 0.003,
     "train_steps_per_second": 0.001
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e8844bb76d8438458fc3e9ef30e981e4c63a3835196663aac3693c5e6a15bfe0
+oid sha256:a3cf4d7c87d918cf4ec9e91d4086e44b37784ce5614f96edb4cf2d11aa85e6df
 size 1976163472

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": -0.030795110389590263,
-    "train_runtime": 10341.1372,
-    "train_samples": 88,
-    "train_samples_per_second": 0.004,
+    "train_loss": -0.007105642557144165,
+    "train_runtime": 13988.242,
+    "train_samples": 104,
+    "train_samples_per_second": 0.003,
     "train_steps_per_second": 0.001
 }

trainer_state.json

@@ -2,7 +2,7 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.22727272727272727,
+  "epoch": 0.19230769230769232,
   "eval_steps": 500,
   "global_step": 10,
   "is_hyper_param_search": false,
@@ -16,21 +16,21 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 74.5,
-      "completions/max_terminated_length": 74.5,
-      "completions/mean_length": 36.875,
-      "completions/mean_terminated_length": 36.875,
-      "completions/min_length": 4.0,
-      "completions/min_terminated_length": 4.0,
-      "epoch": 0.045454545454545456,
+      "completions/max_length": 71.0,
+      "completions/max_terminated_length": 71.0,
+      "completions/mean_length": 30.125,
+      "completions/mean_terminated_length": 30.125,
+      "completions/min_length": 10.0,
+      "completions/min_terminated_length": 10.0,
+      "epoch": 0.038461538461538464,
       "frac_reward_zero_std": 0.0,
-      "grad_norm": 30.32355499267578,
+      "grad_norm": 26.099611282348633,
       "kl": 0.0,
       "learning_rate": 5e-07,
-      "loss": -0.0699,
-      "num_tokens": 1319.0,
-      "reward": 0.08947309106588364,
-      "reward_std": 0.02952927676960826,
+      "loss": -0.1539,
+      "num_tokens": 1265.0,
+      "reward": 0.17112372070550919,
+      "reward_std": 0.06430763006210327,
       "rewards/concensus_correctness_reward_func/mean": 0.0,
       "rewards/concensus_correctness_reward_func/std": 0.0,
       "rewards/consensus_reward_func/mean": 0.0,
@@ -39,8 +39,8 @@
       "rewards/cumulative_reward_2/std": 0.0,
       "rewards/final_correctness_reward_func/mean": 0.0,
       "rewards/final_correctness_reward_func/std": 0.0,
-      "rewards/question_recreation_reward_func/mean": 0.08947309292852879,
-      "rewards/question_recreation_reward_func/std": 0.02744971076026559,
+      "rewards/question_recreation_reward_func/mean": 0.17112372815608978,
+      "rewards/question_recreation_reward_func/std": 0.06673818826675415,
       "rewards/soft_format_reward_func/mean": 0.0,
       "rewards/soft_format_reward_func/std": 0.0,
       "rewards/strict_format_reward_func/mean": 0.0,
@@ -55,22 +55,22 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.125,
-      "completions/max_length": 130.5,
-      "completions/max_terminated_length": 35.5,
-      "completions/mean_length": 45.0,
-      "completions/mean_terminated_length": 16.541666984558105,
-      "completions/min_length": 6.5,
-      "completions/min_terminated_length": 6.5,
-      "epoch": 0.09090909090909091,
-      "frac_reward_zero_std": 0.25,
-      "grad_norm": 71.5344009399414,
-      "kl": 0.0013015866788919084,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 112.0,
+      "completions/max_terminated_length": 112.0,
+      "completions/mean_length": 80.125,
+      "completions/mean_terminated_length": 80.125,
+      "completions/min_length": 53.5,
+      "completions/min_terminated_length": 53.5,
+      "epoch": 0.07692307692307693,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 15.406155586242676,
+      "kl": 0.001192556883324869,
       "learning_rate": 4.415111107797445e-07,
-      "loss": -0.0174,
-      "num_tokens": 2703.0,
-      "reward": 0.08281245455145836,
-      "reward_std": 0.03817772259935737,
+      "loss": -0.0825,
+      "num_tokens": 2930.0,
+      "reward": 0.1593383327126503,
+      "reward_std": 0.04592637158930302,
       "rewards/concensus_correctness_reward_func/mean": 0.0,
       "rewards/concensus_correctness_reward_func/std": 0.0,
       "rewards/consensus_reward_func/mean": 0.0,
@@ -79,8 +79,8 @@
       "rewards/cumulative_reward_2/std": 0.0,
       "rewards/final_correctness_reward_func/mean": 0.0,
       "rewards/final_correctness_reward_func/std": 0.0,
-      "rewards/question_recreation_reward_func/mean": 0.08281246200203896,
-      "rewards/question_recreation_reward_func/std": 0.04765166528522968,
+      "rewards/question_recreation_reward_func/mean": 0.1593383327126503,
+      "rewards/question_recreation_reward_func/std": 0.04842444974929094,
       "rewards/soft_format_reward_func/mean": 0.0,
       "rewards/soft_format_reward_func/std": 0.0,
       "rewards/strict_format_reward_func/mean": 0.0,
@@ -95,22 +95,22 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 177.5,
-      "completions/max_terminated_length": 177.5,
-      "completions/mean_length": 76.375,
-      "completions/mean_terminated_length": 76.375,
-      "completions/min_length": 23.5,
-      "completions/min_terminated_length": 23.5,
-      "epoch": 0.13636363636363635,
+      "completions/clipped_ratio": 0.25,
+      "completions/max_length": 256.0,
+      "completions/max_terminated_length": 161.0,
+      "completions/mean_length": 132.5,
+      "completions/mean_terminated_length": 91.33333587646484,
+      "completions/min_length": 36.0,
+      "completions/min_terminated_length": 36.0,
+      "epoch": 0.11538461538461539,
       "frac_reward_zero_std": 0.0,
-      "grad_norm": 60.853816986083984,
-      "kl": 0.0031654702906962484,
+      "grad_norm": 15.05298900604248,
+      "kl": 0.0028286711749387905,
       "learning_rate": 2.934120444167326e-07,
-      "loss": -0.0248,
-      "num_tokens": 4338.0,
-      "reward": 0.10044452454894781,
-      "reward_std": 0.021916877012699842,
+      "loss": 0.0432,
+      "num_tokens": 5014.0,
+      "reward": 0.07234940817579627,
+      "reward_std": 0.022284463979303837,
       "rewards/concensus_correctness_reward_func/mean": 0.0,
       "rewards/concensus_correctness_reward_func/std": 0.0,
       "rewards/consensus_reward_func/mean": 0.0,
@@ -119,8 +119,8 @@
       "rewards/cumulative_reward_2/std": 0.0,
       "rewards/final_correctness_reward_func/mean": 0.0,
       "rewards/final_correctness_reward_func/std": 0.0,
-      "rewards/question_recreation_reward_func/mean": 0.10044452454894781,
-      "rewards/question_recreation_reward_func/std": 0.024618458934128284,
+      "rewards/question_recreation_reward_func/mean": 0.07234940817579627,
+      "rewards/question_recreation_reward_func/std": 0.02770008658990264,
       "rewards/soft_format_reward_func/mean": 0.0,
       "rewards/soft_format_reward_func/std": 0.0,
       "rewards/strict_format_reward_func/mean": 0.0,
@@ -136,21 +136,21 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.125,
-      "completions/max_length": 210.5,
-      "completions/max_terminated_length": 128.0,
-      "completions/mean_length": 76.5,
-      "completions/mean_terminated_length": 48.70833396911621,
-      "completions/min_length": 4.5,
-      "completions/min_terminated_length": 4.5,
-      "epoch": 0.18181818181818182,
-      "frac_reward_zero_std": 0.25,
-      "grad_norm": 22.12053680419922,
-      "kl": 0.0013654041031259112,
+      "completions/max_length": 174.0,
+      "completions/max_terminated_length": 94.5,
+      "completions/mean_length": 73.875,
+      "completions/mean_terminated_length": 50.375,
+      "completions/min_length": 27.0,
+      "completions/min_terminated_length": 27.0,
+      "epoch": 0.15384615384615385,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 47.132999420166016,
+      "kl": 0.007395870692562312,
       "learning_rate": 1.2500000000000005e-07,
-      "loss": 0.0056,
-      "num_tokens": 5974.0,
-      "reward": 0.029887909069657326,
-      "reward_std": 0.018557347357273102,
+      "loss": 0.0003,
+      "num_tokens": 6629.0,
+      "reward": 0.0922946073114872,
+      "reward_std": 0.03348500933498144,
       "rewards/concensus_correctness_reward_func/mean": 0.0,
       "rewards/concensus_correctness_reward_func/std": 0.0,
       "rewards/consensus_reward_func/mean": 0.0,
@@ -159,8 +159,8 @@
       "rewards/cumulative_reward_2/std": 0.0,
       "rewards/final_correctness_reward_func/mean": 0.0,
       "rewards/final_correctness_reward_func/std": 0.0,
-      "rewards/question_recreation_reward_func/mean": 0.029887909069657326,
-      "rewards/question_recreation_reward_func/std": 0.025635237339884043,
+      "rewards/question_recreation_reward_func/mean": 0.0922946073114872,
+      "rewards/question_recreation_reward_func/std": 0.0378275141119957,
       "rewards/soft_format_reward_func/mean": 0.0,
       "rewards/soft_format_reward_func/std": 0.0,
       "rewards/strict_format_reward_func/mean": 0.0,
@@ -175,22 +175,22 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 123.5,
-      "completions/max_terminated_length": 123.5,
-      "completions/mean_length": 41.625,
-      "completions/mean_terminated_length": 41.625,
-      "completions/min_length": 3.5,
-      "completions/min_terminated_length": 3.5,
-      "epoch": 0.22727272727272727,
-      "frac_reward_zero_std": 0.25,
-      "grad_norm": 64.85562133789062,
-      "kl": 0.0105830222601071,
+      "completions/clipped_ratio": 0.25,
+      "completions/max_length": 256.0,
+      "completions/max_terminated_length": 153.5,
+      "completions/mean_length": 127.875,
+      "completions/mean_terminated_length": 85.16666603088379,
+      "completions/min_length": 35.0,
+      "completions/min_terminated_length": 35.0,
+      "epoch": 0.19230769230769232,
+      "frac_reward_zero_std": 0.0,
+      "grad_norm": 15.468522071838379,
+      "kl": 0.005031221779063344,
       "learning_rate": 1.507684480352292e-08,
-      "loss": -0.0475,
-      "num_tokens": 7331.0,
-      "reward": 0.02319902554154396,
-      "reward_std": 0.008738046046346426,
+      "loss": 0.1575,
+      "num_tokens": 8676.0,
+      "reward": 0.028173266910016537,
+      "reward_std": 0.012423289939761162,
       "rewards/concensus_correctness_reward_func/mean": 0.0,
       "rewards/concensus_correctness_reward_func/std": 0.0,
       "rewards/consensus_reward_func/mean": 0.0,
@@ -199,8 +199,8 @@
       "rewards/cumulative_reward_2/std": 0.0,
       "rewards/final_correctness_reward_func/mean": 0.0,
       "rewards/final_correctness_reward_func/std": 0.0,
-      "rewards/question_recreation_reward_func/mean": 0.02319902554154396,
-      "rewards/question_recreation_reward_func/std": 0.010856612119823694,
+      "rewards/question_recreation_reward_func/mean": 0.028173265978693962,
+      "rewards/question_recreation_reward_func/std": 0.016421337611973286,
       "rewards/soft_format_reward_func/mean": 0.0,
       "rewards/soft_format_reward_func/std": 0.0,
       "rewards/strict_format_reward_func/mean": 0.0,
@@ -210,18 +210,18 @@
       "step": 10
     },
     {
-      "epoch": 0.22727272727272727,
+      "epoch": 0.19230769230769232,
       "step": 10,
       "total_flos": 0.0,
-      "train_loss": -0.030795110389590263,
-      "train_runtime": 10341.1372,
-      "train_samples_per_second": 0.004,
+      "train_loss": -0.007105642557144165,
+      "train_runtime": 13988.242,
+      "train_samples_per_second": 0.003,
       "train_steps_per_second": 0.001
     }
   ],
   "logging_steps": 2,
   "max_steps": 10,
-  "num_input_tokens_seen": 7331,
+  "num_input_tokens_seen": 8676,
   "num_train_epochs": 1,
   "save_steps": 10,
   "stateful_callbacks": {