End of training

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.045195698738098145,
-    "train_runtime": 10476.7072,
-    "train_samples": 35,
+    "train_loss": -0.030795110389590263,
+    "train_runtime": 10341.1372,
+    "train_samples": 88,
     "train_samples_per_second": 0.004,
     "train_steps_per_second": 0.001
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5c7a14d05778bc03a42b9f32f97ff73cba08ae4e173c526b8ea2c4cfdcd76b72
+oid sha256:e8844bb76d8438458fc3e9ef30e981e4c63a3835196663aac3693c5e6a15bfe0
 size 1976163472

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.045195698738098145,
-    "train_runtime": 10476.7072,
-    "train_samples": 35,
+    "train_loss": -0.030795110389590263,
+    "train_runtime": 10341.1372,
+    "train_samples": 88,
     "train_samples_per_second": 0.004,
     "train_steps_per_second": 0.001
 }

trainer_state.json

@@ -2,7 +2,7 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.5714285714285714,
+  "epoch": 0.22727272727272727,
   "eval_steps": 500,
   "global_step": 10,
   "is_hyper_param_search": false,
@@ -15,22 +15,22 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.125,
-      "completions/max_length": 170.0,
-      "completions/max_terminated_length": 110.0,
-      "completions/mean_length": 97.125,
-      "completions/mean_terminated_length": 75.91666793823242,
-      "completions/min_length": 46.0,
-      "completions/min_terminated_length": 46.0,
-      "epoch": 0.11428571428571428,
+      "completions/clipped_ratio": 0.0,
+      "completions/max_length": 74.5,
+      "completions/max_terminated_length": 74.5,
+      "completions/mean_length": 36.875,
+      "completions/mean_terminated_length": 36.875,
+      "completions/min_length": 4.0,
+      "completions/min_terminated_length": 4.0,
+      "epoch": 0.045454545454545456,
       "frac_reward_zero_std": 0.0,
-      "grad_norm": 12.094612121582031,
+      "grad_norm": 30.32355499267578,
       "kl": 0.0,
       "learning_rate": 5e-07,
-      "loss": 0.2019,
-      "num_tokens": 1801.0,
-      "reward": 0.16552798449993134,
-      "reward_std": 0.0639336034655571,
+      "loss": -0.0699,
+      "num_tokens": 1319.0,
+      "reward": 0.08947309106588364,
+      "reward_std": 0.02952927676960826,
       "rewards/concensus_correctness_reward_func/mean": 0.0,
       "rewards/concensus_correctness_reward_func/std": 0.0,
       "rewards/consensus_reward_func/mean": 0.0,
@@ -39,8 +39,8 @@
       "rewards/cumulative_reward_2/std": 0.0,
       "rewards/final_correctness_reward_func/mean": 0.0,
       "rewards/final_correctness_reward_func/std": 0.0,
-      "rewards/question_recreation_reward_func/mean": 0.16552798449993134,
-      "rewards/question_recreation_reward_func/std": 0.06421760097146034,
+      "rewards/question_recreation_reward_func/mean": 0.08947309292852879,
+      "rewards/question_recreation_reward_func/std": 0.02744971076026559,
       "rewards/soft_format_reward_func/mean": 0.0,
       "rewards/soft_format_reward_func/std": 0.0,
       "rewards/strict_format_reward_func/mean": 0.0,
@@ -55,22 +55,22 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.25,
-      "completions/max_length": 256.0,
-      "completions/max_terminated_length": 162.5,
-      "completions/mean_length": 127.0,
-      "completions/mean_terminated_length": 84.0,
-      "completions/min_length": 26.5,
-      "completions/min_terminated_length": 26.5,
-      "epoch": 0.22857142857142856,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 10.76623821258545,
-      "kl": 0.00028998898051213473,
+      "completions/clipped_ratio": 0.125,
+      "completions/max_length": 130.5,
+      "completions/max_terminated_length": 35.5,
+      "completions/mean_length": 45.0,
+      "completions/mean_terminated_length": 16.541666984558105,
+      "completions/min_length": 6.5,
+      "completions/min_terminated_length": 6.5,
+      "epoch": 0.09090909090909091,
+      "frac_reward_zero_std": 0.25,
+      "grad_norm": 71.5344009399414,
+      "kl": 0.0013015866788919084,
       "learning_rate": 4.415111107797445e-07,
-      "loss": 0.2063,
-      "num_tokens": 3841.0,
-      "reward": 0.03576416149735451,
-      "reward_std": 0.00957817304879427,
+      "loss": -0.0174,
+      "num_tokens": 2703.0,
+      "reward": 0.08281245455145836,
+      "reward_std": 0.03817772259935737,
       "rewards/concensus_correctness_reward_func/mean": 0.0,
       "rewards/concensus_correctness_reward_func/std": 0.0,
       "rewards/consensus_reward_func/mean": 0.0,
@@ -79,8 +79,8 @@
       "rewards/cumulative_reward_2/std": 0.0,
       "rewards/final_correctness_reward_func/mean": 0.0,
       "rewards/final_correctness_reward_func/std": 0.0,
-      "rewards/question_recreation_reward_func/mean": 0.03576416149735451,
-      "rewards/question_recreation_reward_func/std": 0.011012929026037455,
+      "rewards/question_recreation_reward_func/mean": 0.08281246200203896,
+      "rewards/question_recreation_reward_func/std": 0.04765166528522968,
       "rewards/soft_format_reward_func/mean": 0.0,
       "rewards/soft_format_reward_func/std": 0.0,
       "rewards/strict_format_reward_func/mean": 0.0,
@@ -96,21 +96,21 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 209.5,
-      "completions/max_terminated_length": 209.5,
-      "completions/mean_length": 108.75,
-      "completions/mean_terminated_length": 108.75,
-      "completions/min_length": 21.5,
-      "completions/min_terminated_length": 21.5,
-      "epoch": 0.34285714285714286,
+      "completions/max_length": 177.5,
+      "completions/max_terminated_length": 177.5,
+      "completions/mean_length": 76.375,
+      "completions/mean_terminated_length": 76.375,
+      "completions/min_length": 23.5,
+      "completions/min_terminated_length": 23.5,
+      "epoch": 0.13636363636363635,
       "frac_reward_zero_std": 0.0,
-      "grad_norm": 82.63467407226562,
-      "kl": 0.02681938411842566,
+      "grad_norm": 60.853816986083984,
+      "kl": 0.0031654702906962484,
       "learning_rate": 2.934120444167326e-07,
-      "loss": 0.0717,
-      "num_tokens": 5735.0,
-      "reward": 0.017660686746239662,
-      "reward_std": 0.009620588505640626,
+      "loss": -0.0248,
+      "num_tokens": 4338.0,
+      "reward": 0.10044452454894781,
+      "reward_std": 0.021916877012699842,
       "rewards/concensus_correctness_reward_func/mean": 0.0,
       "rewards/concensus_correctness_reward_func/std": 0.0,
       "rewards/consensus_reward_func/mean": 0.0,
@@ -119,8 +119,8 @@
       "rewards/cumulative_reward_2/std": 0.0,
       "rewards/final_correctness_reward_func/mean": 0.0,
       "rewards/final_correctness_reward_func/std": 0.0,
-      "rewards/question_recreation_reward_func/mean": 0.017660685814917088,
-      "rewards/question_recreation_reward_func/std": 0.011452750652097166,
+      "rewards/question_recreation_reward_func/mean": 0.10044452454894781,
+      "rewards/question_recreation_reward_func/std": 0.024618458934128284,
       "rewards/soft_format_reward_func/mean": 0.0,
       "rewards/soft_format_reward_func/std": 0.0,
       "rewards/strict_format_reward_func/mean": 0.0,
@@ -135,22 +135,22 @@
       "clip_ratio/low_mean": 0.0,
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
-      "completions/clipped_ratio": 0.0,
-      "completions/max_length": 38.5,
-      "completions/max_terminated_length": 38.5,
-      "completions/mean_length": 22.375,
-      "completions/mean_terminated_length": 22.375,
-      "completions/min_length": 6.0,
-      "completions/min_terminated_length": 6.0,
-      "epoch": 0.45714285714285713,
+      "completions/clipped_ratio": 0.125,
+      "completions/max_length": 210.5,
+      "completions/max_terminated_length": 128.0,
+      "completions/mean_length": 76.5,
+      "completions/mean_terminated_length": 48.70833396911621,
+      "completions/min_length": 4.5,
+      "completions/min_terminated_length": 4.5,
+      "epoch": 0.18181818181818182,
       "frac_reward_zero_std": 0.25,
-      "grad_norm": 48.86420440673828,
-      "kl": 0.04224458080716431,
+      "grad_norm": 22.12053680419922,
+      "kl": 0.0013654041031259112,
       "learning_rate": 1.2500000000000005e-07,
-      "loss": -0.0623,
-      "num_tokens": 6938.0,
-      "reward": 0.01573411887511611,
-      "reward_std": 0.0049291979521512985,
+      "loss": 0.0056,
+      "num_tokens": 5974.0,
+      "reward": 0.029887909069657326,
+      "reward_std": 0.018557347357273102,
       "rewards/concensus_correctness_reward_func/mean": 0.0,
       "rewards/concensus_correctness_reward_func/std": 0.0,
       "rewards/consensus_reward_func/mean": 0.0,
@@ -159,8 +159,8 @@
       "rewards/cumulative_reward_2/std": 0.0,
       "rewards/final_correctness_reward_func/mean": 0.0,
       "rewards/final_correctness_reward_func/std": 0.0,
-      "rewards/question_recreation_reward_func/mean": 0.01573411887511611,
-      "rewards/question_recreation_reward_func/std": 0.0070181540213525295,
+      "rewards/question_recreation_reward_func/mean": 0.029887909069657326,
+      "rewards/question_recreation_reward_func/std": 0.025635237339884043,
       "rewards/soft_format_reward_func/mean": 0.0,
       "rewards/soft_format_reward_func/std": 0.0,
       "rewards/strict_format_reward_func/mean": 0.0,
@@ -176,21 +176,21 @@
       "clip_ratio/low_min": 0.0,
       "clip_ratio/region_mean": 0.0,
       "completions/clipped_ratio": 0.0,
-      "completions/max_length": 67.0,
-      "completions/max_terminated_length": 67.0,
-      "completions/mean_length": 27.375,
-      "completions/mean_terminated_length": 27.375,
-      "completions/min_length": 8.0,
-      "completions/min_terminated_length": 8.0,
-      "epoch": 0.5714285714285714,
-      "frac_reward_zero_std": 0.0,
-      "grad_norm": 30.950077056884766,
-      "kl": 0.22198594151996076,
+      "completions/max_length": 123.5,
+      "completions/max_terminated_length": 123.5,
+      "completions/mean_length": 41.625,
+      "completions/mean_terminated_length": 41.625,
+      "completions/min_length": 3.5,
+      "completions/min_terminated_length": 3.5,
+      "epoch": 0.22727272727272727,
+      "frac_reward_zero_std": 0.25,
+      "grad_norm": 64.85562133789062,
+      "kl": 0.0105830222601071,
       "learning_rate": 1.507684480352292e-08,
-      "loss": -0.1916,
-      "num_tokens": 8181.0,
-      "reward": 0.028241774067282677,
-      "reward_std": 0.009065406396985054,
+      "loss": -0.0475,
+      "num_tokens": 7331.0,
+      "reward": 0.02319902554154396,
+      "reward_std": 0.008738046046346426,
       "rewards/concensus_correctness_reward_func/mean": 0.0,
       "rewards/concensus_correctness_reward_func/std": 0.0,
       "rewards/consensus_reward_func/mean": 0.0,
@@ -199,8 +199,8 @@
       "rewards/cumulative_reward_2/std": 0.0,
       "rewards/final_correctness_reward_func/mean": 0.0,
       "rewards/final_correctness_reward_func/std": 0.0,
-      "rewards/question_recreation_reward_func/mean": 0.028241775929927826,
-      "rewards/question_recreation_reward_func/std": 0.011007396038621664,
+      "rewards/question_recreation_reward_func/mean": 0.02319902554154396,
+      "rewards/question_recreation_reward_func/std": 0.010856612119823694,
       "rewards/soft_format_reward_func/mean": 0.0,
       "rewards/soft_format_reward_func/std": 0.0,
       "rewards/strict_format_reward_func/mean": 0.0,
@@ -210,18 +210,18 @@
       "step": 10
     },
     {
-      "epoch": 0.5714285714285714,
+      "epoch": 0.22727272727272727,
       "step": 10,
       "total_flos": 0.0,
-      "train_loss": 0.045195698738098145,
-      "train_runtime": 10476.7072,
+      "train_loss": -0.030795110389590263,
+      "train_runtime": 10341.1372,
       "train_samples_per_second": 0.004,
       "train_steps_per_second": 0.001
     }
   ],
   "logging_steps": 2,
   "max_steps": 10,
-  "num_input_tokens_seen": 8181,
+  "num_input_tokens_seen": 7331,
   "num_train_epochs": 1,
   "save_steps": 10,
   "stateful_callbacks": {