Model save

README.md

@@ -27,7 +27,7 @@ print(output["generated_text"])
 
 ## Training procedure
 
-[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/haokungui-hong-kong-university-of-science-and-technology/huggingface/runs/f09hw7av) 
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/haokungui-hong-kong-university-of-science-and-technology/huggingface/runs/jk6prsw3) 
 
 
 This model was trained with GRPO, a method introduced in [DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models](https://huggingface.co/papers/2402.03300).
@@ -35,9 +35,9 @@ This model was trained with GRPO, a method introduced in [DeepSeekMath: Pushing
 ### Framework versions
 
 - TRL: 0.16.0.dev0
-- Transformers: 4.49.0.dev0
+- Transformers: 4.50.0.dev0
 - Pytorch: 2.5.1
-- Datasets: 3.3.0
+- Datasets: 3.3.2
 - Tokenizers: 0.21.0
 
 ## Citations

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": -0.00021707314596730205,
-    "train_runtime": 6919.2787,
+    "train_loss": 0.05106422754711118,
+    "train_runtime": 6823.4661,
     "train_samples": 7500,
-    "train_samples_per_second": 1.084,
-    "train_steps_per_second": 0.008
+    "train_samples_per_second": 1.099,
+    "train_steps_per_second": 0.009
 }

config.json

@@ -22,7 +22,7 @@
   "sliding_window": 4096,
   "tie_word_embeddings": false,
   "torch_dtype": "bfloat16",
-  "transformers_version": "4.49.0.dev0",
+  "transformers_version": "4.50.0.dev0",
   "use_cache": false,
   "use_mrope": false,
   "use_sliding_window": false,

generation_config.json

@@ -2,5 +2,5 @@
   "bos_token_id": 151643,
   "eos_token_id": 151643,
   "max_new_tokens": 2048,
-  "transformers_version": "4.49.0.dev0"
+  "transformers_version": "4.50.0.dev0"
 }

model-00001-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d09b07690e14ea60f95ff7c3dc4258f9b72af70769f68446415af4a8a5494d21
+oid sha256:500fdf241a5bd22e2d5b097fb79fbf89bf04ff7aec09676a3c9d431055960c76
 size 4877660776

model-00002-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f5f3dee11356461e7f6a397af882541b234f550fd1de5f7c27460cb487f314e0
+oid sha256:1943eb3f4eda46b078c73ab62c768b0210a292c7835ea0ed779e1f005a834eae
 size 4932751008

model-00003-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9f4d2664767ef7f5583950f6da93e6e0b22fe5d182069746f6170fb483332052
+oid sha256:aace49adaf8c20390da0938e69bd9f53dad03662b47066d5f94eb6effdec3fda
 size 4330865200

model-00004-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:58eca92f8ca49d1a98a0d9d47e9434a04253c1a7f4ee8faf31c95651511d547d
+oid sha256:4503159aee19406f709e504ac4c0e2b530d50b19b0dc3e312ca04a929312c0ac
 size 1089994880

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": -0.00021707314596730205,
-    "train_runtime": 6919.2787,
+    "train_loss": 0.05106422754711118,
+    "train_runtime": 6823.4661,
     "train_samples": 7500,
-    "train_samples_per_second": 1.084,
-    "train_steps_per_second": 0.008
+    "train_samples_per_second": 1.099,
+    "train_steps_per_second": 0.009
 }

trainer_state.json

@@ -9,162 +9,174 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 616.8576179504395,
+      "clip_ratio": 0.0,
+      "completion_length": 613.3375259399414,
       "epoch": 0.08528784648187633,
-      "grad_norm": 3.189967632293701,
-      "kl": 0.0002296924591064453,
+      "grad_norm": 0.5308949947357178,
+      "kl": 0.00026957988739013673,
       "learning_rate": 2.5e-06,
-      "loss": 0.0,
-      "reward": 0.6506696678698063,
-      "reward_std": 0.3288835626095533,
-      "rewards/accuracy_reward": 0.6502232395112515,
+      "loss": 0.0422,
+      "reward": 0.6412946701049804,
+      "reward_std": 0.33645378351211547,
+      "rewards/accuracy_reward": 0.6408482432365418,
       "rewards/format_reward": 0.00044642859138548373,
       "step": 5
     },
     {
-      "completion_length": 616.0665504455567,
+      "clip_ratio": 0.0,
+      "completion_length": 611.9799339294434,
       "epoch": 0.17057569296375266,
-      "grad_norm": 1.5170903205871582,
-      "kl": 0.017084503173828126,
+      "grad_norm": 3.3914427757263184,
+      "kl": 0.002694988250732422,
       "learning_rate": 2.956412726139078e-06,
-      "loss": 0.0007,
-      "reward": 0.7037946730852127,
-      "reward_std": 0.28115556947886944,
-      "rewards/accuracy_reward": 0.7037946730852127,
+      "loss": 0.0675,
+      "reward": 0.7015625357627868,
+      "reward_std": 0.2750119812786579,
+      "rewards/accuracy_reward": 0.7015625357627868,
       "rewards/format_reward": 0.0,
       "step": 10
     },
     {
-      "completion_length": 619.1498023986817,
+      "clip_ratio": 0.0,
+      "completion_length": 598.9616317749023,
       "epoch": 0.255863539445629,
-      "grad_norm": 0.1955304741859436,
-      "kl": 0.015854263305664064,
+      "grad_norm": 0.3315448462963104,
+      "kl": 0.0043125152587890625,
       "learning_rate": 2.7836719084521715e-06,
-      "loss": 0.0006,
-      "reward": 0.7582589656114578,
-      "reward_std": 0.21650202702730895,
-      "rewards/accuracy_reward": 0.7582589656114578,
+      "loss": 0.0621,
+      "reward": 0.764285746216774,
+      "reward_std": 0.21017537489533425,
+      "rewards/accuracy_reward": 0.764285746216774,
       "rewards/format_reward": 0.0,
       "step": 15
     },
     {
-      "completion_length": 609.8542671203613,
+      "clip_ratio": 0.0,
+      "completion_length": 588.717440032959,
       "epoch": 0.3411513859275053,
-      "grad_norm": 0.2464970499277115,
-      "kl": 0.003617095947265625,
+      "grad_norm": 0.8005821704864502,
+      "kl": 0.004384231567382812,
       "learning_rate": 2.4946839873611927e-06,
-      "loss": 0.0001,
-      "reward": 0.7537946790456772,
-      "reward_std": 0.22740934304893018,
-      "rewards/accuracy_reward": 0.7537946790456772,
+      "loss": 0.0533,
+      "reward": 0.7665178924798965,
+      "reward_std": 0.1955874715000391,
+      "rewards/accuracy_reward": 0.7665178924798965,
       "rewards/format_reward": 0.0,
       "step": 20
     },
     {
-      "completion_length": 613.51654586792,
+      "clip_ratio": 0.0,
+      "completion_length": 592.9370819091797,
       "epoch": 0.42643923240938164,
-      "grad_norm": 0.8888379335403442,
-      "kl": 0.005077743530273437,
+      "grad_norm": 0.7205191850662231,
+      "kl": 0.004600143432617188,
       "learning_rate": 2.1156192081791355e-06,
-      "loss": 0.0002,
-      "reward": 0.7627232491970062,
-      "reward_std": 0.18942257491871714,
-      "rewards/accuracy_reward": 0.7627232491970062,
+      "loss": 0.045,
+      "reward": 0.7629464656114578,
+      "reward_std": 0.18942818641662598,
+      "rewards/accuracy_reward": 0.7629464656114578,
       "rewards/format_reward": 0.0,
       "step": 25
     },
     {
-      "completion_length": 612.561190032959,
+      "clip_ratio": 0.0,
+      "completion_length": 598.0611862182617,
       "epoch": 0.511727078891258,
-      "grad_norm": 0.6480019688606262,
-      "kl": 0.0041637420654296875,
+      "grad_norm": 0.6142191886901855,
+      "kl": 0.042702865600585935,
       "learning_rate": 1.6808050203829845e-06,
-      "loss": 0.0002,
-      "reward": 0.7622768208384514,
-      "reward_std": 0.18584611341357232,
-      "rewards/accuracy_reward": 0.7622768208384514,
+      "loss": 0.0428,
+      "reward": 0.7551339700818062,
+      "reward_std": 0.1889859580434859,
+      "rewards/accuracy_reward": 0.7551339700818062,
       "rewards/format_reward": 0.0,
       "step": 30
     },
     {
-      "completion_length": 601.9710105895996,
+      "clip_ratio": 0.0,
+      "completion_length": 593.2520332336426,
       "epoch": 0.5970149253731343,
-      "grad_norm": 0.19176428020000458,
-      "kl": 0.003966903686523438,
+      "grad_norm": 10.091300964355469,
+      "kl": 0.4265655517578125,
       "learning_rate": 1.2296174432791415e-06,
-      "loss": 0.0002,
-      "reward": 0.755357176810503,
-      "reward_std": 0.16201179698109627,
-      "rewards/accuracy_reward": 0.755357176810503,
+      "loss": 0.0673,
+      "reward": 0.7410714656114579,
+      "reward_std": 0.179294466227293,
+      "rewards/accuracy_reward": 0.7410714656114579,
       "rewards/format_reward": 0.0,
       "step": 35
     },
     {
-      "completion_length": 588.7897583007813,
+      "clip_ratio": 0.0,
+      "completion_length": 577.117886352539,
       "epoch": 0.6823027718550106,
-      "grad_norm": 0.5784619450569153,
-      "kl": 0.0035259246826171873,
+      "grad_norm": 81.48939514160156,
+      "kl": 0.06499862670898438,
       "learning_rate": 8.029152419343472e-07,
-      "loss": 0.0001,
-      "reward": 0.7720982477068901,
-      "reward_std": 0.17176652811467646,
-      "rewards/accuracy_reward": 0.7720982477068901,
+      "loss": 0.0492,
+      "reward": 0.7669643267989159,
+      "reward_std": 0.17745222924277188,
+      "rewards/accuracy_reward": 0.7669643267989159,
       "rewards/format_reward": 0.0,
       "step": 40
     },
     {
-      "completion_length": 608.5564971923828,
+      "clip_ratio": 0.0,
+      "completion_length": 594.6520378112793,
       "epoch": 0.767590618336887,
-      "grad_norm": 0.2929813861846924,
-      "kl": 0.0034820556640625,
+      "grad_norm": 59.926361083984375,
+      "kl": 0.22182693481445312,
       "learning_rate": 4.3933982822017883e-07,
-      "loss": 0.0001,
-      "reward": 0.7569196790456771,
-      "reward_std": 0.17854769751429558,
-      "rewards/accuracy_reward": 0.7569196790456771,
+      "loss": 0.0426,
+      "reward": 0.7529018267989158,
+      "reward_std": 0.1793302634730935,
+      "rewards/accuracy_reward": 0.7529018267989158,
       "rewards/format_reward": 0.0,
       "step": 45
     },
     {
-      "completion_length": 609.3294929504394,
+      "clip_ratio": 0.0,
+      "completion_length": 599.8544906616211,
       "epoch": 0.8528784648187633,
-      "grad_norm": 0.8820053339004517,
-      "kl": 0.003635406494140625,
+      "grad_norm": 0.3387981653213501,
+      "kl": 0.171990966796875,
       "learning_rate": 1.718159615201853e-07,
-      "loss": 0.0001,
-      "reward": 0.7529018223285675,
-      "reward_std": 0.18069615559652447,
-      "rewards/accuracy_reward": 0.7529018223285675,
+      "loss": 0.0534,
+      "reward": 0.7386161029338837,
+      "reward_std": 0.18926974572241306,
+      "rewards/accuracy_reward": 0.7386161029338837,
       "rewards/format_reward": 0.0,
       "step": 50
     },
     {
-      "completion_length": 597.9283767700196,
+      "clip_ratio": 0.0,
+      "completion_length": 590.5582893371582,
       "epoch": 0.9381663113006397,
-      "grad_norm": 0.2659400999546051,
-      "kl": 0.0099273681640625,
+      "grad_norm": 7.056384563446045,
+      "kl": 0.15600433349609374,
       "learning_rate": 2.4570139579284723e-08,
-      "loss": 0.0004,
-      "reward": 0.7859375342726708,
-      "reward_std": 0.17906498536467552,
-      "rewards/accuracy_reward": 0.7859375342726708,
+      "loss": 0.0572,
+      "reward": 0.7738839611411095,
+      "reward_std": 0.18812613375484943,
+      "rewards/accuracy_reward": 0.7738839611411095,
       "rewards/format_reward": 0.0,
       "step": 55
     },
     {
-      "completion_length": 592.1878992716471,
+      "clip_ratio": 0.0,
+      "completion_length": 576.9970359802246,
       "epoch": 0.9893390191897654,
-      "kl": 0.0038731892903645835,
-      "reward": 0.772693489988645,
-      "reward_std": 0.1824363498017192,
-      "rewards/accuracy_reward": 0.772693489988645,
+      "kl": 0.025601704915364582,
+      "reward": 0.7589286093910536,
+      "reward_std": 0.1643078220076859,
+      "rewards/accuracy_reward": 0.7589286093910536,
       "rewards/format_reward": 0.0,
       "step": 58,
       "total_flos": 0.0,
-      "train_loss": -0.00021707314596730205,
-      "train_runtime": 6919.2787,
-      "train_samples_per_second": 1.084,
-      "train_steps_per_second": 0.008
+      "train_loss": 0.05106422754711118,
+      "train_runtime": 6823.4661,
+      "train_samples_per_second": 1.099,
+      "train_steps_per_second": 0.009
     }
   ],
   "logging_steps": 5,

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f18f62535d61e36f08936af2cc23013ccba1fa668e22155a15ff711fb1075a93
-size 7608
+oid sha256:f50a1c1b961775d6dcb6bb20b19456022d781be9766c958e14e48133d49616e8
+size 7736