Model save

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,67 @@
+---
+library_name: transformers
+model_name: Qwen2.5-1.5B-Open-R1-GRPO
+tags:
+- generated_from_trainer
+- trl
+- grpo
+licence: license
+---
+
+# Model Card for Qwen2.5-1.5B-Open-R1-GRPO
+
+This model is a fine-tuned version of [None](https://huggingface.co/None).
+It has been trained using [TRL](https://github.com/huggingface/trl).
+
+## Quick start
+
+```python
+from transformers import pipeline
+
+question = "If you had a time machine, but could only go to the past or the future once and never return, which would you choose and why?"
+generator = pipeline("text-generation", model="LuyiCui/Qwen2.5-1.5B-Open-R1-GRPO", device="cuda")
+output = generator([{"role": "user", "content": question}], max_new_tokens=128, return_full_text=False)[0]
+print(output["generated_text"])
+```
+
+## Training procedure
+
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/cuiluyi/huggingface/runs/zkm542w0) 
+
+
+This model was trained with GRPO, a method introduced in [DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models](https://huggingface.co/papers/2402.03300).
+
+### Framework versions
+
+- TRL: 0.15.0.dev0
+- Transformers: 4.49.0.dev0
+- Pytorch: 2.5.1
+- Datasets: 3.2.0
+- Tokenizers: 0.21.0
+
+## Citations
+
+Cite GRPO as:
+
+```bibtex
+@article{zhihong2024deepseekmath,
+    title        = {{DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models}},
+    author       = {Zhihong Shao and Peiyi Wang and Qihao Zhu and Runxin Xu and Junxiao Song and Mingchuan Zhang and Y. K. Li and Y. Wu and Daya Guo},
+    year         = 2024,
+    eprint       = {arXiv:2402.03300},
+}
+
+```
+
+Cite TRL as:
+    
+```bibtex
+@misc{vonwerra2022trl,
+	title        = {{TRL: Transformer Reinforcement Learning}},
+	author       = {Leandro von Werra and Younes Belkada and Lewis Tunstall and Edward Beeching and Tristan Thrush and Nathan Lambert and Shengyi Huang and Kashif Rasul and Quentin Gallouédec},
+	year         = 2020,
+	journal      = {GitHub repository},
+	publisher    = {GitHub},
+	howpublished = {\url{https://github.com/huggingface/trl}}
+}
+```

added_tokens.json

@@ -0,0 +1,24 @@
+{
+  "</tool_call>": 151658,
+  "<tool_call>": 151657,
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
+  "<|endoftext|>": 151643,
+  "<|file_sep|>": 151664,
+  "<|fim_middle|>": 151660,
+  "<|fim_pad|>": 151662,
+  "<|fim_prefix|>": 151659,
+  "<|fim_suffix|>": 151661,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image_pad|>": 151655,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|repo_name|>": 151663,
+  "<|video_pad|>": 151656,
+  "<|vision_end|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vision_start|>": 151652
+}

all_results.json

@@ -0,0 +1,8 @@
+{
+    "total_flos": 0.0,
+    "train_loss": 11.024624033321947,
+    "train_runtime": 8177.44,
+    "train_samples": 2000,
+    "train_samples_per_second": 0.245,
+    "train_steps_per_second": 0.031
+}

config.json

@@ -0,0 +1,29 @@
+{
+  "_name_or_path": "/data/cuiluyi/resources/models/Qwen/Qwen2.5-1.5B-Instruct",
+  "architectures": [
+    "Qwen2ForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 151643,
+  "eos_token_id": 151645,
+  "hidden_act": "silu",
+  "hidden_size": 1536,
+  "initializer_range": 0.02,
+  "intermediate_size": 8960,
+  "max_position_embeddings": 32768,
+  "max_window_layers": 21,
+  "model_type": "qwen2",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 28,
+  "num_key_value_heads": 2,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 1000000.0,
+  "sliding_window": null,
+  "tie_word_embeddings": true,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.49.0.dev0",
+  "use_cache": false,
+  "use_sliding_window": false,
+  "vocab_size": 151936
+}

generation_config.json

@@ -0,0 +1,14 @@
+{
+  "bos_token_id": 151643,
+  "do_sample": true,
+  "eos_token_id": [
+    151645,
+    151643
+  ],
+  "pad_token_id": 151643,
+  "repetition_penalty": 1.1,
+  "temperature": 0.7,
+  "top_k": 20,
+  "top_p": 0.8,
+  "transformers_version": "4.49.0.dev0"
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1585b87c3a015ccc1d0dedcdfb24d2ae2c12339d73e1cb8b79de7fc1ac57d613
+size 3087467144

special_tokens_map.json

@@ -0,0 +1,31 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5eee858c5123a4279c3e1f7b81247343f356ac767940b2692a928ad929543214
+size 11422063

tokenizer_config.json

@@ -0,0 +1,209 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151657": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151658": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151659": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151660": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151661": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151662": {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151663": {
+      "content": "<|repo_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151664": {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "bos_token": null,
+  "chat_template": "{%- if tools %}\n    {{- '<|im_start|>system\\n' }}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- messages[0]['content'] }}\n    {%- else %}\n        {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}\n    {%- endif %}\n    {{- \"\\n\\n# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>\" }}\n    {%- for tool in tools %}\n        {{- \"\\n\" }}\n        {{- tool | tojson }}\n    {%- endfor %}\n    {{- \"\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}\\n</tool_call><|im_end|>\\n\" }}\n{%- else %}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- '<|im_start|>system\\n' + messages[0]['content'] + '<|im_end|>\\n' }}\n    {%- else %}\n        {{- '<|im_start|>system\\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\\n' }}\n    {%- endif %}\n{%- endif %}\n{%- for message in messages %}\n    {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) or (message.role == \"assistant\" and not message.tool_calls) %}\n        {{- '<|im_start|>' + message.role + '\\n' + message.content + '<|im_end|>' + '\\n' }}\n    {%- elif message.role == \"assistant\" %}\n        {{- '<|im_start|>' + message.role }}\n        {%- if message.content %}\n            {{- '\\n' + message.content }}\n        {%- endif %}\n        {%- for tool_call in message.tool_calls %}\n            {%- if tool_call.function is defined %}\n                {%- set tool_call = tool_call.function %}\n            {%- endif %}\n            {{- '\\n<tool_call>\\n{\"name\": \"' }}\n            {{- tool_call.name }}\n            {{- '\", \"arguments\": ' }}\n            {{- tool_call.arguments | tojson }}\n            {{- '}\\n</tool_call>' }}\n        {%- endfor %}\n        {{- '<|im_end|>\\n' }}\n    {%- elif message.role == \"tool\" %}\n        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != \"tool\") %}\n            {{- '<|im_start|>user' }}\n        {%- endif %}\n        {{- '\\n<tool_response>\\n' }}\n        {{- message.content }}\n        {{- '\\n</tool_response>' }}\n        {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n            {{- '<|im_end|>\\n' }}\n        {%- endif %}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '<|im_start|>assistant\\n' }}\n{%- endif %}\n",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": {},
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "padding_side": "left",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

train_results.json

@@ -0,0 +1,8 @@
+{
+    "total_flos": 0.0,
+    "train_loss": 11.024624033321947,
+    "train_runtime": 8177.44,
+    "train_samples": 2000,
+    "train_samples_per_second": 0.245,
+    "train_steps_per_second": 0.031
+}

trainer_state.json

@@ -0,0 +1,720 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "eval_steps": 100,
+  "global_step": 250,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "completion_length": 460.1208465576172,
+      "epoch": 0.02,
+      "grad_norm": 1.802279947354497,
+      "kl": 0.00032548904418945314,
+      "learning_rate": 4.000000000000001e-06,
+      "loss": 0.0,
+      "reward": 0.6041666805744171,
+      "reward_std": 0.4559453740715981,
+      "rewards/accuracy_reward": 0.14583333879709243,
+      "rewards/format_reward": 0.45833334922790525,
+      "step": 5
+    },
+    {
+      "completion_length": 193.0666717529297,
+      "epoch": 0.04,
+      "grad_norm": 2.646688833040104,
+      "kl": 0.115576171875,
+      "learning_rate": 8.000000000000001e-06,
+      "loss": 0.0046,
+      "reward": 0.9041666865348816,
+      "reward_std": 0.29244016036391257,
+      "rewards/accuracy_reward": 0.06250000074505806,
+      "rewards/format_reward": 0.8416666805744171,
+      "step": 10
+    },
+    {
+      "completion_length": 146.40833892822266,
+      "epoch": 0.06,
+      "grad_norm": 3.596725518738137,
+      "kl": 0.151220703125,
+      "learning_rate": 1.2e-05,
+      "loss": 0.006,
+      "reward": 0.9833333611488342,
+      "reward_std": 0.15119582265615464,
+      "rewards/accuracy_reward": 0.02916666716337204,
+      "rewards/format_reward": 0.9541666865348816,
+      "step": 15
+    },
+    {
+      "completion_length": 177.0041717529297,
+      "epoch": 0.08,
+      "grad_norm": 0.904416286564434,
+      "kl": 0.14755859375,
+      "learning_rate": 1.6000000000000003e-05,
+      "loss": 0.0059,
+      "reward": 1.0125000298023223,
+      "reward_std": 0.20182792022824286,
+      "rewards/accuracy_reward": 0.06250000149011611,
+      "rewards/format_reward": 0.950000011920929,
+      "step": 20
+    },
+    {
+      "completion_length": 232.3125030517578,
+      "epoch": 0.1,
+      "grad_norm": 1.1878978579316373,
+      "kl": 0.128955078125,
+      "learning_rate": 2e-05,
+      "loss": 0.0052,
+      "reward": 1.1083333492279053,
+      "reward_std": 0.26786036118865014,
+      "rewards/accuracy_reward": 0.16250000782310964,
+      "rewards/format_reward": 0.9458333492279053,
+      "step": 25
+    },
+    {
+      "completion_length": 278.2250061035156,
+      "epoch": 0.12,
+      "grad_norm": 1.1015274367009757,
+      "kl": 0.11708984375,
+      "learning_rate": 1.9975640502598243e-05,
+      "loss": 0.0047,
+      "reward": 1.0583333432674409,
+      "reward_std": 0.4198418617248535,
+      "rewards/accuracy_reward": 0.17916667386889457,
+      "rewards/format_reward": 0.8791666924953461,
+      "step": 30
+    },
+    {
+      "completion_length": 321.1458450317383,
+      "epoch": 0.14,
+      "grad_norm": 1.4687757867130748,
+      "kl": 0.175,
+      "learning_rate": 1.9902680687415704e-05,
+      "loss": 0.007,
+      "reward": 1.1125000238418579,
+      "reward_std": 0.3428553894162178,
+      "rewards/accuracy_reward": 0.17916667312383652,
+      "rewards/format_reward": 0.9333333611488343,
+      "step": 35
+    },
+    {
+      "completion_length": 403.1916839599609,
+      "epoch": 0.16,
+      "grad_norm": 5.101977047647239,
+      "kl": 0.21806640625,
+      "learning_rate": 1.9781476007338058e-05,
+      "loss": 0.0087,
+      "reward": 1.208333384990692,
+      "reward_std": 0.3654559038579464,
+      "rewards/accuracy_reward": 0.2708333428949118,
+      "rewards/format_reward": 0.9375000178813935,
+      "step": 40
+    },
+    {
+      "completion_length": 373.4916763305664,
+      "epoch": 0.18,
+      "grad_norm": 0.8111764758573549,
+      "kl": 0.27001953125,
+      "learning_rate": 1.961261695938319e-05,
+      "loss": 0.0108,
+      "reward": 1.1541666984558105,
+      "reward_std": 0.3001556023955345,
+      "rewards/accuracy_reward": 0.2000000063329935,
+      "rewards/format_reward": 0.9541666805744171,
+      "step": 45
+    },
+    {
+      "completion_length": 330.6041732788086,
+      "epoch": 0.2,
+      "grad_norm": 1.715639326948892,
+      "kl": 0.2083984375,
+      "learning_rate": 1.9396926207859085e-05,
+      "loss": 0.0083,
+      "reward": 1.2000000476837158,
+      "reward_std": 0.35943536311388014,
+      "rewards/accuracy_reward": 0.2583333417773247,
+      "rewards/format_reward": 0.9416666805744172,
+      "step": 50
+    },
+    {
+      "completion_length": 250.3041778564453,
+      "epoch": 0.22,
+      "grad_norm": 6.204624879520759,
+      "kl": 0.4203125,
+      "learning_rate": 1.913545457642601e-05,
+      "loss": 0.0168,
+      "reward": 1.087500023841858,
+      "reward_std": 0.3333131104707718,
+      "rewards/accuracy_reward": 0.17500001043081284,
+      "rewards/format_reward": 0.912500011920929,
+      "step": 55
+    },
+    {
+      "completion_length": 337.4125091552734,
+      "epoch": 0.24,
+      "grad_norm": 12684.323654415588,
+      "kl": 13734.98203125,
+      "learning_rate": 1.8829475928589272e-05,
+      "loss": 548.7591,
+      "reward": 0.791666692495346,
+      "reward_std": 0.488527849316597,
+      "rewards/accuracy_reward": 0.0958333346992731,
+      "rewards/format_reward": 0.6958333522081375,
+      "step": 60
+    },
+    {
+      "completion_length": 531.7333465576172,
+      "epoch": 0.26,
+      "grad_norm": 8.44517412426292,
+      "kl": 1.90859375,
+      "learning_rate": 1.848048096156426e-05,
+      "loss": 0.0764,
+      "reward": 0.4291666768491268,
+      "reward_std": 0.5041220858693123,
+      "rewards/accuracy_reward": 0.0666666679084301,
+      "rewards/format_reward": 0.3625000067055225,
+      "step": 65
+    },
+    {
+      "completion_length": 289.68334197998047,
+      "epoch": 0.28,
+      "grad_norm": 10.463530390230416,
+      "kl": 1.259765625,
+      "learning_rate": 1.8090169943749477e-05,
+      "loss": 0.0505,
+      "reward": 0.791666692495346,
+      "reward_std": 0.50125632584095,
+      "rewards/accuracy_reward": 0.09583333618938923,
+      "rewards/format_reward": 0.6958333492279053,
+      "step": 70
+    },
+    {
+      "completion_length": 255.75833740234376,
+      "epoch": 0.3,
+      "grad_norm": 3.5149825133274617,
+      "kl": 0.7419921875,
+      "learning_rate": 1.766044443118978e-05,
+      "loss": 0.0297,
+      "reward": 0.9875000238418579,
+      "reward_std": 0.3046964339911938,
+      "rewards/accuracy_reward": 0.08333333432674409,
+      "rewards/format_reward": 0.9041666805744171,
+      "step": 75
+    },
+    {
+      "completion_length": 255.90000610351564,
+      "epoch": 0.32,
+      "grad_norm": 4.453741845912516,
+      "kl": 0.38984375,
+      "learning_rate": 1.7193398003386514e-05,
+      "loss": 0.0156,
+      "reward": 1.075000035762787,
+      "reward_std": 0.20900286808609964,
+      "rewards/accuracy_reward": 0.11250000298023224,
+      "rewards/format_reward": 0.9625000298023224,
+      "step": 80
+    },
+    {
+      "completion_length": 246.9666763305664,
+      "epoch": 0.34,
+      "grad_norm": 13.40541291266616,
+      "kl": 2.200390625,
+      "learning_rate": 1.6691306063588583e-05,
+      "loss": 0.0883,
+      "reward": 1.0416666984558105,
+      "reward_std": 0.3525183081626892,
+      "rewards/accuracy_reward": 0.14166667126119137,
+      "rewards/format_reward": 0.9000000178813934,
+      "step": 85
+    },
+    {
+      "completion_length": 230.99167175292968,
+      "epoch": 0.36,
+      "grad_norm": 3.056906791586755,
+      "kl": 0.9048828125,
+      "learning_rate": 1.6156614753256583e-05,
+      "loss": 0.0362,
+      "reward": 1.1375000476837158,
+      "reward_std": 0.2906723290681839,
+      "rewards/accuracy_reward": 0.18333334103226662,
+      "rewards/format_reward": 0.9541666805744171,
+      "step": 90
+    },
+    {
+      "completion_length": 234.40834045410156,
+      "epoch": 0.38,
+      "grad_norm": 1.307857007062594,
+      "kl": 0.4330078125,
+      "learning_rate": 1.5591929034707468e-05,
+      "loss": 0.0173,
+      "reward": 1.137500035762787,
+      "reward_std": 0.24904104620218276,
+      "rewards/accuracy_reward": 0.18333333730697632,
+      "rewards/format_reward": 0.9541666865348816,
+      "step": 95
+    },
+    {
+      "completion_length": 229.05833892822267,
+      "epoch": 0.4,
+      "grad_norm": 7.704130617990992,
+      "kl": 0.6828125,
+      "learning_rate": 1.5000000000000002e-05,
+      "loss": 0.0273,
+      "reward": 1.066666692495346,
+      "reward_std": 0.46584135591983794,
+      "rewards/accuracy_reward": 0.20416667498648167,
+      "rewards/format_reward": 0.8625000298023224,
+      "step": 100
+    },
+    {
+      "epoch": 0.4,
+      "eval_completion_length": 239.04327627328726,
+      "eval_kl": 6.236778846153846,
+      "eval_loss": 0.2575533390045166,
+      "eval_reward": 0.8733974649355962,
+      "eval_reward_std": 0.48696746505223787,
+      "eval_rewards/accuracy_reward": 0.15705128773473775,
+      "eval_rewards/format_reward": 0.7163461721860446,
+      "eval_runtime": 122.539,
+      "eval_samples_per_second": 0.808,
+      "eval_steps_per_second": 0.024,
+      "step": 100
+    },
+    {
+      "completion_length": 242.44583892822266,
+      "epoch": 0.42,
+      "grad_norm": 11.103519186469361,
+      "kl": 3.2890625,
+      "learning_rate": 1.4383711467890776e-05,
+      "loss": 0.1314,
+      "reward": 0.7958333551883697,
+      "reward_std": 0.49174695312976835,
+      "rewards/accuracy_reward": 0.1125000037252903,
+      "rewards/format_reward": 0.6833333551883698,
+      "step": 105
+    },
+    {
+      "completion_length": 269.5166717529297,
+      "epoch": 0.44,
+      "grad_norm": 10.198571524313941,
+      "kl": 0.8923828125,
+      "learning_rate": 1.3746065934159123e-05,
+      "loss": 0.0357,
+      "reward": 0.837500023841858,
+      "reward_std": 0.43169391751289365,
+      "rewards/accuracy_reward": 0.08750000037252903,
+      "rewards/format_reward": 0.7500000178813935,
+      "step": 110
+    },
+    {
+      "completion_length": 216.58333892822264,
+      "epoch": 0.46,
+      "grad_norm": 105.22697678836317,
+      "kl": 1.768359375,
+      "learning_rate": 1.3090169943749475e-05,
+      "loss": 0.0707,
+      "reward": 0.9041666984558105,
+      "reward_std": 0.4620524659752846,
+      "rewards/accuracy_reward": 0.11250000223517417,
+      "rewards/format_reward": 0.7916666805744171,
+      "step": 115
+    },
+    {
+      "completion_length": 221.1541717529297,
+      "epoch": 0.48,
+      "grad_norm": 6606.088155209928,
+      "kl": 27.155078125,
+      "learning_rate": 1.2419218955996677e-05,
+      "loss": 1.0907,
+      "reward": 1.0500000417232513,
+      "reward_std": 0.3967551693320274,
+      "rewards/accuracy_reward": 0.17916667573153972,
+      "rewards/format_reward": 0.8708333551883698,
+      "step": 120
+    },
+    {
+      "completion_length": 236.00834045410156,
+      "epoch": 0.5,
+      "grad_norm": 8.287123875687264,
+      "kl": 0.7736328125,
+      "learning_rate": 1.1736481776669307e-05,
+      "loss": 0.0309,
+      "reward": 1.1125000417232513,
+      "reward_std": 0.3053982339799404,
+      "rewards/accuracy_reward": 0.1708333384245634,
+      "rewards/format_reward": 0.9416666924953461,
+      "step": 125
+    },
+    {
+      "completion_length": 256.1875061035156,
+      "epoch": 0.52,
+      "grad_norm": 5.02200322357131,
+      "kl": 0.4322265625,
+      "learning_rate": 1.1045284632676535e-05,
+      "loss": 0.0173,
+      "reward": 1.1541667222976684,
+      "reward_std": 0.2508885085582733,
+      "rewards/accuracy_reward": 0.17083333656191826,
+      "rewards/format_reward": 0.9833333492279053,
+      "step": 130
+    },
+    {
+      "completion_length": 276.4541763305664,
+      "epoch": 0.54,
+      "grad_norm": 4.384174961816704,
+      "kl": 0.4177734375,
+      "learning_rate": 1.0348994967025012e-05,
+      "loss": 0.0167,
+      "reward": 1.1083333611488342,
+      "reward_std": 0.2354046382009983,
+      "rewards/accuracy_reward": 0.1375000026077032,
+      "rewards/format_reward": 0.9708333432674408,
+      "step": 135
+    },
+    {
+      "completion_length": 268.2125076293945,
+      "epoch": 0.56,
+      "grad_norm": 3.342401293663943,
+      "kl": 0.35498046875,
+      "learning_rate": 9.651005032974994e-06,
+      "loss": 0.0142,
+      "reward": 1.1041667103767394,
+      "reward_std": 0.2977468356490135,
+      "rewards/accuracy_reward": 0.15416667200624942,
+      "rewards/format_reward": 0.9500000238418579,
+      "step": 140
+    },
+    {
+      "completion_length": 243.7666748046875,
+      "epoch": 0.58,
+      "grad_norm": 10.249167956214594,
+      "kl": 1.1703125,
+      "learning_rate": 8.954715367323468e-06,
+      "loss": 0.0468,
+      "reward": 1.0458333671092988,
+      "reward_std": 0.38689075857400895,
+      "rewards/accuracy_reward": 0.1458333373069763,
+      "rewards/format_reward": 0.9000000178813934,
+      "step": 145
+    },
+    {
+      "completion_length": 226.80834197998047,
+      "epoch": 0.6,
+      "grad_norm": 28.306367181479345,
+      "kl": 0.8587890625,
+      "learning_rate": 8.263518223330698e-06,
+      "loss": 0.0343,
+      "reward": 1.0875000178813934,
+      "reward_std": 0.3584312066435814,
+      "rewards/accuracy_reward": 0.19583333656191826,
+      "rewards/format_reward": 0.8916666865348816,
+      "step": 150
+    },
+    {
+      "completion_length": 210.64167175292968,
+      "epoch": 0.62,
+      "grad_norm": 147.78201107458216,
+      "kl": 1.8443359375,
+      "learning_rate": 7.580781044003324e-06,
+      "loss": 0.0736,
+      "reward": 0.9833333551883697,
+      "reward_std": 0.41491681188344953,
+      "rewards/accuracy_reward": 0.15416667126119138,
+      "rewards/format_reward": 0.8291666865348816,
+      "step": 155
+    },
+    {
+      "completion_length": 247.33334197998047,
+      "epoch": 0.64,
+      "grad_norm": 15.659633952783947,
+      "kl": 2.0666015625,
+      "learning_rate": 6.909830056250527e-06,
+      "loss": 0.0829,
+      "reward": 1.0291667044162751,
+      "reward_std": 0.3954852521419525,
+      "rewards/accuracy_reward": 0.1458333358168602,
+      "rewards/format_reward": 0.8833333551883698,
+      "step": 160
+    },
+    {
+      "completion_length": 209.2166732788086,
+      "epoch": 0.66,
+      "grad_norm": 4.737432858154195,
+      "kl": 0.7224609375,
+      "learning_rate": 6.25393406584088e-06,
+      "loss": 0.0289,
+      "reward": 1.1041667222976685,
+      "reward_std": 0.25414432808756826,
+      "rewards/accuracy_reward": 0.15416666828095912,
+      "rewards/format_reward": 0.950000011920929,
+      "step": 165
+    },
+    {
+      "completion_length": 212.47500610351562,
+      "epoch": 0.68,
+      "grad_norm": 9.622515017916685,
+      "kl": 0.38310546875,
+      "learning_rate": 5.616288532109225e-06,
+      "loss": 0.0153,
+      "reward": 1.1500000476837158,
+      "reward_std": 0.25412100180983543,
+      "rewards/accuracy_reward": 0.17916667461395264,
+      "rewards/format_reward": 0.9708333432674408,
+      "step": 170
+    },
+    {
+      "completion_length": 215.3041732788086,
+      "epoch": 0.7,
+      "grad_norm": 1.614688005147557,
+      "kl": 0.5701171875,
+      "learning_rate": 5.000000000000003e-06,
+      "loss": 0.0228,
+      "reward": 1.137500035762787,
+      "reward_std": 0.22350237593054773,
+      "rewards/accuracy_reward": 0.17083333767950534,
+      "rewards/format_reward": 0.9666666865348816,
+      "step": 175
+    },
+    {
+      "completion_length": 205.10834045410155,
+      "epoch": 0.72,
+      "grad_norm": 1.091336661435543,
+      "kl": 0.5271484375,
+      "learning_rate": 4.408070965292534e-06,
+      "loss": 0.0211,
+      "reward": 1.1125000357627868,
+      "reward_std": 0.3238865025341511,
+      "rewards/accuracy_reward": 0.1625000037252903,
+      "rewards/format_reward": 0.9500000178813934,
+      "step": 180
+    },
+    {
+      "completion_length": 213.24584045410157,
+      "epoch": 0.74,
+      "grad_norm": 2.9050766275055353,
+      "kl": 0.662109375,
+      "learning_rate": 3.8433852467434175e-06,
+      "loss": 0.0265,
+      "reward": 1.1375000476837158,
+      "reward_std": 0.3509949237108231,
+      "rewards/accuracy_reward": 0.20416667200624944,
+      "rewards/format_reward": 0.9333333551883698,
+      "step": 185
+    },
+    {
+      "completion_length": 218.59583587646483,
+      "epoch": 0.76,
+      "grad_norm": 1.5330170710229287,
+      "kl": 0.405859375,
+      "learning_rate": 3.308693936411421e-06,
+      "loss": 0.0162,
+      "reward": 1.1333333492279052,
+      "reward_std": 0.3301138609647751,
+      "rewards/accuracy_reward": 0.18750000409781933,
+      "rewards/format_reward": 0.9458333432674408,
+      "step": 190
+    },
+    {
+      "completion_length": 217.2125045776367,
+      "epoch": 0.78,
+      "grad_norm": 2.9257483023737056,
+      "kl": 0.43125,
+      "learning_rate": 2.8066019966134907e-06,
+      "loss": 0.0172,
+      "reward": 1.1291666984558106,
+      "reward_std": 0.23582691550254822,
+      "rewards/accuracy_reward": 0.15416667200624942,
+      "rewards/format_reward": 0.9750000178813935,
+      "step": 195
+    },
+    {
+      "completion_length": 229.40000610351564,
+      "epoch": 0.8,
+      "grad_norm": 2.9848511807109945,
+      "kl": 0.715234375,
+      "learning_rate": 2.339555568810221e-06,
+      "loss": 0.0286,
+      "reward": 1.108333373069763,
+      "reward_std": 0.31293403804302217,
+      "rewards/accuracy_reward": 0.17500000707805158,
+      "rewards/format_reward": 0.9333333432674408,
+      "step": 200
+    },
+    {
+      "epoch": 0.8,
+      "eval_completion_length": 241.64263857327975,
+      "eval_kl": 0.8054387019230769,
+      "eval_loss": 0.03347592428326607,
+      "eval_reward": 1.1057692628640394,
+      "eval_reward_std": 0.3724508308447324,
+      "eval_rewards/accuracy_reward": 0.19391026347875595,
+      "eval_rewards/format_reward": 0.9118589988121619,
+      "eval_runtime": 148.8747,
+      "eval_samples_per_second": 0.665,
+      "eval_steps_per_second": 0.02,
+      "step": 200
+    },
+    {
+      "completion_length": 272.07500610351565,
+      "epoch": 0.82,
+      "grad_norm": 6.533327129383711,
+      "kl": 1.0576171875,
+      "learning_rate": 1.9098300562505266e-06,
+      "loss": 0.0423,
+      "reward": 1.037500023841858,
+      "reward_std": 0.37259582430124283,
+      "rewards/accuracy_reward": 0.14583333805203438,
+      "rewards/format_reward": 0.8916666865348816,
+      "step": 205
+    },
+    {
+      "completion_length": 308.77500915527344,
+      "epoch": 0.84,
+      "grad_norm": 6.059865062027995,
+      "kl": 0.7416015625,
+      "learning_rate": 1.5195190384357405e-06,
+      "loss": 0.0297,
+      "reward": 1.0541667222976685,
+      "reward_std": 0.4518925681710243,
+      "rewards/accuracy_reward": 0.17916667088866234,
+      "rewards/format_reward": 0.8750000178813935,
+      "step": 210
+    },
+    {
+      "completion_length": 284.2458465576172,
+      "epoch": 0.86,
+      "grad_norm": 7.905885888448849,
+      "kl": 0.6669921875,
+      "learning_rate": 1.1705240714107301e-06,
+      "loss": 0.0266,
+      "reward": 1.0708333492279052,
+      "reward_std": 0.39237469881772996,
+      "rewards/accuracy_reward": 0.18750000484287738,
+      "rewards/format_reward": 0.8833333551883698,
+      "step": 215
+    },
+    {
+      "completion_length": 282.00000762939453,
+      "epoch": 0.88,
+      "grad_norm": 4.37771923360857,
+      "kl": 0.61904296875,
+      "learning_rate": 8.645454235739903e-07,
+      "loss": 0.0248,
+      "reward": 1.0708333671092987,
+      "reward_std": 0.3581776343286037,
+      "rewards/accuracy_reward": 0.16250000484287738,
+      "rewards/format_reward": 0.9083333492279053,
+      "step": 220
+    },
+    {
+      "completion_length": 296.21667327880857,
+      "epoch": 0.9,
+      "grad_norm": 2.6800870053150487,
+      "kl": 0.5798828125,
+      "learning_rate": 6.030737921409169e-07,
+      "loss": 0.0232,
+      "reward": 1.0250000298023223,
+      "reward_std": 0.34061215743422507,
+      "rewards/accuracy_reward": 0.15000000558793544,
+      "rewards/format_reward": 0.8750000119209289,
+      "step": 225
+    },
+    {
+      "completion_length": 266.26250610351565,
+      "epoch": 0.92,
+      "grad_norm": 2.09794383441415,
+      "kl": 0.498046875,
+      "learning_rate": 3.8738304061681107e-07,
+      "loss": 0.0199,
+      "reward": 1.1125000298023224,
+      "reward_std": 0.29474459737539294,
+      "rewards/accuracy_reward": 0.1875000037252903,
+      "rewards/format_reward": 0.9250000178813934,
+      "step": 230
+    },
+    {
+      "completion_length": 237.3041732788086,
+      "epoch": 0.94,
+      "grad_norm": 3.7721249716047582,
+      "kl": 0.3232421875,
+      "learning_rate": 2.1852399266194312e-07,
+      "loss": 0.0129,
+      "reward": 1.1166666924953461,
+      "reward_std": 0.2401700422167778,
+      "rewards/accuracy_reward": 0.15833333805203437,
+      "rewards/format_reward": 0.9583333492279053,
+      "step": 235
+    },
+    {
+      "completion_length": 267.9583404541016,
+      "epoch": 0.96,
+      "grad_norm": 3.3890207151811844,
+      "kl": 0.446875,
+      "learning_rate": 9.731931258429638e-08,
+      "loss": 0.0179,
+      "reward": 1.1541666984558105,
+      "reward_std": 0.35387340411543844,
+      "rewards/accuracy_reward": 0.21250000223517418,
+      "rewards/format_reward": 0.9416666924953461,
+      "step": 240
+    },
+    {
+      "completion_length": 253.70001068115235,
+      "epoch": 0.98,
+      "grad_norm": 1.8424759936451511,
+      "kl": 0.4060546875,
+      "learning_rate": 2.4359497401758026e-08,
+      "loss": 0.0162,
+      "reward": 1.1583333730697631,
+      "reward_std": 0.3411997601389885,
+      "rewards/accuracy_reward": 0.21666667461395264,
+      "rewards/format_reward": 0.9416666805744172,
+      "step": 245
+    },
+    {
+      "completion_length": 263.2708374023438,
+      "epoch": 1.0,
+      "grad_norm": 1.7906114607102377,
+      "kl": 0.4353515625,
+      "learning_rate": 0.0,
+      "loss": 0.0174,
+      "reward": 1.1541666984558105,
+      "reward_std": 0.31376161351799964,
+      "rewards/accuracy_reward": 0.2125000037252903,
+      "rewards/format_reward": 0.9416666865348816,
+      "step": 250
+    },
+    {
+      "epoch": 1.0,
+      "step": 250,
+      "total_flos": 0.0,
+      "train_loss": 11.024624033321947,
+      "train_runtime": 8177.44,
+      "train_samples_per_second": 0.245,
+      "train_steps_per_second": 0.031
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 250,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": false,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 4,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5e3a75ea9214b8718714a127b48b890cc4d75d87b1f89703eb7a46a95a49a921
+size 7480