Model save

README.md

@@ -0,0 +1,60 @@
+---
+license: mit
+base_model: TTTXXX01/My-Zephyr-7B-iter-1
+tags:
+- trl
+- dpo
+- generated_from_trainer
+model-index:
+- name: My-Zephyr-7B-iter-2
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# My-Zephyr-7B-iter-2
+
+This model is a fine-tuned version of [TTTXXX01/My-Zephyr-7B-iter-1](https://huggingface.co/TTTXXX01/My-Zephyr-7B-iter-1) on an unknown dataset.
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 5e-07
+- train_batch_size: 3
+- eval_batch_size: 3
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 3
+- gradient_accumulation_steps: 4
+- total_train_batch_size: 36
+- total_eval_batch_size: 9
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: cosine
+- lr_scheduler_warmup_ratio: 0.1
+- num_epochs: 1
+
+### Training results
+
+
+
+### Framework versions
+
+- Transformers 4.41.2
+- Pytorch 2.3.0+cu121
+- Datasets 2.19.1
+- Tokenizers 0.19.1

all_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 0.998234255444379,
+    "total_flos": 0.0,
+    "train_loss": 0.387972928302468,
+    "train_runtime": 9484.4674,
+    "train_samples": 15283,
+    "train_samples_per_second": 1.611,
+    "train_steps_per_second": 0.045
+}

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "transformers_version": "4.41.2"
+}

model-00001-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:58dd6b9d56e576aa1e3ab7798619b56c8a1de7e5c10f4c6b706525bc31235643
+size 4943162336

model-00002-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:786e6441ec3b70ad264c210512f28d009108f33c30092e122606005461d1f3d6
+size 4999819336

model-00003-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:91ba2bf89b573c5283d35013cb1413bffe3ddb3c70b3c52b5bd7f31ff5a932ef
+size 4540516344

model.safetensors.index.json

@@ -0,0 +1,298 @@
+{
+  "metadata": {
+    "total_size": 14483464192
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00003-of-00003.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.norm.weight": "model-00003-of-00003.safetensors"
+  }
+}

train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 0.998234255444379,
+    "total_flos": 0.0,
+    "train_loss": 0.387972928302468,
+    "train_runtime": 9484.4674,
+    "train_samples": 15283,
+    "train_samples_per_second": 1.611,
+    "train_steps_per_second": 0.045
+}

trainer_state.json

@@ -0,0 +1,773 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.998234255444379,
+  "eval_steps": 500,
+  "global_step": 424,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.002354326074161271,
+      "grad_norm": 387.14608138046685,
+      "learning_rate": 1.1627906976744186e-08,
+      "logits/chosen": 0.9922162890434265,
+      "logits/rejected": 2.5945053100585938,
+      "logps/chosen": -671.7749633789062,
+      "logps/pi_response": -362.8890380859375,
+      "logps/ref_response": -362.8890380859375,
+      "logps/rejected": -872.0834350585938,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 1
+    },
+    {
+      "epoch": 0.023543260741612712,
+      "grad_norm": 416.60085993891914,
+      "learning_rate": 1.1627906976744186e-07,
+      "logits/chosen": 0.5027725696563721,
+      "logits/rejected": 1.1612285375595093,
+      "logps/chosen": -615.636474609375,
+      "logps/pi_response": -373.8717346191406,
+      "logps/ref_response": -371.5905456542969,
+      "logps/rejected": -773.8812255859375,
+      "loss": 0.68,
+      "rewards/accuracies": 0.5185185074806213,
+      "rewards/chosen": 0.0004265241150278598,
+      "rewards/margins": 0.02714625373482704,
+      "rewards/rejected": -0.026719728484749794,
+      "step": 10
+    },
+    {
+      "epoch": 0.047086521483225424,
+      "grad_norm": 146.4782737758625,
+      "learning_rate": 2.3255813953488372e-07,
+      "logits/chosen": 1.0533249378204346,
+      "logits/rejected": 1.2860088348388672,
+      "logps/chosen": -583.3096923828125,
+      "logps/pi_response": -664.9867553710938,
+      "logps/ref_response": -384.66943359375,
+      "logps/rejected": -1088.1510009765625,
+      "loss": 0.5167,
+      "rewards/accuracies": 0.8500000834465027,
+      "rewards/chosen": -0.05585538223385811,
+      "rewards/margins": 2.6645126342773438,
+      "rewards/rejected": -2.7203681468963623,
+      "step": 20
+    },
+    {
+      "epoch": 0.07062978222483814,
+      "grad_norm": 193.36310820542772,
+      "learning_rate": 3.4883720930232557e-07,
+      "logits/chosen": 1.5033495426177979,
+      "logits/rejected": 2.0693583488464355,
+      "logps/chosen": -675.2313232421875,
+      "logps/pi_response": -1286.485107421875,
+      "logps/ref_response": -382.0600280761719,
+      "logps/rejected": -1842.0443115234375,
+      "loss": 0.3477,
+      "rewards/accuracies": 0.8333333730697632,
+      "rewards/chosen": -1.2492175102233887,
+      "rewards/margins": 8.74349594116211,
+      "rewards/rejected": -9.99271297454834,
+      "step": 30
+    },
+    {
+      "epoch": 0.09417304296645085,
+      "grad_norm": 161.8114972801858,
+      "learning_rate": 4.6511627906976743e-07,
+      "logits/chosen": 2.231173515319824,
+      "logits/rejected": 2.243076801300049,
+      "logps/chosen": -888.1588745117188,
+      "logps/pi_response": -1944.935546875,
+      "logps/ref_response": -385.018798828125,
+      "logps/rejected": -2694.44189453125,
+      "loss": 0.5111,
+      "rewards/accuracies": 0.8666666746139526,
+      "rewards/chosen": -3.3236842155456543,
+      "rewards/margins": 15.406656265258789,
+      "rewards/rejected": -18.7303409576416,
+      "step": 40
+    },
+    {
+      "epoch": 0.11771630370806356,
+      "grad_norm": 73.27022369826706,
+      "learning_rate": 4.995836722963699e-07,
+      "logits/chosen": 1.7006012201309204,
+      "logits/rejected": 2.5465452671051025,
+      "logps/chosen": -736.0626220703125,
+      "logps/pi_response": -1658.806640625,
+      "logps/ref_response": -418.013671875,
+      "logps/rejected": -2036.8658447265625,
+      "loss": 0.3367,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -1.4946272373199463,
+      "rewards/margins": 11.121105194091797,
+      "rewards/rejected": -12.615732192993164,
+      "step": 50
+    },
+    {
+      "epoch": 0.14125956444967627,
+      "grad_norm": 197.275331081517,
+      "learning_rate": 4.975478535699678e-07,
+      "logits/chosen": 2.0557973384857178,
+      "logits/rejected": 2.7715961933135986,
+      "logps/chosen": -786.7219848632812,
+      "logps/pi_response": -1516.2132568359375,
+      "logps/ref_response": -400.8934326171875,
+      "logps/rejected": -1846.4228515625,
+      "loss": 0.3929,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -1.6042267084121704,
+      "rewards/margins": 9.362323760986328,
+      "rewards/rejected": -10.966550827026367,
+      "step": 60
+    },
+    {
+      "epoch": 0.164802825191289,
+      "grad_norm": 125.32869675723494,
+      "learning_rate": 4.938298919762906e-07,
+      "logits/chosen": 2.2630465030670166,
+      "logits/rejected": 2.313655138015747,
+      "logps/chosen": -874.7604370117188,
+      "logps/pi_response": -1775.0299072265625,
+      "logps/ref_response": -406.5614013671875,
+      "logps/rejected": -2185.577392578125,
+      "loss": 0.3204,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -2.6973066329956055,
+      "rewards/margins": 11.22050666809082,
+      "rewards/rejected": -13.917811393737793,
+      "step": 70
+    },
+    {
+      "epoch": 0.1883460859329017,
+      "grad_norm": 183.2640323450364,
+      "learning_rate": 4.884550518934592e-07,
+      "logits/chosen": 2.544085741043091,
+      "logits/rejected": 2.2697596549987793,
+      "logps/chosen": -814.3087158203125,
+      "logps/pi_response": -1254.173583984375,
+      "logps/ref_response": -385.601318359375,
+      "logps/rejected": -1756.57421875,
+      "loss": 0.4552,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -2.785598039627075,
+      "rewards/margins": 6.5690412521362305,
+      "rewards/rejected": -9.354639053344727,
+      "step": 80
+    },
+    {
+      "epoch": 0.21188934667451442,
+      "grad_norm": 117.0357298014428,
+      "learning_rate": 4.814598565584062e-07,
+      "logits/chosen": 3.02203369140625,
+      "logits/rejected": 3.5366241931915283,
+      "logps/chosen": -908.7738037109375,
+      "logps/pi_response": -1195.170166015625,
+      "logps/ref_response": -383.25390625,
+      "logps/rejected": -1815.823974609375,
+      "loss": 0.3526,
+      "rewards/accuracies": 0.85833340883255,
+      "rewards/chosen": -3.5764949321746826,
+      "rewards/margins": 6.11873722076416,
+      "rewards/rejected": -9.695232391357422,
+      "step": 90
+    },
+    {
+      "epoch": 0.23543260741612712,
+      "grad_norm": 398.94556345700823,
+      "learning_rate": 4.7289183988333603e-07,
+      "logits/chosen": 2.4340739250183105,
+      "logits/rejected": 3.0718626976013184,
+      "logps/chosen": -979.5059814453125,
+      "logps/pi_response": -1823.2095947265625,
+      "logps/ref_response": -389.8714294433594,
+      "logps/rejected": -2347.225341796875,
+      "loss": 0.5772,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -4.193310260772705,
+      "rewards/margins": 10.396108627319336,
+      "rewards/rejected": -14.5894193649292,
+      "step": 100
+    },
+    {
+      "epoch": 0.2589758681577399,
+      "grad_norm": 122.8282681969783,
+      "learning_rate": 4.6280922345219255e-07,
+      "logits/chosen": 2.0978169441223145,
+      "logits/rejected": 2.4143967628479004,
+      "logps/chosen": -942.2001953125,
+      "logps/pi_response": -1763.4075927734375,
+      "logps/ref_response": -404.2528381347656,
+      "logps/rejected": -2334.311279296875,
+      "loss": 0.2675,
+      "rewards/accuracies": 0.8916667103767395,
+      "rewards/chosen": -3.5824508666992188,
+      "rewards/margins": 11.474252700805664,
+      "rewards/rejected": -15.056703567504883,
+      "step": 110
+    },
+    {
+      "epoch": 0.28251912889935255,
+      "grad_norm": 114.42881059618138,
+      "learning_rate": 4.512805208920118e-07,
+      "logits/chosen": 1.876939058303833,
+      "logits/rejected": 1.8963384628295898,
+      "logps/chosen": -1116.2161865234375,
+      "logps/pi_response": -2194.660888671875,
+      "logps/ref_response": -382.1685485839844,
+      "logps/rejected": -2665.172607421875,
+      "loss": 0.3802,
+      "rewards/accuracies": 0.8833333849906921,
+      "rewards/chosen": -5.01099157333374,
+      "rewards/margins": 14.220196723937988,
+      "rewards/rejected": -19.231189727783203,
+      "step": 120
+    },
+    {
+      "epoch": 0.30606238964096527,
+      "grad_norm": 289.6702886782038,
+      "learning_rate": 4.383840723075488e-07,
+      "logits/chosen": 0.6433286070823669,
+      "logits/rejected": 1.8898694515228271,
+      "logps/chosen": -1148.931884765625,
+      "logps/pi_response": -1483.739990234375,
+      "logps/ref_response": -384.0348815917969,
+      "logps/rejected": -2114.025634765625,
+      "loss": 0.5347,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -5.955471515655518,
+      "rewards/margins": 6.998837947845459,
+      "rewards/rejected": -12.954309463500977,
+      "step": 130
+    },
+    {
+      "epoch": 0.329605650382578,
+      "grad_norm": 68.08348369771358,
+      "learning_rate": 4.2420751194279604e-07,
+      "logits/chosen": -0.47634777426719666,
+      "logits/rejected": 0.15769393742084503,
+      "logps/chosen": -1069.734619140625,
+      "logps/pi_response": -1077.26025390625,
+      "logps/ref_response": -385.5350646972656,
+      "logps/rejected": -1771.5843505859375,
+      "loss": 0.2586,
+      "rewards/accuracies": 0.9083333015441895,
+      "rewards/chosen": -4.997823238372803,
+      "rewards/margins": 4.333145618438721,
+      "rewards/rejected": -9.330968856811523,
+      "step": 140
+    },
+    {
+      "epoch": 0.3531489111241907,
+      "grad_norm": 163.9369240446969,
+      "learning_rate": 4.0884717268675306e-07,
+      "logits/chosen": -0.20242293179035187,
+      "logits/rejected": 0.2356901615858078,
+      "logps/chosen": -1135.7840576171875,
+      "logps/pi_response": -1698.245361328125,
+      "logps/ref_response": -393.395263671875,
+      "logps/rejected": -2191.13671875,
+      "loss": 0.2882,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -5.520671367645264,
+      "rewards/margins": 8.618199348449707,
+      "rewards/rejected": -14.138870239257812,
+      "step": 150
+    },
+    {
+      "epoch": 0.3766921718658034,
+      "grad_norm": 150.38185986907246,
+      "learning_rate": 3.9240743146996427e-07,
+      "logits/chosen": -0.22249138355255127,
+      "logits/rejected": 0.3604862689971924,
+      "logps/chosen": -981.8104248046875,
+      "logps/pi_response": -1485.242919921875,
+      "logps/ref_response": -374.290283203125,
+      "logps/rejected": -2042.9921875,
+      "loss": 0.3641,
+      "rewards/accuracies": 0.8250001072883606,
+      "rewards/chosen": -3.9730944633483887,
+      "rewards/margins": 8.374366760253906,
+      "rewards/rejected": -12.347461700439453,
+      "step": 160
+    },
+    {
+      "epoch": 0.4002354326074161,
+      "grad_norm": 80.54150444296407,
+      "learning_rate": 3.75e-07,
+      "logits/chosen": -0.11508075892925262,
+      "logits/rejected": 0.29167068004608154,
+      "logps/chosen": -1000.4583740234375,
+      "logps/pi_response": -1468.6123046875,
+      "logps/ref_response": -418.64288330078125,
+      "logps/rejected": -2010.297607421875,
+      "loss": 0.3769,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -4.033980369567871,
+      "rewards/margins": 7.807894229888916,
+      "rewards/rejected": -11.841875076293945,
+      "step": 170
+    },
+    {
+      "epoch": 0.42377869334902885,
+      "grad_norm": 157.0982901424276,
+      "learning_rate": 3.5674316565549227e-07,
+      "logits/chosen": 0.3534625172615051,
+      "logits/rejected": 0.6805320382118225,
+      "logps/chosen": -1109.6680908203125,
+      "logps/pi_response": -1332.754638671875,
+      "logps/ref_response": -410.44232177734375,
+      "logps/rejected": -1955.057861328125,
+      "loss": 0.2771,
+      "rewards/accuracies": 0.8666666746139526,
+      "rewards/chosen": -5.245041370391846,
+      "rewards/margins": 6.281018257141113,
+      "rewards/rejected": -11.526060104370117,
+      "step": 180
+    },
+    {
+      "epoch": 0.44732195409064157,
+      "grad_norm": 138.9366989622839,
+      "learning_rate": 3.377609876970194e-07,
+      "logits/chosen": -0.06883997470140457,
+      "logits/rejected": 0.5686254501342773,
+      "logps/chosen": -1120.6253662109375,
+      "logps/pi_response": -1725.830810546875,
+      "logps/ref_response": -391.46697998046875,
+      "logps/rejected": -2330.28564453125,
+      "loss": 0.3201,
+      "rewards/accuracies": 0.85833340883255,
+      "rewards/chosen": -5.111655235290527,
+      "rewards/margins": 10.163948059082031,
+      "rewards/rejected": -15.275602340698242,
+      "step": 190
+    },
+    {
+      "epoch": 0.47086521483225424,
+      "grad_norm": 74.34016252556022,
+      "learning_rate": 3.1818245425676556e-07,
+      "logits/chosen": 0.038237761706113815,
+      "logits/rejected": 0.7730900049209595,
+      "logps/chosen": -1035.312744140625,
+      "logps/pi_response": -1930.384765625,
+      "logps/ref_response": -400.8553771972656,
+      "logps/rejected": -2589.25439453125,
+      "loss": 0.3094,
+      "rewards/accuracies": 0.9250000715255737,
+      "rewards/chosen": -4.300232410430908,
+      "rewards/margins": 12.978002548217773,
+      "rewards/rejected": -17.278234481811523,
+      "step": 200
+    },
+    {
+      "epoch": 0.49440847557386697,
+      "grad_norm": 135.89742493179222,
+      "learning_rate": 2.981406058353988e-07,
+      "logits/chosen": 1.6664092540740967,
+      "logits/rejected": 1.5270541906356812,
+      "logps/chosen": -1253.89013671875,
+      "logps/pi_response": -3124.22021484375,
+      "logps/ref_response": -380.658447265625,
+      "logps/rejected": -3518.0625,
+      "loss": 0.31,
+      "rewards/accuracies": 0.8916667103767395,
+      "rewards/chosen": -6.066841125488281,
+      "rewards/margins": 21.756336212158203,
+      "rewards/rejected": -27.823177337646484,
+      "step": 210
+    },
+    {
+      "epoch": 0.5179517363154797,
+      "grad_norm": 367.00195352780696,
+      "learning_rate": 2.7777163126220113e-07,
+      "logits/chosen": 1.1289093494415283,
+      "logits/rejected": 1.753485918045044,
+      "logps/chosen": -1161.3385009765625,
+      "logps/pi_response": -2382.952880859375,
+      "logps/ref_response": -397.78643798828125,
+      "logps/rejected": -2924.198974609375,
+      "loss": 0.3399,
+      "rewards/accuracies": 0.8416668176651001,
+      "rewards/chosen": -5.906828880310059,
+      "rewards/margins": 15.091941833496094,
+      "rewards/rejected": -20.998775482177734,
+      "step": 220
+    },
+    {
+      "epoch": 0.5414949970570924,
+      "grad_norm": 429.21214818334255,
+      "learning_rate": 2.5721394226160456e-07,
+      "logits/chosen": 0.7405129671096802,
+      "logits/rejected": 1.092500925064087,
+      "logps/chosen": -1127.665771484375,
+      "logps/pi_response": -3084.66748046875,
+      "logps/ref_response": -385.5183410644531,
+      "logps/rejected": -3698.11083984375,
+      "loss": 0.3142,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -5.245604515075684,
+      "rewards/margins": 23.772628784179688,
+      "rewards/rejected": -29.018238067626953,
+      "step": 230
+    },
+    {
+      "epoch": 0.5650382577987051,
+      "grad_norm": 143.78300847624018,
+      "learning_rate": 2.3660723291465753e-07,
+      "logits/chosen": 0.8486520648002625,
+      "logits/rejected": 1.15015709400177,
+      "logps/chosen": -1044.281982421875,
+      "logps/pi_response": -2387.38623046875,
+      "logps/ref_response": -387.73004150390625,
+      "logps/rejected": -2920.59375,
+      "loss": 0.2255,
+      "rewards/accuracies": 0.9583333730697632,
+      "rewards/chosen": -4.885871887207031,
+      "rewards/margins": 15.968719482421875,
+      "rewards/rejected": -20.854591369628906,
+      "step": 240
+    },
+    {
+      "epoch": 0.5885815185403178,
+      "grad_norm": 125.17444301250116,
+      "learning_rate": 2.1609153040659357e-07,
+      "logits/chosen": 1.349198341369629,
+      "logits/rejected": 1.477452278137207,
+      "logps/chosen": -1202.511962890625,
+      "logps/pi_response": -4804.65576171875,
+      "logps/ref_response": -403.5562438964844,
+      "logps/rejected": -4992.322265625,
+      "loss": 0.2998,
+      "rewards/accuracies": 0.8333333134651184,
+      "rewards/chosen": -5.66055154800415,
+      "rewards/margins": 36.13991928100586,
+      "rewards/rejected": -41.800472259521484,
+      "step": 250
+    },
+    {
+      "epoch": 0.6121247792819305,
+      "grad_norm": 75.45331010998015,
+      "learning_rate": 1.9580624351088174e-07,
+      "logits/chosen": 1.8510982990264893,
+      "logits/rejected": 1.8215391635894775,
+      "logps/chosen": -1237.7900390625,
+      "logps/pi_response": -4366.1884765625,
+      "logps/ref_response": -402.7124328613281,
+      "logps/rejected": -4625.4326171875,
+      "loss": 0.2256,
+      "rewards/accuracies": 0.908333420753479,
+      "rewards/chosen": -6.537631034851074,
+      "rewards/margins": 31.707489013671875,
+      "rewards/rejected": -38.245121002197266,
+      "step": 260
+    },
+    {
+      "epoch": 0.6356680400235433,
+      "grad_norm": 121.70692272560186,
+      "learning_rate": 1.7588921527552315e-07,
+      "logits/chosen": 1.641945242881775,
+      "logits/rejected": 1.5603419542312622,
+      "logps/chosen": -1264.4483642578125,
+      "logps/pi_response": -4920.81396484375,
+      "logps/ref_response": -420.87274169921875,
+      "logps/rejected": -5146.81005859375,
+      "loss": 0.7912,
+      "rewards/accuracies": 0.9000000953674316,
+      "rewards/chosen": -6.559077262878418,
+      "rewards/margins": 36.41227722167969,
+      "rewards/rejected": -42.97135925292969,
+      "step": 270
+    },
+    {
+      "epoch": 0.659211300765156,
+      "grad_norm": 121.24373782517053,
+      "learning_rate": 1.564757863488017e-07,
+      "logits/chosen": 1.3223627805709839,
+      "logits/rejected": 2.0051231384277344,
+      "logps/chosen": -1246.6629638671875,
+      "logps/pi_response": -4004.585205078125,
+      "logps/ref_response": -386.71160888671875,
+      "logps/rejected": -4359.3427734375,
+      "loss": 0.1897,
+      "rewards/accuracies": 0.9333333969116211,
+      "rewards/chosen": -6.822081089019775,
+      "rewards/margins": 27.98373794555664,
+      "rewards/rejected": -34.80581283569336,
+      "step": 280
+    },
+    {
+      "epoch": 0.6827545615067687,
+      "grad_norm": 830.0337931185638,
+      "learning_rate": 1.3769787530939818e-07,
+      "logits/chosen": 1.528591275215149,
+      "logits/rejected": 2.117349147796631,
+      "logps/chosen": -1309.528564453125,
+      "logps/pi_response": -5731.9384765625,
+      "logps/ref_response": -398.12322998046875,
+      "logps/rejected": -5667.63916015625,
+      "loss": 0.297,
+      "rewards/accuracies": 0.9166668057441711,
+      "rewards/chosen": -7.466167449951172,
+      "rewards/margins": 41.12438201904297,
+      "rewards/rejected": -48.590553283691406,
+      "step": 290
+    },
+    {
+      "epoch": 0.7062978222483814,
+      "grad_norm": 152.99540073883847,
+      "learning_rate": 1.19683082250231e-07,
+      "logits/chosen": 1.3115044832229614,
+      "logits/rejected": 1.6584991216659546,
+      "logps/chosen": -1225.9017333984375,
+      "logps/pi_response": -4866.2646484375,
+      "logps/ref_response": -388.7275085449219,
+      "logps/rejected": -4971.64306640625,
+      "loss": 0.2874,
+      "rewards/accuracies": 0.8833333253860474,
+      "rewards/chosen": -6.632820129394531,
+      "rewards/margins": 34.09852600097656,
+      "rewards/rejected": -40.731346130371094,
+      "step": 300
+    },
+    {
+      "epoch": 0.7298410829899941,
+      "grad_norm": 119.56456897387905,
+      "learning_rate": 1.0255382170737015e-07,
+      "logits/chosen": 1.3735281229019165,
+      "logits/rejected": 1.32486891746521,
+      "logps/chosen": -1208.898193359375,
+      "logps/pi_response": -5530.7060546875,
+      "logps/ref_response": -391.98046875,
+      "logps/rejected": -6051.8076171875,
+      "loss": 0.591,
+      "rewards/accuracies": 0.8666666746139526,
+      "rewards/chosen": -6.081249713897705,
+      "rewards/margins": 46.203948974609375,
+      "rewards/rejected": -52.28520584106445,
+      "step": 310
+    },
+    {
+      "epoch": 0.7533843437316068,
+      "grad_norm": 108.85771777397696,
+      "learning_rate": 8.642649082596692e-08,
+      "logits/chosen": 1.5552794933319092,
+      "logits/rejected": 1.771057367324829,
+      "logps/chosen": -1233.845947265625,
+      "logps/pi_response": -5965.13623046875,
+      "logps/ref_response": -382.88055419921875,
+      "logps/rejected": -6220.9912109375,
+      "loss": 0.4376,
+      "rewards/accuracies": 0.8916667699813843,
+      "rewards/chosen": -6.246255397796631,
+      "rewards/margins": 48.06293487548828,
+      "rewards/rejected": -54.30919647216797,
+      "step": 320
+    },
+    {
+      "epoch": 0.7769276044732195,
+      "grad_norm": 64.98595778509755,
+      "learning_rate": 7.141067841569633e-08,
+      "logits/chosen": 1.0594632625579834,
+      "logits/rejected": 1.5874229669570923,
+      "logps/chosen": -1108.4306640625,
+      "logps/pi_response": -3888.35400390625,
+      "logps/ref_response": -377.4826354980469,
+      "logps/rejected": -4457.27685546875,
+      "loss": 0.6749,
+      "rewards/accuracies": 0.908333420753479,
+      "rewards/chosen": -5.92837381362915,
+      "rewards/margins": 30.60651206970215,
+      "rewards/rejected": -36.534889221191406,
+      "step": 330
+    },
+    {
+      "epoch": 0.8004708652148322,
+      "grad_norm": 114.35171172829054,
+      "learning_rate": 5.7608420270357614e-08,
+      "logits/chosen": 1.8669030666351318,
+      "logits/rejected": 2.0596258640289307,
+      "logps/chosen": -1213.849609375,
+      "logps/pi_response": -4912.9462890625,
+      "logps/ref_response": -396.6376037597656,
+      "logps/rejected": -4539.1298828125,
+      "loss": 0.435,
+      "rewards/accuracies": 0.9000000953674316,
+      "rewards/chosen": -6.412614345550537,
+      "rewards/margins": 30.942264556884766,
+      "rewards/rejected": -37.354881286621094,
+      "step": 340
+    },
+    {
+      "epoch": 0.824014125956445,
+      "grad_norm": 179.13336254225945,
+      "learning_rate": 4.511350581190129e-08,
+      "logits/chosen": 1.3449863195419312,
+      "logits/rejected": 1.5248844623565674,
+      "logps/chosen": -1232.9034423828125,
+      "logps/pi_response": -6418.89697265625,
+      "logps/ref_response": -406.46099853515625,
+      "logps/rejected": -5906.890625,
+      "loss": 0.2574,
+      "rewards/accuracies": 0.85833340883255,
+      "rewards/chosen": -6.55679178237915,
+      "rewards/margins": 44.44318771362305,
+      "rewards/rejected": -50.99998092651367,
+      "step": 350
+    },
+    {
+      "epoch": 0.8475573866980577,
+      "grad_norm": 100.07236020794019,
+      "learning_rate": 3.401084077039293e-08,
+      "logits/chosen": 1.0567926168441772,
+      "logits/rejected": 1.0804252624511719,
+      "logps/chosen": -1290.2373046875,
+      "logps/pi_response": -5767.3994140625,
+      "logps/ref_response": -370.0479431152344,
+      "logps/rejected": -6238.93115234375,
+      "loss": 0.182,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -6.525579929351807,
+      "rewards/margins": 48.529579162597656,
+      "rewards/rejected": -55.0551643371582,
+      "step": 360
+    },
+    {
+      "epoch": 0.8711006474396704,
+      "grad_norm": 154.01389939852425,
+      "learning_rate": 2.4375870230643413e-08,
+      "logits/chosen": 1.2523785829544067,
+      "logits/rejected": 0.6351985931396484,
+      "logps/chosen": -1255.387451171875,
+      "logps/pi_response": -7876.88427734375,
+      "logps/ref_response": -419.7919006347656,
+      "logps/rejected": -7547.12353515625,
+      "loss": 0.8119,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -6.398593902587891,
+      "rewards/margins": 61.322898864746094,
+      "rewards/rejected": -67.72149658203125,
+      "step": 370
+    },
+    {
+      "epoch": 0.8946439081812831,
+      "grad_norm": 78.3832326659616,
+      "learning_rate": 1.627406596603359e-08,
+      "logits/chosen": 1.103989839553833,
+      "logits/rejected": 1.358469843864441,
+      "logps/chosen": -1181.985595703125,
+      "logps/pi_response": -6574.1669921875,
+      "logps/ref_response": -373.62188720703125,
+      "logps/rejected": -7001.06103515625,
+      "loss": 0.1925,
+      "rewards/accuracies": 0.89166659116745,
+      "rewards/chosen": -6.272666931152344,
+      "rewards/margins": 55.82216262817383,
+      "rewards/rejected": -62.094825744628906,
+      "step": 380
+    },
+    {
+      "epoch": 0.9181871689228959,
+      "grad_norm": 127.93583130286875,
+      "learning_rate": 9.760481543214128e-09,
+      "logits/chosen": 1.224596619606018,
+      "logits/rejected": 1.153881311416626,
+      "logps/chosen": -1271.9290771484375,
+      "logps/pi_response": -6838.2158203125,
+      "logps/ref_response": -396.5231628417969,
+      "logps/rejected": -7279.1904296875,
+      "loss": 0.8634,
+      "rewards/accuracies": 0.85833340883255,
+      "rewards/chosen": -6.516213417053223,
+      "rewards/margins": 58.329864501953125,
+      "rewards/rejected": -64.84608459472656,
+      "step": 390
+    },
+    {
+      "epoch": 0.9417304296645085,
+      "grad_norm": 70.22000102721056,
+      "learning_rate": 4.879378220843666e-09,
+      "logits/chosen": 0.9867501258850098,
+      "logits/rejected": 1.4171621799468994,
+      "logps/chosen": -1219.076416015625,
+      "logps/pi_response": -6917.1083984375,
+      "logps/ref_response": -392.57037353515625,
+      "logps/rejected": -6101.5478515625,
+      "loss": 0.225,
+      "rewards/accuracies": 0.9166666865348816,
+      "rewards/chosen": -6.3670525550842285,
+      "rewards/margins": 46.422969818115234,
+      "rewards/rejected": -52.7900276184082,
+      "step": 400
+    },
+    {
+      "epoch": 0.9652736904061212,
+      "grad_norm": 138.07733003631637,
+      "learning_rate": 1.6639241844659535e-09,
+      "logits/chosen": 1.2070242166519165,
+      "logits/rejected": 1.469334363937378,
+      "logps/chosen": -1205.38525390625,
+      "logps/pi_response": -5451.77392578125,
+      "logps/ref_response": -356.38177490234375,
+      "logps/rejected": -5381.77587890625,
+      "loss": 0.2645,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -6.463148593902588,
+      "rewards/margins": 39.27529525756836,
+      "rewards/rejected": -45.73843765258789,
+      "step": 410
+    },
+    {
+      "epoch": 0.9888169511477339,
+      "grad_norm": 52.54825181887789,
+      "learning_rate": 1.359691612926872e-10,
+      "logits/chosen": 1.1104196310043335,
+      "logits/rejected": 0.860076904296875,
+      "logps/chosen": -1356.6370849609375,
+      "logps/pi_response": -6781.0595703125,
+      "logps/ref_response": -390.4653015136719,
+      "logps/rejected": -7239.3955078125,
+      "loss": 0.3053,
+      "rewards/accuracies": 0.9333333969116211,
+      "rewards/chosen": -7.076499938964844,
+      "rewards/margins": 57.4049072265625,
+      "rewards/rejected": -64.48140716552734,
+      "step": 420
+    },
+    {
+      "epoch": 0.998234255444379,
+      "step": 424,
+      "total_flos": 0.0,
+      "train_loss": 0.387972928302468,
+      "train_runtime": 9484.4674,
+      "train_samples_per_second": 1.611,
+      "train_steps_per_second": 0.045
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 424,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 100,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 3,
+  "trial_name": null,
+  "trial_params": null
+}