TX

README.md

@@ -0,0 +1,61 @@
+---
+license: apache-2.0
+base_model: alignment-handbook/zephyr-7b-sft-full
+tags:
+- alignment_handbook-handbook
+- generated_from_trainer
+datasets:
+- HuggingFaceH4/ultrafeedback_binarized
+model-index:
+- name: zephyr-7b-Cal-DPO-1-001-False
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# zephyr-7b-Cal-DPO-1-001-False
+
+This model is a fine-tuned version of [alignment-handbook/zephyr-7b-sft-full](https://huggingface.co/alignment-handbook/zephyr-7b-sft-full) on the HuggingFaceH4/ultrafeedback_binarized dataset.
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 5e-07
+- train_batch_size: 2
+- eval_batch_size: 4
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 4
+- gradient_accumulation_steps: 16
+- total_train_batch_size: 128
+- total_eval_batch_size: 16
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: cosine
+- lr_scheduler_warmup_ratio: 0.1
+- num_epochs: 1
+
+### Training results
+
+
+
+### Framework versions
+
+- Transformers 4.40.2
+- Pytorch 2.1.2+cu121
+- Datasets 2.14.6
+- Tokenizers 0.19.1

all_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 0.998691442030882,
+    "total_flos": 0.0,
+    "train_loss": 114854.6911687631,
+    "train_runtime": 7479.76,
+    "train_samples": 61135,
+    "train_samples_per_second": 8.173,
+    "train_steps_per_second": 0.064
+}

config.json

@@ -0,0 +1,26 @@
+{
+  "_name_or_path": "alignment-handbook/zephyr-7b-sft-full",
+  "architectures": [
+    "MistralForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 32768,
+  "model_type": "mistral",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "rms_norm_eps": 1e-05,
+  "rope_theta": 10000.0,
+  "sliding_window": 4096,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.40.2",
+  "use_cache": true,
+  "vocab_size": 32000
+}

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "transformers_version": "4.40.2"
+}

model-00001-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:df402522469c33d7b32e6064252f24e75e040ee32cfbefef658e8d6fa4394537
+size 4943162336

model-00002-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fd928c1f875ee4afd3ba9062bd626b6c05213a718f9aab283124b1d283402fbc
+size 4999819336

model-00003-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:809b7b96bffdc6766d94c595181101d7598014523583db1f9a9e0ca45ea933f7
+size 4540516344

model.safetensors.index.json

@@ -0,0 +1,298 @@
+{
+  "metadata": {
+    "total_size": 14483464192
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00003-of-00003.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.norm.weight": "model-00003-of-00003.safetensors"
+  }
+}

special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,43 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [],
+  "bos_token": "<s>",
+  "chat_template": "{% for message in messages %}\n{% if message['role'] == 'user' %}\n{{ '<|user|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'system' %}\n{{ '<|system|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'assistant' %}\n{{ '<|assistant|>\n'  + message['content'] + eos_token }}\n{% endif %}\n{% if loop.last and add_generation_prompt %}\n{{ '<|assistant|>' }}\n{% endif %}\n{% endfor %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "legacy": true,
+  "model_max_length": 2048,
+  "pad_token": "</s>",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}

train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 0.998691442030882,
+    "total_flos": 0.0,
+    "train_loss": 114854.6911687631,
+    "train_runtime": 7479.76,
+    "train_samples": 61135,
+    "train_samples_per_second": 8.173,
+    "train_steps_per_second": 0.064
+}

trainer_state.json

@@ -0,0 +1,1455 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.998691442030882,
+  "eval_steps": 500,
+  "global_step": 477,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.010468463752944255,
+      "grad_norm": 522503.4218003003,
+      "learning_rate": 5.208333333333333e-08,
+      "logits/chosen": -2.7708499431610107,
+      "logits/rejected": -2.7286171913146973,
+      "logps/chosen": -253.5421142578125,
+      "logps/rejected": -244.00076293945312,
+      "loss": 125000.2375,
+      "rewards/accuracies": 0.44999998807907104,
+      "rewards/chosen": -253.5421142578125,
+      "rewards/margins": -9.541345596313477,
+      "rewards/rejected": -244.00076293945312,
+      "step": 5
+    },
+    {
+      "epoch": 0.02093692750588851,
+      "grad_norm": 409413.4458768586,
+      "learning_rate": 1.0416666666666667e-07,
+      "logits/chosen": -2.7510223388671875,
+      "logits/rejected": -2.7252488136291504,
+      "logps/chosen": -268.2208557128906,
+      "logps/rejected": -243.4846649169922,
+      "loss": 124997.8,
+      "rewards/accuracies": 0.44999998807907104,
+      "rewards/chosen": -268.2208557128906,
+      "rewards/margins": -24.73613929748535,
+      "rewards/rejected": -243.4846649169922,
+      "step": 10
+    },
+    {
+      "epoch": 0.031405391258832765,
+      "grad_norm": 414950.26842560945,
+      "learning_rate": 1.5624999999999999e-07,
+      "logits/chosen": -2.730611562728882,
+      "logits/rejected": -2.6970362663269043,
+      "logps/chosen": -236.9029541015625,
+      "logps/rejected": -273.0437316894531,
+      "loss": 124982.75,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -236.9029541015625,
+      "rewards/margins": 36.140785217285156,
+      "rewards/rejected": -273.0437316894531,
+      "step": 15
+    },
+    {
+      "epoch": 0.04187385501177702,
+      "grad_norm": 718250.9758832247,
+      "learning_rate": 2.0833333333333333e-07,
+      "logits/chosen": -2.802532434463501,
+      "logits/rejected": -2.7090563774108887,
+      "logps/chosen": -301.2122802734375,
+      "logps/rejected": -269.02008056640625,
+      "loss": 124929.4,
+      "rewards/accuracies": 0.46875,
+      "rewards/chosen": -301.2122802734375,
+      "rewards/margins": -32.19221496582031,
+      "rewards/rejected": -269.02008056640625,
+      "step": 20
+    },
+    {
+      "epoch": 0.05234231876472128,
+      "grad_norm": 444864.8327529458,
+      "learning_rate": 2.604166666666667e-07,
+      "logits/chosen": -2.796382188796997,
+      "logits/rejected": -2.7728114128112793,
+      "logps/chosen": -274.8233947753906,
+      "logps/rejected": -246.075927734375,
+      "loss": 124820.5,
+      "rewards/accuracies": 0.4437499940395355,
+      "rewards/chosen": -274.8233947753906,
+      "rewards/margins": -28.747509002685547,
+      "rewards/rejected": -246.075927734375,
+      "step": 25
+    },
+    {
+      "epoch": 0.06281078251766553,
+      "grad_norm": 458498.66919273353,
+      "learning_rate": 3.1249999999999997e-07,
+      "logits/chosen": -2.7689995765686035,
+      "logits/rejected": -2.7268056869506836,
+      "logps/chosen": -240.1889190673828,
+      "logps/rejected": -226.0777587890625,
+      "loss": 124675.575,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": -240.1889190673828,
+      "rewards/margins": -14.111166000366211,
+      "rewards/rejected": -226.0777587890625,
+      "step": 30
+    },
+    {
+      "epoch": 0.07327924627060979,
+      "grad_norm": 450902.00202128256,
+      "learning_rate": 3.645833333333333e-07,
+      "logits/chosen": -2.7725298404693604,
+      "logits/rejected": -2.6864283084869385,
+      "logps/chosen": -274.01287841796875,
+      "logps/rejected": -278.74652099609375,
+      "loss": 124375.275,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": -274.01287841796875,
+      "rewards/margins": 4.733607292175293,
+      "rewards/rejected": -278.74652099609375,
+      "step": 35
+    },
+    {
+      "epoch": 0.08374771002355404,
+      "grad_norm": 417057.7715157791,
+      "learning_rate": 4.1666666666666667e-07,
+      "logits/chosen": -2.7862133979797363,
+      "logits/rejected": -2.6889536380767822,
+      "logps/chosen": -294.39178466796875,
+      "logps/rejected": -261.9639892578125,
+      "loss": 123992.45,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": -294.39178466796875,
+      "rewards/margins": -32.427791595458984,
+      "rewards/rejected": -261.9639892578125,
+      "step": 40
+    },
+    {
+      "epoch": 0.0942161737764983,
+      "grad_norm": 466508.70788908063,
+      "learning_rate": 4.6874999999999996e-07,
+      "logits/chosen": -2.7797024250030518,
+      "logits/rejected": -2.731426477432251,
+      "logps/chosen": -278.8966979980469,
+      "logps/rejected": -264.8343200683594,
+      "loss": 123537.05,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": -278.8966979980469,
+      "rewards/margins": -14.062413215637207,
+      "rewards/rejected": -264.8343200683594,
+      "step": 45
+    },
+    {
+      "epoch": 0.10468463752944256,
+      "grad_norm": 559270.5129260804,
+      "learning_rate": 4.999731868769026e-07,
+      "logits/chosen": -2.708888053894043,
+      "logits/rejected": -2.639033079147339,
+      "logps/chosen": -292.9398498535156,
+      "logps/rejected": -284.3505554199219,
+      "loss": 122464.5875,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -292.9398498535156,
+      "rewards/margins": -8.58930492401123,
+      "rewards/rejected": -284.3505554199219,
+      "step": 50
+    },
+    {
+      "epoch": 0.11515310128238682,
+      "grad_norm": 521515.212841857,
+      "learning_rate": 4.996716052911017e-07,
+      "logits/chosen": -2.685326099395752,
+      "logits/rejected": -2.643601894378662,
+      "logps/chosen": -283.38983154296875,
+      "logps/rejected": -281.60528564453125,
+      "loss": 122310.025,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -283.38983154296875,
+      "rewards/margins": -1.784542441368103,
+      "rewards/rejected": -281.60528564453125,
+      "step": 55
+    },
+    {
+      "epoch": 0.12562156503533106,
+      "grad_norm": 729100.4219221757,
+      "learning_rate": 4.990353313429303e-07,
+      "logits/chosen": -2.724234104156494,
+      "logits/rejected": -2.681450605392456,
+      "logps/chosen": -270.0491027832031,
+      "logps/rejected": -281.71820068359375,
+      "loss": 121167.4375,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -270.0491027832031,
+      "rewards/margins": 11.669103622436523,
+      "rewards/rejected": -281.71820068359375,
+      "step": 60
+    },
+    {
+      "epoch": 0.1360900287882753,
+      "grad_norm": 961680.3751689971,
+      "learning_rate": 4.980652179769217e-07,
+      "logits/chosen": -2.718280792236328,
+      "logits/rejected": -2.597512722015381,
+      "logps/chosen": -310.9617004394531,
+      "logps/rejected": -293.2528381347656,
+      "loss": 119799.15,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -310.9617004394531,
+      "rewards/margins": -17.708873748779297,
+      "rewards/rejected": -293.2528381347656,
+      "step": 65
+    },
+    {
+      "epoch": 0.14655849254121958,
+      "grad_norm": 783593.0959446996,
+      "learning_rate": 4.967625656594781e-07,
+      "logits/chosen": -2.6035261154174805,
+      "logits/rejected": -2.5565543174743652,
+      "logps/chosen": -290.8232421875,
+      "logps/rejected": -287.1130065917969,
+      "loss": 119810.2,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -290.8232421875,
+      "rewards/margins": -3.710245132446289,
+      "rewards/rejected": -287.1130065917969,
+      "step": 70
+    },
+    {
+      "epoch": 0.15702695629416383,
+      "grad_norm": 745801.2137469078,
+      "learning_rate": 4.951291206355559e-07,
+      "logits/chosen": -2.7136483192443848,
+      "logits/rejected": -2.6402573585510254,
+      "logps/chosen": -291.23773193359375,
+      "logps/rejected": -318.0777587890625,
+      "loss": 119278.1375,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -291.23773193359375,
+      "rewards/margins": 26.84000015258789,
+      "rewards/rejected": -318.0777587890625,
+      "step": 75
+    },
+    {
+      "epoch": 0.16749542004710807,
+      "grad_norm": 907046.1955290881,
+      "learning_rate": 4.93167072587771e-07,
+      "logits/chosen": -2.718100070953369,
+      "logits/rejected": -2.6574056148529053,
+      "logps/chosen": -292.93060302734375,
+      "logps/rejected": -307.5615234375,
+      "loss": 119745.725,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -292.93060302734375,
+      "rewards/margins": 14.630932807922363,
+      "rewards/rejected": -307.5615234375,
+      "step": 80
+    },
+    {
+      "epoch": 0.17796388380005235,
+      "grad_norm": 860624.0897837883,
+      "learning_rate": 4.908790517010636e-07,
+      "logits/chosen": -2.757253408432007,
+      "logits/rejected": -2.69834566116333,
+      "logps/chosen": -295.21539306640625,
+      "logps/rejected": -327.061767578125,
+      "loss": 118244.975,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -295.21539306640625,
+      "rewards/margins": 31.846384048461914,
+      "rewards/rejected": -327.061767578125,
+      "step": 85
+    },
+    {
+      "epoch": 0.1884323475529966,
+      "grad_norm": 1180769.397532202,
+      "learning_rate": 4.882681251368548e-07,
+      "logits/chosen": -2.7516987323760986,
+      "logits/rejected": -2.7340774536132812,
+      "logps/chosen": -311.9580993652344,
+      "logps/rejected": -325.9947814941406,
+      "loss": 118380.25,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -311.9580993652344,
+      "rewards/margins": 14.036661148071289,
+      "rewards/rejected": -325.9947814941406,
+      "step": 90
+    },
+    {
+      "epoch": 0.19890081130594087,
+      "grad_norm": 1435209.2727647137,
+      "learning_rate": 4.853377929214243e-07,
+      "logits/chosen": -2.7984097003936768,
+      "logits/rejected": -2.713646411895752,
+      "logps/chosen": -308.3005676269531,
+      "logps/rejected": -312.81201171875,
+      "loss": 116816.6,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": -308.3005676269531,
+      "rewards/margins": 4.5114593505859375,
+      "rewards/rejected": -312.81201171875,
+      "step": 95
+    },
+    {
+      "epoch": 0.2093692750588851,
+      "grad_norm": 1728193.4126168345,
+      "learning_rate": 4.820919832540181e-07,
+      "logits/chosen": -2.70627498626709,
+      "logits/rejected": -2.6122021675109863,
+      "logps/chosen": -283.68487548828125,
+      "logps/rejected": -284.87298583984375,
+      "loss": 118125.7625,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -283.68487548828125,
+      "rewards/margins": 1.188114881515503,
+      "rewards/rejected": -284.87298583984375,
+      "step": 100
+    },
+    {
+      "epoch": 0.21983773881182936,
+      "grad_norm": 1249095.5568791889,
+      "learning_rate": 4.785350472409791e-07,
+      "logits/chosen": -2.555546283721924,
+      "logits/rejected": -2.499777317047119,
+      "logps/chosen": -262.59674072265625,
+      "logps/rejected": -337.804931640625,
+      "loss": 116391.2,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -262.59674072265625,
+      "rewards/margins": 75.20821380615234,
+      "rewards/rejected": -337.804931640625,
+      "step": 105
+    },
+    {
+      "epoch": 0.23030620256477363,
+      "grad_norm": 1324636.8678255256,
+      "learning_rate": 4.7467175306295647e-07,
+      "logits/chosen": -2.2604401111602783,
+      "logits/rejected": -2.107639789581299,
+      "logps/chosen": -314.15606689453125,
+      "logps/rejected": -354.67578125,
+      "loss": 115764.325,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -314.15606689453125,
+      "rewards/margins": 40.519737243652344,
+      "rewards/rejected": -354.67578125,
+      "step": 110
+    },
+    {
+      "epoch": 0.24077466631771788,
+      "grad_norm": 2088676.398368818,
+      "learning_rate": 4.70507279583015e-07,
+      "logits/chosen": -2.022336959838867,
+      "logits/rejected": -1.7513967752456665,
+      "logps/chosen": -331.6026611328125,
+      "logps/rejected": -372.3267517089844,
+      "loss": 115322.7375,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -331.6026611328125,
+      "rewards/margins": 40.724082946777344,
+      "rewards/rejected": -372.3267517089844,
+      "step": 115
+    },
+    {
+      "epoch": 0.2512431300706621,
+      "grad_norm": 1807091.2744565222,
+      "learning_rate": 4.6604720940421207e-07,
+      "logits/chosen": -1.804964303970337,
+      "logits/rejected": -1.5805766582489014,
+      "logps/chosen": -315.6778869628906,
+      "logps/rejected": -387.36260986328125,
+      "loss": 115043.6,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -315.6778869628906,
+      "rewards/margins": 71.68473052978516,
+      "rewards/rejected": -387.36260986328125,
+      "step": 120
+    },
+    {
+      "epoch": 0.26171159382360637,
+      "grad_norm": 1582536.6614139078,
+      "learning_rate": 4.612975213859487e-07,
+      "logits/chosen": -2.094388961791992,
+      "logits/rejected": -1.9840008020401,
+      "logps/chosen": -285.61419677734375,
+      "logps/rejected": -304.2431335449219,
+      "loss": 116704.675,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -285.61419677734375,
+      "rewards/margins": 18.628963470458984,
+      "rewards/rejected": -304.2431335449219,
+      "step": 125
+    },
+    {
+      "epoch": 0.2721800575765506,
+      "grad_norm": 1405438.3906735398,
+      "learning_rate": 4.5626458262912735e-07,
+      "logits/chosen": -2.012509822845459,
+      "logits/rejected": -1.8286924362182617,
+      "logps/chosen": -295.90277099609375,
+      "logps/rejected": -341.51812744140625,
+      "loss": 116832.5,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -295.90277099609375,
+      "rewards/margins": 45.61534881591797,
+      "rewards/rejected": -341.51812744140625,
+      "step": 130
+    },
+    {
+      "epoch": 0.2826485213294949,
+      "grad_norm": 2166459.423278389,
+      "learning_rate": 4.5095513994085974e-07,
+      "logits/chosen": -1.7868998050689697,
+      "logits/rejected": -1.1601707935333252,
+      "logps/chosen": -296.5223693847656,
+      "logps/rejected": -369.28326416015625,
+      "loss": 115705.0,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -296.5223693847656,
+      "rewards/margins": 72.76090240478516,
+      "rewards/rejected": -369.28326416015625,
+      "step": 135
+    },
+    {
+      "epoch": 0.29311698508243916,
+      "grad_norm": 1788707.2451489503,
+      "learning_rate": 4.453763107901675e-07,
+      "logits/chosen": -1.2310272455215454,
+      "logits/rejected": -0.7306966781616211,
+      "logps/chosen": -284.04351806640625,
+      "logps/rejected": -365.6307678222656,
+      "loss": 112320.4,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -284.04351806640625,
+      "rewards/margins": 81.58724975585938,
+      "rewards/rejected": -365.6307678222656,
+      "step": 140
+    },
+    {
+      "epoch": 0.3035854488353834,
+      "grad_norm": 2512886.744124689,
+      "learning_rate": 4.395355737667985e-07,
+      "logits/chosen": -1.370542049407959,
+      "logits/rejected": -0.847873330116272,
+      "logps/chosen": -295.51165771484375,
+      "logps/rejected": -344.68634033203125,
+      "loss": 116938.0875,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -295.51165771484375,
+      "rewards/margins": 49.174705505371094,
+      "rewards/rejected": -344.68634033203125,
+      "step": 145
+    },
+    {
+      "epoch": 0.31405391258832765,
+      "grad_norm": 2459660.1241130554,
+      "learning_rate": 4.3344075855595097e-07,
+      "logits/chosen": -1.8194392919540405,
+      "logits/rejected": -1.4853712320327759,
+      "logps/chosen": -293.786865234375,
+      "logps/rejected": -332.8251953125,
+      "loss": 112010.725,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -293.786865234375,
+      "rewards/margins": 39.03831100463867,
+      "rewards/rejected": -332.8251953125,
+      "step": 150
+    },
+    {
+      "epoch": 0.3245223763412719,
+      "grad_norm": 4807009.727933751,
+      "learning_rate": 4.271000354423425e-07,
+      "logits/chosen": -1.3470885753631592,
+      "logits/rejected": -0.8607180714607239,
+      "logps/chosen": -337.14300537109375,
+      "logps/rejected": -391.9334716796875,
+      "loss": 115519.375,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -337.14300537109375,
+      "rewards/margins": 54.79045867919922,
+      "rewards/rejected": -391.9334716796875,
+      "step": 155
+    },
+    {
+      "epoch": 0.33499084009421615,
+      "grad_norm": 2699245.2043745625,
+      "learning_rate": 4.2052190435769554e-07,
+      "logits/chosen": -1.7011409997940063,
+      "logits/rejected": -1.335893988609314,
+      "logps/chosen": -274.394287109375,
+      "logps/rejected": -330.22100830078125,
+      "loss": 114255.0875,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -274.394287109375,
+      "rewards/margins": 55.82673263549805,
+      "rewards/rejected": -330.22100830078125,
+      "step": 160
+    },
+    {
+      "epoch": 0.34545930384716045,
+      "grad_norm": 840943.395175269,
+      "learning_rate": 4.137151834863213e-07,
+      "logits/chosen": -2.163696527481079,
+      "logits/rejected": -1.7751731872558594,
+      "logps/chosen": -273.88092041015625,
+      "logps/rejected": -331.56085205078125,
+      "loss": 116034.3125,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -273.88092041015625,
+      "rewards/margins": 57.679969787597656,
+      "rewards/rejected": -331.56085205078125,
+      "step": 165
+    },
+    {
+      "epoch": 0.3559277676001047,
+      "grad_norm": 1277262.6579787417,
+      "learning_rate": 4.0668899744407567e-07,
+      "logits/chosen": -1.8461663722991943,
+      "logits/rejected": -1.260231852531433,
+      "logps/chosen": -303.04083251953125,
+      "logps/rejected": -325.74371337890625,
+      "loss": 114209.3125,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -303.04083251953125,
+      "rewards/margins": 22.702842712402344,
+      "rewards/rejected": -325.74371337890625,
+      "step": 170
+    },
+    {
+      "epoch": 0.36639623135304894,
+      "grad_norm": 1135243.1010669412,
+      "learning_rate": 3.994527650465352e-07,
+      "logits/chosen": -1.029491901397705,
+      "logits/rejected": -0.0737704187631607,
+      "logps/chosen": -286.8111877441406,
+      "logps/rejected": -299.5812072753906,
+      "loss": 115237.225,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -286.8111877441406,
+      "rewards/margins": 12.770014762878418,
+      "rewards/rejected": -299.5812072753906,
+      "step": 175
+    },
+    {
+      "epoch": 0.3768646951059932,
+      "grad_norm": 1925685.8821038436,
+      "learning_rate": 3.920161866827889e-07,
+      "logits/chosen": -0.454359769821167,
+      "logits/rejected": -0.02063882350921631,
+      "logps/chosen": -297.41461181640625,
+      "logps/rejected": -369.3190002441406,
+      "loss": 113950.1,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -297.41461181640625,
+      "rewards/margins": 71.90437316894531,
+      "rewards/rejected": -369.3190002441406,
+      "step": 180
+    },
+    {
+      "epoch": 0.38733315885893743,
+      "grad_norm": 2698235.0873295115,
+      "learning_rate": 3.8438923131177237e-07,
+      "logits/chosen": -0.5734198689460754,
+      "logits/rejected": 0.27511700987815857,
+      "logps/chosen": -316.64971923828125,
+      "logps/rejected": -350.3505554199219,
+      "loss": 113131.425,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -316.64971923828125,
+      "rewards/margins": 33.700801849365234,
+      "rewards/rejected": -350.3505554199219,
+      "step": 185
+    },
+    {
+      "epoch": 0.39780162261188173,
+      "grad_norm": 2263250.8365393346,
+      "learning_rate": 3.765821230985757e-07,
+      "logits/chosen": -0.4672353267669678,
+      "logits/rejected": -0.028242725878953934,
+      "logps/chosen": -283.13873291015625,
+      "logps/rejected": -305.73968505859375,
+      "loss": 115869.625,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -283.13873291015625,
+      "rewards/margins": 22.600967407226562,
+      "rewards/rejected": -305.73968505859375,
+      "step": 190
+    },
+    {
+      "epoch": 0.408270086364826,
+      "grad_norm": 2336859.8229430458,
+      "learning_rate": 3.6860532770864005e-07,
+      "logits/chosen": -0.3446078598499298,
+      "logits/rejected": 0.17580614984035492,
+      "logps/chosen": -345.1635437011719,
+      "logps/rejected": -373.7042236328125,
+      "loss": 115610.2375,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -345.1635437011719,
+      "rewards/margins": 28.540664672851562,
+      "rewards/rejected": -373.7042236328125,
+      "step": 195
+    },
+    {
+      "epoch": 0.4187385501177702,
+      "grad_norm": 2875002.890975931,
+      "learning_rate": 3.604695382782159e-07,
+      "logits/chosen": -0.3627869188785553,
+      "logits/rejected": -0.14643223583698273,
+      "logps/chosen": -305.51885986328125,
+      "logps/rejected": -352.6678466796875,
+      "loss": 116295.75,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -305.51885986328125,
+      "rewards/margins": 47.14897537231445,
+      "rewards/rejected": -352.6678466796875,
+      "step": 200
+    },
+    {
+      "epoch": 0.42920701387071447,
+      "grad_norm": 1400861.091712678,
+      "learning_rate": 3.5218566107988867e-07,
+      "logits/chosen": -0.8571730852127075,
+      "logits/rejected": -0.25482162833213806,
+      "logps/chosen": -296.7934875488281,
+      "logps/rejected": -329.7360534667969,
+      "loss": 114386.15,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -296.7934875488281,
+      "rewards/margins": 32.94261932373047,
+      "rewards/rejected": -329.7360534667969,
+      "step": 205
+    },
+    {
+      "epoch": 0.4396754776236587,
+      "grad_norm": 2247391.450727784,
+      "learning_rate": 3.4376480090239047e-07,
+      "logits/chosen": -0.7361810803413391,
+      "logits/rejected": -0.26914888620376587,
+      "logps/chosen": -324.780517578125,
+      "logps/rejected": -346.3909606933594,
+      "loss": 115300.375,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": -324.780517578125,
+      "rewards/margins": 21.61046028137207,
+      "rewards/rejected": -346.3909606933594,
+      "step": 210
+    },
+    {
+      "epoch": 0.45014394137660296,
+      "grad_norm": 1942071.0568026109,
+      "learning_rate": 3.3521824616429284e-07,
+      "logits/chosen": -0.7115877270698547,
+      "logits/rejected": -0.16935975849628448,
+      "logps/chosen": -306.1327819824219,
+      "logps/rejected": -375.35125732421875,
+      "loss": 113532.725,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -306.1327819824219,
+      "rewards/margins": 69.21846008300781,
+      "rewards/rejected": -375.35125732421875,
+      "step": 215
+    },
+    {
+      "epoch": 0.46061240512954726,
+      "grad_norm": 1880893.803266421,
+      "learning_rate": 3.265574537815398e-07,
+      "logits/chosen": -0.7726610898971558,
+      "logits/rejected": 0.11326490342617035,
+      "logps/chosen": -315.4923095703125,
+      "logps/rejected": -381.5898132324219,
+      "loss": 113337.5125,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -315.4923095703125,
+      "rewards/margins": 66.09752655029297,
+      "rewards/rejected": -381.5898132324219,
+      "step": 220
+    },
+    {
+      "epoch": 0.4710808688824915,
+      "grad_norm": 1945629.7722817352,
+      "learning_rate": 3.1779403380910425e-07,
+      "logits/chosen": -0.5554154515266418,
+      "logits/rejected": 0.3164888024330139,
+      "logps/chosen": -310.39068603515625,
+      "logps/rejected": -347.70709228515625,
+      "loss": 112926.525,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -310.39068603515625,
+      "rewards/margins": 37.316429138183594,
+      "rewards/rejected": -347.70709228515625,
+      "step": 225
+    },
+    {
+      "epoch": 0.48154933263543576,
+      "grad_norm": 1616156.0747634866,
+      "learning_rate": 3.0893973387735683e-07,
+      "logits/chosen": -0.8544542193412781,
+      "logits/rejected": 0.16107910871505737,
+      "logps/chosen": -327.91864013671875,
+      "logps/rejected": -367.1671447753906,
+      "loss": 112795.225,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -327.91864013671875,
+      "rewards/margins": 39.24848175048828,
+      "rewards/rejected": -367.1671447753906,
+      "step": 230
+    },
+    {
+      "epoch": 0.49201779638838,
+      "grad_norm": 1973066.2630310976,
+      "learning_rate": 3.000064234440111e-07,
+      "logits/chosen": -0.6506497263908386,
+      "logits/rejected": 0.023212980479002,
+      "logps/chosen": -294.9061279296875,
+      "logps/rejected": -359.06591796875,
+      "loss": 113001.925,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -294.9061279296875,
+      "rewards/margins": 64.1597671508789,
+      "rewards/rejected": -359.06591796875,
+      "step": 235
+    },
+    {
+      "epoch": 0.5024862601413242,
+      "grad_norm": 2122255.375505896,
+      "learning_rate": 2.910060778827554e-07,
+      "logits/chosen": -0.884017288684845,
+      "logits/rejected": -0.20214924216270447,
+      "logps/chosen": -304.61309814453125,
+      "logps/rejected": -344.1042175292969,
+      "loss": 112705.575,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -304.61309814453125,
+      "rewards/margins": 39.491153717041016,
+      "rewards/rejected": -344.1042175292969,
+      "step": 240
+    },
+    {
+      "epoch": 0.5129547238942685,
+      "grad_norm": 1750645.3017169987,
+      "learning_rate": 2.8195076242990116e-07,
+      "logits/chosen": -0.5472243428230286,
+      "logits/rejected": 0.13178296387195587,
+      "logps/chosen": -284.5224609375,
+      "logps/rejected": -338.53375244140625,
+      "loss": 114159.5,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -284.5224609375,
+      "rewards/margins": 54.01129150390625,
+      "rewards/rejected": -338.53375244140625,
+      "step": 245
+    },
+    {
+      "epoch": 0.5234231876472127,
+      "grad_norm": 2085184.1734165123,
+      "learning_rate": 2.7285261601056697e-07,
+      "logits/chosen": -0.4131326675415039,
+      "logits/rejected": 0.3880499005317688,
+      "logps/chosen": -274.2272033691406,
+      "logps/rejected": -340.6448669433594,
+      "loss": 114531.3375,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -274.2272033691406,
+      "rewards/margins": 66.4177017211914,
+      "rewards/rejected": -340.6448669433594,
+      "step": 250
+    },
+    {
+      "epoch": 0.533891651400157,
+      "grad_norm": 2304054.790659312,
+      "learning_rate": 2.6372383496608186e-07,
+      "logits/chosen": -0.17990997433662415,
+      "logits/rejected": 0.9603084325790405,
+      "logps/chosen": -331.6031188964844,
+      "logps/rejected": -412.93817138671875,
+      "loss": 111151.35,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -331.6031188964844,
+      "rewards/margins": 81.3350830078125,
+      "rewards/rejected": -412.93817138671875,
+      "step": 255
+    },
+    {
+      "epoch": 0.5443601151531012,
+      "grad_norm": 1940084.13729787,
+      "learning_rate": 2.5457665670441937e-07,
+      "logits/chosen": -0.2644157409667969,
+      "logits/rejected": 0.6259841918945312,
+      "logps/chosen": -258.68670654296875,
+      "logps/rejected": -366.4393615722656,
+      "loss": 112844.35,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -258.68670654296875,
+      "rewards/margins": 107.7526626586914,
+      "rewards/rejected": -366.4393615722656,
+      "step": 260
+    },
+    {
+      "epoch": 0.5548285789060455,
+      "grad_norm": 1521044.9308265387,
+      "learning_rate": 2.454233432955807e-07,
+      "logits/chosen": -0.47308096289634705,
+      "logits/rejected": 0.20548152923583984,
+      "logps/chosen": -286.80633544921875,
+      "logps/rejected": -340.56756591796875,
+      "loss": 112564.5375,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -286.80633544921875,
+      "rewards/margins": 53.76123809814453,
+      "rewards/rejected": -340.56756591796875,
+      "step": 265
+    },
+    {
+      "epoch": 0.5652970426589898,
+      "grad_norm": 1873803.7134507762,
+      "learning_rate": 2.3627616503391812e-07,
+      "logits/chosen": -0.5675733685493469,
+      "logits/rejected": 0.14759287238121033,
+      "logps/chosen": -301.10040283203125,
+      "logps/rejected": -370.1958312988281,
+      "loss": 114260.7375,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -301.10040283203125,
+      "rewards/margins": 69.09541320800781,
+      "rewards/rejected": -370.1958312988281,
+      "step": 270
+    },
+    {
+      "epoch": 0.575765506411934,
+      "grad_norm": 2362564.8339613965,
+      "learning_rate": 2.2714738398943308e-07,
+      "logits/chosen": -0.6608392000198364,
+      "logits/rejected": 0.4885464608669281,
+      "logps/chosen": -315.5157775878906,
+      "logps/rejected": -383.5878601074219,
+      "loss": 111772.275,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -315.5157775878906,
+      "rewards/margins": 68.07210540771484,
+      "rewards/rejected": -383.5878601074219,
+      "step": 275
+    },
+    {
+      "epoch": 0.5862339701648783,
+      "grad_norm": 1807476.8899846808,
+      "learning_rate": 2.1804923757009882e-07,
+      "logits/chosen": -0.5532770752906799,
+      "logits/rejected": 0.43744096159935,
+      "logps/chosen": -315.10870361328125,
+      "logps/rejected": -358.2113952636719,
+      "loss": 112631.6,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -315.10870361328125,
+      "rewards/margins": 43.10268020629883,
+      "rewards/rejected": -358.2113952636719,
+      "step": 280
+    },
+    {
+      "epoch": 0.5967024339178225,
+      "grad_norm": 2237959.642261439,
+      "learning_rate": 2.089939221172446e-07,
+      "logits/chosen": -0.46047574281692505,
+      "logits/rejected": 0.012615549378097057,
+      "logps/chosen": -293.84136962890625,
+      "logps/rejected": -370.7669372558594,
+      "loss": 112695.3,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -293.84136962890625,
+      "rewards/margins": 76.92558288574219,
+      "rewards/rejected": -370.7669372558594,
+      "step": 285
+    },
+    {
+      "epoch": 0.6071708976707668,
+      "grad_norm": 1554217.5345301214,
+      "learning_rate": 1.9999357655598891e-07,
+      "logits/chosen": -0.5754300355911255,
+      "logits/rejected": 0.33466261625289917,
+      "logps/chosen": -303.59539794921875,
+      "logps/rejected": -346.04412841796875,
+      "loss": 112625.2125,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -303.59539794921875,
+      "rewards/margins": 42.44871520996094,
+      "rewards/rejected": -346.04412841796875,
+      "step": 290
+    },
+    {
+      "epoch": 0.6176393614237111,
+      "grad_norm": 2521456.999656128,
+      "learning_rate": 1.9106026612264315e-07,
+      "logits/chosen": -0.687907338142395,
+      "logits/rejected": 0.15663442015647888,
+      "logps/chosen": -279.992919921875,
+      "logps/rejected": -332.9551696777344,
+      "loss": 114791.675,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -279.992919921875,
+      "rewards/margins": 52.962249755859375,
+      "rewards/rejected": -332.9551696777344,
+      "step": 295
+    },
+    {
+      "epoch": 0.6281078251766553,
+      "grad_norm": 3193163.1405664305,
+      "learning_rate": 1.8220596619089573e-07,
+      "logits/chosen": -0.7139306664466858,
+      "logits/rejected": -0.06598733365535736,
+      "logps/chosen": -264.813232421875,
+      "logps/rejected": -318.2864685058594,
+      "loss": 112895.5875,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -264.813232421875,
+      "rewards/margins": 53.47322463989258,
+      "rewards/rejected": -318.2864685058594,
+      "step": 300
+    },
+    {
+      "epoch": 0.6385762889295996,
+      "grad_norm": 2896442.2441057838,
+      "learning_rate": 1.7344254621846017e-07,
+      "logits/chosen": -0.5134263634681702,
+      "logits/rejected": 0.272099107503891,
+      "logps/chosen": -293.1112060546875,
+      "logps/rejected": -366.1026306152344,
+      "loss": 111599.1625,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -293.1112060546875,
+      "rewards/margins": 72.99146270751953,
+      "rewards/rejected": -366.1026306152344,
+      "step": 305
+    },
+    {
+      "epoch": 0.6490447526825438,
+      "grad_norm": 1410328.2640552768,
+      "learning_rate": 1.647817538357072e-07,
+      "logits/chosen": -0.5094229578971863,
+      "logits/rejected": -0.08179249614477158,
+      "logps/chosen": -262.71868896484375,
+      "logps/rejected": -373.6757507324219,
+      "loss": 110514.8625,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -262.71868896484375,
+      "rewards/margins": 110.95704650878906,
+      "rewards/rejected": -373.6757507324219,
+      "step": 310
+    },
+    {
+      "epoch": 0.6595132164354881,
+      "grad_norm": 4370337.480639941,
+      "learning_rate": 1.562351990976095e-07,
+      "logits/chosen": -0.5516433715820312,
+      "logits/rejected": 0.3842604160308838,
+      "logps/chosen": -299.07208251953125,
+      "logps/rejected": -386.3035583496094,
+      "loss": 111146.3375,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -299.07208251953125,
+      "rewards/margins": 87.2314682006836,
+      "rewards/rejected": -386.3035583496094,
+      "step": 315
+    },
+    {
+      "epoch": 0.6699816801884323,
+      "grad_norm": 8860713.857687093,
+      "learning_rate": 1.478143389201113e-07,
+      "logits/chosen": -0.3542352616786957,
+      "logits/rejected": 0.15903237462043762,
+      "logps/chosen": -277.9214782714844,
+      "logps/rejected": -365.1288146972656,
+      "loss": 112815.525,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -277.9214782714844,
+      "rewards/margins": 87.20732879638672,
+      "rewards/rejected": -365.1288146972656,
+      "step": 320
+    },
+    {
+      "epoch": 0.6804501439413766,
+      "grad_norm": 1908239.1278667776,
+      "learning_rate": 1.3953046172178413e-07,
+      "logits/chosen": -0.5815322399139404,
+      "logits/rejected": 0.323032021522522,
+      "logps/chosen": -323.28436279296875,
+      "logps/rejected": -390.740234375,
+      "loss": 112318.4,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -323.28436279296875,
+      "rewards/margins": 67.45586395263672,
+      "rewards/rejected": -390.740234375,
+      "step": 325
+    },
+    {
+      "epoch": 0.6909186076943209,
+      "grad_norm": 2119739.073105214,
+      "learning_rate": 1.3139467229135998e-07,
+      "logits/chosen": -0.668155312538147,
+      "logits/rejected": 0.13567589223384857,
+      "logps/chosen": -295.8731384277344,
+      "logps/rejected": -347.6330261230469,
+      "loss": 111457.7,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -295.8731384277344,
+      "rewards/margins": 51.7598762512207,
+      "rewards/rejected": -347.6330261230469,
+      "step": 330
+    },
+    {
+      "epoch": 0.7013870714472651,
+      "grad_norm": 1927999.1666317754,
+      "learning_rate": 1.2341787690142435e-07,
+      "logits/chosen": -0.8385793566703796,
+      "logits/rejected": -0.16031041741371155,
+      "logps/chosen": -299.18060302734375,
+      "logps/rejected": -360.9104919433594,
+      "loss": 112842.25,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -299.18060302734375,
+      "rewards/margins": 61.72993087768555,
+      "rewards/rejected": -360.9104919433594,
+      "step": 335
+    },
+    {
+      "epoch": 0.7118555352002094,
+      "grad_norm": 2176227.517015282,
+      "learning_rate": 1.1561076868822755e-07,
+      "logits/chosen": -0.7431761622428894,
+      "logits/rejected": 0.11613886058330536,
+      "logps/chosen": -331.2103271484375,
+      "logps/rejected": -360.3314208984375,
+      "loss": 111618.6375,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -331.2103271484375,
+      "rewards/margins": 29.121089935302734,
+      "rewards/rejected": -360.3314208984375,
+      "step": 340
+    },
+    {
+      "epoch": 0.7223239989531536,
+      "grad_norm": 2699423.898748738,
+      "learning_rate": 1.0798381331721107e-07,
+      "logits/chosen": -0.7944961786270142,
+      "logits/rejected": 0.2726106643676758,
+      "logps/chosen": -324.1297302246094,
+      "logps/rejected": -362.2281188964844,
+      "loss": 111899.3375,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -324.1297302246094,
+      "rewards/margins": 38.09839630126953,
+      "rewards/rejected": -362.2281188964844,
+      "step": 345
+    },
+    {
+      "epoch": 0.7327924627060979,
+      "grad_norm": 2320601.542393335,
+      "learning_rate": 1.0054723495346482e-07,
+      "logits/chosen": -0.2713714838027954,
+      "logits/rejected": 0.22024962306022644,
+      "logps/chosen": -288.8630676269531,
+      "logps/rejected": -365.87994384765625,
+      "loss": 113524.075,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -288.8630676269531,
+      "rewards/margins": 77.01687622070312,
+      "rewards/rejected": -365.87994384765625,
+      "step": 350
+    },
+    {
+      "epoch": 0.7432609264590422,
+      "grad_norm": 1815809.2243324781,
+      "learning_rate": 9.331100255592436e-08,
+      "logits/chosen": -0.2964836657047272,
+      "logits/rejected": 0.5565654039382935,
+      "logps/chosen": -277.563232421875,
+      "logps/rejected": -366.7838134765625,
+      "loss": 111167.625,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -277.563232421875,
+      "rewards/margins": 89.22059631347656,
+      "rewards/rejected": -366.7838134765625,
+      "step": 355
+    },
+    {
+      "epoch": 0.7537293902119864,
+      "grad_norm": 1971255.6245351008,
+      "learning_rate": 8.628481651367875e-08,
+      "logits/chosen": -0.7251734137535095,
+      "logits/rejected": 0.05562416836619377,
+      "logps/chosen": -304.60357666015625,
+      "logps/rejected": -355.7652587890625,
+      "loss": 109653.975,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -304.60357666015625,
+      "rewards/margins": 51.16161346435547,
+      "rewards/rejected": -355.7652587890625,
+      "step": 360
+    },
+    {
+      "epoch": 0.7641978539649307,
+      "grad_norm": 1754900.7076158663,
+      "learning_rate": 7.947809564230445e-08,
+      "logits/chosen": -0.45269888639450073,
+      "logits/rejected": 0.3153532147407532,
+      "logps/chosen": -322.9766845703125,
+      "logps/rejected": -375.75787353515625,
+      "loss": 114208.8,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -322.9766845703125,
+      "rewards/margins": 52.78118896484375,
+      "rewards/rejected": -375.75787353515625,
+      "step": 365
+    },
+    {
+      "epoch": 0.7746663177178749,
+      "grad_norm": 2147071.573360362,
+      "learning_rate": 7.289996455765748e-08,
+      "logits/chosen": -0.43487948179244995,
+      "logits/rejected": 0.42789965867996216,
+      "logps/chosen": -311.0076599121094,
+      "logps/rejected": -392.62152099609375,
+      "loss": 112070.675,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -311.0076599121094,
+      "rewards/margins": 81.61390686035156,
+      "rewards/rejected": -392.62152099609375,
+      "step": 370
+    },
+    {
+      "epoch": 0.7851347814708192,
+      "grad_norm": 1709073.4926486812,
+      "learning_rate": 6.655924144404906e-08,
+      "logits/chosen": -0.30552688241004944,
+      "logits/rejected": -0.23342788219451904,
+      "logps/chosen": -273.34320068359375,
+      "logps/rejected": -361.71856689453125,
+      "loss": 110961.7625,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -273.34320068359375,
+      "rewards/margins": 88.37535095214844,
+      "rewards/rejected": -361.71856689453125,
+      "step": 375
+    },
+    {
+      "epoch": 0.7956032452237635,
+      "grad_norm": 2722896.7250441867,
+      "learning_rate": 6.046442623320145e-08,
+      "logits/chosen": -0.41763028502464294,
+      "logits/rejected": 0.005231809802353382,
+      "logps/chosen": -307.29693603515625,
+      "logps/rejected": -393.7823181152344,
+      "loss": 111852.7,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -307.29693603515625,
+      "rewards/margins": 86.4853744506836,
+      "rewards/rejected": -393.7823181152344,
+      "step": 380
+    },
+    {
+      "epoch": 0.8060717089767077,
+      "grad_norm": 1745300.231202083,
+      "learning_rate": 5.4623689209832484e-08,
+      "logits/chosen": -0.718239426612854,
+      "logits/rejected": -0.026650678366422653,
+      "logps/chosen": -292.75494384765625,
+      "logps/rejected": -368.7347717285156,
+      "loss": 112271.8625,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -292.75494384765625,
+      "rewards/margins": 75.97982788085938,
+      "rewards/rejected": -368.7347717285156,
+      "step": 385
+    },
+    {
+      "epoch": 0.816540172729652,
+      "grad_norm": 2201936.896729985,
+      "learning_rate": 4.904486005914027e-08,
+      "logits/chosen": -0.9130474328994751,
+      "logits/rejected": 0.17849242687225342,
+      "logps/chosen": -299.4166259765625,
+      "logps/rejected": -363.7040100097656,
+      "loss": 110293.55,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -299.4166259765625,
+      "rewards/margins": 64.2873764038086,
+      "rewards/rejected": -363.7040100097656,
+      "step": 390
+    },
+    {
+      "epoch": 0.8270086364825961,
+      "grad_norm": 1965017.8644740335,
+      "learning_rate": 4.373541737087263e-08,
+      "logits/chosen": -0.7144075036048889,
+      "logits/rejected": 0.521481454372406,
+      "logps/chosen": -278.52801513671875,
+      "logps/rejected": -330.89202880859375,
+      "loss": 109924.925,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -278.52801513671875,
+      "rewards/margins": 52.3640251159668,
+      "rewards/rejected": -330.89202880859375,
+      "step": 395
+    },
+    {
+      "epoch": 0.8374771002355405,
+      "grad_norm": 2507424.209111181,
+      "learning_rate": 3.8702478614051345e-08,
+      "logits/chosen": -0.6051787734031677,
+      "logits/rejected": 0.003916704561561346,
+      "logps/chosen": -281.8033447265625,
+      "logps/rejected": -354.2074279785156,
+      "loss": 113172.8375,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -281.8033447265625,
+      "rewards/margins": 72.404052734375,
+      "rewards/rejected": -354.2074279785156,
+      "step": 400
+    },
+    {
+      "epoch": 0.8479455639884846,
+      "grad_norm": 2422379.6896536425,
+      "learning_rate": 3.3952790595787986e-08,
+      "logits/chosen": -0.6138535737991333,
+      "logits/rejected": -0.022265207022428513,
+      "logps/chosen": -283.72808837890625,
+      "logps/rejected": -363.5780944824219,
+      "loss": 112426.925,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -283.72808837890625,
+      "rewards/margins": 79.85002899169922,
+      "rewards/rejected": -363.5780944824219,
+      "step": 405
+    },
+    {
+      "epoch": 0.8584140277414289,
+      "grad_norm": 2043445.172936004,
+      "learning_rate": 2.9492720416985e-08,
+      "logits/chosen": -0.7124518752098083,
+      "logits/rejected": -0.10028399527072906,
+      "logps/chosen": -277.91021728515625,
+      "logps/rejected": -341.11846923828125,
+      "loss": 112355.5875,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -277.91021728515625,
+      "rewards/margins": 63.208221435546875,
+      "rewards/rejected": -341.11846923828125,
+      "step": 410
+    },
+    {
+      "epoch": 0.8688824914943732,
+      "grad_norm": 1929711.202147663,
+      "learning_rate": 2.5328246937043525e-08,
+      "logits/chosen": -0.5992367267608643,
+      "logits/rejected": 0.41105660796165466,
+      "logps/chosen": -325.5799560546875,
+      "logps/rejected": -357.8695983886719,
+      "loss": 110899.8875,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -325.5799560546875,
+      "rewards/margins": 32.289649963378906,
+      "rewards/rejected": -357.8695983886719,
+      "step": 415
+    },
+    {
+      "epoch": 0.8793509552473174,
+      "grad_norm": 1596356.9629490953,
+      "learning_rate": 2.1464952759020856e-08,
+      "logits/chosen": -0.666754961013794,
+      "logits/rejected": -0.003174859331920743,
+      "logps/chosen": -269.9102478027344,
+      "logps/rejected": -335.4573974609375,
+      "loss": 111847.95,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -269.9102478027344,
+      "rewards/margins": 65.54713439941406,
+      "rewards/rejected": -335.4573974609375,
+      "step": 420
+    },
+    {
+      "epoch": 0.8898194190002617,
+      "grad_norm": 1777896.953612317,
+      "learning_rate": 1.7908016745981856e-08,
+      "logits/chosen": -0.5347632765769958,
+      "logits/rejected": 0.15892794728279114,
+      "logps/chosen": -334.6045227050781,
+      "logps/rejected": -374.3296813964844,
+      "loss": 111778.7375,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -334.6045227050781,
+      "rewards/margins": 39.72520065307617,
+      "rewards/rejected": -374.3296813964844,
+      "step": 425
+    },
+    {
+      "epoch": 0.9002878827532059,
+      "grad_norm": 1803597.2077325762,
+      "learning_rate": 1.4662207078575684e-08,
+      "logits/chosen": -0.5875300765037537,
+      "logits/rejected": 0.1222870722413063,
+      "logps/chosen": -282.852783203125,
+      "logps/rejected": -356.6959228515625,
+      "loss": 111337.7875,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -282.852783203125,
+      "rewards/margins": 73.84312438964844,
+      "rewards/rejected": -356.6959228515625,
+      "step": 430
+    },
+    {
+      "epoch": 0.9107563465061502,
+      "grad_norm": 2178153.4810825037,
+      "learning_rate": 1.1731874863145142e-08,
+      "logits/chosen": -0.4485841691493988,
+      "logits/rejected": 0.1706048995256424,
+      "logps/chosen": -246.25521850585938,
+      "logps/rejected": -336.5995178222656,
+      "loss": 112746.675,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -246.25521850585938,
+      "rewards/margins": 90.34431457519531,
+      "rewards/rejected": -336.5995178222656,
+      "step": 435
+    },
+    {
+      "epoch": 0.9212248102590945,
+      "grad_norm": 2249926.728268449,
+      "learning_rate": 9.12094829893642e-09,
+      "logits/chosen": -0.23983442783355713,
+      "logits/rejected": 0.3341619372367859,
+      "logps/chosen": -297.7192077636719,
+      "logps/rejected": -405.0928649902344,
+      "loss": 112266.5125,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -297.7192077636719,
+      "rewards/margins": 107.37367248535156,
+      "rewards/rejected": -405.0928649902344,
+      "step": 440
+    },
+    {
+      "epoch": 0.9316932740120387,
+      "grad_norm": 2093448.7772529407,
+      "learning_rate": 6.832927412229017e-09,
+      "logits/chosen": -0.5585839152336121,
+      "logits/rejected": 0.15140660107135773,
+      "logps/chosen": -316.87335205078125,
+      "logps/rejected": -385.70098876953125,
+      "loss": 112199.175,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -316.87335205078125,
+      "rewards/margins": 68.82764434814453,
+      "rewards/rejected": -385.70098876953125,
+      "step": 445
+    },
+    {
+      "epoch": 0.942161737764983,
+      "grad_norm": 2219595.1116894637,
+      "learning_rate": 4.8708793644441086e-09,
+      "logits/chosen": -0.3891911506652832,
+      "logits/rejected": 0.024116378277540207,
+      "logps/chosen": -270.6603088378906,
+      "logps/rejected": -366.87481689453125,
+      "loss": 110792.1375,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -270.6603088378906,
+      "rewards/margins": 96.21451568603516,
+      "rewards/rejected": -366.87481689453125,
+      "step": 450
+    },
+    {
+      "epoch": 0.9526302015179272,
+      "grad_norm": 2056046.2252654296,
+      "learning_rate": 3.2374343405217884e-09,
+      "logits/chosen": -0.3999771475791931,
+      "logits/rejected": 0.3102295994758606,
+      "logps/chosen": -297.7739562988281,
+      "logps/rejected": -349.3359069824219,
+      "loss": 111786.575,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -297.7739562988281,
+      "rewards/margins": 51.561973571777344,
+      "rewards/rejected": -349.3359069824219,
+      "step": 455
+    },
+    {
+      "epoch": 0.9630986652708715,
+      "grad_norm": 1596322.4991768566,
+      "learning_rate": 1.9347820230782295e-09,
+      "logits/chosen": -0.33234018087387085,
+      "logits/rejected": 0.4286741614341736,
+      "logps/chosen": -302.89599609375,
+      "logps/rejected": -369.64752197265625,
+      "loss": 110058.4875,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -302.89599609375,
+      "rewards/margins": 66.75153350830078,
+      "rewards/rejected": -369.64752197265625,
+      "step": 460
+    },
+    {
+      "epoch": 0.9735671290238157,
+      "grad_norm": 2030583.2053086923,
+      "learning_rate": 9.64668657069706e-10,
+      "logits/chosen": -0.7010616064071655,
+      "logits/rejected": 0.4352653920650482,
+      "logps/chosen": -332.9463195800781,
+      "logps/rejected": -413.53594970703125,
+      "loss": 109834.325,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -332.9463195800781,
+      "rewards/margins": 80.58956146240234,
+      "rewards/rejected": -413.53594970703125,
+      "step": 465
+    },
+    {
+      "epoch": 0.98403559277676,
+      "grad_norm": 1771708.6547032476,
+      "learning_rate": 3.2839470889836627e-10,
+      "logits/chosen": -0.673961877822876,
+      "logits/rejected": 0.0036758959759026766,
+      "logps/chosen": -312.06292724609375,
+      "logps/rejected": -398.3697814941406,
+      "loss": 112512.225,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -312.06292724609375,
+      "rewards/margins": 86.30685424804688,
+      "rewards/rejected": -398.3697814941406,
+      "step": 470
+    },
+    {
+      "epoch": 0.9945040565297043,
+      "grad_norm": 1840231.2095069257,
+      "learning_rate": 2.6813123097352287e-11,
+      "logits/chosen": -0.7123033404350281,
+      "logits/rejected": 0.20311255753040314,
+      "logps/chosen": -341.65423583984375,
+      "logps/rejected": -375.7768859863281,
+      "loss": 111344.2125,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -341.65423583984375,
+      "rewards/margins": 34.122650146484375,
+      "rewards/rejected": -375.7768859863281,
+      "step": 475
+    },
+    {
+      "epoch": 0.998691442030882,
+      "step": 477,
+      "total_flos": 0.0,
+      "train_loss": 114854.6911687631,
+      "train_runtime": 7479.76,
+      "train_samples_per_second": 8.173,
+      "train_steps_per_second": 0.064
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 477,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 1000000,
+  "total_flos": 0.0,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:08209d240fb25d6d3c95b9bfdbccd0a07021fe1ab4913d65aad01d1bfa225462
+size 7224