Model save

README.md

@@ -1,20 +1,17 @@
 ---
 base_model: google/gemma-7b
-datasets:
-- argilla/dpo-mix-7k
 library_name: transformers
-model_name: google/gemma-7b
+model_name: gemma-7b-silvia-basic-5e-5-v4
 tags:
 - generated_from_trainer
-- alignment-handbook
 - trl
 - orpo
 licence: license
 ---
 
-# Model Card for google/gemma-7b
+# Model Card for gemma-7b-silvia-basic-5e-5-v4
 
-This model is a fine-tuned version of [google/gemma-7b](https://huggingface.co/google/gemma-7b) on the [['argilla/dpo-mix-7k']](https://huggingface.co/datasets/['argilla/dpo-mix-7k']) dataset.
+This model is a fine-tuned version of [google/gemma-7b](https://huggingface.co/google/gemma-7b).
 It has been trained using [TRL](https://github.com/huggingface/trl).
 
 ## Quick start
@@ -30,7 +27,7 @@ print(output["generated_text"])
 
 ## Training procedure
 
-[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/silvias/huggingface/runs/jxlh26x2) 
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/silvias/huggingface/runs/0lavtiko) 
 
 
 This model was trained with ORPO, a method introduced in [ORPO: Monolithic Preference Optimization without Reference Model](https://huggingface.co/papers/2403.07691).

all_results.json

@@ -1,9 +1,9 @@
 {
     "epoch": 2.9765925925925925,
     "total_flos": 0.0,
-    "train_loss": -61.895723918127636,
-    "train_runtime": 9291.6158,
+    "train_loss": -125.51550612676711,
+    "train_runtime": 9350.753,
     "train_samples": 6750,
-    "train_samples_per_second": 2.179,
+    "train_samples_per_second": 2.166,
     "train_steps_per_second": 0.034
 }

train_results.json

@@ -1,9 +1,9 @@
 {
     "epoch": 2.9765925925925925,
     "total_flos": 0.0,
-    "train_loss": -61.895723918127636,
-    "train_runtime": 9291.6158,
+    "train_loss": -125.51550612676711,
+    "train_runtime": 9350.753,
     "train_samples": 6750,
-    "train_samples_per_second": 2.179,
+    "train_samples_per_second": 2.166,
     "train_steps_per_second": 0.034
 }

trainer_state.json

@@ -10,1019 +10,1019 @@
   "log_history": [
     {
       "epoch": 0.047407407407407405,
-      "grad_norm": 1309.7264404296875,
+      "grad_norm": 1346.035400390625,
       "learning_rate": 7.8125e-06,
-      "log_odds_chosen": -1.0092629194259644,
-      "log_odds_ratio": -10.180293083190918,
-      "logps/chosen": -21.700639724731445,
-      "logps/rejected": -20.691608428955078,
-      "loss": 710.1368,
-      "nll_loss": 9.67548656463623,
+      "log_odds_chosen": -1.0068690776824951,
+      "log_odds_ratio": -10.175436019897461,
+      "logps/chosen": -21.69235610961914,
+      "logps/rejected": -20.685726165771484,
+      "loss": 709.8242,
+      "nll_loss": 9.674795150756836,
       "rewards/accuracies": 0.484375,
-      "rewards/chosen": -10.850319862365723,
-      "rewards/margins": -0.5045148134231567,
-      "rewards/rejected": -10.345804214477539,
+      "rewards/chosen": -10.84617805480957,
+      "rewards/margins": -0.5033147931098938,
+      "rewards/rejected": -10.342863082885742,
       "step": 5
     },
     {
       "epoch": 0.09481481481481481,
-      "grad_norm": 1233.22119140625,
+      "grad_norm": 1455.1102294921875,
       "learning_rate": 1.5625e-05,
-      "log_odds_chosen": -2.799683094024658,
-      "log_odds_ratio": -10.153043746948242,
-      "logps/chosen": -20.964075088500977,
-      "logps/rejected": -18.164752960205078,
-      "loss": 714.4481,
-      "nll_loss": 8.683107376098633,
+      "log_odds_chosen": -2.8008556365966797,
+      "log_odds_ratio": -10.15751838684082,
+      "logps/chosen": -20.97389793395996,
+      "logps/rejected": -18.17340660095215,
+      "loss": 714.7821,
+      "nll_loss": 8.685011863708496,
       "rewards/accuracies": 0.44062501192092896,
-      "rewards/chosen": -10.482037544250488,
-      "rewards/margins": -1.3996615409851074,
-      "rewards/rejected": -9.082376480102539,
+      "rewards/chosen": -10.48694896697998,
+      "rewards/margins": -1.4002447128295898,
+      "rewards/rejected": -9.086703300476074,
       "step": 10
     },
     {
       "epoch": 0.14222222222222222,
-      "grad_norm": 664.7484741210938,
+      "grad_norm": 638.308837890625,
       "learning_rate": 2.34375e-05,
-      "log_odds_chosen": -1.274759292602539,
-      "log_odds_ratio": -10.432046890258789,
-      "logps/chosen": -20.83938980102539,
-      "logps/rejected": -19.563129425048828,
-      "loss": 686.9441,
-      "nll_loss": 8.075501441955566,
-      "rewards/accuracies": 0.515625,
-      "rewards/chosen": -10.419694900512695,
-      "rewards/margins": -0.638128936290741,
-      "rewards/rejected": -9.781564712524414,
+      "log_odds_chosen": -1.2725921869277954,
+      "log_odds_ratio": -10.43528938293457,
+      "logps/chosen": -20.847124099731445,
+      "logps/rejected": -19.573030471801758,
+      "loss": 687.155,
+      "nll_loss": 8.07744312286377,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -10.423562049865723,
+      "rewards/margins": -0.6370473504066467,
+      "rewards/rejected": -9.786515235900879,
       "step": 15
     },
     {
       "epoch": 0.18962962962962962,
-      "grad_norm": 5581.04345703125,
+      "grad_norm": 5364.12744140625,
       "learning_rate": 3.125e-05,
-      "log_odds_chosen": -3.752788543701172,
-      "log_odds_ratio": -8.436421394348145,
-      "logps/chosen": -15.843017578125,
-      "logps/rejected": -12.092793464660645,
-      "loss": 565.9146,
-      "nll_loss": 6.499529838562012,
-      "rewards/accuracies": 0.4375,
-      "rewards/chosen": -7.9215087890625,
-      "rewards/margins": -1.8751119375228882,
-      "rewards/rejected": -6.046396732330322,
+      "log_odds_chosen": -3.7516868114471436,
+      "log_odds_ratio": -8.449677467346191,
+      "logps/chosen": -15.886372566223145,
+      "logps/rejected": -12.137221336364746,
+      "loss": 567.2934,
+      "nll_loss": 6.516029357910156,
+      "rewards/accuracies": 0.4312500059604645,
+      "rewards/chosen": -7.943186283111572,
+      "rewards/margins": -1.8745750188827515,
+      "rewards/rejected": -6.068610668182373,
       "step": 20
     },
     {
       "epoch": 0.23703703703703705,
-      "grad_norm": 271.88677978515625,
+      "grad_norm": 732.103271484375,
       "learning_rate": 3.90625e-05,
-      "log_odds_chosen": -0.09555625915527344,
-      "log_odds_ratio": -1.5934648513793945,
-      "logps/chosen": -3.681891918182373,
-      "logps/rejected": -3.571626663208008,
-      "loss": 120.1892,
-      "nll_loss": 3.53655743598938,
-      "rewards/accuracies": 0.5562499761581421,
-      "rewards/chosen": -1.8409459590911865,
-      "rewards/margins": -0.055132925510406494,
-      "rewards/rejected": -1.785813331604004,
+      "log_odds_chosen": -0.05239830166101456,
+      "log_odds_ratio": -1.5554496049880981,
+      "logps/chosen": -3.6205036640167236,
+      "logps/rejected": -3.5546963214874268,
+      "loss": 117.5423,
+      "nll_loss": 3.4973304271698,
+      "rewards/accuracies": 0.5531250238418579,
+      "rewards/chosen": -1.8102518320083618,
+      "rewards/margins": -0.03290349990129471,
+      "rewards/rejected": -1.7773481607437134,
       "step": 25
     },
     {
       "epoch": 0.28444444444444444,
-      "grad_norm": 225.77731323242188,
+      "grad_norm": 235.78294372558594,
       "learning_rate": 4.6875e-05,
-      "log_odds_chosen": 0.05429550260305405,
-      "log_odds_ratio": -0.8730447888374329,
-      "logps/chosen": -1.855459213256836,
-      "logps/rejected": -1.9073245525360107,
-      "loss": 58.9805,
-      "nll_loss": 2.5173518657684326,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": -0.927729606628418,
-      "rewards/margins": 0.025932852178812027,
-      "rewards/rejected": -0.9536622762680054,
+      "log_odds_chosen": -0.004375058226287365,
+      "log_odds_ratio": -0.9642303586006165,
+      "logps/chosen": -2.1026854515075684,
+      "logps/rejected": -2.0986106395721436,
+      "loss": 67.608,
+      "nll_loss": 2.820711851119995,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -1.0513427257537842,
+      "rewards/margins": -0.002037324011325836,
+      "rewards/rejected": -1.0493053197860718,
       "step": 30
     },
     {
       "epoch": 0.33185185185185184,
-      "grad_norm": 330.9626770019531,
+      "grad_norm": 201.821533203125,
       "learning_rate": 4.998613757348784e-05,
-      "log_odds_chosen": 0.21338006854057312,
-      "log_odds_ratio": -0.8034284710884094,
-      "logps/chosen": -1.7032153606414795,
-      "logps/rejected": -1.8913615942001343,
-      "loss": 52.0844,
-      "nll_loss": 2.3274359703063965,
-      "rewards/accuracies": 0.518750011920929,
-      "rewards/chosen": -0.8516076803207397,
-      "rewards/margins": 0.09407330304384232,
-      "rewards/rejected": -0.9456807971000671,
+      "log_odds_chosen": 0.23235782980918884,
+      "log_odds_ratio": -0.796781063079834,
+      "logps/chosen": -1.6877104043960571,
+      "logps/rejected": -1.896958589553833,
+      "loss": 51.2266,
+      "nll_loss": 2.3120052814483643,
+      "rewards/accuracies": 0.5093749761581421,
+      "rewards/chosen": -0.8438552021980286,
+      "rewards/margins": 0.10462428629398346,
+      "rewards/rejected": -0.9484792947769165,
       "step": 35
     },
     {
       "epoch": 0.37925925925925924,
-      "grad_norm": 175.3074493408203,
+      "grad_norm": 501.2691345214844,
       "learning_rate": 4.990147841143462e-05,
-      "log_odds_chosen": 0.31794872879981995,
-      "log_odds_ratio": -0.7034906148910522,
-      "logps/chosen": -1.5063345432281494,
-      "logps/rejected": -1.7724323272705078,
-      "loss": 45.107,
-      "nll_loss": 2.3017280101776123,
-      "rewards/accuracies": 0.590624988079071,
-      "rewards/chosen": -0.7531672716140747,
-      "rewards/margins": 0.1330489218235016,
-      "rewards/rejected": -0.8862161636352539,
+      "log_odds_chosen": 0.3352593779563904,
+      "log_odds_ratio": -0.7122059464454651,
+      "logps/chosen": -1.5017939805984497,
+      "logps/rejected": -1.7836459875106812,
+      "loss": 44.764,
+      "nll_loss": 2.343039035797119,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.7508969902992249,
+      "rewards/margins": 0.1409260481595993,
+      "rewards/rejected": -0.8918229937553406,
       "step": 40
     },
     {
       "epoch": 0.4266666666666667,
-      "grad_norm": 148.9108123779297,
+      "grad_norm": 224.84271240234375,
       "learning_rate": 4.97401218720448e-05,
-      "log_odds_chosen": 0.32661986351013184,
-      "log_odds_ratio": -0.695238471031189,
-      "logps/chosen": -1.4421089887619019,
-      "logps/rejected": -1.732181191444397,
-      "loss": 42.7725,
-      "nll_loss": 2.121166944503784,
-      "rewards/accuracies": 0.6031249761581421,
-      "rewards/chosen": -0.7210544943809509,
-      "rewards/margins": 0.14503604173660278,
-      "rewards/rejected": -0.8660905957221985,
+      "log_odds_chosen": 0.3514587879180908,
+      "log_odds_ratio": -0.6827880144119263,
+      "logps/chosen": -1.4357855319976807,
+      "logps/rejected": -1.7441604137420654,
+      "loss": 42.3725,
+      "nll_loss": 2.1640477180480957,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -0.7178927659988403,
+      "rewards/margins": 0.15418751537799835,
+      "rewards/rejected": -0.8720802068710327,
       "step": 45
     },
     {
       "epoch": 0.4740740740740741,
-      "grad_norm": 168.20802307128906,
+      "grad_norm": 208.4950714111328,
       "learning_rate": 4.9502564938797946e-05,
-      "log_odds_chosen": 0.41916948556900024,
-      "log_odds_ratio": -0.6709465980529785,
-      "logps/chosen": -1.3388197422027588,
-      "logps/rejected": -1.7024009227752686,
-      "loss": 38.517,
-      "nll_loss": 2.0670645236968994,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": -0.6694098711013794,
-      "rewards/margins": 0.18179059028625488,
-      "rewards/rejected": -0.8512004613876343,
+      "log_odds_chosen": 0.3455984592437744,
+      "log_odds_ratio": -0.6899778842926025,
+      "logps/chosen": -1.336740493774414,
+      "logps/rejected": -1.6336427927017212,
+      "loss": 39.3189,
+      "nll_loss": 2.020599842071533,
+      "rewards/accuracies": 0.6156250238418579,
+      "rewards/chosen": -0.668370246887207,
+      "rewards/margins": 0.14845120906829834,
+      "rewards/rejected": -0.8168213963508606,
       "step": 50
     },
     {
       "epoch": 0.5214814814814814,
-      "grad_norm": 1289.98828125,
+      "grad_norm": 1214.6304931640625,
       "learning_rate": 4.918953929490768e-05,
-      "log_odds_chosen": 1.8812118768692017,
-      "log_odds_ratio": -0.7291940450668335,
-      "logps/chosen": -1.6740214824676514,
-      "logps/rejected": -3.4779555797576904,
-      "loss": 26.985,
-      "nll_loss": 2.3563015460968018,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": -0.8370107412338257,
-      "rewards/margins": 0.9019671678543091,
-      "rewards/rejected": -1.7389777898788452,
+      "log_odds_chosen": 1.7929474115371704,
+      "log_odds_ratio": -0.681054413318634,
+      "logps/chosen": -1.559952735900879,
+      "logps/rejected": -3.2753818035125732,
+      "loss": 24.7409,
+      "nll_loss": 2.3052754402160645,
+      "rewards/accuracies": 0.628125011920929,
+      "rewards/chosen": -0.7799763679504395,
+      "rewards/margins": 0.8577146530151367,
+      "rewards/rejected": -1.6376909017562866,
       "step": 55
     },
     {
       "epoch": 0.5688888888888889,
-      "grad_norm": 846.8933715820312,
+      "grad_norm": 1253.04541015625,
       "learning_rate": 4.88020090697132e-05,
-      "log_odds_chosen": 5.034865379333496,
-      "log_odds_ratio": -0.7896591424942017,
-      "logps/chosen": -1.8219846487045288,
-      "logps/rejected": -6.7804718017578125,
-      "loss": -18.5626,
-      "nll_loss": 2.4730238914489746,
-      "rewards/accuracies": 0.653124988079071,
-      "rewards/chosen": -0.9109923243522644,
-      "rewards/margins": 2.479243755340576,
-      "rewards/rejected": -3.3902359008789062,
+      "log_odds_chosen": 6.016274452209473,
+      "log_odds_ratio": -0.6842174530029297,
+      "logps/chosen": -2.042940616607666,
+      "logps/rejected": -7.971026420593262,
+      "loss": -26.6321,
+      "nll_loss": 2.7467756271362305,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -1.021470308303833,
+      "rewards/margins": 2.964043140411377,
+      "rewards/rejected": -3.985513210296631,
       "step": 60
     },
     {
       "epoch": 0.6162962962962963,
-      "grad_norm": 1050.5416259765625,
+      "grad_norm": 22529.046875,
       "learning_rate": 4.834116786912897e-05,
-      "log_odds_chosen": 7.387457370758057,
-      "log_odds_ratio": -2.9080045223236084,
-      "logps/chosen": -5.713994026184082,
-      "logps/rejected": -13.041893005371094,
-      "loss": 67.5015,
-      "nll_loss": 5.906378746032715,
-      "rewards/accuracies": 0.6312500238418579,
-      "rewards/chosen": -2.856997013092041,
-      "rewards/margins": 3.6639492511749268,
-      "rewards/rejected": -6.520946502685547,
+      "log_odds_chosen": 13.827421188354492,
+      "log_odds_ratio": -6.549749851226807,
+      "logps/chosen": -11.553215026855469,
+      "logps/rejected": -25.299327850341797,
+      "loss": 152.311,
+      "nll_loss": 9.787839889526367,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -5.776607513427734,
+      "rewards/margins": 6.873056888580322,
+      "rewards/rejected": -12.649663925170898,
       "step": 65
     },
     {
       "epoch": 0.6637037037037037,
-      "grad_norm": 14261.298828125,
+      "grad_norm": 6729.6181640625,
       "learning_rate": 4.7808435099299045e-05,
-      "log_odds_chosen": 14.964570999145508,
-      "log_odds_ratio": -1.2665787935256958,
-      "logps/chosen": -2.980807304382324,
-      "logps/rejected": -17.873931884765625,
-      "loss": -140.3602,
-      "nll_loss": 3.395730495452881,
-      "rewards/accuracies": 0.6656249761581421,
-      "rewards/chosen": -1.490403652191162,
-      "rewards/margins": 7.446562767028809,
-      "rewards/rejected": -8.936965942382812,
+      "log_odds_chosen": 9.947161674499512,
+      "log_odds_ratio": -1.2594565153121948,
+      "logps/chosen": -2.880189895629883,
+      "logps/rejected": -12.729331970214844,
+      "loss": -62.2908,
+      "nll_loss": 3.341439723968506,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -1.4400949478149414,
+      "rewards/margins": 4.9245710372924805,
+      "rewards/rejected": -6.364665985107422,
       "step": 70
     },
     {
       "epoch": 0.7111111111111111,
-      "grad_norm": 3282.506591796875,
+      "grad_norm": 9763.5361328125,
       "learning_rate": 4.720545159477922e-05,
-      "log_odds_chosen": 11.172109603881836,
-      "log_odds_ratio": -1.5487353801727295,
-      "logps/chosen": -2.9426088333129883,
-      "logps/rejected": -14.05163860321045,
-      "loss": -81.785,
-      "nll_loss": 3.614391326904297,
-      "rewards/accuracies": 0.643750011920929,
-      "rewards/chosen": -1.4713044166564941,
-      "rewards/margins": 5.554513931274414,
-      "rewards/rejected": -7.025819301605225,
+      "log_odds_chosen": 9.978219985961914,
+      "log_odds_ratio": -2.5337777137756348,
+      "logps/chosen": -4.444361686706543,
+      "logps/rejected": -14.362889289855957,
+      "loss": -14.9101,
+      "nll_loss": 4.244600296020508,
+      "rewards/accuracies": 0.640625,
+      "rewards/chosen": -2.2221808433532715,
+      "rewards/margins": 4.959264278411865,
+      "rewards/rejected": -7.1814446449279785,
       "step": 75
     },
     {
       "epoch": 0.7585185185185185,
-      "grad_norm": 22710.70703125,
+      "grad_norm": 19205.619140625,
       "learning_rate": 4.653407456471222e-05,
-      "log_odds_chosen": 14.214700698852539,
-      "log_odds_ratio": -3.5922465324401855,
-      "logps/chosen": -5.844705104827881,
-      "logps/rejected": -20.002582550048828,
-      "loss": -37.6675,
-      "nll_loss": 6.309117794036865,
-      "rewards/accuracies": 0.59375,
-      "rewards/chosen": -2.9223525524139404,
-      "rewards/margins": 7.078938961029053,
-      "rewards/rejected": -10.001291275024414,
+      "log_odds_chosen": 9.231340408325195,
+      "log_odds_ratio": -3.549084186553955,
+      "logps/chosen": -9.340559959411621,
+      "logps/rejected": -18.532567977905273,
+      "loss": 153.5262,
+      "nll_loss": 9.000253677368164,
+      "rewards/accuracies": 0.5843750238418579,
+      "rewards/chosen": -4.6702799797058105,
+      "rewards/margins": 4.596003532409668,
+      "rewards/rejected": -9.266283988952637,
       "step": 80
     },
     {
       "epoch": 0.8059259259259259,
-      "grad_norm": 4206.48681640625,
+      "grad_norm": 1391.7034912109375,
       "learning_rate": 4.579637187256222e-05,
-      "log_odds_chosen": 11.754661560058594,
-      "log_odds_ratio": -2.568394184112549,
-      "logps/chosen": -5.506086826324463,
-      "logps/rejected": -17.211210250854492,
-      "loss": -9.382,
-      "nll_loss": 5.795596122741699,
-      "rewards/accuracies": 0.6156250238418579,
-      "rewards/chosen": -2.7530434131622314,
-      "rewards/margins": 5.852560997009277,
-      "rewards/rejected": -8.605605125427246,
+      "log_odds_chosen": 6.83356237411499,
+      "log_odds_ratio": -0.8187046051025391,
+      "logps/chosen": -1.6173499822616577,
+      "logps/rejected": -8.405004501342773,
+      "loss": -55.3577,
+      "nll_loss": 2.6606650352478027,
+      "rewards/accuracies": 0.609375,
+      "rewards/chosen": -0.8086749911308289,
+      "rewards/margins": 3.393826961517334,
+      "rewards/rejected": -4.202502250671387,
       "step": 85
     },
     {
       "epoch": 0.8533333333333334,
-      "grad_norm": 2450.24169921875,
+      "grad_norm": 714.6627197265625,
       "learning_rate": 4.499461566702685e-05,
-      "log_odds_chosen": 10.724493980407715,
-      "log_odds_ratio": -0.7071830034255981,
-      "logps/chosen": -1.4572128057479858,
-      "logps/rejected": -12.113273620605469,
-      "loss": -122.1039,
-      "nll_loss": 2.4590225219726562,
+      "log_odds_chosen": 12.91296100616455,
+      "log_odds_ratio": -0.7300605177879333,
+      "logps/chosen": -3.667490005493164,
+      "logps/rejected": -16.505706787109375,
+      "loss": -86.1328,
+      "nll_loss": 4.034636974334717,
       "rewards/accuracies": 0.6156250238418579,
-      "rewards/chosen": -0.7286064028739929,
-      "rewards/margins": 5.328030109405518,
-      "rewards/rejected": -6.056636810302734,
+      "rewards/chosen": -1.833745002746582,
+      "rewards/margins": 6.419107913970947,
+      "rewards/rejected": -8.252853393554688,
       "step": 90
     },
     {
       "epoch": 0.9007407407407407,
-      "grad_norm": 2608.7001953125,
+      "grad_norm": 5335.16162109375,
       "learning_rate": 4.413127538374411e-05,
-      "log_odds_chosen": 16.54291534423828,
-      "log_odds_ratio": -3.4148666858673096,
-      "logps/chosen": -4.583926200866699,
-      "logps/rejected": -21.081348419189453,
-      "loss": -115.7481,
-      "nll_loss": 4.339598655700684,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": -2.2919631004333496,
-      "rewards/margins": 8.248711585998535,
-      "rewards/rejected": -10.540674209594727,
+      "log_odds_chosen": 21.759021759033203,
+      "log_odds_ratio": -5.81619930267334,
+      "logps/chosen": -8.227999687194824,
+      "logps/rejected": -29.910602569580078,
+      "loss": -80.9532,
+      "nll_loss": 7.397497653961182,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -4.113999843597412,
+      "rewards/margins": 10.841300010681152,
+      "rewards/rejected": -14.955301284790039,
       "step": 95
     },
     {
       "epoch": 0.9481481481481482,
-      "grad_norm": 1113.737548828125,
+      "grad_norm": 9169.87890625,
       "learning_rate": 4.320901013934887e-05,
-      "log_odds_chosen": 20.849918365478516,
-      "log_odds_ratio": -2.3039841651916504,
-      "logps/chosen": -3.297405242919922,
-      "logps/rejected": -24.073766708374023,
-      "loss": -224.7481,
-      "nll_loss": 4.907585144042969,
-      "rewards/accuracies": 0.6187499761581421,
-      "rewards/chosen": -1.648702621459961,
-      "rewards/margins": 10.38818073272705,
-      "rewards/rejected": -12.036883354187012,
+      "log_odds_chosen": 21.403528213500977,
+      "log_odds_ratio": -4.2520060539245605,
+      "logps/chosen": -6.323123931884766,
+      "logps/rejected": -27.63702964782715,
+      "loss": -135.6809,
+      "nll_loss": 5.874188423156738,
+      "rewards/accuracies": 0.684374988079071,
+      "rewards/chosen": -3.161561965942383,
+      "rewards/margins": 10.656951904296875,
+      "rewards/rejected": -13.818514823913574,
       "step": 100
     },
     {
       "epoch": 0.9955555555555555,
-      "grad_norm": 564.52294921875,
+      "grad_norm": 15093.3359375,
       "learning_rate": 4.223066054130568e-05,
-      "log_odds_chosen": 21.292091369628906,
-      "log_odds_ratio": -1.3454216718673706,
-      "logps/chosen": -3.835742473602295,
-      "logps/rejected": -25.064594268798828,
-      "loss": -214.725,
-      "nll_loss": 4.128335475921631,
-      "rewards/accuracies": 0.609375,
-      "rewards/chosen": -1.9178712368011475,
-      "rewards/margins": 10.614426612854004,
-      "rewards/rejected": -12.532297134399414,
+      "log_odds_chosen": 21.14289093017578,
+      "log_odds_ratio": -2.737898349761963,
+      "logps/chosen": -6.241971015930176,
+      "logps/rejected": -27.327472686767578,
+      "loss": -135.2989,
+      "nll_loss": 6.4709601402282715,
+      "rewards/accuracies": 0.640625,
+      "rewards/chosen": -3.120985507965088,
+      "rewards/margins": 10.542750358581543,
+      "rewards/rejected": -13.663736343383789,
       "step": 105
     },
     {
       "epoch": 1.037925925925926,
-      "grad_norm": 2678.355224609375,
+      "grad_norm": 2860.319580078125,
       "learning_rate": 4.1199239938743797e-05,
-      "log_odds_chosen": 17.971054077148438,
-      "log_odds_ratio": -0.902182936668396,
-      "logps/chosen": -1.6016716957092285,
-      "logps/rejected": -19.46338653564453,
-      "loss": -207.4208,
-      "nll_loss": 2.815678358078003,
+      "log_odds_chosen": 14.374427795410156,
+      "log_odds_ratio": -0.5818507075309753,
+      "logps/chosen": -2.059671401977539,
+      "logps/rejected": -16.356985092163086,
+      "loss": -143.7255,
+      "nll_loss": 2.7815299034118652,
       "rewards/accuracies": 0.6538461446762085,
-      "rewards/chosen": -0.8008358478546143,
-      "rewards/margins": 8.930856704711914,
-      "rewards/rejected": -9.731693267822266,
+      "rewards/chosen": -1.0298357009887695,
+      "rewards/margins": 7.148656368255615,
+      "rewards/rejected": -8.178492546081543,
       "step": 110
     },
     {
       "epoch": 1.0853333333333333,
-      "grad_norm": 16211.197265625,
+      "grad_norm": 32810.50390625,
       "learning_rate": 4.0117925141242174e-05,
-      "log_odds_chosen": 32.54540252685547,
-      "log_odds_ratio": -7.060112953186035,
-      "logps/chosen": -16.32876205444336,
-      "logps/rejected": -48.71499252319336,
-      "loss": 7.6217,
-      "nll_loss": 15.108541488647461,
-      "rewards/accuracies": 0.6968749761581421,
-      "rewards/chosen": -8.16438102722168,
-      "rewards/margins": 16.193115234375,
-      "rewards/rejected": -24.35749626159668,
+      "log_odds_chosen": 27.462366104125977,
+      "log_odds_ratio": -3.54345703125,
+      "logps/chosen": -7.290637016296387,
+      "logps/rejected": -34.6160888671875,
+      "loss": -200.7838,
+      "nll_loss": 7.086569786071777,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -3.6453185081481934,
+      "rewards/margins": 13.662725448608398,
+      "rewards/rejected": -17.30804443359375,
       "step": 115
     },
     {
       "epoch": 1.1327407407407408,
-      "grad_norm": 24504.349609375,
+      "grad_norm": 2398.583984375,
       "learning_rate": 3.899004663415084e-05,
-      "log_odds_chosen": 28.460607528686523,
-      "log_odds_ratio": -6.060556888580322,
-      "logps/chosen": -10.85397720336914,
-      "logps/rejected": -39.16749954223633,
-      "loss": -102.1427,
-      "nll_loss": 10.644952774047852,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": -5.42698860168457,
-      "rewards/margins": 14.156761169433594,
-      "rewards/rejected": -19.583749771118164,
+      "log_odds_chosen": 17.45090675354004,
+      "log_odds_ratio": -4.304950714111328,
+      "logps/chosen": -7.110805511474609,
+      "logps/rejected": -24.452959060668945,
+      "loss": -47.5812,
+      "nll_loss": 7.3393707275390625,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -3.5554027557373047,
+      "rewards/margins": 8.671076774597168,
+      "rewards/rejected": -12.226479530334473,
       "step": 120
     },
     {
       "epoch": 1.1801481481481482,
-      "grad_norm": 40890.17578125,
+      "grad_norm": 23893.76171875,
       "learning_rate": 3.781907832058587e-05,
-      "log_odds_chosen": 20.61638832092285,
-      "log_odds_ratio": -6.12774133682251,
-      "logps/chosen": -11.241783142089844,
-      "logps/rejected": -31.751750946044922,
-      "loss": 34.3804,
-      "nll_loss": 11.169973373413086,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": -5.620891571044922,
-      "rewards/margins": 10.254984855651855,
-      "rewards/rejected": -15.875875473022461,
+      "log_odds_chosen": 24.439823150634766,
+      "log_odds_ratio": -5.230821132659912,
+      "logps/chosen": -8.570024490356445,
+      "logps/rejected": -32.90158462524414,
+      "loss": -112.3321,
+      "nll_loss": 8.563321113586426,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.285012245178223,
+      "rewards/margins": 12.165781021118164,
+      "rewards/rejected": -16.45079231262207,
       "step": 125
     },
     {
       "epoch": 1.2275555555555555,
-      "grad_norm": 4595.94677734375,
+      "grad_norm": 2984.347900390625,
       "learning_rate": 3.660862682169282e-05,
-      "log_odds_chosen": 15.926393508911133,
-      "log_odds_ratio": -2.1731228828430176,
-      "logps/chosen": -4.010047912597656,
-      "logps/rejected": -19.802104949951172,
-      "loss": -121.6234,
-      "nll_loss": 4.461260795593262,
-      "rewards/accuracies": 0.6468750238418579,
-      "rewards/chosen": -2.005023956298828,
-      "rewards/margins": 7.896027565002441,
-      "rewards/rejected": -9.901052474975586,
+      "log_odds_chosen": 31.73781967163086,
+      "log_odds_ratio": -3.9809155464172363,
+      "logps/chosen": -10.971330642700195,
+      "logps/rejected": -42.56734085083008,
+      "loss": -150.9388,
+      "nll_loss": 10.558366775512695,
+      "rewards/accuracies": 0.690625011920929,
+      "rewards/chosen": -5.485665321350098,
+      "rewards/margins": 15.798006057739258,
+      "rewards/rejected": -21.28367042541504,
       "step": 130
     },
     {
       "epoch": 1.274962962962963,
-      "grad_norm": 7289.251953125,
+      "grad_norm": 8554.173828125,
       "learning_rate": 3.5362420368134356e-05,
-      "log_odds_chosen": 23.66111946105957,
-      "log_odds_ratio": -1.2839603424072266,
-      "logps/chosen": -2.44439959526062,
-      "logps/rejected": -25.9698429107666,
-      "loss": -294.9251,
-      "nll_loss": 3.672980546951294,
-      "rewards/accuracies": 0.71875,
-      "rewards/chosen": -1.22219979763031,
-      "rewards/margins": 11.762721061706543,
-      "rewards/rejected": -12.9849214553833,
+      "log_odds_chosen": 28.130645751953125,
+      "log_odds_ratio": -1.710423231124878,
+      "logps/chosen": -3.9046826362609863,
+      "logps/rejected": -31.89151382446289,
+      "loss": -319.8829,
+      "nll_loss": 3.7381324768066406,
+      "rewards/accuracies": 0.6968749761581421,
+      "rewards/chosen": -1.9523413181304932,
+      "rewards/margins": 13.993415832519531,
+      "rewards/rejected": -15.945756912231445,
       "step": 135
     },
     {
       "epoch": 1.3223703703703704,
-      "grad_norm": 35099.68359375,
+      "grad_norm": 4571.39404296875,
       "learning_rate": 3.408429731701635e-05,
-      "log_odds_chosen": 22.592227935791016,
-      "log_odds_ratio": -3.179147958755493,
-      "logps/chosen": -5.041490077972412,
-      "logps/rejected": -27.51487159729004,
-      "loss": -195.6627,
-      "nll_loss": 5.91623592376709,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": -2.520745038986206,
-      "rewards/margins": 11.236689567565918,
-      "rewards/rejected": -13.75743579864502,
+      "log_odds_chosen": 23.899776458740234,
+      "log_odds_ratio": -3.7175846099853516,
+      "logps/chosen": -5.324652671813965,
+      "logps/rejected": -29.102670669555664,
+      "loss": -207.4437,
+      "nll_loss": 5.260149955749512,
+      "rewards/accuracies": 0.6781250238418579,
+      "rewards/chosen": -2.6623263359069824,
+      "rewards/margins": 11.889008522033691,
+      "rewards/rejected": -14.551335334777832,
       "step": 140
     },
     {
       "epoch": 1.3697777777777778,
-      "grad_norm": 4246.3798828125,
+      "grad_norm": 2103.40673828125,
       "learning_rate": 3.2778194329621104e-05,
-      "log_odds_chosen": 20.304393768310547,
-      "log_odds_ratio": -2.3136236667633057,
-      "logps/chosen": -3.2953147888183594,
-      "logps/rejected": -23.475563049316406,
-      "loss": -214.76,
-      "nll_loss": 4.836905002593994,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": -1.6476573944091797,
-      "rewards/margins": 10.090123176574707,
-      "rewards/rejected": -11.737781524658203,
+      "log_odds_chosen": 20.076154708862305,
+      "log_odds_ratio": -2.8479440212249756,
+      "logps/chosen": -3.987905979156494,
+      "logps/rejected": -23.93975257873535,
+      "loss": -188.7188,
+      "nll_loss": 6.649003505706787,
+      "rewards/accuracies": 0.684374988079071,
+      "rewards/chosen": -1.993952989578247,
+      "rewards/margins": 9.975922584533691,
+      "rewards/rejected": -11.969876289367676,
       "step": 145
     },
     {
       "epoch": 1.417185185185185,
-      "grad_norm": 112606.7421875,
+      "grad_norm": 21103.2109375,
       "learning_rate": 3.144813424636031e-05,
-      "log_odds_chosen": 28.99297523498535,
-      "log_odds_ratio": -4.194870948791504,
-      "logps/chosen": -7.332457542419434,
-      "logps/rejected": -36.18639373779297,
-      "loss": -223.9742,
-      "nll_loss": 9.322813034057617,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": -3.666228771209717,
-      "rewards/margins": 14.426966667175293,
-      "rewards/rejected": -18.093196868896484,
+      "log_odds_chosen": 26.536422729492188,
+      "log_odds_ratio": -3.6103949546813965,
+      "logps/chosen": -7.1077470779418945,
+      "logps/rejected": -33.501373291015625,
+      "loss": -191.7946,
+      "nll_loss": 10.428869247436523,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -3.5538735389709473,
+      "rewards/margins": 13.196813583374023,
+      "rewards/rejected": -16.750686645507812,
       "step": 150
     },
     {
       "epoch": 1.4645925925925927,
-      "grad_norm": 57207.05859375,
+      "grad_norm": 20648.404296875,
       "learning_rate": 3.0098213696293542e-05,
-      "log_odds_chosen": 27.4794864654541,
-      "log_odds_ratio": -6.622164726257324,
-      "logps/chosen": -13.51585578918457,
-      "logps/rejected": -40.87260818481445,
-      "loss": -2.1912,
-      "nll_loss": 16.61840057373047,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": -6.757927894592285,
-      "rewards/margins": 13.678375244140625,
-      "rewards/rejected": -20.436304092407227,
+      "log_odds_chosen": 30.618228912353516,
+      "log_odds_ratio": -8.441522598266602,
+      "logps/chosen": -22.582386016845703,
+      "logps/rejected": -53.13446044921875,
+      "loss": 236.5034,
+      "nll_loss": 25.505130767822266,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -11.291193008422852,
+      "rewards/margins": 15.276036262512207,
+      "rewards/rejected": -26.567230224609375,
       "step": 155
     },
     {
       "epoch": 1.512,
-      "grad_norm": 22721.037109375,
+      "grad_norm": 6142.369140625,
       "learning_rate": 2.8732590479375165e-05,
-      "log_odds_chosen": 26.662273406982422,
-      "log_odds_ratio": -6.387181282043457,
-      "logps/chosen": -10.372232437133789,
-      "logps/rejected": -36.92156219482422,
-      "loss": -90.2426,
-      "nll_loss": 9.191670417785645,
-      "rewards/accuracies": 0.643750011920929,
-      "rewards/chosen": -5.1861162185668945,
-      "rewards/margins": 13.274665832519531,
-      "rewards/rejected": -18.46078109741211,
+      "log_odds_chosen": 28.4414005279541,
+      "log_odds_ratio": -4.836407661437988,
+      "logps/chosen": -10.202522277832031,
+      "logps/rejected": -38.521305084228516,
+      "loss": -124.1278,
+      "nll_loss": 7.826313018798828,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -5.101261138916016,
+      "rewards/margins": 14.159391403198242,
+      "rewards/rejected": -19.260652542114258,
       "step": 160
     },
     {
       "epoch": 1.5594074074074074,
-      "grad_norm": 8884.1220703125,
+      "grad_norm": 887.5225219726562,
       "learning_rate": 2.7355470760292956e-05,
-      "log_odds_chosen": 22.65861701965332,
-      "log_odds_ratio": -3.268022060394287,
-      "logps/chosen": -4.845406532287598,
-      "logps/rejected": -27.359100341796875,
-      "loss": -202.281,
-      "nll_loss": 5.931666374206543,
-      "rewards/accuracies": 0.684374988079071,
-      "rewards/chosen": -2.422703266143799,
-      "rewards/margins": 11.256847381591797,
-      "rewards/rejected": -13.679550170898438,
+      "log_odds_chosen": 15.544232368469238,
+      "log_odds_ratio": -2.6855194568634033,
+      "logps/chosen": -3.4005138874053955,
+      "logps/rejected": -18.811342239379883,
+      "loss": -135.4756,
+      "nll_loss": 4.0409064292907715,
+      "rewards/accuracies": 0.671875,
+      "rewards/chosen": -1.7002569437026978,
+      "rewards/margins": 7.705413818359375,
+      "rewards/rejected": -9.405671119689941,
       "step": 165
     },
     {
       "epoch": 1.6068148148148147,
-      "grad_norm": 10747.482421875,
+      "grad_norm": 1129.25048828125,
       "learning_rate": 2.597109611334169e-05,
-      "log_odds_chosen": 24.835664749145508,
-      "log_odds_ratio": -5.039122104644775,
-      "logps/chosen": -8.912939071655273,
-      "logps/rejected": -33.64850616455078,
-      "loss": -108.0337,
-      "nll_loss": 9.482011795043945,
-      "rewards/accuracies": 0.6468750238418579,
-      "rewards/chosen": -4.456469535827637,
-      "rewards/margins": 12.367780685424805,
-      "rewards/rejected": -16.82425308227539,
+      "log_odds_chosen": 17.921756744384766,
+      "log_odds_ratio": -2.1972568035125732,
+      "logps/chosen": -3.035590171813965,
+      "logps/rejected": -20.839210510253906,
+      "loss": -185.0614,
+      "nll_loss": 3.198902130126953,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -1.5177950859069824,
+      "rewards/margins": 8.901809692382812,
+      "rewards/rejected": -10.419605255126953,
       "step": 170
     },
     {
       "epoch": 1.6542222222222223,
-      "grad_norm": 25721.013671875,
+      "grad_norm": 6776.86669921875,
       "learning_rate": 2.458373045823404e-05,
-      "log_odds_chosen": 17.206180572509766,
-      "log_odds_ratio": -5.200994968414307,
-      "logps/chosen": -10.323943138122559,
-      "logps/rejected": -27.417028427124023,
-      "loss": 59.6029,
-      "nll_loss": 11.46679973602295,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": -5.161971569061279,
-      "rewards/margins": 8.546542167663574,
-      "rewards/rejected": -13.708514213562012,
+      "log_odds_chosen": 19.142057418823242,
+      "log_odds_ratio": -4.326524257659912,
+      "logps/chosen": -8.229945182800293,
+      "logps/rejected": -27.229846954345703,
+      "loss": -37.5736,
+      "nll_loss": 9.928422927856445,
+      "rewards/accuracies": 0.6968749761581421,
+      "rewards/chosen": -4.1149725914001465,
+      "rewards/margins": 9.499950408935547,
+      "rewards/rejected": -13.614923477172852,
       "step": 175
     },
     {
       "epoch": 1.7016296296296296,
-      "grad_norm": 5182.34716796875,
+      "grad_norm": 6735.8046875,
       "learning_rate": 2.3197646927086697e-05,
-      "log_odds_chosen": 17.235403060913086,
-      "log_odds_ratio": -5.827118873596191,
-      "logps/chosen": -9.327442169189453,
-      "logps/rejected": -26.453588485717773,
-      "loss": 26.7508,
-      "nll_loss": 10.854573249816895,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": -4.663721084594727,
-      "rewards/margins": 8.56307315826416,
-      "rewards/rejected": -13.226794242858887,
+      "log_odds_chosen": 21.28363609313965,
+      "log_odds_ratio": -3.7137675285339355,
+      "logps/chosen": -6.68829345703125,
+      "logps/rejected": -27.832469940185547,
+      "loss": -121.5285,
+      "nll_loss": 10.662646293640137,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -3.344146728515625,
+      "rewards/margins": 10.572088241577148,
+      "rewards/rejected": -13.916234970092773,
       "step": 180
     },
     {
       "epoch": 1.749037037037037,
-      "grad_norm": 1122.926513671875,
+      "grad_norm": 980.9172973632812,
       "learning_rate": 2.1817114703032176e-05,
-      "log_odds_chosen": 12.183832168579102,
-      "log_odds_ratio": -3.2944283485412598,
-      "logps/chosen": -4.907401084899902,
-      "logps/rejected": -17.006296157836914,
-      "loss": -34.3607,
-      "nll_loss": 4.914076328277588,
-      "rewards/accuracies": 0.640625,
-      "rewards/chosen": -2.453700542449951,
-      "rewards/margins": 6.049447059631348,
-      "rewards/rejected": -8.503148078918457,
+      "log_odds_chosen": 11.027368545532227,
+      "log_odds_ratio": -1.5065878629684448,
+      "logps/chosen": -2.5348973274230957,
+      "logps/rejected": -13.437909126281738,
+      "loss": -90.6843,
+      "nll_loss": 4.682984828948975,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -1.2674486637115479,
+      "rewards/margins": 5.4515061378479,
+      "rewards/rejected": -6.718954563140869,
       "step": 185
     },
     {
       "epoch": 1.7964444444444445,
-      "grad_norm": 8495.58984375,
+      "grad_norm": 2794.48828125,
       "learning_rate": 2.0446385870993467e-05,
-      "log_odds_chosen": 27.890201568603516,
-      "log_odds_ratio": -4.537432670593262,
-      "logps/chosen": -6.876814365386963,
-      "logps/rejected": -34.62470245361328,
-      "loss": -220.5601,
-      "nll_loss": 7.243471622467041,
-      "rewards/accuracies": 0.690625011920929,
-      "rewards/chosen": -3.4384071826934814,
-      "rewards/margins": 13.873942375183105,
-      "rewards/rejected": -17.31235122680664,
+      "log_odds_chosen": 25.16632080078125,
+      "log_odds_ratio": -1.736579179763794,
+      "logps/chosen": -2.6034204959869385,
+      "logps/rejected": -27.604034423828125,
+      "loss": -313.2717,
+      "nll_loss": 4.227630138397217,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -1.3017102479934692,
+      "rewards/margins": 12.5003080368042,
+      "rewards/rejected": -13.802017211914062,
       "step": 190
     },
     {
       "epoch": 1.8438518518518519,
-      "grad_norm": 21205.203125,
+      "grad_norm": 18748.30859375,
       "learning_rate": 1.9089682321121834e-05,
-      "log_odds_chosen": 28.1275577545166,
-      "log_odds_ratio": -10.069475173950195,
-      "logps/chosen": -13.262001037597656,
-      "logps/rejected": -41.2625732421875,
-      "loss": -20.6418,
-      "nll_loss": 14.18327522277832,
-      "rewards/accuracies": 0.671875,
-      "rewards/chosen": -6.631000518798828,
-      "rewards/margins": 14.000287055969238,
-      "rewards/rejected": -20.63128662109375,
+      "log_odds_chosen": 25.935617446899414,
+      "log_odds_ratio": -5.700267791748047,
+      "logps/chosen": -7.655577182769775,
+      "logps/rejected": -33.431495666503906,
+      "loss": -164.2176,
+      "nll_loss": 11.73558235168457,
+      "rewards/accuracies": 0.703125,
+      "rewards/chosen": -3.8277885913848877,
+      "rewards/margins": 12.887959480285645,
+      "rewards/rejected": -16.715747833251953,
       "step": 195
     },
     {
       "epoch": 1.8912592592592592,
-      "grad_norm": 74072.71875,
+      "grad_norm": 9071.26953125,
       "learning_rate": 1.775118274523545e-05,
-      "log_odds_chosen": 39.34962844848633,
-      "log_odds_ratio": -16.783084869384766,
-      "logps/chosen": -29.612789154052734,
-      "logps/rejected": -68.8023452758789,
-      "loss": 324.4527,
-      "nll_loss": 29.834827423095703,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": -14.806394577026367,
-      "rewards/margins": 19.59477424621582,
-      "rewards/rejected": -34.40117263793945,
+      "log_odds_chosen": 29.912353515625,
+      "log_odds_ratio": -3.8106212615966797,
+      "logps/chosen": -7.143023490905762,
+      "logps/rejected": -36.88776397705078,
+      "loss": -243.6502,
+      "nll_loss": 12.543132781982422,
+      "rewards/accuracies": 0.715624988079071,
+      "rewards/chosen": -3.571511745452881,
+      "rewards/margins": 14.872370719909668,
+      "rewards/rejected": -18.44388198852539,
       "step": 200
     },
     {
       "epoch": 1.9386666666666668,
-      "grad_norm": 395353.21875,
+      "grad_norm": 5131.81005859375,
       "learning_rate": 1.643500976631037e-05,
-      "log_odds_chosen": 42.678619384765625,
-      "log_odds_ratio": -18.052345275878906,
-      "logps/chosen": -42.023616790771484,
-      "logps/rejected": -84.55274963378906,
-      "loss": 668.5252,
-      "nll_loss": 37.75287628173828,
+      "log_odds_chosen": 21.7247314453125,
+      "log_odds_ratio": -4.9020280838012695,
+      "logps/chosen": -8.960885047912598,
+      "logps/rejected": -30.520559310913086,
+      "loss": -54.6864,
+      "nll_loss": 11.589077949523926,
       "rewards/accuracies": 0.71875,
-      "rewards/chosen": -21.011808395385742,
-      "rewards/margins": 21.264570236206055,
-      "rewards/rejected": -42.27637481689453,
+      "rewards/chosen": -4.480442523956299,
+      "rewards/margins": 10.779837608337402,
+      "rewards/rejected": -15.260279655456543,
       "step": 205
     },
     {
       "epoch": 1.986074074074074,
-      "grad_norm": 80640.96875,
+      "grad_norm": 6081.4013671875,
       "learning_rate": 1.514521724066537e-05,
-      "log_odds_chosen": 39.148033142089844,
-      "log_odds_ratio": -19.23377799987793,
-      "logps/chosen": -32.76858901977539,
-      "logps/rejected": -71.80208587646484,
-      "loss": 426.931,
-      "nll_loss": 26.356372833251953,
-      "rewards/accuracies": 0.668749988079071,
-      "rewards/chosen": -16.384294509887695,
-      "rewards/margins": 19.51674461364746,
-      "rewards/rejected": -35.90104293823242,
+      "log_odds_chosen": 26.899578094482422,
+      "log_odds_ratio": -4.448848724365234,
+      "logps/chosen": -6.3961944580078125,
+      "logps/rejected": -33.168235778808594,
+      "loss": -220.8887,
+      "nll_loss": 8.339305877685547,
+      "rewards/accuracies": 0.6656249761581421,
+      "rewards/chosen": -3.1980972290039062,
+      "rewards/margins": 13.386022567749023,
+      "rewards/rejected": -16.584117889404297,
       "step": 210
     },
     {
       "epoch": 2.0284444444444443,
-      "grad_norm": 34031.30859375,
+      "grad_norm": 13102.798828125,
       "learning_rate": 1.3885777771950348e-05,
-      "log_odds_chosen": 39.754032135009766,
-      "log_odds_ratio": -14.231975555419922,
-      "logps/chosen": -18.812442779541016,
-      "logps/rejected": -58.38700485229492,
-      "loss": -24.6149,
-      "nll_loss": 17.645992279052734,
-      "rewards/accuracies": 0.6958041787147522,
-      "rewards/chosen": -9.406221389770508,
-      "rewards/margins": 19.787281036376953,
-      "rewards/rejected": -29.19350242614746,
+      "log_odds_chosen": 31.693286895751953,
+      "log_odds_ratio": -7.093617916107178,
+      "logps/chosen": -9.377802848815918,
+      "logps/rejected": -40.8663330078125,
+      "loss": -178.6939,
+      "nll_loss": 11.103485107421875,
+      "rewards/accuracies": 0.7062937021255493,
+      "rewards/chosen": -4.688901424407959,
+      "rewards/margins": 15.744266510009766,
+      "rewards/rejected": -20.43316650390625,
       "step": 215
     },
     {
       "epoch": 2.075851851851852,
-      "grad_norm": 16462.95703125,
+      "grad_norm": 16916.310546875,
       "learning_rate": 1.2660570475395683e-05,
-      "log_odds_chosen": 28.321569442749023,
-      "log_odds_ratio": -13.535270690917969,
-      "logps/chosen": -21.737045288085938,
-      "logps/rejected": -49.909423828125,
-      "loss": 248.0299,
-      "nll_loss": 19.69781494140625,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": -10.868522644042969,
-      "rewards/margins": 14.086189270019531,
-      "rewards/rejected": -24.9547119140625,
+      "log_odds_chosen": 29.3436222076416,
+      "log_odds_ratio": -5.789451599121094,
+      "logps/chosen": -14.939682006835938,
+      "logps/rejected": -44.14417266845703,
+      "loss": 14.2524,
+      "nll_loss": 22.04985809326172,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -7.469841003417969,
+      "rewards/margins": 14.60224437713623,
+      "rewards/rejected": -22.072086334228516,
       "step": 220
     },
     {
       "epoch": 2.1232592592592594,
-      "grad_norm": 17380.919921875,
+      "grad_norm": 17463.740234375,
       "learning_rate": 1.1473369030008974e-05,
-      "log_odds_chosen": 41.30133819580078,
-      "log_odds_ratio": -5.992632865905762,
-      "logps/chosen": -9.869775772094727,
-      "logps/rejected": -51.008033752441406,
-      "loss": -338.6483,
-      "nll_loss": 9.406192779541016,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": -4.934887886047363,
-      "rewards/margins": 20.569129943847656,
-      "rewards/rejected": -25.504016876220703,
+      "log_odds_chosen": 36.83652877807617,
+      "log_odds_ratio": -3.7113094329833984,
+      "logps/chosen": -9.069849967956543,
+      "logps/rejected": -45.71887969970703,
+      "loss": -291.8311,
+      "nll_loss": 16.01699447631836,
+      "rewards/accuracies": 0.7093750238418579,
+      "rewards/chosen": -4.5349249839782715,
+      "rewards/margins": 18.324514389038086,
+      "rewards/rejected": -22.859439849853516,
       "step": 225
     },
     {
       "epoch": 2.1706666666666665,
-      "grad_norm": 2604.8720703125,
+      "grad_norm": 2066.959228515625,
       "learning_rate": 1.0327830055518842e-05,
-      "log_odds_chosen": 40.362274169921875,
-      "log_odds_ratio": -9.142515182495117,
-      "logps/chosen": -14.164266586303711,
-      "logps/rejected": -54.35051345825195,
-      "loss": -186.1042,
-      "nll_loss": 13.237195014953613,
-      "rewards/accuracies": 0.684374988079071,
-      "rewards/chosen": -7.0821332931518555,
-      "rewards/margins": 20.093120574951172,
-      "rewards/rejected": -27.175256729125977,
+      "log_odds_chosen": 34.293540954589844,
+      "log_odds_ratio": -2.4719889163970947,
+      "logps/chosen": -6.306262016296387,
+      "logps/rejected": -40.41852951049805,
+      "loss": -340.089,
+      "nll_loss": 12.120941162109375,
+      "rewards/accuracies": 0.703125,
+      "rewards/chosen": -3.1531310081481934,
+      "rewards/margins": 17.056133270263672,
+      "rewards/rejected": -20.209264755249023,
       "step": 230
     },
     {
       "epoch": 2.218074074074074,
-      "grad_norm": 164766.734375,
+      "grad_norm": 3184.418212890625,
       "learning_rate": 9.227481849865235e-06,
-      "log_odds_chosen": 45.14253234863281,
-      "log_odds_ratio": -7.5437116622924805,
-      "logps/chosen": -11.828727722167969,
-      "logps/rejected": -56.79478073120117,
-      "loss": -336.8467,
-      "nll_loss": 12.229296684265137,
-      "rewards/accuracies": 0.721875011920929,
-      "rewards/chosen": -5.914363861083984,
-      "rewards/margins": 22.483028411865234,
-      "rewards/rejected": -28.397390365600586,
+      "log_odds_chosen": 36.27004623413086,
+      "log_odds_ratio": -2.707547903060913,
+      "logps/chosen": -3.9607880115509033,
+      "logps/rejected": -40.02045440673828,
+      "loss": -446.1736,
+      "nll_loss": 6.836036682128906,
+      "rewards/accuracies": 0.734375,
+      "rewards/chosen": -1.9803940057754517,
+      "rewards/margins": 18.02983283996582,
+      "rewards/rejected": -20.01022720336914,
       "step": 235
     },
     {
       "epoch": 2.2654814814814817,
-      "grad_norm": 81824.2890625,
+      "grad_norm": 12459.7412109375,
       "learning_rate": 8.175713521924978e-06,
-      "log_odds_chosen": 40.605594635009766,
-      "log_odds_ratio": -11.402399063110352,
-      "logps/chosen": -26.724740982055664,
-      "logps/rejected": -67.15190124511719,
-      "loss": 212.4567,
-      "nll_loss": 23.99557113647461,
-      "rewards/accuracies": 0.721875011920929,
-      "rewards/chosen": -13.362370491027832,
-      "rewards/margins": 20.213586807250977,
-      "rewards/rejected": -33.575950622558594,
+      "log_odds_chosen": 31.65945816040039,
+      "log_odds_ratio": -2.4659223556518555,
+      "logps/chosen": -4.006608963012695,
+      "logps/rejected": -35.472171783447266,
+      "loss": -371.1167,
+      "nll_loss": 7.588781833648682,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -2.0033044815063477,
+      "rewards/margins": 15.732783317565918,
+      "rewards/rejected": -17.736085891723633,
       "step": 240
     },
     {
       "epoch": 2.3128888888888888,
-      "grad_norm": 14960.7578125,
+      "grad_norm": 10665.607421875,
       "learning_rate": 7.1757645529443665e-06,
-      "log_odds_chosen": 55.09406661987305,
-      "log_odds_ratio": -10.749176979064941,
-      "logps/chosen": -19.03034019470215,
-      "logps/rejected": -73.93666076660156,
-      "loss": -264.6577,
-      "nll_loss": 19.223167419433594,
-      "rewards/accuracies": 0.7593749761581421,
-      "rewards/chosen": -9.515170097351074,
-      "rewards/margins": 27.45315933227539,
-      "rewards/rejected": -36.96833038330078,
+      "log_odds_chosen": 41.71519088745117,
+      "log_odds_ratio": -2.4143545627593994,
+      "logps/chosen": -5.24094295501709,
+      "logps/rejected": -46.76899337768555,
+      "loss": -492.2525,
+      "nll_loss": 14.035673141479492,
+      "rewards/accuracies": 0.765625,
+      "rewards/chosen": -2.620471477508545,
+      "rewards/margins": 20.764026641845703,
+      "rewards/rejected": -23.384496688842773,
       "step": 245
     },
     {
       "epoch": 2.3602962962962963,
-      "grad_norm": 17807.7265625,
+      "grad_norm": 15119.1689453125,
       "learning_rate": 6.230714818829733e-06,
-      "log_odds_chosen": 45.54753875732422,
-      "log_odds_ratio": -7.856490135192871,
-      "logps/chosen": -11.773348808288574,
-      "logps/rejected": -57.12473678588867,
-      "loss": -344.7895,
-      "nll_loss": 12.78067684173584,
+      "log_odds_chosen": 35.395755767822266,
+      "log_odds_ratio": -1.310185194015503,
+      "logps/chosen": -5.026707649230957,
+      "logps/rejected": -40.237152099609375,
+      "loss": -398.4109,
+      "nll_loss": 9.918493270874023,
       "rewards/accuracies": 0.7093750238418579,
-      "rewards/chosen": -5.886674404144287,
-      "rewards/margins": 22.67569351196289,
-      "rewards/rejected": -28.562368392944336,
+      "rewards/chosen": -2.5133538246154785,
+      "rewards/margins": 17.605222702026367,
+      "rewards/rejected": -20.118576049804688,
       "step": 250
     },
     {
       "epoch": 2.407703703703704,
-      "grad_norm": 104304.0390625,
+      "grad_norm": 5417.455078125,
       "learning_rate": 5.343475104027743e-06,
-      "log_odds_chosen": 47.276451110839844,
-      "log_odds_ratio": -8.991453170776367,
-      "logps/chosen": -13.746038436889648,
-      "logps/rejected": -60.85129928588867,
-      "loss": -309.5943,
-      "nll_loss": 14.252038955688477,
-      "rewards/accuracies": 0.7281249761581421,
-      "rewards/chosen": -6.873019218444824,
-      "rewards/margins": 23.552631378173828,
-      "rewards/rejected": -30.425649642944336,
+      "log_odds_chosen": 34.123165130615234,
+      "log_odds_ratio": -2.2660939693450928,
+      "logps/chosen": -6.2752251625061035,
+      "logps/rejected": -40.19994354248047,
+      "loss": -337.6817,
+      "nll_loss": 11.942907333374023,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -3.1376125812530518,
+      "rewards/margins": 16.962358474731445,
+      "rewards/rejected": -20.099971771240234,
       "step": 255
     },
     {
       "epoch": 2.455111111111111,
-      "grad_norm": 38147.03125,
+      "grad_norm": 5251.4404296875,
       "learning_rate": 4.516778136213037e-06,
-      "log_odds_chosen": 40.15815734863281,
-      "log_odds_ratio": -7.543868064880371,
-      "logps/chosen": -11.01616096496582,
-      "logps/rejected": -50.988121032714844,
-      "loss": -282.8974,
-      "nll_loss": 12.92004108428955,
-      "rewards/accuracies": 0.731249988079071,
-      "rewards/chosen": -5.50808048248291,
-      "rewards/margins": 19.985979080200195,
-      "rewards/rejected": -25.494060516357422,
+      "log_odds_chosen": 29.794910430908203,
+      "log_odds_ratio": -3.3600330352783203,
+      "logps/chosen": -5.655725955963135,
+      "logps/rejected": -35.23265838623047,
+      "loss": -288.0842,
+      "nll_loss": 10.556007385253906,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -2.8278629779815674,
+      "rewards/margins": 14.78846549987793,
+      "rewards/rejected": -17.616329193115234,
       "step": 260
     },
     {
       "epoch": 2.5025185185185186,
-      "grad_norm": 24916.00390625,
+      "grad_norm": 17512.310546875,
       "learning_rate": 3.7531701693965554e-06,
-      "log_odds_chosen": 50.416419982910156,
-      "log_odds_ratio": -8.888219833374023,
-      "logps/chosen": -14.691574096679688,
-      "logps/rejected": -64.95278930664062,
-      "loss": -330.3818,
-      "nll_loss": 14.175634384155273,
-      "rewards/accuracies": 0.6937500238418579,
-      "rewards/chosen": -7.345787048339844,
-      "rewards/margins": 25.1306095123291,
-      "rewards/rejected": -32.47639465332031,
+      "log_odds_chosen": 43.79105758666992,
+      "log_odds_ratio": -4.43022346496582,
+      "logps/chosen": -8.150279998779297,
+      "logps/rejected": -51.75994110107422,
+      "loss": -432.5288,
+      "nll_loss": 18.760528564453125,
+      "rewards/accuracies": 0.715624988079071,
+      "rewards/chosen": -4.075139999389648,
+      "rewards/margins": 21.804828643798828,
+      "rewards/rejected": -25.87997055053711,
       "step": 265
     },
     {
       "epoch": 2.549925925925926,
-      "grad_norm": 17820.6953125,
+      "grad_norm": 16464.80859375,
       "learning_rate": 3.055003141378948e-06,
-      "log_odds_chosen": 56.49353790283203,
-      "log_odds_ratio": -4.24954891204834,
-      "logps/chosen": -7.077002048492432,
-      "logps/rejected": -63.38737106323242,
-      "loss": -670.0222,
-      "nll_loss": 8.986559867858887,
-      "rewards/accuracies": 0.734375,
-      "rewards/chosen": -3.538501024246216,
-      "rewards/margins": 28.155181884765625,
-      "rewards/rejected": -31.69368553161621,
+      "log_odds_chosen": 45.24549102783203,
+      "log_odds_ratio": -4.265998840332031,
+      "logps/chosen": -9.220375061035156,
+      "logps/rejected": -54.259765625,
+      "loss": -420.995,
+      "nll_loss": 17.882661819458008,
+      "rewards/accuracies": 0.7281249761581421,
+      "rewards/chosen": -4.610187530517578,
+      "rewards/margins": 22.51969337463379,
+      "rewards/rejected": -27.1298828125,
       "step": 270
     },
     {
       "epoch": 2.5973333333333333,
-      "grad_norm": 21286.701171875,
+      "grad_norm": 10890.8203125,
       "learning_rate": 2.424427429704365e-06,
-      "log_odds_chosen": 45.696632385253906,
-      "log_odds_ratio": -8.857467651367188,
-      "logps/chosen": -13.247896194458008,
-      "logps/rejected": -58.740142822265625,
-      "loss": -299.5551,
-      "nll_loss": 12.918680191040039,
-      "rewards/accuracies": 0.778124988079071,
-      "rewards/chosen": -6.623948097229004,
-      "rewards/margins": 22.746124267578125,
-      "rewards/rejected": -29.370071411132812,
+      "log_odds_chosen": 35.03772735595703,
+      "log_odds_ratio": -4.138024806976318,
+      "logps/chosen": -8.157249450683594,
+      "logps/rejected": -42.98029327392578,
+      "loss": -291.3517,
+      "nll_loss": 13.331751823425293,
+      "rewards/accuracies": 0.765625,
+      "rewards/chosen": -4.078624725341797,
+      "rewards/margins": 17.411518096923828,
+      "rewards/rejected": -21.49014663696289,
       "step": 275
     },
     {
       "epoch": 2.644740740740741,
-      "grad_norm": 50187.1171875,
+      "grad_norm": 12637.064453125,
       "learning_rate": 1.8633852284264508e-06,
-      "log_odds_chosen": 55.4704704284668,
-      "log_odds_ratio": -6.3292717933654785,
-      "logps/chosen": -11.759183883666992,
-      "logps/rejected": -67.04707336425781,
-      "loss": -503.517,
-      "nll_loss": 12.290529251098633,
-      "rewards/accuracies": 0.734375,
-      "rewards/chosen": -5.879591941833496,
-      "rewards/margins": 27.643945693969727,
-      "rewards/rejected": -33.523536682128906,
+      "log_odds_chosen": 41.4226188659668,
+      "log_odds_ratio": -3.3395819664001465,
+      "logps/chosen": -5.7105512619018555,
+      "logps/rejected": -46.93029022216797,
+      "loss": -472.0542,
+      "nll_loss": 11.638737678527832,
+      "rewards/accuracies": 0.7281249761581421,
+      "rewards/chosen": -2.8552756309509277,
+      "rewards/margins": 20.609867095947266,
+      "rewards/rejected": -23.465145111083984,
       "step": 280
     },
     {
       "epoch": 2.6921481481481484,
-      "grad_norm": 114461.5,
+      "grad_norm": 18065.2109375,
       "learning_rate": 1.3736045660864034e-06,
-      "log_odds_chosen": 41.011329650878906,
-      "log_odds_ratio": -8.477639198303223,
-      "logps/chosen": -12.918500900268555,
-      "logps/rejected": -53.76482391357422,
-      "loss": -236.1638,
-      "nll_loss": 13.538793563842773,
-      "rewards/accuracies": 0.7250000238418579,
-      "rewards/chosen": -6.459250450134277,
-      "rewards/margins": 20.423160552978516,
-      "rewards/rejected": -26.88241195678711,
+      "log_odds_chosen": 33.556556701660156,
+      "log_odds_ratio": -4.451475620269775,
+      "logps/chosen": -7.034272193908691,
+      "logps/rejected": -40.41550064086914,
+      "loss": -304.8266,
+      "nll_loss": 12.904256820678711,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -3.5171360969543457,
+      "rewards/margins": 16.690614700317383,
+      "rewards/rejected": -20.20775032043457,
       "step": 285
     },
     {
       "epoch": 2.7395555555555555,
-      "grad_norm": 26927.990234375,
+      "grad_norm": 13026.53515625,
       "learning_rate": 9.565939833279192e-07,
-      "log_odds_chosen": 40.3164176940918,
-      "log_odds_ratio": -15.15136432647705,
-      "logps/chosen": -22.780040740966797,
-      "logps/rejected": -62.926292419433594,
-      "loss": 90.1159,
-      "nll_loss": 20.46670913696289,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": -11.390020370483398,
-      "rewards/margins": 20.073129653930664,
-      "rewards/rejected": -31.463146209716797,
+      "log_odds_chosen": 35.401798248291016,
+      "log_odds_ratio": -3.9767117500305176,
+      "logps/chosen": -7.844551086425781,
+      "logps/rejected": -43.053428649902344,
+      "loss": -308.0105,
+      "nll_loss": 11.636465072631836,
+      "rewards/accuracies": 0.721875011920929,
+      "rewards/chosen": -3.9222755432128906,
+      "rewards/margins": 17.604434967041016,
+      "rewards/rejected": -21.526714324951172,
       "step": 290
     },
     {
       "epoch": 2.786962962962963,
-      "grad_norm": 10974.4052734375,
+      "grad_norm": 7266.41943359375,
       "learning_rate": 6.136378865420872e-07,
-      "log_odds_chosen": 47.6572380065918,
-      "log_odds_ratio": -7.081920623779297,
-      "logps/chosen": -10.659754753112793,
-      "logps/rejected": -58.14702224731445,
-      "loss": -414.5048,
-      "nll_loss": 11.201601028442383,
-      "rewards/accuracies": 0.7093750238418579,
-      "rewards/chosen": -5.3298773765563965,
-      "rewards/margins": 23.743627548217773,
-      "rewards/rejected": -29.073511123657227,
+      "log_odds_chosen": 32.378849029541016,
+      "log_odds_ratio": -2.3196144104003906,
+      "logps/chosen": -5.068166255950928,
+      "logps/rejected": -37.264400482177734,
+      "loss": -348.6297,
+      "nll_loss": 9.656694412231445,
+      "rewards/accuracies": 0.715624988079071,
+      "rewards/chosen": -2.534083127975464,
+      "rewards/margins": 16.098114013671875,
+      "rewards/rejected": -18.632200241088867,
       "step": 295
     },
     {
       "epoch": 2.83437037037037,
-      "grad_norm": 42624.33203125,
+      "grad_norm": 6610.1474609375,
       "learning_rate": 3.45792591853214e-07,
-      "log_odds_chosen": 52.529335021972656,
-      "log_odds_ratio": -6.441692352294922,
-      "logps/chosen": -9.135614395141602,
-      "logps/rejected": -61.50330352783203,
-      "loss": -541.3402,
-      "nll_loss": 10.765565872192383,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": -4.567807197570801,
-      "rewards/margins": 26.1838436126709,
-      "rewards/rejected": -30.751651763916016,
+      "log_odds_chosen": 41.97068786621094,
+      "log_odds_ratio": -1.2807286977767944,
+      "logps/chosen": -3.7067933082580566,
+      "logps/rejected": -45.47202682495117,
+      "loss": -545.1357,
+      "nll_loss": 8.846769332885742,
+      "rewards/accuracies": 0.7406250238418579,
+      "rewards/chosen": -1.8533966541290283,
+      "rewards/margins": 20.88261604309082,
+      "rewards/rejected": -22.736013412475586,
       "step": 300
     },
     {
       "epoch": 2.8817777777777778,
-      "grad_norm": 8348.119140625,
+      "grad_norm": 8774.75390625,
       "learning_rate": 1.538830716302092e-07,
-      "log_odds_chosen": 44.924232482910156,
-      "log_odds_ratio": -9.396774291992188,
-      "logps/chosen": -14.924581527709961,
-      "logps/rejected": -59.64234161376953,
-      "loss": -233.7471,
-      "nll_loss": 14.424337387084961,
-      "rewards/accuracies": 0.7124999761581421,
-      "rewards/chosen": -7.4622907638549805,
-      "rewards/margins": 22.3588809967041,
-      "rewards/rejected": -29.821170806884766,
+      "log_odds_chosen": 37.96703338623047,
+      "log_odds_ratio": -2.458489179611206,
+      "logps/chosen": -4.410444736480713,
+      "logps/rejected": -42.13166046142578,
+      "loss": -457.7542,
+      "nll_loss": 8.809645652770996,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -2.2052223682403564,
+      "rewards/margins": 18.860610961914062,
+      "rewards/rejected": -21.06583023071289,
       "step": 305
     },
     {
       "epoch": 2.9291851851851853,
-      "grad_norm": 8581.84765625,
+      "grad_norm": 5141.3994140625,
       "learning_rate": 3.8500413544415025e-08,
-      "log_odds_chosen": 50.05999755859375,
-      "log_odds_ratio": -14.670018196105957,
-      "logps/chosen": -20.66399574279785,
-      "logps/rejected": -70.54561614990234,
-      "loss": -132.5917,
-      "nll_loss": 18.822933197021484,
-      "rewards/accuracies": 0.7437499761581421,
-      "rewards/chosen": -10.331997871398926,
-      "rewards/margins": 24.940807342529297,
-      "rewards/rejected": -35.27280807495117,
+      "log_odds_chosen": 43.835670471191406,
+      "log_odds_ratio": -3.51432466506958,
+      "logps/chosen": -6.511708736419678,
+      "logps/rejected": -50.12602996826172,
+      "loss": -484.2083,
+      "nll_loss": 11.03735637664795,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -3.255854368209839,
+      "rewards/margins": 21.807161331176758,
+      "rewards/rejected": -25.06301498413086,
       "step": 310
     },
     {
       "epoch": 2.9765925925925925,
-      "grad_norm": 8825.96875,
+      "grad_norm": 12163.37890625,
       "learning_rate": 0.0,
-      "log_odds_chosen": 60.088844299316406,
-      "log_odds_ratio": -5.067689418792725,
-      "logps/chosen": -8.742526054382324,
-      "logps/rejected": -68.63340759277344,
-      "loss": -673.9993,
-      "nll_loss": 10.168830871582031,
+      "log_odds_chosen": 37.13484573364258,
+      "log_odds_ratio": -1.1676459312438965,
+      "logps/chosen": -3.6658377647399902,
+      "logps/rejected": -40.594600677490234,
+      "loss": -469.246,
+      "nll_loss": 8.678117752075195,
       "rewards/accuracies": 0.75,
-      "rewards/chosen": -4.371263027191162,
-      "rewards/margins": 29.945446014404297,
-      "rewards/rejected": -34.31670379638672,
+      "rewards/chosen": -1.8329188823699951,
+      "rewards/margins": 18.46438217163086,
+      "rewards/rejected": -20.297300338745117,
       "step": 315
     },
     {
       "epoch": 2.9765925925925925,
       "step": 315,
       "total_flos": 0.0,
-      "train_loss": -61.895723918127636,
-      "train_runtime": 9291.6158,
-      "train_samples_per_second": 2.179,
+      "train_loss": -125.51550612676711,
+      "train_runtime": 9350.753,
+      "train_samples_per_second": 2.166,
       "train_steps_per_second": 0.034
     }
   ],