{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 2.986666666666667,
  "eval_steps": 500,
  "global_step": 168,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.17777777777777778,
      "grad_norm": 7.6314873695373535,
      "learning_rate": 2.9411764705882355e-06,
      "logits/chosen": -0.6844646334648132,
      "logits/rejected": -0.35283079743385315,
      "logps/chosen": -2.9458744525909424,
      "logps/rejected": -3.7147631645202637,
      "loss": 2.9904,
      "odds_ratio_loss": 25.87075424194336,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.2945874333381653,
      "rewards/margins": 0.07688888907432556,
      "rewards/rejected": -0.37147635221481323,
      "sft_loss": 0.40333542227745056,
      "step": 10
    },
    {
      "epoch": 0.35555555555555557,
      "grad_norm": 9.168193817138672,
      "learning_rate": 4.995131923687488e-06,
      "logits/chosen": -0.3836471438407898,
      "logits/rejected": 0.008832636289298534,
      "logps/chosen": -2.8614296913146973,
      "logps/rejected": -3.626382350921631,
      "loss": 2.9085,
      "odds_ratio_loss": 25.276187896728516,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.2861429452896118,
      "rewards/margins": 0.07649530470371246,
      "rewards/rejected": -0.3626382648944855,
      "sft_loss": 0.3808932900428772,
      "step": 20
    },
    {
      "epoch": 0.5333333333333333,
      "grad_norm": 8.867982864379883,
      "learning_rate": 4.90911473983908e-06,
      "logits/chosen": -0.5513006448745728,
      "logits/rejected": -0.1812809407711029,
      "logps/chosen": -2.488701343536377,
      "logps/rejected": -3.2581515312194824,
      "loss": 2.5338,
      "odds_ratio_loss": 22.105247497558594,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.2488701343536377,
      "rewards/margins": 0.07694502174854279,
      "rewards/rejected": -0.3258151412010193,
      "sft_loss": 0.32325050234794617,
      "step": 30
    },
    {
      "epoch": 0.7111111111111111,
      "grad_norm": 6.930299282073975,
      "learning_rate": 4.71919261421297e-06,
      "logits/chosen": -0.542073667049408,
      "logits/rejected": -0.3160572946071625,
      "logps/chosen": -1.9436540603637695,
      "logps/rejected": -2.893744707107544,
      "loss": 1.9838,
      "odds_ratio_loss": 17.531436920166016,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -0.19436539709568024,
      "rewards/margins": 0.09500905126333237,
      "rewards/rejected": -0.289374440908432,
      "sft_loss": 0.2306901514530182,
      "step": 40
    },
    {
      "epoch": 0.8888888888888888,
      "grad_norm": 5.411482334136963,
      "learning_rate": 4.43355687413747e-06,
      "logits/chosen": -0.3556787371635437,
      "logits/rejected": -0.13702432811260223,
      "logps/chosen": -1.401972770690918,
      "logps/rejected": -2.2697913646698,
      "loss": 1.4458,
      "odds_ratio_loss": 12.927480697631836,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.1401972770690918,
      "rewards/margins": 0.08678187429904938,
      "rewards/rejected": -0.2269791066646576,
      "sft_loss": 0.15308110415935516,
      "step": 50
    },
    {
      "epoch": 1.0666666666666667,
      "grad_norm": 3.6417393684387207,
      "learning_rate": 4.064526968101844e-06,
      "logits/chosen": -0.08338828384876251,
      "logits/rejected": 0.10747692734003067,
      "logps/chosen": -1.0365681648254395,
      "logps/rejected": -2.0373435020446777,
      "loss": 1.0768,
      "odds_ratio_loss": 9.728913307189941,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.10365680605173111,
      "rewards/margins": 0.1000775545835495,
      "rewards/rejected": -0.2037343531847,
      "sft_loss": 0.10390937328338623,
      "step": 60
    },
    {
      "epoch": 1.2444444444444445,
      "grad_norm": 4.931552886962891,
      "learning_rate": 3.6280191288478437e-06,
      "logits/chosen": -0.3539902865886688,
      "logits/rejected": -0.07786711305379868,
      "logps/chosen": -0.7550846338272095,
      "logps/rejected": -1.6581230163574219,
      "loss": 0.7984,
      "odds_ratio_loss": 7.2239203453063965,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.07550846040248871,
      "rewards/margins": 0.09030384570360184,
      "rewards/rejected": -0.16581231355667114,
      "sft_loss": 0.07596994936466217,
      "step": 70
    },
    {
      "epoch": 1.4222222222222223,
      "grad_norm": 4.805271148681641,
      "learning_rate": 3.142859907420615e-06,
      "logits/chosen": 0.25180932879447937,
      "logits/rejected": 0.5221790671348572,
      "logps/chosen": -0.5656775236129761,
      "logps/rejected": -1.4922306537628174,
      "loss": 0.6071,
      "odds_ratio_loss": 5.452211380004883,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.05656775087118149,
      "rewards/margins": 0.09265531599521637,
      "rewards/rejected": -0.14922307431697845,
      "sft_loss": 0.06183191016316414,
      "step": 80
    },
    {
      "epoch": 1.6,
      "grad_norm": 3.405627965927124,
      "learning_rate": 2.629974185404951e-06,
      "logits/chosen": -0.19811873137950897,
      "logits/rejected": 0.17863574624061584,
      "logps/chosen": -0.47559866309165955,
      "logps/rejected": -1.4184300899505615,
      "loss": 0.5238,
      "odds_ratio_loss": 4.697209358215332,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.04755987599492073,
      "rewards/margins": 0.09428314864635468,
      "rewards/rejected": -0.1418430209159851,
      "sft_loss": 0.0540793314576149,
      "step": 90
    },
    {
      "epoch": 1.7777777777777777,
      "grad_norm": 3.66274356842041,
      "learning_rate": 2.1114826863194882e-06,
      "logits/chosen": -0.39210397005081177,
      "logits/rejected": 0.007559856865555048,
      "logps/chosen": -0.3131728768348694,
      "logps/rejected": -1.3085200786590576,
      "loss": 0.3503,
      "odds_ratio_loss": 3.154588222503662,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.03131728619337082,
      "rewards/margins": 0.09953471273183823,
      "rewards/rejected": -0.13085201382637024,
      "sft_loss": 0.03485164791345596,
      "step": 100
    },
    {
      "epoch": 1.9555555555555557,
      "grad_norm": 2.7570531368255615,
      "learning_rate": 1.6097479104361328e-06,
      "logits/chosen": -0.7070721387863159,
      "logits/rejected": -0.22419829666614532,
      "logps/chosen": -0.27365046739578247,
      "logps/rejected": -1.0460803508758545,
      "loss": 0.322,
      "odds_ratio_loss": 2.940398693084717,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.027365049347281456,
      "rewards/margins": 0.07724298536777496,
      "rewards/rejected": -0.10460802167654037,
      "sft_loss": 0.027918804436922073,
      "step": 110
    },
    {
      "epoch": 2.1333333333333333,
      "grad_norm": 2.4038069248199463,
      "learning_rate": 1.1464096417858821e-06,
      "logits/chosen": -0.5680712461471558,
      "logits/rejected": -0.28739961981773376,
      "logps/chosen": -0.2213420867919922,
      "logps/rejected": -1.1936922073364258,
      "loss": 0.2601,
      "odds_ratio_loss": 2.3274455070495605,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.0221342071890831,
      "rewards/margins": 0.09723500907421112,
      "rewards/rejected": -0.11936922371387482,
      "sft_loss": 0.02737104333937168,
      "step": 120
    },
    {
      "epoch": 2.311111111111111,
      "grad_norm": 1.1880685091018677,
      "learning_rate": 7.414516258630245e-07,
      "logits/chosen": -0.5090128779411316,
      "logits/rejected": -0.18801167607307434,
      "logps/chosen": -0.20155474543571472,
      "logps/rejected": -1.0635544061660767,
      "loss": 0.2454,
      "odds_ratio_loss": 2.2017226219177246,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.02015547640621662,
      "rewards/margins": 0.08619997650384903,
      "rewards/rejected": -0.1063554510474205,
      "sft_loss": 0.02521040476858616,
      "step": 130
    },
    {
      "epoch": 2.488888888888889,
      "grad_norm": 6.385291576385498,
      "learning_rate": 4.123396721497977e-07,
      "logits/chosen": -0.4758281111717224,
      "logits/rejected": 0.05975949764251709,
      "logps/chosen": -0.21704864501953125,
      "logps/rejected": -1.0010992288589478,
      "loss": 0.2694,
      "odds_ratio_loss": 2.4016900062561035,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.021704863756895065,
      "rewards/margins": 0.07840504497289658,
      "rewards/rejected": -0.10010991245508194,
      "sft_loss": 0.029203277081251144,
      "step": 140
    },
    {
      "epoch": 2.6666666666666665,
      "grad_norm": 3.057594060897827,
      "learning_rate": 1.7326835503629542e-07,
      "logits/chosen": -0.66801917552948,
      "logits/rejected": -0.19290518760681152,
      "logps/chosen": -0.17285582423210144,
      "logps/rejected": -1.0114670991897583,
      "loss": 0.2015,
      "odds_ratio_loss": 1.7902212142944336,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.017285581678152084,
      "rewards/margins": 0.08386112749576569,
      "rewards/rejected": -0.10114671289920807,
      "sft_loss": 0.02243014983832836,
      "step": 150
    },
    {
      "epoch": 2.8444444444444446,
      "grad_norm": 2.3622488975524902,
      "learning_rate": 3.4548802869627806e-08,
      "logits/chosen": -0.27373093366622925,
      "logits/rejected": 0.14009419083595276,
      "logps/chosen": -0.1832386553287506,
      "logps/rejected": -0.950977623462677,
      "loss": 0.2228,
      "odds_ratio_loss": 2.0095624923706055,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.018323864787817,
      "rewards/margins": 0.07677389681339264,
      "rewards/rejected": -0.09509776532649994,
      "sft_loss": 0.021797562018036842,
      "step": 160
    },
    {
      "epoch": 2.986666666666667,
      "step": 168,
      "total_flos": 3.471912421559501e+16,
      "train_loss": 1.00575195536727,
      "train_runtime": 639.1511,
      "train_samples_per_second": 4.224,
      "train_steps_per_second": 0.263
    }
  ],
  "logging_steps": 10,
  "max_steps": 168,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 3,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 3.471912421559501e+16,
  "train_batch_size": 2,
  "trial_name": null,
  "trial_params": null
}