{ "best_metric": 0.4997052252292633, "best_model_checkpoint": "saves/sycophancy/Llama-8B-3.1-Instruct/kto-100/checkpoint-100", "epoch": 9.777777777777779, "eval_steps": 50, "global_step": 110, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "epoch": 0.8888888888888888, "grad_norm": 0.6491909027099609, "kl": 3.4562063217163086, "learning_rate": 4.5454545454545455e-06, "logits/chosen": -6489723.7894736845, "logits/rejected": -7186845.714285715, "logps/chosen": -19.671953703227796, "logps/rejected": -20.928447905040922, "loss": 0.5005, "rewards/chosen": -0.001196424623853282, "rewards/margins": -0.004711177834965531, "rewards/rejected": 0.0035147532111122494, "step": 10 }, { "epoch": 1.7777777777777777, "grad_norm": 0.8529065251350403, "kl": 4.441736221313477, "learning_rate": 4.8987324340362445e-06, "logits/chosen": -6480347.428571428, "logits/rejected": -7280319.2289156625, "logps/chosen": -20.52882127637987, "logps/rejected": -20.018041403896838, "loss": 0.4998, "rewards/chosen": -0.0007010871036486192, "rewards/margins": 0.0022902732504876525, "rewards/rejected": -0.0029913603541362717, "step": 20 }, { "epoch": 2.6666666666666665, "grad_norm": 0.6697812676429749, "kl": 2.5247769355773926, "learning_rate": 4.559191453574582e-06, "logits/chosen": -6102143.1466666665, "logits/rejected": -7669873.694117647, "logps/chosen": -19.55682291666667, "logps/rejected": -20.801958869485293, "loss": 0.499, "rewards/chosen": 0.0057321461041768395, "rewards/margins": 0.00854761147031597, "rewards/rejected": -0.0028154653661391312, "step": 30 }, { "epoch": 3.5555555555555554, "grad_norm": 0.9257539510726929, "kl": 2.3885769844055176, "learning_rate": 4.014024217844167e-06, "logits/chosen": -6635654.320987654, "logits/rejected": -7498068.253164557, "logps/chosen": -17.21456344039352, "logps/rejected": -20.309491507614716, "loss": 0.4977, "rewards/chosen": 0.014323502411077053, "rewards/margins": 0.01919812530833383, "rewards/rejected": -0.004874622897256779, "step": 40 }, { "epoch": 4.444444444444445, "grad_norm": 0.8241211771965027, "kl": 1.9659223556518555, "learning_rate": 3.3176699082935546e-06, "logits/chosen": -6149508.155844156, "logits/rejected": -7053594.21686747, "logps/chosen": -20.374757444703732, "logps/rejected": -20.175387095256024, "loss": 0.4944, "rewards/chosen": 0.031391688755580356, "rewards/margins": 0.04555150604904621, "rewards/rejected": -0.014159817293465856, "step": 50 }, { "epoch": 4.444444444444445, "eval_logits/chosen": -5154306.285714285, "eval_logits/rejected": -7920785.333333333, "eval_logps/chosen": -17.13888440813337, "eval_logps/rejected": -23.718454996744793, "eval_loss": 0.501809298992157, "eval_rewards/chosen": -0.0014386344701051712, "eval_rewards/margins": -0.02036647933224837, "eval_rewards/rejected": 0.0189278448621432, "eval_runtime": 3.9297, "eval_samples_per_second": 5.089, "eval_steps_per_second": 2.545, "kl": 0.07577133178710938, "step": 50 }, { "epoch": 5.333333333333333, "grad_norm": 0.7947250008583069, "kl": 2.237136125564575, "learning_rate": 2.53966490958702e-06, "logits/chosen": -6028127.05882353, "logits/rejected": -7365730.782608695, "logps/chosen": -20.794239717371322, "logps/rejected": -20.906398607336957, "loss": 0.4925, "rewards/chosen": 0.03433014364803538, "rewards/margins": 0.06258551818330574, "rewards/rejected": -0.02825537453527036, "step": 60 }, { "epoch": 6.222222222222222, "grad_norm": 1.0035609006881714, "kl": 0.19663143157958984, "learning_rate": 1.7576990616793139e-06, "logits/chosen": -5932590.8292682925, "logits/rejected": -7592413.538461538, "logps/chosen": -19.798485732660062, "logps/rejected": -20.841768704927883, "loss": 0.4885, "rewards/chosen": 0.041047639963103504, "rewards/margins": 0.0920561089822842, "rewards/rejected": -0.05100846901918069, "step": 70 }, { "epoch": 7.111111111111111, "grad_norm": 1.3625916242599487, "kl": 1.2758498191833496, "learning_rate": 1.049857726072005e-06, "logits/chosen": -6768110.08, "logits/rejected": -6676467.9529411765, "logps/chosen": -17.8654541015625, "logps/rejected": -21.48829704733456, "loss": 0.4841, "rewards/chosen": 0.053166917165120446, "rewards/margins": 0.1267875899520575, "rewards/rejected": -0.07362067278693704, "step": 80 }, { "epoch": 8.0, "grad_norm": 1.2151498794555664, "kl": 1.8016748428344727, "learning_rate": 4.868243561723535e-07, "logits/chosen": -5652549.818181818, "logits/rejected": -7915004.915662651, "logps/chosen": -19.880209390219157, "logps/rejected": -21.150170016001507, "loss": 0.4834, "rewards/chosen": 0.05748867059682871, "rewards/margins": 0.1355446326193573, "rewards/rejected": -0.0780559620225286, "step": 90 }, { "epoch": 8.88888888888889, "grad_norm": 1.0270472764968872, "kl": 1.7680234909057617, "learning_rate": 1.2482220564763669e-07, "logits/chosen": -6533376.432432433, "logits/rejected": -6922219.162790698, "logps/chosen": -19.65522065033784, "logps/rejected": -21.008502339207848, "loss": 0.4809, "rewards/chosen": 0.0715047024391793, "rewards/margins": 0.15263205215962408, "rewards/rejected": -0.08112734972044479, "step": 100 }, { "epoch": 8.88888888888889, "eval_logits/chosen": -5053702.857142857, "eval_logits/rejected": -7957526.666666667, "eval_logps/chosen": -17.07437242780413, "eval_logps/rejected": -23.829910278320312, "eval_loss": 0.4997052252292633, "eval_rewards/chosen": 0.005012544138090951, "eval_rewards/margins": -0.0027697735599109107, "eval_rewards/rejected": 0.007782317698001862, "eval_runtime": 3.937, "eval_samples_per_second": 5.08, "eval_steps_per_second": 2.54, "kl": 0.0, "step": 100 }, { "epoch": 9.777777777777779, "grad_norm": 0.8866751194000244, "kl": 1.2843537330627441, "learning_rate": 0.0, "logits/chosen": -6480412.307692308, "logits/rejected": -8124753.1707317075, "logps/chosen": -19.448489458133015, "logps/rejected": -22.091482767244663, "loss": 0.4805, "rewards/chosen": 0.06529526221446502, "rewards/margins": 0.1558387591139535, "rewards/rejected": -0.0905434968994885, "step": 110 }, { "epoch": 9.777777777777779, "step": 110, "total_flos": 1.0124196800495616e+16, "train_loss": 0.4910273031754927, "train_runtime": 636.4533, "train_samples_per_second": 2.828, "train_steps_per_second": 0.173 } ], "logging_steps": 10, "max_steps": 110, "num_input_tokens_seen": 0, "num_train_epochs": 10, "save_steps": 50, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": true, "should_training_stop": true }, "attributes": {} } }, "total_flos": 1.0124196800495616e+16, "train_batch_size": 2, "trial_name": null, "trial_params": null }