Llama-3.1-8B-Instruct-KTO-100 / trainer_state.json
chchen's picture
End of training
8c0a70b verified
Raw History Blame Contribute Delete
7.94 kB
{
"best_metric": 0.4997052252292633,
"best_model_checkpoint": "saves/sycophancy/Llama-8B-3.1-Instruct/kto-100/checkpoint-100",
"epoch": 9.777777777777779,
"eval_steps": 50,
"global_step": 110,
"is_hyper_param_search": false,
"is_local_process_zero": true,
"is_world_process_zero": true,
"log_history": [
{
"epoch": 0.8888888888888888,
"grad_norm": 0.6491909027099609,
"kl": 3.4562063217163086,
"learning_rate": 4.5454545454545455e-06,
"logits/chosen": -6489723.7894736845,
"logits/rejected": -7186845.714285715,
"logps/chosen": -19.671953703227796,
"logps/rejected": -20.928447905040922,
"loss": 0.5005,
"rewards/chosen": -0.001196424623853282,
"rewards/margins": -0.004711177834965531,
"rewards/rejected": 0.0035147532111122494,
"step": 10
},
{
"epoch": 1.7777777777777777,
"grad_norm": 0.8529065251350403,
"kl": 4.441736221313477,
"learning_rate": 4.8987324340362445e-06,
"logits/chosen": -6480347.428571428,
"logits/rejected": -7280319.2289156625,
"logps/chosen": -20.52882127637987,
"logps/rejected": -20.018041403896838,
"loss": 0.4998,
"rewards/chosen": -0.0007010871036486192,
"rewards/margins": 0.0022902732504876525,
"rewards/rejected": -0.0029913603541362717,
"step": 20
},
{
"epoch": 2.6666666666666665,
"grad_norm": 0.6697812676429749,
"kl": 2.5247769355773926,
"learning_rate": 4.559191453574582e-06,
"logits/chosen": -6102143.1466666665,
"logits/rejected": -7669873.694117647,
"logps/chosen": -19.55682291666667,
"logps/rejected": -20.801958869485293,
"loss": 0.499,
"rewards/chosen": 0.0057321461041768395,
"rewards/margins": 0.00854761147031597,
"rewards/rejected": -0.0028154653661391312,
"step": 30
},
{
"epoch": 3.5555555555555554,
"grad_norm": 0.9257539510726929,
"kl": 2.3885769844055176,
"learning_rate": 4.014024217844167e-06,
"logits/chosen": -6635654.320987654,
"logits/rejected": -7498068.253164557,
"logps/chosen": -17.21456344039352,
"logps/rejected": -20.309491507614716,
"loss": 0.4977,
"rewards/chosen": 0.014323502411077053,
"rewards/margins": 0.01919812530833383,
"rewards/rejected": -0.004874622897256779,
"step": 40
},
{
"epoch": 4.444444444444445,
"grad_norm": 0.8241211771965027,
"kl": 1.9659223556518555,
"learning_rate": 3.3176699082935546e-06,
"logits/chosen": -6149508.155844156,
"logits/rejected": -7053594.21686747,
"logps/chosen": -20.374757444703732,
"logps/rejected": -20.175387095256024,
"loss": 0.4944,
"rewards/chosen": 0.031391688755580356,
"rewards/margins": 0.04555150604904621,
"rewards/rejected": -0.014159817293465856,
"step": 50
},
{
"epoch": 4.444444444444445,
"eval_logits/chosen": -5154306.285714285,
"eval_logits/rejected": -7920785.333333333,
"eval_logps/chosen": -17.13888440813337,
"eval_logps/rejected": -23.718454996744793,
"eval_loss": 0.501809298992157,
"eval_rewards/chosen": -0.0014386344701051712,
"eval_rewards/margins": -0.02036647933224837,
"eval_rewards/rejected": 0.0189278448621432,
"eval_runtime": 3.9297,
"eval_samples_per_second": 5.089,
"eval_steps_per_second": 2.545,
"kl": 0.07577133178710938,
"step": 50
},
{
"epoch": 5.333333333333333,
"grad_norm": 0.7947250008583069,
"kl": 2.237136125564575,
"learning_rate": 2.53966490958702e-06,
"logits/chosen": -6028127.05882353,
"logits/rejected": -7365730.782608695,
"logps/chosen": -20.794239717371322,
"logps/rejected": -20.906398607336957,
"loss": 0.4925,
"rewards/chosen": 0.03433014364803538,
"rewards/margins": 0.06258551818330574,
"rewards/rejected": -0.02825537453527036,
"step": 60
},
{
"epoch": 6.222222222222222,
"grad_norm": 1.0035609006881714,
"kl": 0.19663143157958984,
"learning_rate": 1.7576990616793139e-06,
"logits/chosen": -5932590.8292682925,
"logits/rejected": -7592413.538461538,
"logps/chosen": -19.798485732660062,
"logps/rejected": -20.841768704927883,
"loss": 0.4885,
"rewards/chosen": 0.041047639963103504,
"rewards/margins": 0.0920561089822842,
"rewards/rejected": -0.05100846901918069,
"step": 70
},
{
"epoch": 7.111111111111111,
"grad_norm": 1.3625916242599487,
"kl": 1.2758498191833496,
"learning_rate": 1.049857726072005e-06,
"logits/chosen": -6768110.08,
"logits/rejected": -6676467.9529411765,
"logps/chosen": -17.8654541015625,
"logps/rejected": -21.48829704733456,
"loss": 0.4841,
"rewards/chosen": 0.053166917165120446,
"rewards/margins": 0.1267875899520575,
"rewards/rejected": -0.07362067278693704,
"step": 80
},
{
"epoch": 8.0,
"grad_norm": 1.2151498794555664,
"kl": 1.8016748428344727,
"learning_rate": 4.868243561723535e-07,
"logits/chosen": -5652549.818181818,
"logits/rejected": -7915004.915662651,
"logps/chosen": -19.880209390219157,
"logps/rejected": -21.150170016001507,
"loss": 0.4834,
"rewards/chosen": 0.05748867059682871,
"rewards/margins": 0.1355446326193573,
"rewards/rejected": -0.0780559620225286,
"step": 90
},
{
"epoch": 8.88888888888889,
"grad_norm": 1.0270472764968872,
"kl": 1.7680234909057617,
"learning_rate": 1.2482220564763669e-07,
"logits/chosen": -6533376.432432433,
"logits/rejected": -6922219.162790698,
"logps/chosen": -19.65522065033784,
"logps/rejected": -21.008502339207848,
"loss": 0.4809,
"rewards/chosen": 0.0715047024391793,
"rewards/margins": 0.15263205215962408,
"rewards/rejected": -0.08112734972044479,
"step": 100
},
{
"epoch": 8.88888888888889,
"eval_logits/chosen": -5053702.857142857,
"eval_logits/rejected": -7957526.666666667,
"eval_logps/chosen": -17.07437242780413,
"eval_logps/rejected": -23.829910278320312,
"eval_loss": 0.4997052252292633,
"eval_rewards/chosen": 0.005012544138090951,
"eval_rewards/margins": -0.0027697735599109107,
"eval_rewards/rejected": 0.007782317698001862,
"eval_runtime": 3.937,
"eval_samples_per_second": 5.08,
"eval_steps_per_second": 2.54,
"kl": 0.0,
"step": 100
},
{
"epoch": 9.777777777777779,
"grad_norm": 0.8866751194000244,
"kl": 1.2843537330627441,
"learning_rate": 0.0,
"logits/chosen": -6480412.307692308,
"logits/rejected": -8124753.1707317075,
"logps/chosen": -19.448489458133015,
"logps/rejected": -22.091482767244663,
"loss": 0.4805,
"rewards/chosen": 0.06529526221446502,
"rewards/margins": 0.1558387591139535,
"rewards/rejected": -0.0905434968994885,
"step": 110
},
{
"epoch": 9.777777777777779,
"step": 110,
"total_flos": 1.0124196800495616e+16,
"train_loss": 0.4910273031754927,
"train_runtime": 636.4533,
"train_samples_per_second": 2.828,
"train_steps_per_second": 0.173
}
],
"logging_steps": 10,
"max_steps": 110,
"num_input_tokens_seen": 0,
"num_train_epochs": 10,
"save_steps": 50,
"stateful_callbacks": {
"TrainerControl": {
"args": {
"should_epoch_stop": false,
"should_evaluate": false,
"should_log": false,
"should_save": true,
"should_training_stop": true
},
"attributes": {}
}
},
"total_flos": 1.0124196800495616e+16,
"train_batch_size": 2,
"trial_name": null,
"trial_params": null
}