Instructions to use chchen/Llama-3.1-8B-Instruct-KTO-100 with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- PEFT
How to use chchen/Llama-3.1-8B-Instruct-KTO-100 with PEFT:
from peft import PeftModel from transformers import AutoModelForCausalLM base_model = AutoModelForCausalLM.from_pretrained("meta-llama/Meta-Llama-3.1-8B-Instruct") model = PeftModel.from_pretrained(base_model, "chchen/Llama-3.1-8B-Instruct-KTO-100") - Notebooks
- Google Colab
- Kaggle
Download trainer_state.json from chchen/Llama-3.1-8B-Instruct-KTO-100: direct link, hf CLI and curl.
- Browser
- Download file 7.94 kB
-
https://huggingface.co/chchen/Llama-3.1-8B-Instruct-KTO-100/resolve/main/trainer_state.json
- Command line
-
hf download hf://chchen/Llama-3.1-8B-Instruct-KTO-100/trainer_state.json
-
curl -L -o trainer_state.json https://huggingface.co/chchen/Llama-3.1-8B-Instruct-KTO-100/resolve/main/trainer_state.json
7.94 kB
| { | |
| "best_metric": 0.4997052252292633, | |
| "best_model_checkpoint": "saves/sycophancy/Llama-8B-3.1-Instruct/kto-100/checkpoint-100", | |
| "epoch": 9.777777777777779, | |
| "eval_steps": 50, | |
| "global_step": 110, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "epoch": 0.8888888888888888, | |
| "grad_norm": 0.6491909027099609, | |
| "kl": 3.4562063217163086, | |
| "learning_rate": 4.5454545454545455e-06, | |
| "logits/chosen": -6489723.7894736845, | |
| "logits/rejected": -7186845.714285715, | |
| "logps/chosen": -19.671953703227796, | |
| "logps/rejected": -20.928447905040922, | |
| "loss": 0.5005, | |
| "rewards/chosen": -0.001196424623853282, | |
| "rewards/margins": -0.004711177834965531, | |
| "rewards/rejected": 0.0035147532111122494, | |
| "step": 10 | |
| }, | |
| { | |
| "epoch": 1.7777777777777777, | |
| "grad_norm": 0.8529065251350403, | |
| "kl": 4.441736221313477, | |
| "learning_rate": 4.8987324340362445e-06, | |
| "logits/chosen": -6480347.428571428, | |
| "logits/rejected": -7280319.2289156625, | |
| "logps/chosen": -20.52882127637987, | |
| "logps/rejected": -20.018041403896838, | |
| "loss": 0.4998, | |
| "rewards/chosen": -0.0007010871036486192, | |
| "rewards/margins": 0.0022902732504876525, | |
| "rewards/rejected": -0.0029913603541362717, | |
| "step": 20 | |
| }, | |
| { | |
| "epoch": 2.6666666666666665, | |
| "grad_norm": 0.6697812676429749, | |
| "kl": 2.5247769355773926, | |
| "learning_rate": 4.559191453574582e-06, | |
| "logits/chosen": -6102143.1466666665, | |
| "logits/rejected": -7669873.694117647, | |
| "logps/chosen": -19.55682291666667, | |
| "logps/rejected": -20.801958869485293, | |
| "loss": 0.499, | |
| "rewards/chosen": 0.0057321461041768395, | |
| "rewards/margins": 0.00854761147031597, | |
| "rewards/rejected": -0.0028154653661391312, | |
| "step": 30 | |
| }, | |
| { | |
| "epoch": 3.5555555555555554, | |
| "grad_norm": 0.9257539510726929, | |
| "kl": 2.3885769844055176, | |
| "learning_rate": 4.014024217844167e-06, | |
| "logits/chosen": -6635654.320987654, | |
| "logits/rejected": -7498068.253164557, | |
| "logps/chosen": -17.21456344039352, | |
| "logps/rejected": -20.309491507614716, | |
| "loss": 0.4977, | |
| "rewards/chosen": 0.014323502411077053, | |
| "rewards/margins": 0.01919812530833383, | |
| "rewards/rejected": -0.004874622897256779, | |
| "step": 40 | |
| }, | |
| { | |
| "epoch": 4.444444444444445, | |
| "grad_norm": 0.8241211771965027, | |
| "kl": 1.9659223556518555, | |
| "learning_rate": 3.3176699082935546e-06, | |
| "logits/chosen": -6149508.155844156, | |
| "logits/rejected": -7053594.21686747, | |
| "logps/chosen": -20.374757444703732, | |
| "logps/rejected": -20.175387095256024, | |
| "loss": 0.4944, | |
| "rewards/chosen": 0.031391688755580356, | |
| "rewards/margins": 0.04555150604904621, | |
| "rewards/rejected": -0.014159817293465856, | |
| "step": 50 | |
| }, | |
| { | |
| "epoch": 4.444444444444445, | |
| "eval_logits/chosen": -5154306.285714285, | |
| "eval_logits/rejected": -7920785.333333333, | |
| "eval_logps/chosen": -17.13888440813337, | |
| "eval_logps/rejected": -23.718454996744793, | |
| "eval_loss": 0.501809298992157, | |
| "eval_rewards/chosen": -0.0014386344701051712, | |
| "eval_rewards/margins": -0.02036647933224837, | |
| "eval_rewards/rejected": 0.0189278448621432, | |
| "eval_runtime": 3.9297, | |
| "eval_samples_per_second": 5.089, | |
| "eval_steps_per_second": 2.545, | |
| "kl": 0.07577133178710938, | |
| "step": 50 | |
| }, | |
| { | |
| "epoch": 5.333333333333333, | |
| "grad_norm": 0.7947250008583069, | |
| "kl": 2.237136125564575, | |
| "learning_rate": 2.53966490958702e-06, | |
| "logits/chosen": -6028127.05882353, | |
| "logits/rejected": -7365730.782608695, | |
| "logps/chosen": -20.794239717371322, | |
| "logps/rejected": -20.906398607336957, | |
| "loss": 0.4925, | |
| "rewards/chosen": 0.03433014364803538, | |
| "rewards/margins": 0.06258551818330574, | |
| "rewards/rejected": -0.02825537453527036, | |
| "step": 60 | |
| }, | |
| { | |
| "epoch": 6.222222222222222, | |
| "grad_norm": 1.0035609006881714, | |
| "kl": 0.19663143157958984, | |
| "learning_rate": 1.7576990616793139e-06, | |
| "logits/chosen": -5932590.8292682925, | |
| "logits/rejected": -7592413.538461538, | |
| "logps/chosen": -19.798485732660062, | |
| "logps/rejected": -20.841768704927883, | |
| "loss": 0.4885, | |
| "rewards/chosen": 0.041047639963103504, | |
| "rewards/margins": 0.0920561089822842, | |
| "rewards/rejected": -0.05100846901918069, | |
| "step": 70 | |
| }, | |
| { | |
| "epoch": 7.111111111111111, | |
| "grad_norm": 1.3625916242599487, | |
| "kl": 1.2758498191833496, | |
| "learning_rate": 1.049857726072005e-06, | |
| "logits/chosen": -6768110.08, | |
| "logits/rejected": -6676467.9529411765, | |
| "logps/chosen": -17.8654541015625, | |
| "logps/rejected": -21.48829704733456, | |
| "loss": 0.4841, | |
| "rewards/chosen": 0.053166917165120446, | |
| "rewards/margins": 0.1267875899520575, | |
| "rewards/rejected": -0.07362067278693704, | |
| "step": 80 | |
| }, | |
| { | |
| "epoch": 8.0, | |
| "grad_norm": 1.2151498794555664, | |
| "kl": 1.8016748428344727, | |
| "learning_rate": 4.868243561723535e-07, | |
| "logits/chosen": -5652549.818181818, | |
| "logits/rejected": -7915004.915662651, | |
| "logps/chosen": -19.880209390219157, | |
| "logps/rejected": -21.150170016001507, | |
| "loss": 0.4834, | |
| "rewards/chosen": 0.05748867059682871, | |
| "rewards/margins": 0.1355446326193573, | |
| "rewards/rejected": -0.0780559620225286, | |
| "step": 90 | |
| }, | |
| { | |
| "epoch": 8.88888888888889, | |
| "grad_norm": 1.0270472764968872, | |
| "kl": 1.7680234909057617, | |
| "learning_rate": 1.2482220564763669e-07, | |
| "logits/chosen": -6533376.432432433, | |
| "logits/rejected": -6922219.162790698, | |
| "logps/chosen": -19.65522065033784, | |
| "logps/rejected": -21.008502339207848, | |
| "loss": 0.4809, | |
| "rewards/chosen": 0.0715047024391793, | |
| "rewards/margins": 0.15263205215962408, | |
| "rewards/rejected": -0.08112734972044479, | |
| "step": 100 | |
| }, | |
| { | |
| "epoch": 8.88888888888889, | |
| "eval_logits/chosen": -5053702.857142857, | |
| "eval_logits/rejected": -7957526.666666667, | |
| "eval_logps/chosen": -17.07437242780413, | |
| "eval_logps/rejected": -23.829910278320312, | |
| "eval_loss": 0.4997052252292633, | |
| "eval_rewards/chosen": 0.005012544138090951, | |
| "eval_rewards/margins": -0.0027697735599109107, | |
| "eval_rewards/rejected": 0.007782317698001862, | |
| "eval_runtime": 3.937, | |
| "eval_samples_per_second": 5.08, | |
| "eval_steps_per_second": 2.54, | |
| "kl": 0.0, | |
| "step": 100 | |
| }, | |
| { | |
| "epoch": 9.777777777777779, | |
| "grad_norm": 0.8866751194000244, | |
| "kl": 1.2843537330627441, | |
| "learning_rate": 0.0, | |
| "logits/chosen": -6480412.307692308, | |
| "logits/rejected": -8124753.1707317075, | |
| "logps/chosen": -19.448489458133015, | |
| "logps/rejected": -22.091482767244663, | |
| "loss": 0.4805, | |
| "rewards/chosen": 0.06529526221446502, | |
| "rewards/margins": 0.1558387591139535, | |
| "rewards/rejected": -0.0905434968994885, | |
| "step": 110 | |
| }, | |
| { | |
| "epoch": 9.777777777777779, | |
| "step": 110, | |
| "total_flos": 1.0124196800495616e+16, | |
| "train_loss": 0.4910273031754927, | |
| "train_runtime": 636.4533, | |
| "train_samples_per_second": 2.828, | |
| "train_steps_per_second": 0.173 | |
| } | |
| ], | |
| "logging_steps": 10, | |
| "max_steps": 110, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 10, | |
| "save_steps": 50, | |
| "stateful_callbacks": { | |
| "TrainerControl": { | |
| "args": { | |
| "should_epoch_stop": false, | |
| "should_evaluate": false, | |
| "should_log": false, | |
| "should_save": true, | |
| "should_training_stop": true | |
| }, | |
| "attributes": {} | |
| } | |
| }, | |
| "total_flos": 1.0124196800495616e+16, | |
| "train_batch_size": 2, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |