End of training

README.md

@@ -3,9 +3,10 @@ base_model: google/gemma-2-9b-it
 library_name: peft
 license: gemma
 tags:
+- llama-factory
+- lora
 - trl
 - dpo
-- llama-factory
 - generated_from_trainer
 model-index:
 - name: Gemma-2-9B-It-SAA
@@ -17,7 +18,19 @@ should probably proofread and complete it, then remove this comment. -->
 
 # Gemma-2-9B-It-SAA
 
-This model is a fine-tuned version of [google/gemma-2-9b-it](https://huggingface.co/google/gemma-2-9b-it) on an unknown dataset.
+This model is a fine-tuned version of [google/gemma-2-9b-it](https://huggingface.co/google/gemma-2-9b-it) on the bct_non_cot_dpo_1000 dataset.
+It achieves the following results on the evaluation set:
+- Loss: 0.2066
+- Rewards/chosen: -0.0162
+- Rewards/rejected: -0.0908
+- Rewards/accuracies: 0.8300
+- Rewards/margins: 0.0746
+- Logps/rejected: -0.9081
+- Logps/chosen: -0.1623
+- Logits/rejected: 0.1790
+- Logits/chosen: -0.2699
+- Sft Loss: 0.0178
+- Odds Ratio Loss: 1.8881
 
 ## Model description
 

all_results.json

@@ -0,0 +1,22 @@
+{
+    "epoch": 2.986666666666667,
+    "eval_logits/chosen": -0.26991021633148193,
+    "eval_logits/rejected": 0.17896275222301483,
+    "eval_logps/chosen": -0.16233521699905396,
+    "eval_logps/rejected": -0.90810227394104,
+    "eval_loss": 0.20663729310035706,
+    "eval_odds_ratio_loss": 1.8880867958068848,
+    "eval_rewards/accuracies": 0.8299999833106995,
+    "eval_rewards/chosen": -0.016233522444963455,
+    "eval_rewards/margins": 0.0745767131447792,
+    "eval_rewards/rejected": -0.09081023186445236,
+    "eval_runtime": 6.9404,
+    "eval_samples_per_second": 14.408,
+    "eval_sft_loss": 0.017828578129410744,
+    "eval_steps_per_second": 7.204,
+    "total_flos": 3.471912421559501e+16,
+    "train_loss": 1.00575195536727,
+    "train_runtime": 639.1511,
+    "train_samples_per_second": 4.224,
+    "train_steps_per_second": 0.263
+}

eval_results.json

@@ -0,0 +1,17 @@
+{
+    "epoch": 2.986666666666667,
+    "eval_logits/chosen": -0.26991021633148193,
+    "eval_logits/rejected": 0.17896275222301483,
+    "eval_logps/chosen": -0.16233521699905396,
+    "eval_logps/rejected": -0.90810227394104,
+    "eval_loss": 0.20663729310035706,
+    "eval_odds_ratio_loss": 1.8880867958068848,
+    "eval_rewards/accuracies": 0.8299999833106995,
+    "eval_rewards/chosen": -0.016233522444963455,
+    "eval_rewards/margins": 0.0745767131447792,
+    "eval_rewards/rejected": -0.09081023186445236,
+    "eval_runtime": 6.9404,
+    "eval_samples_per_second": 14.408,
+    "eval_sft_loss": 0.017828578129410744,
+    "eval_steps_per_second": 7.204
+}

train_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 2.986666666666667,
+    "total_flos": 3.471912421559501e+16,
+    "train_loss": 1.00575195536727,
+    "train_runtime": 639.1511,
+    "train_samples_per_second": 4.224,
+    "train_steps_per_second": 0.263
+}

trainer_state.json

@@ -0,0 +1,314 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 2.986666666666667,
+  "eval_steps": 500,
+  "global_step": 168,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.17777777777777778,
+      "grad_norm": 7.6314873695373535,
+      "learning_rate": 2.9411764705882355e-06,
+      "logits/chosen": -0.6844646334648132,
+      "logits/rejected": -0.35283079743385315,
+      "logps/chosen": -2.9458744525909424,
+      "logps/rejected": -3.7147631645202637,
+      "loss": 2.9904,
+      "odds_ratio_loss": 25.87075424194336,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.2945874333381653,
+      "rewards/margins": 0.07688888907432556,
+      "rewards/rejected": -0.37147635221481323,
+      "sft_loss": 0.40333542227745056,
+      "step": 10
+    },
+    {
+      "epoch": 0.35555555555555557,
+      "grad_norm": 9.168193817138672,
+      "learning_rate": 4.995131923687488e-06,
+      "logits/chosen": -0.3836471438407898,
+      "logits/rejected": 0.008832636289298534,
+      "logps/chosen": -2.8614296913146973,
+      "logps/rejected": -3.626382350921631,
+      "loss": 2.9085,
+      "odds_ratio_loss": 25.276187896728516,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -0.2861429452896118,
+      "rewards/margins": 0.07649530470371246,
+      "rewards/rejected": -0.3626382648944855,
+      "sft_loss": 0.3808932900428772,
+      "step": 20
+    },
+    {
+      "epoch": 0.5333333333333333,
+      "grad_norm": 8.867982864379883,
+      "learning_rate": 4.90911473983908e-06,
+      "logits/chosen": -0.5513006448745728,
+      "logits/rejected": -0.1812809407711029,
+      "logps/chosen": -2.488701343536377,
+      "logps/rejected": -3.2581515312194824,
+      "loss": 2.5338,
+      "odds_ratio_loss": 22.105247497558594,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -0.2488701343536377,
+      "rewards/margins": 0.07694502174854279,
+      "rewards/rejected": -0.3258151412010193,
+      "sft_loss": 0.32325050234794617,
+      "step": 30
+    },
+    {
+      "epoch": 0.7111111111111111,
+      "grad_norm": 6.930299282073975,
+      "learning_rate": 4.71919261421297e-06,
+      "logits/chosen": -0.542073667049408,
+      "logits/rejected": -0.3160572946071625,
+      "logps/chosen": -1.9436540603637695,
+      "logps/rejected": -2.893744707107544,
+      "loss": 1.9838,
+      "odds_ratio_loss": 17.531436920166016,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": -0.19436539709568024,
+      "rewards/margins": 0.09500905126333237,
+      "rewards/rejected": -0.289374440908432,
+      "sft_loss": 0.2306901514530182,
+      "step": 40
+    },
+    {
+      "epoch": 0.8888888888888888,
+      "grad_norm": 5.411482334136963,
+      "learning_rate": 4.43355687413747e-06,
+      "logits/chosen": -0.3556787371635437,
+      "logits/rejected": -0.13702432811260223,
+      "logps/chosen": -1.401972770690918,
+      "logps/rejected": -2.2697913646698,
+      "loss": 1.4458,
+      "odds_ratio_loss": 12.927480697631836,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -0.1401972770690918,
+      "rewards/margins": 0.08678187429904938,
+      "rewards/rejected": -0.2269791066646576,
+      "sft_loss": 0.15308110415935516,
+      "step": 50
+    },
+    {
+      "epoch": 1.0666666666666667,
+      "grad_norm": 3.6417393684387207,
+      "learning_rate": 4.064526968101844e-06,
+      "logits/chosen": -0.08338828384876251,
+      "logits/rejected": 0.10747692734003067,
+      "logps/chosen": -1.0365681648254395,
+      "logps/rejected": -2.0373435020446777,
+      "loss": 1.0768,
+      "odds_ratio_loss": 9.728913307189941,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -0.10365680605173111,
+      "rewards/margins": 0.1000775545835495,
+      "rewards/rejected": -0.2037343531847,
+      "sft_loss": 0.10390937328338623,
+      "step": 60
+    },
+    {
+      "epoch": 1.2444444444444445,
+      "grad_norm": 4.931552886962891,
+      "learning_rate": 3.6280191288478437e-06,
+      "logits/chosen": -0.3539902865886688,
+      "logits/rejected": -0.07786711305379868,
+      "logps/chosen": -0.7550846338272095,
+      "logps/rejected": -1.6581230163574219,
+      "loss": 0.7984,
+      "odds_ratio_loss": 7.2239203453063965,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.07550846040248871,
+      "rewards/margins": 0.09030384570360184,
+      "rewards/rejected": -0.16581231355667114,
+      "sft_loss": 0.07596994936466217,
+      "step": 70
+    },
+    {
+      "epoch": 1.4222222222222223,
+      "grad_norm": 4.805271148681641,
+      "learning_rate": 3.142859907420615e-06,
+      "logits/chosen": 0.25180932879447937,
+      "logits/rejected": 0.5221790671348572,
+      "logps/chosen": -0.5656775236129761,
+      "logps/rejected": -1.4922306537628174,
+      "loss": 0.6071,
+      "odds_ratio_loss": 5.452211380004883,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -0.05656775087118149,
+      "rewards/margins": 0.09265531599521637,
+      "rewards/rejected": -0.14922307431697845,
+      "sft_loss": 0.06183191016316414,
+      "step": 80
+    },
+    {
+      "epoch": 1.6,
+      "grad_norm": 3.405627965927124,
+      "learning_rate": 2.629974185404951e-06,
+      "logits/chosen": -0.19811873137950897,
+      "logits/rejected": 0.17863574624061584,
+      "logps/chosen": -0.47559866309165955,
+      "logps/rejected": -1.4184300899505615,
+      "loss": 0.5238,
+      "odds_ratio_loss": 4.697209358215332,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -0.04755987599492073,
+      "rewards/margins": 0.09428314864635468,
+      "rewards/rejected": -0.1418430209159851,
+      "sft_loss": 0.0540793314576149,
+      "step": 90
+    },
+    {
+      "epoch": 1.7777777777777777,
+      "grad_norm": 3.66274356842041,
+      "learning_rate": 2.1114826863194882e-06,
+      "logits/chosen": -0.39210397005081177,
+      "logits/rejected": 0.007559856865555048,
+      "logps/chosen": -0.3131728768348694,
+      "logps/rejected": -1.3085200786590576,
+      "loss": 0.3503,
+      "odds_ratio_loss": 3.154588222503662,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.03131728619337082,
+      "rewards/margins": 0.09953471273183823,
+      "rewards/rejected": -0.13085201382637024,
+      "sft_loss": 0.03485164791345596,
+      "step": 100
+    },
+    {
+      "epoch": 1.9555555555555557,
+      "grad_norm": 2.7570531368255615,
+      "learning_rate": 1.6097479104361328e-06,
+      "logits/chosen": -0.7070721387863159,
+      "logits/rejected": -0.22419829666614532,
+      "logps/chosen": -0.27365046739578247,
+      "logps/rejected": -1.0460803508758545,
+      "loss": 0.322,
+      "odds_ratio_loss": 2.940398693084717,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.027365049347281456,
+      "rewards/margins": 0.07724298536777496,
+      "rewards/rejected": -0.10460802167654037,
+      "sft_loss": 0.027918804436922073,
+      "step": 110
+    },
+    {
+      "epoch": 2.1333333333333333,
+      "grad_norm": 2.4038069248199463,
+      "learning_rate": 1.1464096417858821e-06,
+      "logits/chosen": -0.5680712461471558,
+      "logits/rejected": -0.28739961981773376,
+      "logps/chosen": -0.2213420867919922,
+      "logps/rejected": -1.1936922073364258,
+      "loss": 0.2601,
+      "odds_ratio_loss": 2.3274455070495605,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -0.0221342071890831,
+      "rewards/margins": 0.09723500907421112,
+      "rewards/rejected": -0.11936922371387482,
+      "sft_loss": 0.02737104333937168,
+      "step": 120
+    },
+    {
+      "epoch": 2.311111111111111,
+      "grad_norm": 1.1880685091018677,
+      "learning_rate": 7.414516258630245e-07,
+      "logits/chosen": -0.5090128779411316,
+      "logits/rejected": -0.18801167607307434,
+      "logps/chosen": -0.20155474543571472,
+      "logps/rejected": -1.0635544061660767,
+      "loss": 0.2454,
+      "odds_ratio_loss": 2.2017226219177246,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -0.02015547640621662,
+      "rewards/margins": 0.08619997650384903,
+      "rewards/rejected": -0.1063554510474205,
+      "sft_loss": 0.02521040476858616,
+      "step": 130
+    },
+    {
+      "epoch": 2.488888888888889,
+      "grad_norm": 6.385291576385498,
+      "learning_rate": 4.123396721497977e-07,
+      "logits/chosen": -0.4758281111717224,
+      "logits/rejected": 0.05975949764251709,
+      "logps/chosen": -0.21704864501953125,
+      "logps/rejected": -1.0010992288589478,
+      "loss": 0.2694,
+      "odds_ratio_loss": 2.4016900062561035,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -0.021704863756895065,
+      "rewards/margins": 0.07840504497289658,
+      "rewards/rejected": -0.10010991245508194,
+      "sft_loss": 0.029203277081251144,
+      "step": 140
+    },
+    {
+      "epoch": 2.6666666666666665,
+      "grad_norm": 3.057594060897827,
+      "learning_rate": 1.7326835503629542e-07,
+      "logits/chosen": -0.66801917552948,
+      "logits/rejected": -0.19290518760681152,
+      "logps/chosen": -0.17285582423210144,
+      "logps/rejected": -1.0114670991897583,
+      "loss": 0.2015,
+      "odds_ratio_loss": 1.7902212142944336,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": -0.017285581678152084,
+      "rewards/margins": 0.08386112749576569,
+      "rewards/rejected": -0.10114671289920807,
+      "sft_loss": 0.02243014983832836,
+      "step": 150
+    },
+    {
+      "epoch": 2.8444444444444446,
+      "grad_norm": 2.3622488975524902,
+      "learning_rate": 3.4548802869627806e-08,
+      "logits/chosen": -0.27373093366622925,
+      "logits/rejected": 0.14009419083595276,
+      "logps/chosen": -0.1832386553287506,
+      "logps/rejected": -0.950977623462677,
+      "loss": 0.2228,
+      "odds_ratio_loss": 2.0095624923706055,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -0.018323864787817,
+      "rewards/margins": 0.07677389681339264,
+      "rewards/rejected": -0.09509776532649994,
+      "sft_loss": 0.021797562018036842,
+      "step": 160
+    },
+    {
+      "epoch": 2.986666666666667,
+      "step": 168,
+      "total_flos": 3.471912421559501e+16,
+      "train_loss": 1.00575195536727,
+      "train_runtime": 639.1511,
+      "train_samples_per_second": 4.224,
+      "train_steps_per_second": 0.263
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 168,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 3.471912421559501e+16,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}