End of training

README.md

@@ -1,6 +1,6 @@
 ---
 base_model: LGAI-EXAONE/EXAONE-3.5-2.4B-Instruct
-datasets: testing_layer_29_Matrix_mlp.c_proj
+datasets: testing_layer_29_Matrix_attn.attention.v_proj
 library_name: transformers
 model_name: output_exaone_tucker_rank16_newdata_b1_g2_gradient2_e1_b512
 tags:
@@ -12,7 +12,7 @@ licence: license
 
 # Model Card for output_exaone_tucker_rank16_newdata_b1_g2_gradient2_e1_b512
 
-This model is a fine-tuned version of [LGAI-EXAONE/EXAONE-3.5-2.4B-Instruct](https://huggingface.co/LGAI-EXAONE/EXAONE-3.5-2.4B-Instruct) on the [testing_layer_29_Matrix_mlp.c_proj](https://huggingface.co/datasets/testing_layer_29_Matrix_mlp.c_proj) dataset.
+This model is a fine-tuned version of [LGAI-EXAONE/EXAONE-3.5-2.4B-Instruct](https://huggingface.co/LGAI-EXAONE/EXAONE-3.5-2.4B-Instruct) on the [testing_layer_29_Matrix_attn.attention.v_proj](https://huggingface.co/datasets/testing_layer_29_Matrix_attn.attention.v_proj) dataset.
 It has been trained using [TRL](https://github.com/huggingface/trl).
 
 ## Quick start

all_results.json

@@ -1,15 +1,15 @@
 {
-    "eval_accuracy": 0.8085339415160145,
-    "eval_loss": 1.681640625,
-    "eval_runtime": 927.2318,
+    "eval_accuracy": 0.8098589776031488,
+    "eval_loss": 1.64453125,
+    "eval_runtime": 929.8656,
     "eval_samples": 25971,
-    "eval_samples_per_second": 28.009,
-    "eval_steps_per_second": 14.005,
-    "perplexity": 5.374366061438169,
-    "total_flos": 9.3961326985583e+17,
-    "train_loss": 1.7179553695028122,
-    "train_runtime": 11634.1539,
+    "eval_samples_per_second": 27.93,
+    "eval_steps_per_second": 13.965,
+    "perplexity": 5.178581877668445,
+    "total_flos": 9.323627298076754e+17,
+    "train_loss": 1.6535549103175464,
+    "train_runtime": 12676.64,
     "train_samples": 294393,
-    "train_samples_per_second": 25.304,
-    "train_steps_per_second": 6.326
+    "train_samples_per_second": 23.223,
+    "train_steps_per_second": 5.806
 }

eval_layer_index_29_mat_attn.attention.v_proj_results.json

@@ -1,9 +1,9 @@
 {
-    "eval_accuracy": 0.7482185646263703,
-    "eval_loss": 5.08984375,
-    "eval_runtime": 179.6951,
-    "eval_samples": 5103,
-    "eval_samples_per_second": 28.398,
-    "eval_steps_per_second": 14.202,
-    "perplexity": 162.3644906197382
+    "eval_accuracy": 0.8098589776031488,
+    "eval_loss": 1.64453125,
+    "eval_runtime": 929.8656,
+    "eval_samples": 25971,
+    "eval_samples_per_second": 27.93,
+    "eval_steps_per_second": 13.965,
+    "perplexity": 5.178581877668445
 }

model-00001-of-00002.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b8f8f30ae6f98975b62efe02993f798af7fee6c3dd2a00c9213c8e4750246109
-size 4701521288
+oid sha256:33daf942257ad740d4a824433373dda5c7962948834476018ead06d5b07aed4e
+size 4669288872

model.safetensors.index.json

@@ -1,6 +1,6 @@
 {
   "metadata": {
-    "total_size": 5225777664
+    "total_size": 5193544192
   },
   "weight_map": {
     "lm_head.weight": "model-00002-of-00002.safetensors",
@@ -202,10 +202,18 @@
     "transformer.h.28.mlp.c_fc_0.weight": "model-00001-of-00002.safetensors",
     "transformer.h.28.mlp.c_fc_1.weight": "model-00001-of-00002.safetensors",
     "transformer.h.28.mlp.c_proj.weight": "model-00001-of-00002.safetensors",
-    "transformer.h.29.attn.attention.k_proj.weight": "model-00001-of-00002.safetensors",
-    "transformer.h.29.attn.attention.out_proj.weight": "model-00001-of-00002.safetensors",
-    "transformer.h.29.attn.attention.q_proj.weight": "model-00001-of-00002.safetensors",
-    "transformer.h.29.attn.attention.v_proj.weight": "model-00001-of-00002.safetensors",
+    "transformer.h.29.attn.attention.k_proj.tucker_core": "model-00001-of-00002.safetensors",
+    "transformer.h.29.attn.attention.k_proj.tucker_factors.0": "model-00001-of-00002.safetensors",
+    "transformer.h.29.attn.attention.k_proj.tucker_factors.1": "model-00001-of-00002.safetensors",
+    "transformer.h.29.attn.attention.out_proj.tucker_core": "model-00001-of-00002.safetensors",
+    "transformer.h.29.attn.attention.out_proj.tucker_factors.0": "model-00001-of-00002.safetensors",
+    "transformer.h.29.attn.attention.out_proj.tucker_factors.1": "model-00001-of-00002.safetensors",
+    "transformer.h.29.attn.attention.q_proj.tucker_core": "model-00001-of-00002.safetensors",
+    "transformer.h.29.attn.attention.q_proj.tucker_factors.0": "model-00001-of-00002.safetensors",
+    "transformer.h.29.attn.attention.q_proj.tucker_factors.1": "model-00001-of-00002.safetensors",
+    "transformer.h.29.attn.attention.v_proj.tucker_core": "model-00001-of-00002.safetensors",
+    "transformer.h.29.attn.attention.v_proj.tucker_factors.0": "model-00001-of-00002.safetensors",
+    "transformer.h.29.attn.attention.v_proj.tucker_factors.1": "model-00001-of-00002.safetensors",
     "transformer.h.29.ln_1.weight": "model-00001-of-00002.safetensors",
     "transformer.h.29.ln_2.weight": "model-00001-of-00002.safetensors",
     "transformer.h.29.mlp.c_fc_0.tucker_core": "model-00001-of-00002.safetensors",

runs/Mar13_16-43-27_demo2/events.out.tfevents.1741924372.demo2.205134.2

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0ecc320895a0fae2409fdf471e1245f49351199840c95bf7e5b280729b70f19b
+size 47428

runs/Mar13_16-43-27_demo2/events.out.tfevents.1741938009.demo2.205134.3

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fe40159d9ee722502c84615c7f0078d5c12c2828737b8fa3233051f8b0fb1e3a
+size 481

train_layer_index_29_mat_attn.attention.v_proj_results.json

@@ -1,8 +1,8 @@
 {
-    "total_flos": 22519612440576.0,
-    "train_loss": 5.291015625,
-    "train_runtime": 0.3705,
-    "train_samples": 76256,
-    "train_samples_per_second": 21.592,
-    "train_steps_per_second": 5.398
+    "total_flos": 9.323627298076754e+17,
+    "train_loss": 1.6535549103175464,
+    "train_runtime": 12676.64,
+    "train_samples": 294393,
+    "train_samples_per_second": 23.223,
+    "train_steps_per_second": 5.806
 }

trainer_state.json

@@ -10,1189 +10,1189 @@
   "log_history": [
     {
       "epoch": 0.006793616717732019,
-      "grad_norm": 2.0208959579467773,
-      "learning_rate": 0.0004966371368787195,
-      "loss": 2.0427,
-      "mean_token_accuracy": 0.5520137556195259,
+      "grad_norm": 0.4056604504585266,
+      "learning_rate": 0.0004966303432158483,
+      "loss": 1.7077,
+      "mean_token_accuracy": 0.5955310500562191,
       "step": 500
     },
     {
       "epoch": 0.013587233435464039,
-      "grad_norm": 1.8117527961730957,
-      "learning_rate": 0.0004932403054430827,
-      "loss": 1.8742,
-      "mean_token_accuracy": 0.5727833653092385,
+      "grad_norm": 0.3505840599536896,
+      "learning_rate": 0.0004932335117802115,
+      "loss": 1.7028,
+      "mean_token_accuracy": 0.5971505789160728,
       "step": 1000
     },
     {
       "epoch": 0.020380850153196056,
-      "grad_norm": 1.5713497400283813,
-      "learning_rate": 0.0004898434740074459,
-      "loss": 1.8367,
-      "mean_token_accuracy": 0.5783296891450882,
+      "grad_norm": 0.4287165105342865,
+      "learning_rate": 0.0004898366803445745,
+      "loss": 1.6931,
+      "mean_token_accuracy": 0.5984280071258545,
       "step": 1500
     },
     {
       "epoch": 0.027174466870928077,
-      "grad_norm": 1.6636950969696045,
-      "learning_rate": 0.00048644664257180905,
-      "loss": 1.8238,
-      "mean_token_accuracy": 0.5810909342765808,
+      "grad_norm": 0.3165971040725708,
+      "learning_rate": 0.0004864398489089378,
+      "loss": 1.6953,
+      "mean_token_accuracy": 0.59935999122262,
       "step": 2000
     },
     {
       "epoch": 0.033968083588660096,
-      "grad_norm": 1.525341272354126,
-      "learning_rate": 0.0004830498111361722,
-      "loss": 1.8096,
-      "mean_token_accuracy": 0.5805237484574318,
+      "grad_norm": 0.3369743824005127,
+      "learning_rate": 0.0004830430174733009,
+      "loss": 1.6911,
+      "mean_token_accuracy": 0.5979570625722408,
       "step": 2500
     },
     {
       "epoch": 0.04076170030639211,
-      "grad_norm": 1.7869559526443481,
-      "learning_rate": 0.00047965297970053533,
-      "loss": 1.8145,
-      "mean_token_accuracy": 0.5809020410776138,
+      "grad_norm": 0.5177591443061829,
+      "learning_rate": 0.0004796461860376641,
+      "loss": 1.7042,
+      "mean_token_accuracy": 0.5975428151786327,
       "step": 3000
     },
     {
       "epoch": 0.04755531702412413,
-      "grad_norm": 1.389845848083496,
-      "learning_rate": 0.0004762561482648985,
-      "loss": 1.7937,
-      "mean_token_accuracy": 0.5843603150844574,
+      "grad_norm": 0.36950188875198364,
+      "learning_rate": 0.00047624935460202725,
+      "loss": 1.6894,
+      "mean_token_accuracy": 0.5996730836033821,
       "step": 3500
     },
     {
       "epoch": 0.054348933741856155,
-      "grad_norm": 1.1970040798187256,
-      "learning_rate": 0.00047285931682926167,
-      "loss": 1.7926,
-      "mean_token_accuracy": 0.5820418976843357,
+      "grad_norm": 0.35214412212371826,
+      "learning_rate": 0.00047285252316639037,
+      "loss": 1.6939,
+      "mean_token_accuracy": 0.5964787036180497,
       "step": 4000
     },
     {
       "epoch": 0.06114255045958817,
-      "grad_norm": 1.4389044046401978,
-      "learning_rate": 0.00046946248539362484,
-      "loss": 1.7622,
-      "mean_token_accuracy": 0.5891672801971436,
+      "grad_norm": 0.37886953353881836,
+      "learning_rate": 0.0004694692790564961,
+      "loss": 1.6659,
+      "mean_token_accuracy": 0.6038953081071377,
       "step": 4500
     },
     {
       "epoch": 0.06793616717732019,
-      "grad_norm": 1.177388072013855,
-      "learning_rate": 0.000466065653957988,
-      "loss": 1.7738,
-      "mean_token_accuracy": 0.5879195163547992,
+      "grad_norm": 0.34965798258781433,
+      "learning_rate": 0.00046607244762085926,
+      "loss": 1.6818,
+      "mean_token_accuracy": 0.6013828701674938,
       "step": 5000
     },
     {
       "epoch": 0.07472978389505221,
-      "grad_norm": 1.3432203531265259,
-      "learning_rate": 0.00046268240984809374,
-      "loss": 1.7767,
-      "mean_token_accuracy": 0.5867721286118031,
+      "grad_norm": 0.3864031434059143,
+      "learning_rate": 0.000462689203510965,
+      "loss": 1.6851,
+      "mean_token_accuracy": 0.6005209331214428,
       "step": 5500
     },
     {
       "epoch": 0.08152340061278422,
-      "grad_norm": 1.360933542251587,
-      "learning_rate": 0.0004592855784124569,
-      "loss": 1.7767,
-      "mean_token_accuracy": 0.5867527797520161,
+      "grad_norm": 0.3812890946865082,
+      "learning_rate": 0.00045929237207532816,
+      "loss": 1.6881,
+      "mean_token_accuracy": 0.6001524120569229,
       "step": 6000
     },
     {
       "epoch": 0.08831701733051625,
-      "grad_norm": 1.1908655166625977,
-      "learning_rate": 0.0004559023343025626,
-      "loss": 1.7606,
-      "mean_token_accuracy": 0.5915117806196213,
+      "grad_norm": 0.31558525562286377,
+      "learning_rate": 0.0004559091279654339,
+      "loss": 1.6742,
+      "mean_token_accuracy": 0.604626697331667,
       "step": 6500
     },
     {
       "epoch": 0.09511063404824827,
-      "grad_norm": 1.1951971054077148,
-      "learning_rate": 0.00045250550286692574,
-      "loss": 1.7629,
-      "mean_token_accuracy": 0.5892836402058601,
+      "grad_norm": 0.42938554286956787,
+      "learning_rate": 0.00045251229652979705,
+      "loss": 1.6772,
+      "mean_token_accuracy": 0.6019361851513386,
       "step": 7000
     },
     {
       "epoch": 0.10190425076598028,
-      "grad_norm": 1.207234501838684,
-      "learning_rate": 0.00044912225875703147,
-      "loss": 1.7618,
-      "mean_token_accuracy": 0.5882115732729435,
+      "grad_norm": 0.3015531003475189,
+      "learning_rate": 0.0004491290524199027,
+      "loss": 1.6783,
+      "mean_token_accuracy": 0.6007610514461994,
       "step": 7500
     },
     {
       "epoch": 0.10869786748371231,
-      "grad_norm": 1.1731985807418823,
-      "learning_rate": 0.0004457254273213946,
-      "loss": 1.753,
-      "mean_token_accuracy": 0.5895786600708961,
+      "grad_norm": 0.3297825753688812,
+      "learning_rate": 0.0004457322209842659,
+      "loss": 1.6714,
+      "mean_token_accuracy": 0.6026990222036839,
       "step": 8000
     },
     {
       "epoch": 0.11549148420144432,
-      "grad_norm": 1.3629071712493896,
-      "learning_rate": 0.0004423421832115003,
-      "loss": 1.7627,
-      "mean_token_accuracy": 0.587433840662241,
+      "grad_norm": 0.41950032114982605,
+      "learning_rate": 0.0004423489768743716,
+      "loss": 1.6823,
+      "mean_token_accuracy": 0.5994728994369507,
       "step": 8500
     },
     {
       "epoch": 0.12228510091917634,
-      "grad_norm": 1.2118574380874634,
-      "learning_rate": 0.0004389453517758635,
-      "loss": 1.7529,
-      "mean_token_accuracy": 0.5890023435950279,
+      "grad_norm": 0.39976397156715393,
+      "learning_rate": 0.0004389521454387347,
+      "loss": 1.6715,
+      "mean_token_accuracy": 0.6015009447336197,
       "step": 9000
     },
     {
       "epoch": 0.12907871763690837,
-      "grad_norm": 1.1447389125823975,
-      "learning_rate": 0.0004355621076659692,
-      "loss": 1.7433,
-      "mean_token_accuracy": 0.5922272650897503,
+      "grad_norm": 0.40663132071495056,
+      "learning_rate": 0.00043556890132884045,
+      "loss": 1.6644,
+      "mean_token_accuracy": 0.6042043321728706,
       "step": 9500
     },
     {
       "epoch": 0.13587233435464038,
-      "grad_norm": 1.1409871578216553,
-      "learning_rate": 0.00043216527623033237,
-      "loss": 1.7479,
-      "mean_token_accuracy": 0.5903287023007869,
+      "grad_norm": 0.3542971611022949,
+      "learning_rate": 0.0004321720698932036,
+      "loss": 1.6705,
+      "mean_token_accuracy": 0.6018945582509041,
       "step": 10000
     },
     {
       "epoch": 0.1426659510723724,
-      "grad_norm": 1.2514444589614868,
-      "learning_rate": 0.00042878203212043804,
-      "loss": 1.7471,
-      "mean_token_accuracy": 0.5892516044676304,
+      "grad_norm": 0.3548875153064728,
+      "learning_rate": 0.00042878882578330934,
+      "loss": 1.6686,
+      "mean_token_accuracy": 0.601027511626482,
       "step": 10500
     },
     {
       "epoch": 0.14945956779010441,
-      "grad_norm": 1.3417694568634033,
-      "learning_rate": 0.00042538520068480126,
-      "loss": 1.7324,
-      "mean_token_accuracy": 0.5957142559885978,
+      "grad_norm": 0.4442589581012726,
+      "learning_rate": 0.0004253919943476725,
+      "loss": 1.6563,
+      "mean_token_accuracy": 0.6067300595343113,
       "step": 11000
     },
     {
       "epoch": 0.15625318450783643,
-      "grad_norm": 1.3369439840316772,
-      "learning_rate": 0.00042200195657490693,
-      "loss": 1.7428,
-      "mean_token_accuracy": 0.5911534672677516,
+      "grad_norm": 0.4506297707557678,
+      "learning_rate": 0.0004220087502377782,
+      "loss": 1.6668,
+      "mean_token_accuracy": 0.6026452116966248,
       "step": 11500
     },
     {
       "epoch": 0.16304680122556844,
-      "grad_norm": 1.2490959167480469,
-      "learning_rate": 0.00041860512513927005,
-      "loss": 1.7318,
-      "mean_token_accuracy": 0.5937872271835803,
+      "grad_norm": 0.3492109775543213,
+      "learning_rate": 0.0004186119188021414,
+      "loss": 1.6555,
+      "mean_token_accuracy": 0.6048386556506157,
       "step": 12000
     },
     {
       "epoch": 0.1698404179433005,
-      "grad_norm": 1.2229673862457275,
-      "learning_rate": 0.0004152218810293758,
-      "loss": 1.7429,
-      "mean_token_accuracy": 0.5896593699157238,
+      "grad_norm": 0.30816155672073364,
+      "learning_rate": 0.0004152286746922471,
+      "loss": 1.6679,
+      "mean_token_accuracy": 0.6008234451115131,
       "step": 12500
     },
     {
       "epoch": 0.1766340346610325,
-      "grad_norm": 1.3295321464538574,
-      "learning_rate": 0.00041182504959373894,
-      "loss": 1.7633,
-      "mean_token_accuracy": 0.5877331503927707,
+      "grad_norm": 0.37013736367225647,
+      "learning_rate": 0.0004118318432566102,
+      "loss": 1.6894,
+      "mean_token_accuracy": 0.5981535095870495,
       "step": 13000
     },
     {
       "epoch": 0.18342765137876452,
-      "grad_norm": 0.9519675970077515,
-      "learning_rate": 0.00040844180548384466,
-      "loss": 1.7573,
-      "mean_token_accuracy": 0.5893017112016677,
+      "grad_norm": 0.30707740783691406,
+      "learning_rate": 0.00040844859914671597,
+      "loss": 1.6838,
+      "mean_token_accuracy": 0.5997290171980858,
       "step": 13500
     },
     {
       "epoch": 0.19022126809649653,
-      "grad_norm": 1.448428750038147,
-      "learning_rate": 0.00040504497404820783,
-      "loss": 1.7314,
-      "mean_token_accuracy": 0.5936685436069965,
+      "grad_norm": 0.522889256477356,
+      "learning_rate": 0.0004050517677110791,
+      "loss": 1.6597,
+      "mean_token_accuracy": 0.6042063582837581,
       "step": 14000
     },
     {
       "epoch": 0.19701488481422855,
-      "grad_norm": 1.3305081129074097,
-      "learning_rate": 0.00040166172993831356,
-      "loss": 1.7293,
-      "mean_token_accuracy": 0.5929152786135673,
+      "grad_norm": 0.44870564341545105,
+      "learning_rate": 0.0004016685236011848,
+      "loss": 1.656,
+      "mean_token_accuracy": 0.603478528469801,
       "step": 14500
     },
     {
       "epoch": 0.20380850153196056,
-      "grad_norm": 1.2279210090637207,
-      "learning_rate": 0.00039826489850267673,
-      "loss": 1.728,
-      "mean_token_accuracy": 0.5941076972186565,
+      "grad_norm": 0.3704949617385864,
+      "learning_rate": 0.000398271692165548,
+      "loss": 1.6551,
+      "mean_token_accuracy": 0.605626079261303,
       "step": 15000
     },
     {
       "epoch": 0.21060211824969258,
-      "grad_norm": 0.9746572375297546,
-      "learning_rate": 0.0003948816543927824,
-      "loss": 1.7452,
-      "mean_token_accuracy": 0.5927870056033134,
+      "grad_norm": 0.3166308104991913,
+      "learning_rate": 0.0003948884480556537,
+      "loss": 1.6727,
+      "mean_token_accuracy": 0.6039620104730129,
       "step": 15500
     },
     {
       "epoch": 0.21739573496742462,
-      "grad_norm": 1.3720732927322388,
-      "learning_rate": 0.0003914848229571456,
-      "loss": 1.7259,
-      "mean_token_accuracy": 0.5940872375369072,
+      "grad_norm": 0.4356388449668884,
+      "learning_rate": 0.00039149161662001687,
+      "loss": 1.6545,
+      "mean_token_accuracy": 0.605135948419571,
       "step": 16000
     },
     {
       "epoch": 0.22418935168515663,
-      "grad_norm": 1.2336689233779907,
-      "learning_rate": 0.0003881015788472513,
-      "loss": 1.7365,
-      "mean_token_accuracy": 0.5929380969107151,
+      "grad_norm": 0.3973088264465332,
+      "learning_rate": 0.00038810837251012254,
+      "loss": 1.6657,
+      "mean_token_accuracy": 0.6032147918045521,
       "step": 16500
     },
     {
       "epoch": 0.23098296840288865,
-      "grad_norm": 1.0067358016967773,
-      "learning_rate": 0.0003847047474116144,
-      "loss": 1.7373,
-      "mean_token_accuracy": 0.5914620462656021,
+      "grad_norm": 0.3286292552947998,
+      "learning_rate": 0.00038471154107448576,
+      "loss": 1.6667,
+      "mean_token_accuracy": 0.6019928269386291,
       "step": 17000
     },
     {
       "epoch": 0.23777658512062066,
-      "grad_norm": 1.2364681959152222,
-      "learning_rate": 0.0003813215033017202,
-      "loss": 1.7298,
-      "mean_token_accuracy": 0.5921443883776665,
+      "grad_norm": 0.347025990486145,
+      "learning_rate": 0.00038132829696459143,
+      "loss": 1.6615,
+      "mean_token_accuracy": 0.6018516709804534,
       "step": 17500
     },
     {
       "epoch": 0.24457020183835268,
-      "grad_norm": 1.186237096786499,
-      "learning_rate": 0.0003779246718660833,
-      "loss": 1.7336,
-      "mean_token_accuracy": 0.5918942322134971,
+      "grad_norm": 0.32694211602211,
+      "learning_rate": 0.0003779314655289546,
+      "loss": 1.664,
+      "mean_token_accuracy": 0.6017192153930664,
       "step": 18000
     },
     {
       "epoch": 0.2513638185560847,
-      "grad_norm": 1.2172770500183105,
-      "learning_rate": 0.000374541427756189,
-      "loss": 1.7321,
-      "mean_token_accuracy": 0.592568263143301,
+      "grad_norm": 0.3692508041858673,
+      "learning_rate": 0.0003745482214190603,
+      "loss": 1.6626,
+      "mean_token_accuracy": 0.6029216103553772,
       "step": 18500
     },
     {
       "epoch": 0.25815743527381674,
-      "grad_norm": 1.074506402015686,
-      "learning_rate": 0.0003711445963205522,
-      "loss": 1.726,
-      "mean_token_accuracy": 0.5930768192112446,
+      "grad_norm": 0.3326375186443329,
+      "learning_rate": 0.00037115138998342344,
+      "loss": 1.6579,
+      "mean_token_accuracy": 0.6034600695967675,
       "step": 19000
     },
     {
       "epoch": 0.2649510519915487,
-      "grad_norm": 1.3433698415756226,
+      "grad_norm": 0.48608893156051636,
       "learning_rate": 0.0003677681458735292,
-      "loss": 1.7252,
-      "mean_token_accuracy": 0.5934848180115223,
+      "loss": 1.6579,
+      "mean_token_accuracy": 0.6037410476505757,
       "step": 19500
     },
     {
       "epoch": 0.27174466870928077,
-      "grad_norm": 1.0935838222503662,
+      "grad_norm": 0.3404929041862488,
       "learning_rate": 0.00036437131443789233,
-      "loss": 1.7276,
-      "mean_token_accuracy": 0.5916450168192386,
+      "loss": 1.6593,
+      "mean_token_accuracy": 0.6018497814834118,
       "step": 20000
     },
     {
       "epoch": 0.27853828542701276,
-      "grad_norm": 1.5121397972106934,
-      "learning_rate": 0.0003609744830022555,
-      "loss": 1.738,
-      "mean_token_accuracy": 0.5906197059154511,
+      "grad_norm": 0.42589041590690613,
+      "learning_rate": 0.00036098807032799806,
+      "loss": 1.6702,
+      "mean_token_accuracy": 0.6010032378137111,
       "step": 20500
     },
     {
       "epoch": 0.2853319021447448,
-      "grad_norm": 1.2155652046203613,
-      "learning_rate": 0.0003575776515666186,
-      "loss": 1.7221,
-      "mean_token_accuracy": 0.5948817262351513,
+      "grad_norm": 0.42454850673675537,
+      "learning_rate": 0.00035759123889236123,
+      "loss": 1.6545,
+      "mean_token_accuracy": 0.6045684849619866,
       "step": 21000
     },
     {
       "epoch": 0.29212551886247684,
-      "grad_norm": 1.0498167276382446,
-      "learning_rate": 0.0003541944074567244,
-      "loss": 1.7346,
-      "mean_token_accuracy": 0.5901701744496822,
+      "grad_norm": 0.36944735050201416,
+      "learning_rate": 0.0003542079947824669,
+      "loss": 1.6666,
+      "mean_token_accuracy": 0.6005178530812263,
       "step": 21500
     },
     {
       "epoch": 0.29891913558020883,
-      "grad_norm": 0.9237672686576843,
-      "learning_rate": 0.0003507975760210875,
-      "loss": 1.7265,
-      "mean_token_accuracy": 0.5947049247324466,
+      "grad_norm": 0.2879849374294281,
+      "learning_rate": 0.0003508111633468301,
+      "loss": 1.6607,
+      "mean_token_accuracy": 0.6042315942049027,
       "step": 22000
     },
     {
       "epoch": 0.30571275229794087,
-      "grad_norm": 1.235968828201294,
-      "learning_rate": 0.00034741433191119324,
-      "loss": 1.7359,
-      "mean_token_accuracy": 0.5912741076052189,
+      "grad_norm": 0.3621261417865753,
+      "learning_rate": 0.0003474279192369358,
+      "loss": 1.6695,
+      "mean_token_accuracy": 0.600696035951376,
       "step": 22500
     },
     {
       "epoch": 0.31250636901567286,
-      "grad_norm": 1.3003114461898804,
-      "learning_rate": 0.0003440175004755564,
-      "loss": 1.7165,
-      "mean_token_accuracy": 0.596044946283102,
+      "grad_norm": 0.46004965901374817,
+      "learning_rate": 0.00034403108780129896,
+      "loss": 1.6489,
+      "mean_token_accuracy": 0.6064540636539459,
       "step": 23000
     },
     {
       "epoch": 0.3192999857334049,
-      "grad_norm": 0.9103087186813354,
-      "learning_rate": 0.00034063425636566213,
-      "loss": 1.7145,
-      "mean_token_accuracy": 0.5962611735165119,
+      "grad_norm": 0.2892068922519684,
+      "learning_rate": 0.0003406478436914047,
+      "loss": 1.6491,
+      "mean_token_accuracy": 0.6061714172661304,
       "step": 23500
     },
     {
       "epoch": 0.3260936024511369,
-      "grad_norm": 1.3072879314422607,
-      "learning_rate": 0.0003372374249300253,
-      "loss": 1.7182,
-      "mean_token_accuracy": 0.5958844081163407,
+      "grad_norm": 0.37619903683662415,
+      "learning_rate": 0.0003372510122557678,
+      "loss": 1.6541,
+      "mean_token_accuracy": 0.6054886623620986,
       "step": 24000
     },
     {
       "epoch": 0.33288721916886893,
-      "grad_norm": 1.122262716293335,
-      "learning_rate": 0.00033385418082013097,
-      "loss": 1.7277,
-      "mean_token_accuracy": 0.5924356364905834,
+      "grad_norm": 0.3387802839279175,
+      "learning_rate": 0.0003338677681458736,
+      "loss": 1.6634,
+      "mean_token_accuracy": 0.60178011739254,
       "step": 24500
     },
     {
       "epoch": 0.339680835886601,
-      "grad_norm": 1.0892757177352905,
-      "learning_rate": 0.0003304573493844942,
-      "loss": 1.7168,
-      "mean_token_accuracy": 0.5951408152878285,
+      "grad_norm": 0.4234718084335327,
+      "learning_rate": 0.0003304709367102367,
+      "loss": 1.6514,
+      "mean_token_accuracy": 0.6051099980771542,
       "step": 25000
     },
     {
       "epoch": 0.34647445260433296,
-      "grad_norm": 1.3095890283584595,
-      "learning_rate": 0.00032707410527459986,
-      "loss": 1.7103,
-      "mean_token_accuracy": 0.5958214187920093,
+      "grad_norm": 0.46013298630714417,
+      "learning_rate": 0.0003270876926003424,
+      "loss": 1.6465,
+      "mean_token_accuracy": 0.6049905767738819,
       "step": 25500
     },
     {
       "epoch": 0.353268069322065,
-      "grad_norm": 0.9692500233650208,
-      "learning_rate": 0.000323677273838963,
-      "loss": 1.7127,
-      "mean_token_accuracy": 0.5952938674986362,
+      "grad_norm": 0.2997068166732788,
+      "learning_rate": 0.0003236908611647056,
+      "loss": 1.6466,
+      "mean_token_accuracy": 0.6054481092095375,
       "step": 26000
     },
     {
       "epoch": 0.360061686039797,
-      "grad_norm": 1.0970525741577148,
-      "learning_rate": 0.00032029402972906876,
-      "loss": 1.7251,
-      "mean_token_accuracy": 0.5936492686867714,
+      "grad_norm": 0.3821622133255005,
+      "learning_rate": 0.00032030761705481125,
+      "loss": 1.6623,
+      "mean_token_accuracy": 0.6025521129369735,
       "step": 26500
     },
     {
       "epoch": 0.36685530275752903,
-      "grad_norm": 1.2200478315353394,
-      "learning_rate": 0.00031689719829343187,
-      "loss": 1.7082,
-      "mean_token_accuracy": 0.5961000478565693,
+      "grad_norm": 0.455049604177475,
+      "learning_rate": 0.0003169107856191744,
+      "loss": 1.6449,
+      "mean_token_accuracy": 0.606242949217558,
       "step": 27000
     },
     {
       "epoch": 0.373648919475261,
-      "grad_norm": 1.0182169675827026,
-      "learning_rate": 0.0003135139541835376,
-      "loss": 1.7038,
-      "mean_token_accuracy": 0.5973086108565331,
+      "grad_norm": 0.3485479950904846,
+      "learning_rate": 0.00031352754150928015,
+      "loss": 1.6413,
+      "mean_token_accuracy": 0.6063379021286964,
       "step": 27500
     },
     {
       "epoch": 0.38044253619299306,
-      "grad_norm": 0.9256690740585327,
-      "learning_rate": 0.00031011712274790076,
-      "loss": 1.7208,
-      "mean_token_accuracy": 0.593766670614481,
+      "grad_norm": 0.29450029134750366,
+      "learning_rate": 0.0003101307100736433,
+      "loss": 1.6571,
+      "mean_token_accuracy": 0.6033727302551269,
       "step": 28000
     },
     {
       "epoch": 0.3872361529107251,
-      "grad_norm": 1.1710305213928223,
-      "learning_rate": 0.0003067270849751352,
-      "loss": 1.73,
-      "mean_token_accuracy": 0.594021442502737,
+      "grad_norm": 0.37222492694854736,
+      "learning_rate": 0.00030674746596374904,
+      "loss": 1.6672,
+      "mean_token_accuracy": 0.603064342647791,
       "step": 28500
     },
     {
       "epoch": 0.3940297696284571,
-      "grad_norm": 1.1612814664840698,
-      "learning_rate": 0.00030333704720236966,
-      "loss": 1.7146,
-      "mean_token_accuracy": 0.5956000906527043,
+      "grad_norm": 0.39230677485466003,
+      "learning_rate": 0.00030335063452811216,
+      "loss": 1.6526,
+      "mean_token_accuracy": 0.6050780023932457,
       "step": 29000
     },
     {
       "epoch": 0.40082338634618914,
-      "grad_norm": 1.2125431299209595,
-      "learning_rate": 0.00029994021576673283,
-      "loss": 1.7076,
-      "mean_token_accuracy": 0.5957852031290531,
+      "grad_norm": 0.5070363879203796,
+      "learning_rate": 0.00029996739041821793,
+      "loss": 1.6447,
+      "mean_token_accuracy": 0.6044168398082256,
       "step": 29500
     },
     {
       "epoch": 0.4076170030639211,
-      "grad_norm": 1.2176785469055176,
-      "learning_rate": 0.0002965569716568385,
-      "loss": 1.7312,
-      "mean_token_accuracy": 0.5945550291240216,
+      "grad_norm": 0.40913066267967224,
+      "learning_rate": 0.00029657055898258105,
+      "loss": 1.6692,
+      "mean_token_accuracy": 0.6040610507428646,
       "step": 30000
     },
     {
       "epoch": 0.41441061978165317,
-      "grad_norm": 1.5686943531036377,
-      "learning_rate": 0.00029316014022120167,
-      "loss": 1.7171,
-      "mean_token_accuracy": 0.5942135262489319,
+      "grad_norm": 0.4924214482307434,
+      "learning_rate": 0.0002931873148726867,
+      "loss": 1.6559,
+      "mean_token_accuracy": 0.6027441952824593,
       "step": 30500
     },
     {
       "epoch": 0.42120423649938515,
-      "grad_norm": 1.0116132497787476,
-      "learning_rate": 0.0002897768961113074,
-      "loss": 1.6985,
-      "mean_token_accuracy": 0.5982498173713684,
+      "grad_norm": 0.29630807042121887,
+      "learning_rate": 0.00028979048343704994,
+      "loss": 1.6356,
+      "mean_token_accuracy": 0.6075075562298298,
       "step": 31000
     },
     {
       "epoch": 0.4279978532171172,
-      "grad_norm": 0.9668023586273193,
-      "learning_rate": 0.0002863800646756705,
-      "loss": 1.7078,
-      "mean_token_accuracy": 0.5950165801644325,
+      "grad_norm": 0.32989051938056946,
+      "learning_rate": 0.0002864072393271556,
+      "loss": 1.6474,
+      "mean_token_accuracy": 0.6034396274387837,
       "step": 31500
     },
     {
       "epoch": 0.43479146993484924,
-      "grad_norm": 1.2987300157546997,
-      "learning_rate": 0.00028299682056577623,
-      "loss": 1.7288,
-      "mean_token_accuracy": 0.5917907827198505,
+      "grad_norm": 0.3934209942817688,
+      "learning_rate": 0.0002830104078915188,
+      "loss": 1.6682,
+      "mean_token_accuracy": 0.6012586470246315,
       "step": 32000
     },
     {
       "epoch": 0.4415850866525812,
-      "grad_norm": 1.1014894247055054,
-      "learning_rate": 0.0002795999891301394,
-      "loss": 1.7207,
-      "mean_token_accuracy": 0.5922197285294533,
+      "grad_norm": 0.370421439409256,
+      "learning_rate": 0.0002796271637816245,
+      "loss": 1.6597,
+      "mean_token_accuracy": 0.6011568746566772,
       "step": 32500
     },
     {
       "epoch": 0.44837870337031327,
-      "grad_norm": 1.0592535734176636,
-      "learning_rate": 0.0002762167450202451,
-      "loss": 1.7043,
-      "mean_token_accuracy": 0.5975054582953453,
+      "grad_norm": 0.3357086777687073,
+      "learning_rate": 0.0002762303323459877,
+      "loss": 1.645,
+      "mean_token_accuracy": 0.6061246747374535,
       "step": 33000
     },
     {
       "epoch": 0.45517232008804526,
-      "grad_norm": 1.084416151046753,
-      "learning_rate": 0.0002728199135846083,
-      "loss": 1.7153,
-      "mean_token_accuracy": 0.59563816472888,
+      "grad_norm": 0.3402880132198334,
+      "learning_rate": 0.0002728470882360934,
+      "loss": 1.6548,
+      "mean_token_accuracy": 0.6037880809903144,
       "step": 33500
     },
     {
       "epoch": 0.4619659368057773,
-      "grad_norm": 1.0490503311157227,
-      "learning_rate": 0.000269436669474714,
-      "loss": 1.7049,
-      "mean_token_accuracy": 0.5960744704604148,
+      "grad_norm": 0.3954179883003235,
+      "learning_rate": 0.0002694502568004565,
+      "loss": 1.6445,
+      "mean_token_accuracy": 0.6050299068689347,
       "step": 34000
     },
     {
       "epoch": 0.4687595535235093,
-      "grad_norm": 1.3198046684265137,
-      "learning_rate": 0.0002660398380390772,
-      "loss": 1.6951,
-      "mean_token_accuracy": 0.6000102725625038,
+      "grad_norm": 0.5619150400161743,
+      "learning_rate": 0.0002660670126905623,
+      "loss": 1.636,
+      "mean_token_accuracy": 0.6086498996317387,
       "step": 34500
     },
     {
       "epoch": 0.47555317024124133,
-      "grad_norm": 1.1859066486358643,
-      "learning_rate": 0.00026265659392918285,
-      "loss": 1.7025,
-      "mean_token_accuracy": 0.5961538652777671,
+      "grad_norm": 0.44257017970085144,
+      "learning_rate": 0.0002626701812549254,
+      "loss": 1.6428,
+      "mean_token_accuracy": 0.6053203675448895,
       "step": 35000
     },
     {
       "epoch": 0.4823467869589734,
-      "grad_norm": 1.0394221544265747,
-      "learning_rate": 0.000259259762493546,
-      "loss": 1.695,
-      "mean_token_accuracy": 0.599614026337862,
+      "grad_norm": 0.39008113741874695,
+      "learning_rate": 0.0002592869371450311,
+      "loss": 1.635,
+      "mean_token_accuracy": 0.6085173624753952,
       "step": 35500
     },
     {
       "epoch": 0.48914040367670536,
-      "grad_norm": 1.1829257011413574,
-      "learning_rate": 0.00025587651838365175,
-      "loss": 1.7086,
-      "mean_token_accuracy": 0.5964278879761696,
+      "grad_norm": 0.44204434752464294,
+      "learning_rate": 0.0002558901057093943,
+      "loss": 1.6491,
+      "mean_token_accuracy": 0.6055251418352127,
       "step": 36000
     },
     {
       "epoch": 0.4959340203944374,
-      "grad_norm": 1.0766243934631348,
-      "learning_rate": 0.00025247968694801486,
-      "loss": 1.7,
-      "mean_token_accuracy": 0.5977904357016086,
+      "grad_norm": 0.3880552053451538,
+      "learning_rate": 0.00025250686159949997,
+      "loss": 1.6408,
+      "mean_token_accuracy": 0.6063489430248737,
       "step": 36500
     },
     {
       "epoch": 0.5027276371121694,
-      "grad_norm": 1.1784560680389404,
-      "learning_rate": 0.0002490964428381206,
-      "loss": 1.715,
-      "mean_token_accuracy": 0.5959987764656544,
+      "grad_norm": 0.4222866892814636,
+      "learning_rate": 0.00024911003016386314,
+      "loss": 1.6559,
+      "mean_token_accuracy": 0.6046599761843682,
       "step": 37000
     },
     {
       "epoch": 0.5095212538299014,
-      "grad_norm": 1.1208888292312622,
-      "learning_rate": 0.00024569961140248376,
-      "loss": 1.7046,
-      "mean_token_accuracy": 0.5964538981616497,
+      "grad_norm": 0.3555832803249359,
+      "learning_rate": 0.00024572678605396886,
+      "loss": 1.6459,
+      "mean_token_accuracy": 0.6055832685828209,
       "step": 37500
     },
     {
       "epoch": 0.5163148705476335,
-      "grad_norm": 1.0677920579910278,
-      "learning_rate": 0.00024231636729258948,
-      "loss": 1.7001,
-      "mean_token_accuracy": 0.5974933596849441,
+      "grad_norm": 0.3376443088054657,
+      "learning_rate": 0.000242329954618332,
+      "loss": 1.6416,
+      "mean_token_accuracy": 0.6064145671725273,
       "step": 38000
     },
     {
       "epoch": 0.5231084872653654,
-      "grad_norm": 0.9215283393859863,
-      "learning_rate": 0.00023891953585695265,
-      "loss": 1.7105,
-      "mean_token_accuracy": 0.596855210095644,
+      "grad_norm": 0.26011911034584045,
+      "learning_rate": 0.00023894671050843776,
+      "loss": 1.6529,
+      "mean_token_accuracy": 0.6054728911519051,
       "step": 38500
     },
     {
       "epoch": 0.5299021039830974,
-      "grad_norm": 1.23247492313385,
-      "learning_rate": 0.00023553629174705835,
-      "loss": 1.711,
-      "mean_token_accuracy": 0.5971770713925362,
+      "grad_norm": 0.47712820768356323,
+      "learning_rate": 0.0002355498790728009,
+      "loss": 1.6528,
+      "mean_token_accuracy": 0.6053745166957378,
       "step": 39000
     },
     {
       "epoch": 0.5366957207008295,
-      "grad_norm": 1.07834792137146,
-      "learning_rate": 0.00023213946031142152,
-      "loss": 1.6993,
-      "mean_token_accuracy": 0.5984128168821334,
+      "grad_norm": 0.26808470487594604,
+      "learning_rate": 0.0002321666349629066,
+      "loss": 1.6415,
+      "mean_token_accuracy": 0.6073252769708634,
       "step": 39500
     },
     {
       "epoch": 0.5434893374185615,
-      "grad_norm": 1.1771501302719116,
-      "learning_rate": 0.00022875621620152724,
-      "loss": 1.6937,
-      "mean_token_accuracy": 0.6004006116986275,
+      "grad_norm": 0.5174965858459473,
+      "learning_rate": 0.00022876980352726976,
+      "loss": 1.636,
+      "mean_token_accuracy": 0.6094177706837655,
       "step": 40000
     },
     {
       "epoch": 0.5502829541362936,
-      "grad_norm": 1.2967567443847656,
-      "learning_rate": 0.00022535938476589038,
-      "loss": 1.7038,
-      "mean_token_accuracy": 0.5961601254045963,
+      "grad_norm": 0.36289429664611816,
+      "learning_rate": 0.0002253865594173755,
+      "loss": 1.6456,
+      "mean_token_accuracy": 0.6048986793458462,
       "step": 40500
     },
     {
       "epoch": 0.5570765708540255,
-      "grad_norm": 1.1175864934921265,
-      "learning_rate": 0.00022197614065599608,
-      "loss": 1.716,
-      "mean_token_accuracy": 0.5941783719956875,
+      "grad_norm": 0.3834798038005829,
+      "learning_rate": 0.00022198972798173863,
+      "loss": 1.659,
+      "mean_token_accuracy": 0.6029848347008229,
       "step": 41000
     },
     {
       "epoch": 0.5638701875717576,
-      "grad_norm": 1.380516529083252,
-      "learning_rate": 0.00021857930922035927,
-      "loss": 1.6936,
-      "mean_token_accuracy": 0.5974556840360165,
+      "grad_norm": 0.42929068207740784,
+      "learning_rate": 0.00021860648387184435,
+      "loss": 1.637,
+      "mean_token_accuracy": 0.6060945318639278,
       "step": 41500
     },
     {
       "epoch": 0.5706638042894896,
-      "grad_norm": 1.1645792722702026,
-      "learning_rate": 0.00021519606511046497,
-      "loss": 1.6964,
-      "mean_token_accuracy": 0.5985994503200054,
+      "grad_norm": 0.37213245034217834,
+      "learning_rate": 0.00021520965243620752,
+      "loss": 1.6412,
+      "mean_token_accuracy": 0.6067064755856991,
       "step": 42000
     },
     {
       "epoch": 0.5774574210072216,
-      "grad_norm": 1.3061178922653198,
-      "learning_rate": 0.0002117992336748281,
-      "loss": 1.7065,
-      "mean_token_accuracy": 0.5961524350643158,
+      "grad_norm": 0.430021733045578,
+      "learning_rate": 0.00021182640832631322,
+      "loss": 1.6492,
+      "mean_token_accuracy": 0.6046372224390507,
       "step": 42500
     },
     {
       "epoch": 0.5842510377249537,
-      "grad_norm": 1.1523349285125732,
-      "learning_rate": 0.0002084159895649338,
-      "loss": 1.7001,
-      "mean_token_accuracy": 0.5993789280354976,
+      "grad_norm": 0.533805787563324,
+      "learning_rate": 0.00020842957689067636,
+      "loss": 1.6438,
+      "mean_token_accuracy": 0.6081936603784561,
       "step": 43000
     },
     {
       "epoch": 0.5910446544426856,
-      "grad_norm": 1.1238070726394653,
-      "learning_rate": 0.000205019158129297,
-      "loss": 1.724,
-      "mean_token_accuracy": 0.5929664156734943,
+      "grad_norm": 0.4355255365371704,
+      "learning_rate": 0.00020504633278078209,
+      "loss": 1.6681,
+      "mean_token_accuracy": 0.6013061693012715,
       "step": 43500
     },
     {
       "epoch": 0.5978382711604177,
-      "grad_norm": 1.1592812538146973,
-      "learning_rate": 0.0002016359140194027,
-      "loss": 1.7065,
-      "mean_token_accuracy": 0.5956415718495846,
+      "grad_norm": 0.4163975119590759,
+      "learning_rate": 0.00020164950134514526,
+      "loss": 1.6502,
+      "mean_token_accuracy": 0.6033801869153976,
       "step": 44000
     },
     {
       "epoch": 0.6046318878781497,
-      "grad_norm": 1.080269694328308,
-      "learning_rate": 0.00019823908258376585,
-      "loss": 1.6983,
-      "mean_token_accuracy": 0.5975038340389729,
+      "grad_norm": 0.34073227643966675,
+      "learning_rate": 0.00019826625723525095,
+      "loss": 1.6427,
+      "mean_token_accuracy": 0.6056749705076218,
       "step": 44500
     },
     {
       "epoch": 0.6114255045958817,
-      "grad_norm": 1.281002163887024,
-      "learning_rate": 0.0001948558384738716,
-      "loss": 1.6965,
-      "mean_token_accuracy": 0.5972202825844288,
+      "grad_norm": 0.34788087010383606,
+      "learning_rate": 0.00019486942579961412,
+      "loss": 1.6413,
+      "mean_token_accuracy": 0.6051312589347363,
       "step": 45000
     },
     {
       "epoch": 0.6182191213136137,
-      "grad_norm": 1.004654049873352,
-      "learning_rate": 0.00019145900703823474,
-      "loss": 1.7176,
-      "mean_token_accuracy": 0.5956167621910572,
+      "grad_norm": 0.35995861887931824,
+      "learning_rate": 0.00019148618168971984,
+      "loss": 1.6641,
+      "mean_token_accuracy": 0.6028626383543014,
       "step": 45500
     },
     {
       "epoch": 0.6250127380313457,
-      "grad_norm": 1.0295976400375366,
-      "learning_rate": 0.00018807576292834043,
-      "loss": 1.6929,
-      "mean_token_accuracy": 0.5980769891738892,
+      "grad_norm": 0.45607441663742065,
+      "learning_rate": 0.000188089350254083,
+      "loss": 1.6389,
+      "mean_token_accuracy": 0.6055048226416111,
       "step": 46000
     },
     {
       "epoch": 0.6318063547490778,
-      "grad_norm": 1.0710487365722656,
-      "learning_rate": 0.00018467893149270363,
-      "loss": 1.7116,
-      "mean_token_accuracy": 0.5948944690227509,
+      "grad_norm": 0.35824722051620483,
+      "learning_rate": 0.0001847061061441887,
+      "loss": 1.6555,
+      "mean_token_accuracy": 0.6030957559347153,
       "step": 46500
     },
     {
       "epoch": 0.6385999714668098,
-      "grad_norm": 1.1924978494644165,
-      "learning_rate": 0.00018129568738280933,
-      "loss": 1.6896,
-      "mean_token_accuracy": 0.6006084454953671,
+      "grad_norm": 0.34828364849090576,
+      "learning_rate": 0.00018130927470855188,
+      "loss": 1.6369,
+      "mean_token_accuracy": 0.6085557499229908,
       "step": 47000
     },
     {
       "epoch": 0.6453935881845418,
-      "grad_norm": 1.1349835395812988,
-      "learning_rate": 0.00017789885594717247,
-      "loss": 1.706,
-      "mean_token_accuracy": 0.5981652895510197,
+      "grad_norm": 0.4070509076118469,
+      "learning_rate": 0.00017792603059865758,
+      "loss": 1.6535,
+      "mean_token_accuracy": 0.6051889946162701,
       "step": 47500
     },
     {
       "epoch": 0.6521872049022738,
-      "grad_norm": 1.4327572584152222,
-      "learning_rate": 0.0001745156118372782,
-      "loss": 1.7034,
-      "mean_token_accuracy": 0.5959363037645817,
+      "grad_norm": 0.47431492805480957,
+      "learning_rate": 0.00017452919916302075,
+      "loss": 1.6504,
+      "mean_token_accuracy": 0.6033335083127022,
       "step": 48000
     },
     {
       "epoch": 0.6589808216200058,
-      "grad_norm": 1.499800682067871,
-      "learning_rate": 0.00017111878040164136,
-      "loss": 1.696,
-      "mean_token_accuracy": 0.598519076347351,
+      "grad_norm": 0.43468815088272095,
+      "learning_rate": 0.00017114595505312644,
+      "loss": 1.6437,
+      "mean_token_accuracy": 0.606398417443037,
       "step": 48500
     },
     {
       "epoch": 0.6657744383377379,
-      "grad_norm": 1.173693299293518,
-      "learning_rate": 0.00016773553629174706,
-      "loss": 1.6872,
-      "mean_token_accuracy": 0.6003931475877762,
+      "grad_norm": 0.42276880145072937,
+      "learning_rate": 0.0001677491236174896,
+      "loss": 1.6341,
+      "mean_token_accuracy": 0.6078968361318111,
       "step": 49000
     },
     {
       "epoch": 0.6725680550554699,
-      "grad_norm": 1.0841670036315918,
-      "learning_rate": 0.0001643387048561102,
-      "loss": 1.7033,
-      "mean_token_accuracy": 0.5981890520751476,
+      "grad_norm": 0.3050823211669922,
+      "learning_rate": 0.00016436587950759534,
+      "loss": 1.6509,
+      "mean_token_accuracy": 0.605450307816267,
       "step": 49500
     },
     {
       "epoch": 0.679361671773202,
-      "grad_norm": 1.2305493354797363,
-      "learning_rate": 0.00016095546074621593,
-      "loss": 1.7026,
-      "mean_token_accuracy": 0.5967764587402343,
+      "grad_norm": 0.3518511950969696,
+      "learning_rate": 0.00016096904807195848,
+      "loss": 1.6493,
+      "mean_token_accuracy": 0.6049057066738606,
       "step": 50000
     },
     {
       "epoch": 0.6861552884909339,
-      "grad_norm": 1.1358468532562256,
-      "learning_rate": 0.0001575586293105791,
-      "loss": 1.7024,
-      "mean_token_accuracy": 0.5969898876547813,
+      "grad_norm": 0.3663289546966553,
+      "learning_rate": 0.00015758580396206418,
+      "loss": 1.6481,
+      "mean_token_accuracy": 0.6049177789986133,
       "step": 50500
     },
     {
       "epoch": 0.6929489052086659,
-      "grad_norm": 1.3159434795379639,
-      "learning_rate": 0.0001541753852006848,
-      "loss": 1.6995,
-      "mean_token_accuracy": 0.5962181152105331,
+      "grad_norm": 0.47365906834602356,
+      "learning_rate": 0.00015418897252642735,
+      "loss": 1.6463,
+      "mean_token_accuracy": 0.6044630871415139,
       "step": 51000
     },
     {
       "epoch": 0.699742521926398,
-      "grad_norm": 1.27256441116333,
-      "learning_rate": 0.00015077855376504796,
-      "loss": 1.7029,
-      "mean_token_accuracy": 0.5967380773425103,
+      "grad_norm": 0.376928448677063,
+      "learning_rate": 0.00015080572841653307,
+      "loss": 1.6512,
+      "mean_token_accuracy": 0.6040407617986202,
       "step": 51500
     },
     {
       "epoch": 0.70653613864413,
-      "grad_norm": 1.5178892612457275,
-      "learning_rate": 0.00014739530965515369,
-      "loss": 1.6899,
-      "mean_token_accuracy": 0.5981109066009521,
+      "grad_norm": 0.4952867031097412,
+      "learning_rate": 0.0001474088969808962,
+      "loss": 1.6366,
+      "mean_token_accuracy": 0.605654832303524,
       "step": 52000
     },
     {
       "epoch": 0.7133297553618619,
-      "grad_norm": 0.9817212820053101,
-      "learning_rate": 0.00014399847821951683,
-      "loss": 1.6943,
-      "mean_token_accuracy": 0.5980751592516899,
+      "grad_norm": 0.41352519392967224,
+      "learning_rate": 0.00014402565287100193,
+      "loss": 1.6425,
+      "mean_token_accuracy": 0.6056727746725082,
       "step": 52500
     },
     {
       "epoch": 0.720123372079594,
-      "grad_norm": 1.0862836837768555,
-      "learning_rate": 0.00014061523410962255,
-      "loss": 1.7085,
-      "mean_token_accuracy": 0.5958100288212299,
+      "grad_norm": 0.33966323733329773,
+      "learning_rate": 0.0001406288214353651,
+      "loss": 1.6565,
+      "mean_token_accuracy": 0.6033213515579701,
       "step": 53000
     },
     {
       "epoch": 0.726916988797326,
-      "grad_norm": 1.3386940956115723,
-      "learning_rate": 0.00013721840267398572,
-      "loss": 1.6903,
-      "mean_token_accuracy": 0.5972609171569347,
+      "grad_norm": 0.5332663655281067,
+      "learning_rate": 0.0001372455773254708,
+      "loss": 1.6391,
+      "mean_token_accuracy": 0.6053101786971092,
       "step": 53500
     },
     {
       "epoch": 0.7337106055150581,
-      "grad_norm": 1.6551252603530884,
-      "learning_rate": 0.00013383515856409142,
-      "loss": 1.6976,
-      "mean_token_accuracy": 0.5984482218325138,
+      "grad_norm": 0.49498528242111206,
+      "learning_rate": 0.00013384874588983397,
+      "loss": 1.6469,
+      "mean_token_accuracy": 0.6061296546757221,
       "step": 54000
     },
     {
       "epoch": 0.7405042222327901,
-      "grad_norm": 1.1888887882232666,
-      "learning_rate": 0.0001304383271284546,
-      "loss": 1.7019,
-      "mean_token_accuracy": 0.5954872596561909,
+      "grad_norm": 0.3816577196121216,
+      "learning_rate": 0.0001304655017799397,
+      "loss": 1.65,
+      "mean_token_accuracy": 0.6032952789962291,
       "step": 54500
     },
     {
       "epoch": 0.747297838950522,
-      "grad_norm": 1.0421268939971924,
-      "learning_rate": 0.00012705508301856028,
-      "loss": 1.6823,
-      "mean_token_accuracy": 0.5998311370313167,
+      "grad_norm": 0.39760830998420715,
+      "learning_rate": 0.00012706867034430284,
+      "loss": 1.6327,
+      "mean_token_accuracy": 0.60715985378623,
       "step": 55000
     },
     {
       "epoch": 0.7540914556682541,
-      "grad_norm": 1.0341774225234985,
-      "learning_rate": 0.00012365825158292345,
-      "loss": 1.7042,
-      "mean_token_accuracy": 0.5982704365849495,
+      "grad_norm": 0.36706963181495667,
+      "learning_rate": 0.00012368542623440856,
+      "loss": 1.6527,
+      "mean_token_accuracy": 0.6057817209064961,
       "step": 55500
     },
     {
       "epoch": 0.7608850723859861,
-      "grad_norm": 1.222990870475769,
-      "learning_rate": 0.00012027500747302916,
-      "loss": 1.6795,
-      "mean_token_accuracy": 0.6036300667524338,
+      "grad_norm": 0.4897821545600891,
+      "learning_rate": 0.0001202885947987717,
+      "loss": 1.6306,
+      "mean_token_accuracy": 0.610635744035244,
       "step": 56000
     },
     {
       "epoch": 0.7676786891037182,
-      "grad_norm": 1.0490987300872803,
-      "learning_rate": 0.00011687817603739233,
-      "loss": 1.6875,
-      "mean_token_accuracy": 0.597304708212614,
+      "grad_norm": 0.42849886417388916,
+      "learning_rate": 0.00011690535068887743,
+      "loss": 1.6374,
+      "mean_token_accuracy": 0.60488751283288,
       "step": 56500
     },
     {
       "epoch": 0.7744723058214502,
-      "grad_norm": 1.1952784061431885,
-      "learning_rate": 0.00011349493192749803,
-      "loss": 1.6796,
-      "mean_token_accuracy": 0.6028886337280274,
+      "grad_norm": 0.5816629528999329,
+      "learning_rate": 0.00011350851925324058,
+      "loss": 1.6288,
+      "mean_token_accuracy": 0.6102567628026009,
       "step": 57000
     },
     {
       "epoch": 0.7812659225391821,
-      "grad_norm": 1.000869870185852,
-      "learning_rate": 0.0001100981004918612,
-      "loss": 1.7052,
-      "mean_token_accuracy": 0.5968053241670132,
+      "grad_norm": 0.36746883392333984,
+      "learning_rate": 0.00011012527514334629,
+      "loss": 1.6563,
+      "mean_token_accuracy": 0.603935919225216,
       "step": 57500
     },
     {
       "epoch": 0.7880595392569142,
-      "grad_norm": 1.133309006690979,
-      "learning_rate": 0.0001067148563819669,
-      "loss": 1.6965,
-      "mean_token_accuracy": 0.5972890597879886,
+      "grad_norm": 0.4236038327217102,
+      "learning_rate": 0.00010672844370770945,
+      "loss": 1.648,
+      "mean_token_accuracy": 0.6038474014699459,
       "step": 58000
     },
     {
       "epoch": 0.7948531559746462,
-      "grad_norm": 1.2667903900146484,
-      "learning_rate": 0.00010331802494633007,
-      "loss": 1.6786,
-      "mean_token_accuracy": 0.601085704267025,
+      "grad_norm": 0.42980796098709106,
+      "learning_rate": 0.00010334519959781516,
+      "loss": 1.6293,
+      "mean_token_accuracy": 0.6078242086172104,
       "step": 58500
     },
     {
       "epoch": 0.8016467726923783,
-      "grad_norm": 1.0539146661758423,
-      "learning_rate": 9.993478083643577e-05,
-      "loss": 1.7029,
-      "mean_token_accuracy": 0.5962156045734882,
+      "grad_norm": 0.3263840675354004,
+      "learning_rate": 9.994836816217833e-05,
+      "loss": 1.654,
+      "mean_token_accuracy": 0.6031992013156414,
       "step": 59000
     },
     {
       "epoch": 0.8084403894101102,
-      "grad_norm": 1.1023699045181274,
-      "learning_rate": 9.653794940079893e-05,
-      "loss": 1.6731,
-      "mean_token_accuracy": 0.6045135918557644,
+      "grad_norm": 0.4101806879043579,
+      "learning_rate": 9.656512405228402e-05,
+      "loss": 1.6242,
+      "mean_token_accuracy": 0.6107135918736458,
       "step": 59500
     },
     {
       "epoch": 0.8152340061278422,
-      "grad_norm": 1.227386236190796,
-      "learning_rate": 9.315470529090464e-05,
-      "loss": 1.6855,
-      "mean_token_accuracy": 0.5978653418123722,
+      "grad_norm": 0.4116401970386505,
+      "learning_rate": 9.31682926166472e-05,
+      "loss": 1.6371,
+      "mean_token_accuracy": 0.6055142408013344,
       "step": 60000
     },
     {
       "epoch": 0.8220276228455743,
-      "grad_norm": 1.2664766311645508,
-      "learning_rate": 8.976466751813909e-05,
-      "loss": 1.7055,
-      "mean_token_accuracy": 0.5958617155849933,
+      "grad_norm": 0.47866082191467285,
+      "learning_rate": 8.97850485067529e-05,
+      "loss": 1.6563,
+      "mean_token_accuracy": 0.6031313905119896,
       "step": 60500
     },
     {
       "epoch": 0.8288212395633063,
-      "grad_norm": 1.263852834701538,
-      "learning_rate": 8.636783608250224e-05,
-      "loss": 1.6971,
-      "mean_token_accuracy": 0.5953974407315255,
+      "grad_norm": 0.4368223547935486,
+      "learning_rate": 8.638821707111606e-05,
+      "loss": 1.6475,
+      "mean_token_accuracy": 0.6029078653156758,
       "step": 61000
     },
     {
       "epoch": 0.8356148562810384,
-      "grad_norm": 1.1483241319656372,
-      "learning_rate": 8.297100464686541e-05,
-      "loss": 1.6798,
-      "mean_token_accuracy": 0.5995199010372162,
+      "grad_norm": 0.3518359363079071,
+      "learning_rate": 8.300497296122178e-05,
+      "loss": 1.6321,
+      "mean_token_accuracy": 0.6058916521966458,
       "step": 61500
     },
     {
       "epoch": 0.8424084729987703,
-      "grad_norm": 0.8838757872581482,
-      "learning_rate": 7.957417321122856e-05,
-      "loss": 1.6863,
-      "mean_token_accuracy": 0.5994715365469456,
+      "grad_norm": 0.28319722414016724,
+      "learning_rate": 7.960814152558493e-05,
+      "loss": 1.6382,
+      "mean_token_accuracy": 0.6064018523991108,
       "step": 62000
     },
     {
       "epoch": 0.8492020897165024,
-      "grad_norm": 1.0497251749038696,
-      "learning_rate": 7.619092910133428e-05,
-      "loss": 1.6692,
-      "mean_token_accuracy": 0.6024055682122708,
+      "grad_norm": 0.3478228747844696,
+      "learning_rate": 7.622489741569065e-05,
+      "loss": 1.6229,
+      "mean_token_accuracy": 0.6092533214688302,
       "step": 62500
     },
     {
       "epoch": 0.8559957064342344,
-      "grad_norm": 0.9796432852745056,
-      "learning_rate": 7.279409766569745e-05,
-      "loss": 1.6799,
-      "mean_token_accuracy": 0.5993573130667209,
+      "grad_norm": 0.32426026463508606,
+      "learning_rate": 7.28280659800538e-05,
+      "loss": 1.6324,
+      "mean_token_accuracy": 0.6059878776371479,
       "step": 63000
     },
     {
       "epoch": 0.8627893231519664,
-      "grad_norm": 1.2059897184371948,
-      "learning_rate": 6.941085355580315e-05,
-      "loss": 1.6763,
-      "mean_token_accuracy": 0.6016550965607166,
+      "grad_norm": 0.42673128843307495,
+      "learning_rate": 6.944482187015952e-05,
+      "loss": 1.6284,
+      "mean_token_accuracy": 0.6091273921728134,
       "step": 63500
     },
     {
       "epoch": 0.8695829398696985,
-      "grad_norm": 1.184564232826233,
-      "learning_rate": 6.601402212016632e-05,
-      "loss": 1.683,
-      "mean_token_accuracy": 0.5992586967945099,
+      "grad_norm": 0.4263095259666443,
+      "learning_rate": 6.604799043452269e-05,
+      "loss": 1.6373,
+      "mean_token_accuracy": 0.6060122801661492,
       "step": 64000
     },
     {
       "epoch": 0.8763765565874304,
-      "grad_norm": 0.979015588760376,
-      "learning_rate": 6.262398434740074e-05,
-      "loss": 1.6916,
-      "mean_token_accuracy": 0.6002075519263744,
+      "grad_norm": 0.41852718591690063,
+      "learning_rate": 6.26647463246284e-05,
+      "loss": 1.6453,
+      "mean_token_accuracy": 0.6065602390468121,
       "step": 64500
     },
     {
       "epoch": 0.8831701733051625,
-      "grad_norm": 1.4841409921646118,
-      "learning_rate": 5.924074023750645e-05,
-      "loss": 1.6815,
-      "mean_token_accuracy": 0.6010229328274727,
+      "grad_norm": 0.7346561551094055,
+      "learning_rate": 5.926791488899155e-05,
+      "loss": 1.6345,
+      "mean_token_accuracy": 0.6079370491206646,
       "step": 65000
     },
     {
       "epoch": 0.8899637900228945,
-      "grad_norm": 1.0104435682296753,
-      "learning_rate": 5.5843908801869616e-05,
-      "loss": 1.6942,
-      "mean_token_accuracy": 0.5977232045531273,
+      "grad_norm": 0.35610243678092957,
+      "learning_rate": 5.5884670779097254e-05,
+      "loss": 1.6491,
+      "mean_token_accuracy": 0.6041360826790333,
       "step": 65500
     },
     {
       "epoch": 0.8967574067406265,
-      "grad_norm": 1.364670991897583,
-      "learning_rate": 5.244707736623278e-05,
-      "loss": 1.6862,
-      "mean_token_accuracy": 0.5978881956636906,
+      "grad_norm": 0.4822712540626526,
+      "learning_rate": 5.2487839343460424e-05,
+      "loss": 1.6402,
+      "mean_token_accuracy": 0.6046371011734009,
       "step": 66000
     },
     {
       "epoch": 0.9035510234583585,
-      "grad_norm": 1.085087537765503,
-      "learning_rate": 4.905024593059594e-05,
-      "loss": 1.6812,
-      "mean_token_accuracy": 0.5997030725479126,
+      "grad_norm": 0.4815579056739807,
+      "learning_rate": 4.9104595233566134e-05,
+      "loss": 1.6347,
+      "mean_token_accuracy": 0.6062059481143951,
       "step": 66500
     },
     {
       "epoch": 0.9103446401760905,
-      "grad_norm": 0.9562463164329529,
-      "learning_rate": 4.567379548357292e-05,
-      "loss": 1.6848,
-      "mean_token_accuracy": 0.5997031436264515,
+      "grad_norm": 0.3300195038318634,
+      "learning_rate": 4.570776379792929e-05,
+      "loss": 1.6406,
+      "mean_token_accuracy": 0.6060437637567521,
       "step": 67000
     },
     {
       "epoch": 0.9171382568938226,
-      "grad_norm": 1.2848726511001587,
-      "learning_rate": 4.2276964047936085e-05,
-      "loss": 1.6873,
-      "mean_token_accuracy": 0.599891084432602,
+      "grad_norm": 0.5039587616920471,
+      "learning_rate": 4.2324519688035e-05,
+      "loss": 1.6424,
+      "mean_token_accuracy": 0.6062202319502831,
       "step": 67500
     },
     {
       "epoch": 0.9239318736115546,
-      "grad_norm": 1.168323040008545,
-      "learning_rate": 3.888013261229924e-05,
-      "loss": 1.6807,
-      "mean_token_accuracy": 0.599244915753603,
+      "grad_norm": 0.40121036767959595,
+      "learning_rate": 3.892768825239816e-05,
+      "loss": 1.6364,
+      "mean_token_accuracy": 0.6057018077373505,
       "step": 68000
     },
     {
       "epoch": 0.9307254903292866,
-      "grad_norm": 1.1764905452728271,
-      "learning_rate": 3.548330117666241e-05,
-      "loss": 1.6857,
-      "mean_token_accuracy": 0.6015235537290573,
+      "grad_norm": 0.4905497133731842,
+      "learning_rate": 3.554444414250387e-05,
+      "loss": 1.6417,
+      "mean_token_accuracy": 0.6073405523002148,
       "step": 68500
     },
     {
       "epoch": 0.9375191070470186,
-      "grad_norm": 1.2431254386901855,
-      "learning_rate": 3.210005706676812e-05,
-      "loss": 1.6735,
-      "mean_token_accuracy": 0.6007710628509522,
+      "grad_norm": 0.4090203642845154,
+      "learning_rate": 3.2147612706867036e-05,
+      "loss": 1.6299,
+      "mean_token_accuracy": 0.607111094802618,
       "step": 69000
     },
     {
       "epoch": 0.9443127237647506,
-      "grad_norm": 1.4310569763183594,
-      "learning_rate": 2.870322563113128e-05,
-      "loss": 1.6949,
-      "mean_token_accuracy": 0.5974322560429574,
+      "grad_norm": 0.4694749116897583,
+      "learning_rate": 2.8764368596972746e-05,
+      "loss": 1.6522,
+      "mean_token_accuracy": 0.6036750960350037,
       "step": 69500
     },
     {
       "epoch": 0.9511063404824827,
-      "grad_norm": 1.0426503419876099,
-      "learning_rate": 2.531998152123699e-05,
-      "loss": 1.6717,
-      "mean_token_accuracy": 0.6013675058782101,
+      "grad_norm": 0.4072393476963043,
+      "learning_rate": 2.5367537161335906e-05,
+      "loss": 1.6298,
+      "mean_token_accuracy": 0.6072974261939525,
       "step": 70000
     },
     {
       "epoch": 0.9578999572002147,
-      "grad_norm": 1.1059454679489136,
-      "learning_rate": 2.192315008560015e-05,
-      "loss": 1.6838,
-      "mean_token_accuracy": 0.5987102429866791,
+      "grad_norm": 0.3677803874015808,
+      "learning_rate": 2.198429305144162e-05,
+      "loss": 1.6419,
+      "mean_token_accuracy": 0.6047779574692249,
       "step": 70500
     },
     {
       "epoch": 0.9646935739179467,
-      "grad_norm": 1.03636634349823,
-      "learning_rate": 1.8539905975705863e-05,
-      "loss": 1.6863,
-      "mean_token_accuracy": 0.5992070758640766,
+      "grad_norm": 0.3518989384174347,
+      "learning_rate": 1.858746161580478e-05,
+      "loss": 1.6449,
+      "mean_token_accuracy": 0.6055590100586414,
       "step": 71000
     },
     {
       "epoch": 0.9714871906356787,
-      "grad_norm": 0.9837082028388977,
-      "learning_rate": 1.5143074540069023e-05,
-      "loss": 1.678,
-      "mean_token_accuracy": 0.5995057925581933,
+      "grad_norm": 0.32744893431663513,
+      "learning_rate": 1.5204217505910486e-05,
+      "loss": 1.6368,
+      "mean_token_accuracy": 0.6055232758522033,
       "step": 71500
     },
     {
       "epoch": 0.9782808073534107,
-      "grad_norm": 1.1581671237945557,
-      "learning_rate": 1.1759830430174733e-05,
-      "loss": 1.6811,
-      "mean_token_accuracy": 0.6011576275527477,
+      "grad_norm": 0.37632468342781067,
+      "learning_rate": 1.180738607027365e-05,
+      "loss": 1.6407,
+      "mean_token_accuracy": 0.6069048519730568,
       "step": 72000
     },
     {
       "epoch": 0.9850744240711428,
-      "grad_norm": 0.954434335231781,
-      "learning_rate": 8.362998994537894e-06,
-      "loss": 1.6745,
-      "mean_token_accuracy": 0.6010885013043881,
+      "grad_norm": 0.3159102201461792,
+      "learning_rate": 8.424141960379359e-06,
+      "loss": 1.6353,
+      "mean_token_accuracy": 0.6071368261277675,
       "step": 72500
     },
     {
       "epoch": 0.9918680407888748,
-      "grad_norm": 0.984001100063324,
-      "learning_rate": 4.979754884643605e-06,
-      "loss": 1.6786,
-      "mean_token_accuracy": 0.6019023385941982,
+      "grad_norm": 0.3694087564945221,
+      "learning_rate": 5.027310524742521e-06,
+      "loss": 1.6402,
+      "mean_token_accuracy": 0.6074571430683136,
       "step": 73000
     },
     {
       "epoch": 0.9986616575066067,
-      "grad_norm": 1.2359673976898193,
-      "learning_rate": 1.5829234490067665e-06,
-      "loss": 1.6652,
-      "mean_token_accuracy": 0.6032795180380345,
+      "grad_norm": 0.515445351600647,
+      "learning_rate": 1.6440664148482296e-06,
+      "loss": 1.627,
+      "mean_token_accuracy": 0.6084005556106568,
       "step": 73500
     },
     {
       "epoch": 0.9999932063832823,
-      "mean_token_accuracy": 0.5990850239688036,
+      "mean_token_accuracy": 0.6045256301152463,
       "step": 73598,
-      "total_flos": 9.3961326985583e+17,
-      "train_loss": 1.7179553695028122,
-      "train_runtime": 11634.1539,
-      "train_samples_per_second": 25.304,
-      "train_steps_per_second": 6.326
+      "total_flos": 9.323627298076754e+17,
+      "train_loss": 1.6535549103175464,
+      "train_runtime": 12676.64,
+      "train_samples_per_second": 23.223,
+      "train_steps_per_second": 5.806
     }
   ],
   "logging_steps": 500,
@@ -1212,7 +1212,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 9.3961326985583e+17,
+  "total_flos": 9.323627298076754e+17,
   "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null