Add files using upload-large-folder tool

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+wandb/run-20250408_142458-wlfced8t/run-wlfced8t.wandb filter=lfs diff=lfs merge=lfs -text

metrics.json

@@ -0,0 +1,32 @@
+{
+    "PPL": 16.7308349609375,
+    "arc_challenge": {
+        "acc": 0.2150170648464164,
+        "acc_stderr": 0.012005717634133614,
+        "acc_norm": 0.26109215017064846,
+        "acc_norm_stderr": 0.012835523909473867
+    },
+    "arc_easy": {
+        "acc": 0.37836700336700335,
+        "acc_stderr": 0.00995157568333195,
+        "acc_norm": 0.3611111111111111,
+        "acc_norm_stderr": 0.009856013425811242
+    },
+    "hellaswag": {
+        "acc": 0.33519219279028084,
+        "acc_stderr": 0.00471092856998574,
+        "acc_norm": 0.391256721768572,
+        "acc_norm_stderr": 0.004870342592915051
+    },
+    "piqa": {
+        "acc": 0.6142546245919478,
+        "acc_stderr": 0.011357166777524042,
+        "acc_norm": 0.6196953210010882,
+        "acc_norm_stderr": 0.011326620892570317
+    },
+    "winogrande": {
+        "acc": 0.4988161010260458,
+        "acc_stderr": 0.014052446290529019
+    },
+    "QA Avg": 0.4083293973243388
+}

runs/events.out.tfevents.1744122340.816cf5acb821

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b9d2dfc3ed992e856282258efd7e8191ae0bd3e81fd476fcf1e436f0fee2237e
+size 70125

wandb/debug-internal.log

@@ -0,0 +1,16 @@
+{"time":"2025-04-08T14:24:58.122459682Z","level":"INFO","msg":"stream: starting","core version":"0.19.9","symlink path":"ckpts/tinyllama_v1.1/int2-g128/wandb/run-20250408_142458-wlfced8t/logs/debug-core.log"}
+{"time":"2025-04-08T14:24:58.380784982Z","level":"INFO","msg":"created new stream","id":"wlfced8t"}
+{"time":"2025-04-08T14:24:58.380854043Z","level":"INFO","msg":"stream: started","id":"wlfced8t"}
+{"time":"2025-04-08T14:24:58.380871954Z","level":"INFO","msg":"writer: Do: started","stream_id":"wlfced8t"}
+{"time":"2025-04-08T14:24:58.380921165Z","level":"INFO","msg":"sender: started","stream_id":"wlfced8t"}
+{"time":"2025-04-08T14:24:58.380954676Z","level":"INFO","msg":"handler: started","stream_id":"wlfced8t"}
+{"time":"2025-04-08T14:24:58.59144642Z","level":"INFO","msg":"Starting system monitor"}
+{"time":"2025-04-08T15:51:25.126053606Z","level":"INFO","msg":"Stopping system monitor"}
+{"time":"2025-04-08T15:51:25.126119508Z","level":"INFO","msg":"Stopped system monitor"}
+{"time":"2025-04-08T15:51:25.715478158Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2025-04-08T15:51:25.950965444Z","level":"INFO","msg":"handler: operation stats","stats":{}}
+{"time":"2025-04-08T15:51:25.956097933Z","level":"INFO","msg":"stream: closing","id":"wlfced8t"}
+{"time":"2025-04-08T15:51:25.956113153Z","level":"INFO","msg":"handler: closed","stream_id":"wlfced8t"}
+{"time":"2025-04-08T15:51:25.956122893Z","level":"INFO","msg":"writer: Close: closed","stream_id":"wlfced8t"}
+{"time":"2025-04-08T15:51:25.956130383Z","level":"INFO","msg":"sender: closed","stream_id":"wlfced8t"}
+{"time":"2025-04-08T15:51:25.956237656Z","level":"INFO","msg":"stream: closed","id":"wlfced8t"}

wandb/debug.log

@@ -0,0 +1,30 @@
+2025-04-08 14:24:58,116 INFO    MainThread:9420 [wandb_setup.py:_flush():67] Current SDK version is 0.19.9
+2025-04-08 14:24:58,116 INFO    MainThread:9420 [wandb_setup.py:_flush():67] Configure stats pid to 9420
+2025-04-08 14:24:58,116 INFO    MainThread:9420 [wandb_setup.py:_flush():67] Loading settings from /root/.config/wandb/settings
+2025-04-08 14:24:58,116 INFO    MainThread:9420 [wandb_setup.py:_flush():67] Loading settings from /workspace/BitDistiller/train/wandb/settings
+2025-04-08 14:24:58,116 INFO    MainThread:9420 [wandb_setup.py:_flush():67] Loading settings from environment variables
+2025-04-08 14:24:58,116 INFO    MainThread:9420 [wandb_init.py:setup_run_log_directory():662] Logging user logs to ./ckpts/tinyllama_v1.1/int2-g128/wandb/run-20250408_142458-wlfced8t/logs/debug.log
+2025-04-08 14:24:58,117 INFO    MainThread:9420 [wandb_init.py:setup_run_log_directory():663] Logging internal logs to ./ckpts/tinyllama_v1.1/int2-g128/wandb/run-20250408_142458-wlfced8t/logs/debug-internal.log
+2025-04-08 14:24:58,117 INFO    MainThread:9420 [wandb_init.py:init():781] calling init triggers
+2025-04-08 14:24:58,117 INFO    MainThread:9420 [wandb_init.py:init():786] wandb.init called with sweep_config: {}
+config: {'_wandb': {}}
+2025-04-08 14:24:58,117 INFO    MainThread:9420 [wandb_init.py:init():809] starting backend
+2025-04-08 14:24:58,117 INFO    MainThread:9420 [wandb_init.py:init():813] sending inform_init request
+2025-04-08 14:24:58,119 INFO    MainThread:9420 [backend.py:_multiprocessing_setup():101] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2025-04-08 14:24:58,119 INFO    MainThread:9420 [wandb_init.py:init():823] backend started and connected
+2025-04-08 14:24:58,124 INFO    MainThread:9420 [wandb_init.py:init():915] updated telemetry
+2025-04-08 14:24:58,304 INFO    MainThread:9420 [wandb_init.py:init():939] communicating run to backend with 90.0 second timeout
+2025-04-08 14:24:58,589 INFO    MainThread:9420 [wandb_init.py:init():1014] starting run threads in backend
+2025-04-08 14:24:58,680 INFO    MainThread:9420 [wandb_run.py:_console_start():2454] atexit reg
+2025-04-08 14:24:58,681 INFO    MainThread:9420 [wandb_run.py:_redirect():2306] redirect: wrap_raw
+2025-04-08 14:24:58,681 INFO    MainThread:9420 [wandb_run.py:_redirect():2371] Wrapping output streams.
+2025-04-08 14:24:58,681 INFO    MainThread:9420 [wandb_run.py:_redirect():2394] Redirects installed.
+2025-04-08 14:24:58,682 INFO    MainThread:9420 [wandb_init.py:init():1056] run started, returning control to user process
+2025-04-08 14:25:40,038 INFO    MainThread:9420 [wandb_run.py:_config_callback():1327] config_cb None None {'vocab_size': 32001, 'max_position_embeddings': 2048, 'hidden_size': 2048, 'intermediate_size': 5632, 'num_hidden_layers': 22, 'num_attention_heads': 32, 'num_key_value_heads': 4, 'hidden_act': 'silu', 'initializer_range': 0.02, 'rms_norm_eps': 1e-05, 'pretraining_tp': 1, 'use_cache': True, 'rope_theta': 10000.0, 'rope_scaling': None, 'attention_bias': False, 'attention_dropout': 0.0, 'return_dict': True, 'output_hidden_states': False, 'output_attentions': False, 'torchscript': False, 'torch_dtype': 'bfloat16', 'use_bfloat16': False, 'tf_legacy_loss': False, 'pruned_heads': {}, 'tie_word_embeddings': False, 'chunk_size_feed_forward': 0, 'is_encoder_decoder': False, 'is_decoder': False, 'cross_attention_hidden_size': None, 'add_cross_attention': False, 'tie_encoder_decoder': False, 'max_length': 20, 'min_length': 0, 'do_sample': False, 'early_stopping': False, 'num_beams': 1, 'num_beam_groups': 1, 'diversity_penalty': 0.0, 'temperature': 1.0, 'top_k': 50, 'top_p': 1.0, 'typical_p': 1.0, 'repetition_penalty': 1.0, 'length_penalty': 1.0, 'no_repeat_ngram_size': 0, 'encoder_no_repeat_ngram_size': 0, 'bad_words_ids': None, 'num_return_sequences': 1, 'output_scores': False, 'return_dict_in_generate': False, 'forced_bos_token_id': None, 'forced_eos_token_id': None, 'remove_invalid_values': False, 'exponential_decay_length_penalty': None, 'suppress_tokens': None, 'begin_suppress_tokens': None, 'architectures': ['LlamaForCausalLM'], 'finetuning_task': None, 'id2label': {0: 'LABEL_0', 1: 'LABEL_1'}, 'label2id': {'LABEL_0': 0, 'LABEL_1': 1}, 'tokenizer_class': None, 'prefix': None, 'bos_token_id': 1, 'pad_token_id': None, 'eos_token_id': 2, 'sep_token_id': None, 'decoder_start_token_id': None, 'task_specific_params': None, 'problem_type': None, '_name_or_path': '../models/TinyLlama_v1.1/', 'transformers_version': '4.37.0', 'model_type': 'llama', 'output_dir': './ckpts/tinyllama_v1.1/int2-g128/', 'overwrite_output_dir': False, 'do_train': False, 'do_eval': True, 'do_predict': False, 'evaluation_strategy': 'steps', 'prediction_loss_only': False, 'per_device_train_batch_size': 16, 'per_device_eval_batch_size': 16, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 4, 'eval_accumulation_steps': None, 'eval_delay': 0, 'learning_rate': 2e-05, 'weight_decay': 0.0, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 4.0, 'max_steps': -1, 'lr_scheduler_type': 'constant', 'lr_scheduler_kwargs': {}, 'warmup_ratio': 0.0, 'warmup_steps': 0, 'log_level': 'passive', 'log_level_replica': 'warning', 'log_on_each_node': True, 'logging_dir': './ckpts/tinyllama_v1.1/int2-g128/runs/', 'logging_strategy': 'steps', 'logging_first_step': False, 'logging_steps': 1.0, 'logging_nan_inf_filter': True, 'save_strategy': 'steps', 'save_steps': 40, 'save_total_limit': 2, 'save_safetensors': True, 'save_on_each_node': False, 'save_only_model': False, 'no_cuda': False, 'use_cpu': False, 'use_mps_device': False, 'seed': 42, 'data_seed': None, 'jit_mode_eval': False, 'use_ipex': False, 'bf16': True, 'fp16': False, 'fp16_opt_level': 'O1', 'half_precision_backend': 'auto', 'bf16_full_eval': False, 'fp16_full_eval': False, 'tf32': None, 'local_rank': 0, 'ddp_backend': None, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': 40, 'dataloader_num_workers': 0, 'past_index': -1, 'run_name': './ckpts/tinyllama_v1.1/int2-g128/', 'disable_tqdm': False, 'remove_unused_columns': True, 'label_names': None, 'load_best_model_at_end': True, 'metric_for_best_model': 'loss', 'greater_is_better': False, 'ignore_data_skip': False, 'fsdp': [], 'fsdp_min_num_params': 0, 'fsdp_config': {'min_num_params': 0, 'xla': False, 'xla_fsdp_grad_ckpt': False}, 'fsdp_transformer_layer_cls_to_wrap': None, 'deepspeed': 'config/zero.json', 'label_smoothing_factor': 0.0, 'optim': 'adamw_torch', 'optim_args': None, 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['tensorboard', 'wandb'], 'ddp_find_unused_parameters': None, 'ddp_bucket_cap_mb': None, 'ddp_broadcast_buffers': None, 'dataloader_pin_memory': True, 'dataloader_persistent_workers': False, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': False, 'resume_from_checkpoint': None, 'hub_model_id': None, 'hub_strategy': 'every_save', 'hub_token': '<HUB_TOKEN>', 'hub_private_repo': False, 'hub_always_push': False, 'gradient_checkpointing': True, 'gradient_checkpointing_kwargs': None, 'include_inputs_for_metrics': False, 'fp16_backend': 'auto', 'push_to_hub_model_id': None, 'push_to_hub_organization': None, 'push_to_hub_token': '<PUSH_TO_HUB_TOKEN>', 'mp_parameters': '', 'auto_find_batch_size': False, 'full_determinism': False, 'torchdynamo': None, 'ray_scope': 'last', 'ddp_timeout': 1800, 'torch_compile': False, 'torch_compile_backend': None, 'torch_compile_mode': None, 'dispatch_batches': None, 'split_batches': False, 'include_tokens_per_second': False, 'include_num_input_tokens_seen': False, 'neftune_noise_alpha': None, 'cache_dir': None, 'model_max_length': 1024, 'bits': 2, 'q_group_size': 128, 'quant_type': 'int2-asym', 'clip': '../quantization/clip_cache/TinyLlama_v1.1/int2-g128.pt', 'train_kd': True, 'kd_tmp': 1, 'kd_loss_type': 'cakld', 'cakld_steps': 10}
+2025-04-08 15:51:25,124 INFO    MainThread:9420 [wandb_run.py:_finish():2189] finishing run DeepFriedNLP/SNLP_BitDistiller/wlfced8t
+2025-04-08 15:51:25,125 INFO    MainThread:9420 [wandb_run.py:_atexit_cleanup():2419] got exitcode: 0
+2025-04-08 15:51:25,125 INFO    MainThread:9420 [wandb_run.py:_restore():2401] restore
+2025-04-08 15:51:25,125 INFO    MainThread:9420 [wandb_run.py:_restore():2407] restore done
+2025-04-08 15:51:25,953 INFO    MainThread:9420 [wandb_run.py:_footer_history_summary_info():4064] rendering history
+2025-04-08 15:51:25,954 INFO    MainThread:9420 [wandb_run.py:_footer_history_summary_info():4096] rendering summary
+2025-04-08 15:51:25,955 INFO    MainThread:9420 [wandb_run.py:_footer_sync_info():4025] logging synced files

wandb/run-20250408_141746-8llb18c8/logs/debug-core.log

@@ -0,0 +1,13 @@
+{"time":"2025-04-08T14:17:45.949818845Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpp31dfewy/port-6976.txt","pid":6976,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false}
+{"time":"2025-04-08T14:17:45.966044774Z","level":"INFO","msg":"Will exit if parent process dies.","ppid":6976}
+{"time":"2025-04-08T14:17:45.966061155Z","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":46761,"Zone":""}}
+{"time":"2025-04-08T14:17:46.138422649Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:53806"}
+{"time":"2025-04-08T14:17:46.530704428Z","level":"INFO","msg":"handleInformInit: received","streamId":"8llb18c8","id":"127.0.0.1:53806"}
+{"time":"2025-04-08T14:17:46.790763608Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"8llb18c8","id":"127.0.0.1:53806"}
+{"time":"2025-04-08T14:17:47.911560658Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"127.0.0.1:53806"}
+{"time":"2025-04-08T14:17:47.91163659Z","level":"INFO","msg":"connection: closing","id":"127.0.0.1:53806"}
+{"time":"2025-04-08T14:17:47.911668421Z","level":"INFO","msg":"server is shutting down"}
+{"time":"2025-04-08T14:17:47.911737362Z","level":"INFO","msg":"connection: closed successfully","id":"127.0.0.1:53806"}
+{"time":"2025-04-08T14:17:48.092584004Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"127.0.0.1:53806"}
+{"time":"2025-04-08T14:17:48.092971643Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"127.0.0.1:53806"}
+{"time":"2025-04-08T14:17:48.092993873Z","level":"INFO","msg":"server is closed"}

wandb/run-20250408_141746-8llb18c8/logs/debug-internal.log

@@ -0,0 +1,17 @@
+{"time":"2025-04-08T14:17:46.530930213Z","level":"INFO","msg":"stream: starting","core version":"0.19.9","symlink path":"ckpts/tinyllama_v1.1/int2-g128/wandb/run-20250408_141746-8llb18c8/logs/debug-core.log"}
+{"time":"2025-04-08T14:17:46.790699927Z","level":"INFO","msg":"created new stream","id":"8llb18c8"}
+{"time":"2025-04-08T14:17:46.790754738Z","level":"INFO","msg":"stream: started","id":"8llb18c8"}
+{"time":"2025-04-08T14:17:46.790793839Z","level":"INFO","msg":"writer: Do: started","stream_id":"8llb18c8"}
+{"time":"2025-04-08T14:17:46.790821339Z","level":"INFO","msg":"sender: started","stream_id":"8llb18c8"}
+{"time":"2025-04-08T14:17:46.790876061Z","level":"INFO","msg":"handler: started","stream_id":"8llb18c8"}
+{"time":"2025-04-08T14:17:46.909228673Z","level":"ERROR","msg":"HTTP error","status":401,"method":"POST","url":"https://api.wandb.ai/graphql"}
+{"time":"2025-04-08T14:17:46.909315955Z","level":"ERROR","msg":"sender: upsertRun:","error":"failed to upsert bucket: returned error 401: {\"data\":{\"upsertBucket\":null},\"errors\":[{\"message\":\"user is not logged in\",\"path\":[\"upsertBucket\"],\"extensions\":{\"code\":\"PERMISSION_ERROR\"}}]}"}
+{"time":"2025-04-08T14:17:47.911709251Z","level":"INFO","msg":"stream: closing","id":"8llb18c8"}
+{"time":"2025-04-08T14:17:47.911812024Z","level":"ERROR","msg":"sender: upsertConfig: RunRecord is nil"}
+{"time":"2025-04-08T14:17:48.090388056Z","level":"ERROR","msg":"HTTP error","status":404,"method":"POST","url":"https://api.wandb.ai/graphql"}
+{"time":"2025-04-08T14:17:48.090496798Z","level":"ERROR","msg":"runfiles: CreateRunFiles returned error: returned error 404: {\"data\":{\"createRunFiles\":null},\"errors\":[{\"message\":\"run SNLP_BitDistiller/8llb18c8 not found during createRunFiles\",\"path\":[\"createRunFiles\"]}]}"}
+{"time":"2025-04-08T14:17:48.091068171Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2025-04-08T14:17:48.091107982Z","level":"INFO","msg":"handler: closed","stream_id":"8llb18c8"}
+{"time":"2025-04-08T14:17:48.091119232Z","level":"INFO","msg":"writer: Close: closed","stream_id":"8llb18c8"}
+{"time":"2025-04-08T14:17:48.091141302Z","level":"INFO","msg":"sender: closed","stream_id":"8llb18c8"}
+{"time":"2025-04-08T14:17:48.091168643Z","level":"INFO","msg":"stream: closed","id":"8llb18c8"}

wandb/run-20250408_141746-8llb18c8/logs/debug.log

@@ -0,0 +1,16 @@
+2025-04-08 14:17:46,525 INFO    MainThread:6976 [wandb_setup.py:_flush():67] Current SDK version is 0.19.9
+2025-04-08 14:17:46,525 INFO    MainThread:6976 [wandb_setup.py:_flush():67] Configure stats pid to 6976
+2025-04-08 14:17:46,525 INFO    MainThread:6976 [wandb_setup.py:_flush():67] Loading settings from /root/.config/wandb/settings
+2025-04-08 14:17:46,525 INFO    MainThread:6976 [wandb_setup.py:_flush():67] Loading settings from /workspace/BitDistiller/train/wandb/settings
+2025-04-08 14:17:46,525 INFO    MainThread:6976 [wandb_setup.py:_flush():67] Loading settings from environment variables
+2025-04-08 14:17:46,525 INFO    MainThread:6976 [wandb_init.py:setup_run_log_directory():662] Logging user logs to ./ckpts/tinyllama_v1.1/int2-g128/wandb/run-20250408_141746-8llb18c8/logs/debug.log
+2025-04-08 14:17:46,525 INFO    MainThread:6976 [wandb_init.py:setup_run_log_directory():663] Logging internal logs to ./ckpts/tinyllama_v1.1/int2-g128/wandb/run-20250408_141746-8llb18c8/logs/debug-internal.log
+2025-04-08 14:17:46,525 INFO    MainThread:6976 [wandb_init.py:init():781] calling init triggers
+2025-04-08 14:17:46,525 INFO    MainThread:6976 [wandb_init.py:init():786] wandb.init called with sweep_config: {}
+config: {'_wandb': {}}
+2025-04-08 14:17:46,525 INFO    MainThread:6976 [wandb_init.py:init():809] starting backend
+2025-04-08 14:17:46,525 INFO    MainThread:6976 [wandb_init.py:init():813] sending inform_init request
+2025-04-08 14:17:46,528 INFO    MainThread:6976 [backend.py:_multiprocessing_setup():101] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2025-04-08 14:17:46,528 INFO    MainThread:6976 [wandb_init.py:init():823] backend started and connected
+2025-04-08 14:17:46,533 INFO    MainThread:6976 [wandb_init.py:init():915] updated telemetry
+2025-04-08 14:17:46,716 INFO    MainThread:6976 [wandb_init.py:init():939] communicating run to backend with 90.0 second timeout

wandb/run-20250408_142458-wlfced8t/files/config.yaml

@@ -0,0 +1,525 @@
+_name_or_path:
+    value: ../models/TinyLlama_v1.1/
+_wandb:
+    value:
+        cli_version: 0.19.9
+        m:
+            - "1": eval/runtime
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": train/global_step
+              "6":
+                - 3
+              "7": []
+            - "1": eval/samples_per_second
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": eval/steps_per_second
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": train/train_steps_per_second
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": train/train_runtime
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": train/train_samples_per_second
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": train/loss
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": train/epoch
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": eval/loss
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": train/total_flos
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": train/train_loss
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": train/learning_rate
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+        python_version: 3.9.21
+        t:
+            "1":
+                - 1
+                - 5
+                - 11
+                - 49
+                - 51
+                - 53
+                - 55
+                - 71
+                - 98
+            "2":
+                - 1
+                - 5
+                - 11
+                - 49
+                - 51
+                - 53
+                - 55
+                - 71
+                - 98
+            "3":
+                - 2
+                - 7
+                - 13
+                - 15
+                - 23
+                - 55
+                - 66
+            "4": 3.9.21
+            "5": 0.19.9
+            "6": 4.37.0
+            "8":
+                - 5
+            "9":
+                "1": transformers_trainer
+            "12": 0.19.9
+            "13": linux-x86_64
+adafactor:
+    value: false
+adam_beta1:
+    value: 0.9
+adam_beta2:
+    value: 0.999
+adam_epsilon:
+    value: 1e-08
+add_cross_attention:
+    value: false
+architectures:
+    value:
+        - LlamaForCausalLM
+attention_bias:
+    value: false
+attention_dropout:
+    value: 0
+auto_find_batch_size:
+    value: false
+bad_words_ids:
+    value: null
+begin_suppress_tokens:
+    value: null
+bf16:
+    value: true
+bf16_full_eval:
+    value: false
+bits:
+    value: 2
+bos_token_id:
+    value: 1
+cache_dir:
+    value: null
+cakld_steps:
+    value: 10
+chunk_size_feed_forward:
+    value: 0
+clip:
+    value: ../quantization/clip_cache/TinyLlama_v1.1/int2-g128.pt
+cross_attention_hidden_size:
+    value: null
+data_seed:
+    value: null
+dataloader_drop_last:
+    value: false
+dataloader_num_workers:
+    value: 0
+dataloader_persistent_workers:
+    value: false
+dataloader_pin_memory:
+    value: true
+ddp_backend:
+    value: null
+ddp_broadcast_buffers:
+    value: null
+ddp_bucket_cap_mb:
+    value: null
+ddp_find_unused_parameters:
+    value: null
+ddp_timeout:
+    value: 1800
+debug:
+    value: []
+decoder_start_token_id:
+    value: null
+deepspeed:
+    value: config/zero.json
+disable_tqdm:
+    value: false
+dispatch_batches:
+    value: null
+diversity_penalty:
+    value: 0
+do_eval:
+    value: true
+do_predict:
+    value: false
+do_sample:
+    value: false
+do_train:
+    value: false
+early_stopping:
+    value: false
+encoder_no_repeat_ngram_size:
+    value: 0
+eos_token_id:
+    value: 2
+eval_accumulation_steps:
+    value: null
+eval_delay:
+    value: 0
+eval_steps:
+    value: 40
+evaluation_strategy:
+    value: steps
+exponential_decay_length_penalty:
+    value: null
+finetuning_task:
+    value: null
+forced_bos_token_id:
+    value: null
+forced_eos_token_id:
+    value: null
+fp16:
+    value: false
+fp16_backend:
+    value: auto
+fp16_full_eval:
+    value: false
+fp16_opt_level:
+    value: O1
+fsdp:
+    value: []
+fsdp_config:
+    value:
+        min_num_params: 0
+        xla: false
+        xla_fsdp_grad_ckpt: false
+fsdp_min_num_params:
+    value: 0
+fsdp_transformer_layer_cls_to_wrap:
+    value: null
+full_determinism:
+    value: false
+gradient_accumulation_steps:
+    value: 4
+gradient_checkpointing:
+    value: true
+gradient_checkpointing_kwargs:
+    value: null
+greater_is_better:
+    value: false
+group_by_length:
+    value: false
+half_precision_backend:
+    value: auto
+hidden_act:
+    value: silu
+hidden_size:
+    value: 2048
+hub_always_push:
+    value: false
+hub_model_id:
+    value: null
+hub_private_repo:
+    value: false
+hub_strategy:
+    value: every_save
+hub_token:
+    value: <HUB_TOKEN>
+id2label:
+    value:
+        "0": LABEL_0
+        "1": LABEL_1
+ignore_data_skip:
+    value: false
+include_inputs_for_metrics:
+    value: false
+include_num_input_tokens_seen:
+    value: false
+include_tokens_per_second:
+    value: false
+initializer_range:
+    value: 0.02
+intermediate_size:
+    value: 5632
+is_decoder:
+    value: false
+is_encoder_decoder:
+    value: false
+jit_mode_eval:
+    value: false
+kd_loss_type:
+    value: cakld
+kd_tmp:
+    value: 1
+label_names:
+    value: null
+label_smoothing_factor:
+    value: 0
+label2id:
+    value:
+        LABEL_0: 0
+        LABEL_1: 1
+learning_rate:
+    value: 2e-05
+length_column_name:
+    value: length
+length_penalty:
+    value: 1
+load_best_model_at_end:
+    value: true
+local_rank:
+    value: 0
+log_level:
+    value: passive
+log_level_replica:
+    value: warning
+log_on_each_node:
+    value: true
+logging_dir:
+    value: ./ckpts/tinyllama_v1.1/int2-g128/runs/
+logging_first_step:
+    value: false
+logging_nan_inf_filter:
+    value: true
+logging_steps:
+    value: 1
+logging_strategy:
+    value: steps
+lr_scheduler_type:
+    value: constant
+max_grad_norm:
+    value: 1
+max_length:
+    value: 20
+max_position_embeddings:
+    value: 2048
+max_steps:
+    value: -1
+metric_for_best_model:
+    value: loss
+min_length:
+    value: 0
+model_max_length:
+    value: 1024
+model_type:
+    value: llama
+mp_parameters:
+    value: ""
+neftune_noise_alpha:
+    value: null
+no_cuda:
+    value: false
+no_repeat_ngram_size:
+    value: 0
+num_attention_heads:
+    value: 32
+num_beam_groups:
+    value: 1
+num_beams:
+    value: 1
+num_hidden_layers:
+    value: 22
+num_key_value_heads:
+    value: 4
+num_return_sequences:
+    value: 1
+num_train_epochs:
+    value: 4
+optim:
+    value: adamw_torch
+optim_args:
+    value: null
+output_attentions:
+    value: false
+output_dir:
+    value: ./ckpts/tinyllama_v1.1/int2-g128/
+output_hidden_states:
+    value: false
+output_scores:
+    value: false
+overwrite_output_dir:
+    value: false
+pad_token_id:
+    value: null
+past_index:
+    value: -1
+per_device_eval_batch_size:
+    value: 16
+per_device_train_batch_size:
+    value: 16
+per_gpu_eval_batch_size:
+    value: null
+per_gpu_train_batch_size:
+    value: null
+prediction_loss_only:
+    value: false
+prefix:
+    value: null
+pretraining_tp:
+    value: 1
+problem_type:
+    value: null
+push_to_hub:
+    value: false
+push_to_hub_model_id:
+    value: null
+push_to_hub_organization:
+    value: null
+push_to_hub_token:
+    value: <PUSH_TO_HUB_TOKEN>
+q_group_size:
+    value: 128
+quant_type:
+    value: int2-asym
+ray_scope:
+    value: last
+remove_invalid_values:
+    value: false
+remove_unused_columns:
+    value: true
+repetition_penalty:
+    value: 1
+report_to:
+    value:
+        - tensorboard
+        - wandb
+resume_from_checkpoint:
+    value: null
+return_dict:
+    value: true
+return_dict_in_generate:
+    value: false
+rms_norm_eps:
+    value: 1e-05
+rope_scaling:
+    value: null
+rope_theta:
+    value: 10000
+run_name:
+    value: ./ckpts/tinyllama_v1.1/int2-g128/
+save_on_each_node:
+    value: false
+save_only_model:
+    value: false
+save_safetensors:
+    value: true
+save_steps:
+    value: 40
+save_strategy:
+    value: steps
+save_total_limit:
+    value: 2
+seed:
+    value: 42
+sep_token_id:
+    value: null
+skip_memory_metrics:
+    value: true
+split_batches:
+    value: false
+suppress_tokens:
+    value: null
+task_specific_params:
+    value: null
+temperature:
+    value: 1
+tf_legacy_loss:
+    value: false
+tf32:
+    value: null
+tie_encoder_decoder:
+    value: false
+tie_word_embeddings:
+    value: false
+tokenizer_class:
+    value: null
+top_k:
+    value: 50
+top_p:
+    value: 1
+torch_compile:
+    value: false
+torch_compile_backend:
+    value: null
+torch_compile_mode:
+    value: null
+torch_dtype:
+    value: bfloat16
+torchdynamo:
+    value: null
+torchscript:
+    value: false
+tpu_metrics_debug:
+    value: false
+tpu_num_cores:
+    value: null
+train_kd:
+    value: true
+transformers_version:
+    value: 4.37.0
+typical_p:
+    value: 1
+use_bfloat16:
+    value: false
+use_cache:
+    value: true
+use_cpu:
+    value: false
+use_ipex:
+    value: false
+use_legacy_prediction_loop:
+    value: false
+use_mps_device:
+    value: false
+vocab_size:
+    value: 32001
+warmup_ratio:
+    value: 0
+warmup_steps:
+    value: 0
+weight_decay:
+    value: 0

wandb/run-20250408_142458-wlfced8t/files/output.log

@@ -0,0 +1,445 @@
+/workspace/BitDistiller/BitDistillerVenv/lib/python3.9/site-packages/accelerate/accelerator.py:432: FutureWarning: Passing the following arguments to `Accelerator` is deprecated and will be removed in version 1.0 of Accelerate: dict_keys(['dispatch_batches', 'split_batches']). Please pass an `accelerate.DataLoaderConfiguration` instead:
+dataloader_config = DataLoaderConfiguration(dispatch_batches=None, split_batches=False)
+  warnings.warn(
+Using /root/.cache/torch_extensions/py39_cu124 as PyTorch extensions root...
+Creating extension directory /root/.cache/torch_extensions/py39_cu124/cpu_adam...
+Emitting ninja build file /root/.cache/torch_extensions/py39_cu124/cpu_adam/build.ninja...
+Building extension module cpu_adam...
+Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N)
+Loading extension module cpu_adam...
+Time to load cpu_adam op: 23.6294584274292 seconds
+[2025-04-08 14:25:40,026] [WARNING] [lr_schedules.py:683:get_lr] Attempting to get learning rate from scheduler before it has started
+  0%|                                                                             | 0/400 [00:00<?, ?it/s]`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...
+/workspace/BitDistiller/BitDistillerVenv/lib/python3.9/site-packages/torch/_dynamo/eval_frame.py:745: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.5 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.
+  return fn(*args, **kwargs)
+                                                                                                          
+{'loss': 5217.7002, 'learning_rate': 0.0, 'epoch': 0.01}
+{'loss': 5011.7588, 'learning_rate': 2e-05, 'epoch': 0.02}
+{'loss': 4812.8281, 'learning_rate': 2e-05, 'epoch': 0.03}
+{'loss': 1860.1274, 'learning_rate': 2e-05, 'epoch': 0.04}
+{'loss': 1713.5734, 'learning_rate': 2e-05, 'epoch': 0.05}
+{'loss': 1103.2999, 'learning_rate': 2e-05, 'epoch': 0.06}
+{'loss': 675.6068, 'learning_rate': 2e-05, 'epoch': 0.07}
+{'loss': 683.2965, 'learning_rate': 2e-05, 'epoch': 0.08}
+{'loss': 734.9794, 'learning_rate': 2e-05, 'epoch': 0.09}
+{'loss': 585.422, 'learning_rate': 2e-05, 'epoch': 0.1}
+{'loss': 570.1306, 'learning_rate': 2e-05, 'epoch': 0.11}
+{'loss': 599.2966, 'learning_rate': 2e-05, 'epoch': 0.12}
+{'loss': 574.2372, 'learning_rate': 2e-05, 'epoch': 0.13}
+{'loss': 481.7502, 'learning_rate': 2e-05, 'epoch': 0.14}
+{'loss': 531.738, 'learning_rate': 2e-05, 'epoch': 0.15}
+{'loss': 448.2928, 'learning_rate': 2e-05, 'epoch': 0.16}
+{'loss': 490.8231, 'learning_rate': 2e-05, 'epoch': 0.17}
+{'loss': 448.1083, 'learning_rate': 2e-05, 'epoch': 0.18}
+{'loss': 506.6299, 'learning_rate': 2e-05, 'epoch': 0.19}
+{'loss': 432.8912, 'learning_rate': 2e-05, 'epoch': 0.2}
+{'loss': 449.625, 'learning_rate': 2e-05, 'epoch': 0.21}
+{'loss': 588.6897, 'learning_rate': 2e-05, 'epoch': 0.22}
+{'loss': 405.68, 'learning_rate': 2e-05, 'epoch': 0.23}
+{'loss': 516.0303, 'learning_rate': 2e-05, 'epoch': 0.24}
+{'loss': 401.7353, 'learning_rate': 2e-05, 'epoch': 0.25}
+{'loss': 399.9238, 'learning_rate': 2e-05, 'epoch': 0.26}
+{'loss': 351.5388, 'learning_rate': 2e-05, 'epoch': 0.27}
+{'loss': 401.7846, 'learning_rate': 2e-05, 'epoch': 0.28}
+{'loss': 385.5488, 'learning_rate': 2e-05, 'epoch': 0.29}
+{'loss': 378.391, 'learning_rate': 2e-05, 'epoch': 0.3}
+{'loss': 446.2048, 'learning_rate': 2e-05, 'epoch': 0.31}
+{'loss': 354.5841, 'learning_rate': 2e-05, 'epoch': 0.32}
+{'loss': 370.7014, 'learning_rate': 2e-05, 'epoch': 0.33}
+{'loss': 397.8215, 'learning_rate': 2e-05, 'epoch': 0.34}
+{'loss': 400.33, 'learning_rate': 2e-05, 'epoch': 0.35}
+{'loss': 379.8638, 'learning_rate': 2e-05, 'epoch': 0.36}
+{'loss': 282.1729, 'learning_rate': 2e-05, 'epoch': 0.37}
+{'loss': 288.713, 'learning_rate': 2e-05, 'epoch': 0.38}
+{'loss': 362.3285, 'learning_rate': 2e-05, 'epoch': 0.39}
+{'loss': 345.252, 'learning_rate': 2e-05, 'epoch': 0.4}
+  return fn(*args, **kwargs)                                                                              
+{'eval_loss': 349.94061279296875, 'eval_runtime': 91.4064, 'eval_samples_per_second': 17.504, 'eval_steps_per_second': 1.094, 'epoch': 0.4}
+                                                                                                          
+{'loss': 391.5078, 'learning_rate': 2e-05, 'epoch': 0.41}
+{'loss': 332.1484, 'learning_rate': 2e-05, 'epoch': 0.42}
+{'loss': 352.9743, 'learning_rate': 2e-05, 'epoch': 0.43}
+{'loss': 314.8037, 'learning_rate': 2e-05, 'epoch': 0.44}
+{'loss': 386.3977, 'learning_rate': 2e-05, 'epoch': 0.45}
+{'loss': 359.9244, 'learning_rate': 2e-05, 'epoch': 0.46}
+{'loss': 376.9478, 'learning_rate': 2e-05, 'epoch': 0.47}
+{'loss': 307.694, 'learning_rate': 2e-05, 'epoch': 0.48}
+{'loss': 359.4525, 'learning_rate': 2e-05, 'epoch': 0.49}
+{'loss': 319.51, 'learning_rate': 2e-05, 'epoch': 0.5}
+{'loss': 349.2659, 'learning_rate': 2e-05, 'epoch': 0.51}
+{'loss': 332.9238, 'learning_rate': 2e-05, 'epoch': 0.52}
+{'loss': 324.871, 'learning_rate': 2e-05, 'epoch': 0.53}
+{'loss': 305.993, 'learning_rate': 2e-05, 'epoch': 0.54}
+{'loss': 334.1832, 'learning_rate': 2e-05, 'epoch': 0.55}
+{'loss': 393.5037, 'learning_rate': 2e-05, 'epoch': 0.56}
+{'loss': 453.1027, 'learning_rate': 2e-05, 'epoch': 0.57}
+{'loss': 306.5744, 'learning_rate': 2e-05, 'epoch': 0.58}
+{'loss': 343.3282, 'learning_rate': 2e-05, 'epoch': 0.59}
+{'loss': 367.3992, 'learning_rate': 2e-05, 'epoch': 0.6}
+{'loss': 252.5841, 'learning_rate': 2e-05, 'epoch': 0.61}
+{'loss': 332.6815, 'learning_rate': 2e-05, 'epoch': 0.62}
+{'loss': 260.7815, 'learning_rate': 2e-05, 'epoch': 0.63}
+{'loss': 322.11, 'learning_rate': 2e-05, 'epoch': 0.64}
+{'loss': 316.3943, 'learning_rate': 2e-05, 'epoch': 0.65}
+{'loss': 282.9459, 'learning_rate': 2e-05, 'epoch': 0.66}
+{'loss': 389.4407, 'learning_rate': 2e-05, 'epoch': 0.67}
+{'loss': 266.971, 'learning_rate': 2e-05, 'epoch': 0.68}
+{'loss': 309.2886, 'learning_rate': 2e-05, 'epoch': 0.69}
+{'loss': 248.2584, 'learning_rate': 2e-05, 'epoch': 0.7}
+{'loss': 311.3903, 'learning_rate': 2e-05, 'epoch': 0.71}
+{'loss': 281.4355, 'learning_rate': 2e-05, 'epoch': 0.72}
+{'loss': 348.8126, 'learning_rate': 2e-05, 'epoch': 0.73}
+{'loss': 335.4451, 'learning_rate': 2e-05, 'epoch': 0.74}
+{'loss': 252.2587, 'learning_rate': 2e-05, 'epoch': 0.75}
+{'loss': 280.6372, 'learning_rate': 2e-05, 'epoch': 0.76}
+{'loss': 276.1567, 'learning_rate': 2e-05, 'epoch': 0.77}
+{'loss': 295.8489, 'learning_rate': 2e-05, 'epoch': 0.78}
+{'loss': 258.2193, 'learning_rate': 2e-05, 'epoch': 0.79}
+{'loss': 303.292, 'learning_rate': 2e-05, 'epoch': 0.8}
+  return fn(*args, **kwargs)                                                                              
+{'eval_loss': 290.92047119140625, 'eval_runtime': 91.4002, 'eval_samples_per_second': 17.505, 'eval_steps_per_second': 1.094, 'epoch': 0.8}
+                                                                                                          
+{'loss': 292.2969, 'learning_rate': 2e-05, 'epoch': 0.81}
+{'loss': 292.5847, 'learning_rate': 2e-05, 'epoch': 0.82}
+{'loss': 287.2512, 'learning_rate': 2e-05, 'epoch': 0.83}
+{'loss': 274.7219, 'learning_rate': 2e-05, 'epoch': 0.84}
+{'loss': 259.4895, 'learning_rate': 2e-05, 'epoch': 0.85}
+{'loss': 308.7735, 'learning_rate': 2e-05, 'epoch': 0.86}
+{'loss': 358.5125, 'learning_rate': 2e-05, 'epoch': 0.87}
+{'loss': 228.2059, 'learning_rate': 2e-05, 'epoch': 0.88}
+{'loss': 221.5, 'learning_rate': 2e-05, 'epoch': 0.89}
+{'loss': 256.0698, 'learning_rate': 2e-05, 'epoch': 0.9}
+{'loss': 280.6437, 'learning_rate': 2e-05, 'epoch': 0.91}
+{'loss': 265.0312, 'learning_rate': 2e-05, 'epoch': 0.92}
+{'loss': 276.1062, 'learning_rate': 2e-05, 'epoch': 0.93}
+{'loss': 332.605, 'learning_rate': 2e-05, 'epoch': 0.94}
+{'loss': 241.6841, 'learning_rate': 2e-05, 'epoch': 0.95}
+{'loss': 290.7553, 'learning_rate': 2e-05, 'epoch': 0.96}
+{'loss': 242.8853, 'learning_rate': 2e-05, 'epoch': 0.97}
+{'loss': 267.5484, 'learning_rate': 2e-05, 'epoch': 0.98}
+{'loss': 263.2162, 'learning_rate': 2e-05, 'epoch': 0.99}
+{'loss': 226.8806, 'learning_rate': 2e-05, 'epoch': 1.0}
+{'loss': 175.9632, 'learning_rate': 2e-05, 'epoch': 1.01}
+{'loss': 293.8885, 'learning_rate': 2e-05, 'epoch': 1.02}
+{'loss': 264.4063, 'learning_rate': 2e-05, 'epoch': 1.03}
+{'loss': 268.8441, 'learning_rate': 2e-05, 'epoch': 1.04}
+{'loss': 251.1409, 'learning_rate': 2e-05, 'epoch': 1.05}
+{'loss': 222.4796, 'learning_rate': 2e-05, 'epoch': 1.06}
+{'loss': 259.7393, 'learning_rate': 2e-05, 'epoch': 1.07}
+{'loss': 247.0995, 'learning_rate': 2e-05, 'epoch': 1.08}
+{'loss': 228.6188, 'learning_rate': 2e-05, 'epoch': 1.09}
+{'loss': 238.4029, 'learning_rate': 2e-05, 'epoch': 1.1}
+{'loss': 249.7835, 'learning_rate': 2e-05, 'epoch': 1.11}
+{'loss': 255.0745, 'learning_rate': 2e-05, 'epoch': 1.12}
+{'loss': 281.3386, 'learning_rate': 2e-05, 'epoch': 1.13}
+{'loss': 258.1128, 'learning_rate': 2e-05, 'epoch': 1.14}
+{'loss': 258.487, 'learning_rate': 2e-05, 'epoch': 1.15}
+{'loss': 252.1913, 'learning_rate': 2e-05, 'epoch': 1.16}
+{'loss': 222.6366, 'learning_rate': 2e-05, 'epoch': 1.17}
+{'loss': 247.7612, 'learning_rate': 2e-05, 'epoch': 1.18}
+{'loss': 212.2664, 'learning_rate': 2e-05, 'epoch': 1.19}
+{'loss': 260.1885, 'learning_rate': 2e-05, 'epoch': 1.2}
+  return fn(*args, **kwargs)                                                                              
+{'eval_loss': 268.6912841796875, 'eval_runtime': 91.4506, 'eval_samples_per_second': 17.496, 'eval_steps_per_second': 1.093, 'epoch': 1.2}
+                                                                                                          
+{'loss': 225.2303, 'learning_rate': 2e-05, 'epoch': 1.21}
+{'loss': 208.434, 'learning_rate': 2e-05, 'epoch': 1.22}
+{'loss': 260.5778, 'learning_rate': 2e-05, 'epoch': 1.23}
+{'loss': 256.1859, 'learning_rate': 2e-05, 'epoch': 1.24}
+{'loss': 225.3799, 'learning_rate': 2e-05, 'epoch': 1.25}
+{'loss': 242.6659, 'learning_rate': 2e-05, 'epoch': 1.26}
+{'loss': 218.2521, 'learning_rate': 2e-05, 'epoch': 1.27}
+{'loss': 237.711, 'learning_rate': 2e-05, 'epoch': 1.28}
+{'loss': 228.6392, 'learning_rate': 2e-05, 'epoch': 1.29}
+{'loss': 257.0567, 'learning_rate': 2e-05, 'epoch': 1.3}
+{'loss': 225.4318, 'learning_rate': 2e-05, 'epoch': 1.31}
+{'loss': 255.6358, 'learning_rate': 2e-05, 'epoch': 1.32}
+{'loss': 243.6262, 'learning_rate': 2e-05, 'epoch': 1.33}
+{'loss': 235.9305, 'learning_rate': 2e-05, 'epoch': 1.34}
+{'loss': 238.0324, 'learning_rate': 2e-05, 'epoch': 1.35}
+{'loss': 239.2688, 'learning_rate': 2e-05, 'epoch': 1.36}
+{'loss': 234.8799, 'learning_rate': 2e-05, 'epoch': 1.37}
+{'loss': 249.6847, 'learning_rate': 2e-05, 'epoch': 1.38}
+{'loss': 259.0303, 'learning_rate': 2e-05, 'epoch': 1.39}
+{'loss': 230.0663, 'learning_rate': 2e-05, 'epoch': 1.4}
+{'loss': 312.8887, 'learning_rate': 2e-05, 'epoch': 1.41}
+{'loss': 214.6919, 'learning_rate': 2e-05, 'epoch': 1.42}
+{'loss': 204.0403, 'learning_rate': 2e-05, 'epoch': 1.43}
+{'loss': 219.8406, 'learning_rate': 2e-05, 'epoch': 1.44}
+{'loss': 229.476, 'learning_rate': 2e-05, 'epoch': 1.45}
+{'loss': 222.8145, 'learning_rate': 2e-05, 'epoch': 1.46}
+{'loss': 257.3806, 'learning_rate': 2e-05, 'epoch': 1.47}
+{'loss': 206.661, 'learning_rate': 2e-05, 'epoch': 1.48}
+{'loss': 244.2539, 'learning_rate': 2e-05, 'epoch': 1.49}
+{'loss': 219.9999, 'learning_rate': 2e-05, 'epoch': 1.5}
+{'loss': 186.9665, 'learning_rate': 2e-05, 'epoch': 1.51}
+{'loss': 246.9571, 'learning_rate': 2e-05, 'epoch': 1.52}
+{'loss': 296.5907, 'learning_rate': 2e-05, 'epoch': 1.53}
+{'loss': 235.987, 'learning_rate': 2e-05, 'epoch': 1.54}
+{'loss': 232.2841, 'learning_rate': 2e-05, 'epoch': 1.55}
+{'loss': 257.2687, 'learning_rate': 2e-05, 'epoch': 1.56}
+{'loss': 229.2959, 'learning_rate': 2e-05, 'epoch': 1.57}
+{'loss': 204.7547, 'learning_rate': 2e-05, 'epoch': 1.58}
+{'loss': 229.0461, 'learning_rate': 2e-05, 'epoch': 1.59}
+{'loss': 208.6121, 'learning_rate': 2e-05, 'epoch': 1.6}
+  return fn(*args, **kwargs)                                                                              
+{'eval_loss': 251.8457794189453, 'eval_runtime': 91.4542, 'eval_samples_per_second': 17.495, 'eval_steps_per_second': 1.093, 'epoch': 1.6}
+                                                                                                          
+{'loss': 213.7581, 'learning_rate': 2e-05, 'epoch': 1.61}
+{'loss': 250.1387, 'learning_rate': 2e-05, 'epoch': 1.62}
+{'loss': 216.0783, 'learning_rate': 2e-05, 'epoch': 1.63}
+{'loss': 223.1108, 'learning_rate': 2e-05, 'epoch': 1.64}
+{'loss': 223.4337, 'learning_rate': 2e-05, 'epoch': 1.65}
+{'loss': 216.0298, 'learning_rate': 2e-05, 'epoch': 1.66}
+{'loss': 210.1397, 'learning_rate': 2e-05, 'epoch': 1.67}
+{'loss': 255.6102, 'learning_rate': 2e-05, 'epoch': 1.68}
+{'loss': 206.8196, 'learning_rate': 2e-05, 'epoch': 1.69}
+{'loss': 225.3016, 'learning_rate': 2e-05, 'epoch': 1.7}
+{'loss': 204.423, 'learning_rate': 2e-05, 'epoch': 1.71}
+{'loss': 200.3793, 'learning_rate': 2e-05, 'epoch': 1.72}
+{'loss': 254.3165, 'learning_rate': 2e-05, 'epoch': 1.73}
+{'loss': 228.116, 'learning_rate': 2e-05, 'epoch': 1.74}
+{'loss': 215.9781, 'learning_rate': 2e-05, 'epoch': 1.75}
+{'loss': 240.427, 'learning_rate': 2e-05, 'epoch': 1.76}
+{'loss': 285.4974, 'learning_rate': 2e-05, 'epoch': 1.77}
+{'loss': 241.3725, 'learning_rate': 2e-05, 'epoch': 1.78}
+{'loss': 208.2607, 'learning_rate': 2e-05, 'epoch': 1.79}
+{'loss': 189.7236, 'learning_rate': 2e-05, 'epoch': 1.8}
+{'loss': 251.2979, 'learning_rate': 2e-05, 'epoch': 1.81}
+{'loss': 221.9034, 'learning_rate': 2e-05, 'epoch': 1.82}
+{'loss': 212.9315, 'learning_rate': 2e-05, 'epoch': 1.83}
+{'loss': 269.8028, 'learning_rate': 2e-05, 'epoch': 1.84}
+{'loss': 234.7929, 'learning_rate': 2e-05, 'epoch': 1.85}
+{'loss': 232.632, 'learning_rate': 2e-05, 'epoch': 1.86}
+{'loss': 234.5318, 'learning_rate': 2e-05, 'epoch': 1.87}
+{'loss': 217.2522, 'learning_rate': 2e-05, 'epoch': 1.88}
+{'loss': 212.185, 'learning_rate': 2e-05, 'epoch': 1.89}
+{'loss': 193.634, 'learning_rate': 2e-05, 'epoch': 1.9}
+{'loss': 194.4389, 'learning_rate': 2e-05, 'epoch': 1.91}
+{'loss': 239.7684, 'learning_rate': 2e-05, 'epoch': 1.92}
+{'loss': 214.6728, 'learning_rate': 2e-05, 'epoch': 1.93}
+{'loss': 238.8546, 'learning_rate': 2e-05, 'epoch': 1.94}
+{'loss': 252.4766, 'learning_rate': 2e-05, 'epoch': 1.95}
+{'loss': 213.9979, 'learning_rate': 2e-05, 'epoch': 1.96}
+{'loss': 226.8183, 'learning_rate': 2e-05, 'epoch': 1.97}
+{'loss': 178.3002, 'learning_rate': 2e-05, 'epoch': 1.98}
+{'loss': 226.1548, 'learning_rate': 2e-05, 'epoch': 1.99}
+{'loss': 197.3228, 'learning_rate': 2e-05, 'epoch': 2.0}
+  return fn(*args, **kwargs)                                                                              
+{'eval_loss': 234.64894104003906, 'eval_runtime': 91.475, 'eval_samples_per_second': 17.491, 'eval_steps_per_second': 1.093, 'epoch': 2.0}
+                                                                                                          
+{'loss': 177.7395, 'learning_rate': 2e-05, 'epoch': 2.01}
+{'loss': 207.6397, 'learning_rate': 2e-05, 'epoch': 2.02}
+{'loss': 234.7, 'learning_rate': 2e-05, 'epoch': 2.03}
+{'loss': 161.9427, 'learning_rate': 2e-05, 'epoch': 2.04}
+{'loss': 152.5065, 'learning_rate': 2e-05, 'epoch': 2.05}
+{'loss': 209.7177, 'learning_rate': 2e-05, 'epoch': 2.06}
+{'loss': 180.4935, 'learning_rate': 2e-05, 'epoch': 2.07}
+{'loss': 189.8789, 'learning_rate': 2e-05, 'epoch': 2.08}
+{'loss': 179.2588, 'learning_rate': 2e-05, 'epoch': 2.09}
+{'loss': 199.4266, 'learning_rate': 2e-05, 'epoch': 2.1}
+{'loss': 217.2014, 'learning_rate': 2e-05, 'epoch': 2.11}
+{'loss': 194.8955, 'learning_rate': 2e-05, 'epoch': 2.12}
+{'loss': 199.1684, 'learning_rate': 2e-05, 'epoch': 2.13}
+{'loss': 184.8403, 'learning_rate': 2e-05, 'epoch': 2.14}
+{'loss': 181.7052, 'learning_rate': 2e-05, 'epoch': 2.15}
+{'loss': 178.4005, 'learning_rate': 2e-05, 'epoch': 2.16}
+{'loss': 166.0668, 'learning_rate': 2e-05, 'epoch': 2.17}
+{'loss': 200.5868, 'learning_rate': 2e-05, 'epoch': 2.18}
+{'loss': 164.0996, 'learning_rate': 2e-05, 'epoch': 2.19}
+{'loss': 215.2086, 'learning_rate': 2e-05, 'epoch': 2.2}
+{'loss': 182.4766, 'learning_rate': 2e-05, 'epoch': 2.21}
+{'loss': 265.9225, 'learning_rate': 2e-05, 'epoch': 2.22}
+{'loss': 170.2314, 'learning_rate': 2e-05, 'epoch': 2.23}
+{'loss': 168.8915, 'learning_rate': 2e-05, 'epoch': 2.24}
+{'loss': 202.6422, 'learning_rate': 2e-05, 'epoch': 2.25}
+{'loss': 193.6124, 'learning_rate': 2e-05, 'epoch': 2.26}
+{'loss': 191.7759, 'learning_rate': 2e-05, 'epoch': 2.27}
+{'loss': 172.5128, 'learning_rate': 2e-05, 'epoch': 2.28}
+{'loss': 174.589, 'learning_rate': 2e-05, 'epoch': 2.29}
+{'loss': 190.2146, 'learning_rate': 2e-05, 'epoch': 2.3}
+{'loss': 206.5455, 'learning_rate': 2e-05, 'epoch': 2.31}
+{'loss': 212.3613, 'learning_rate': 2e-05, 'epoch': 2.32}
+{'loss': 196.8155, 'learning_rate': 2e-05, 'epoch': 2.33}
+{'loss': 175.7169, 'learning_rate': 2e-05, 'epoch': 2.34}
+{'loss': 246.1433, 'learning_rate': 2e-05, 'epoch': 2.35}
+{'loss': 273.7065, 'learning_rate': 2e-05, 'epoch': 2.36}
+{'loss': 158.33, 'learning_rate': 2e-05, 'epoch': 2.37}
+{'loss': 159.6902, 'learning_rate': 2e-05, 'epoch': 2.38}
+{'loss': 260.6693, 'learning_rate': 2e-05, 'epoch': 2.39}
+{'loss': 191.4345, 'learning_rate': 2e-05, 'epoch': 2.4}
+  return fn(*args, **kwargs)                                                                              
+{'eval_loss': 228.75088500976562, 'eval_runtime': 91.7222, 'eval_samples_per_second': 17.444, 'eval_steps_per_second': 1.09, 'epoch': 2.4}
+                                                                                                          
+{'loss': 194.6978, 'learning_rate': 2e-05, 'epoch': 2.41}
+{'loss': 170.113, 'learning_rate': 2e-05, 'epoch': 2.42}
+{'loss': 206.9311, 'learning_rate': 2e-05, 'epoch': 2.43}
+{'loss': 195.2319, 'learning_rate': 2e-05, 'epoch': 2.44}
+{'loss': 163.4541, 'learning_rate': 2e-05, 'epoch': 2.45}
+{'loss': 194.2114, 'learning_rate': 2e-05, 'epoch': 2.46}
+{'loss': 204.1492, 'learning_rate': 2e-05, 'epoch': 2.47}
+{'loss': 202.1168, 'learning_rate': 2e-05, 'epoch': 2.48}
+{'loss': 188.9232, 'learning_rate': 2e-05, 'epoch': 2.49}
+{'loss': 183.1904, 'learning_rate': 2e-05, 'epoch': 2.5}
+{'loss': 171.6944, 'learning_rate': 2e-05, 'epoch': 2.51}
+{'loss': 218.1628, 'learning_rate': 2e-05, 'epoch': 2.52}
+{'loss': 178.1614, 'learning_rate': 2e-05, 'epoch': 2.53}
+{'loss': 175.8137, 'learning_rate': 2e-05, 'epoch': 2.54}
+{'loss': 176.3016, 'learning_rate': 2e-05, 'epoch': 2.55}
+{'loss': 195.611, 'learning_rate': 2e-05, 'epoch': 2.56}
+{'loss': 154.6473, 'learning_rate': 2e-05, 'epoch': 2.57}
+{'loss': 175.7625, 'learning_rate': 2e-05, 'epoch': 2.58}
+{'loss': 180.9702, 'learning_rate': 2e-05, 'epoch': 2.59}
+{'loss': 172.006, 'learning_rate': 2e-05, 'epoch': 2.6}
+{'loss': 166.616, 'learning_rate': 2e-05, 'epoch': 2.61}
+{'loss': 205.9087, 'learning_rate': 2e-05, 'epoch': 2.62}
+{'loss': 195.5401, 'learning_rate': 2e-05, 'epoch': 2.63}
+{'loss': 182.7327, 'learning_rate': 2e-05, 'epoch': 2.64}
+{'loss': 187.6268, 'learning_rate': 2e-05, 'epoch': 2.65}
+{'loss': 150.9506, 'learning_rate': 2e-05, 'epoch': 2.66}
+{'loss': 187.1612, 'learning_rate': 2e-05, 'epoch': 2.67}
+{'loss': 199.4861, 'learning_rate': 2e-05, 'epoch': 2.68}
+{'loss': 197.6736, 'learning_rate': 2e-05, 'epoch': 2.69}
+{'loss': 204.7334, 'learning_rate': 2e-05, 'epoch': 2.7}
+{'loss': 186.2923, 'learning_rate': 2e-05, 'epoch': 2.71}
+{'loss': 191.4558, 'learning_rate': 2e-05, 'epoch': 2.72}
+{'loss': 195.1405, 'learning_rate': 2e-05, 'epoch': 2.73}
+{'loss': 193.3551, 'learning_rate': 2e-05, 'epoch': 2.74}
+{'loss': 191.0934, 'learning_rate': 2e-05, 'epoch': 2.75}
+{'loss': 181.389, 'learning_rate': 2e-05, 'epoch': 2.76}
+{'loss': 175.3716, 'learning_rate': 2e-05, 'epoch': 2.77}
+{'loss': 172.3194, 'learning_rate': 2e-05, 'epoch': 2.78}
+{'loss': 210.0355, 'learning_rate': 2e-05, 'epoch': 2.79}
+{'loss': 151.5427, 'learning_rate': 2e-05, 'epoch': 2.8}
+  return fn(*args, **kwargs)                                                                              
+{'eval_loss': 220.8135528564453, 'eval_runtime': 91.7827, 'eval_samples_per_second': 17.432, 'eval_steps_per_second': 1.09, 'epoch': 2.8}
+                                                                                                          
+{'loss': 216.8114, 'learning_rate': 2e-05, 'epoch': 2.81}
+{'loss': 204.5617, 'learning_rate': 2e-05, 'epoch': 2.82}
+{'loss': 170.5889, 'learning_rate': 2e-05, 'epoch': 2.83}
+{'loss': 207.3868, 'learning_rate': 2e-05, 'epoch': 2.84}
+{'loss': 181.0243, 'learning_rate': 2e-05, 'epoch': 2.85}
+{'loss': 181.3605, 'learning_rate': 2e-05, 'epoch': 2.86}
+{'loss': 151.9068, 'learning_rate': 2e-05, 'epoch': 2.87}
+{'loss': 181.1088, 'learning_rate': 2e-05, 'epoch': 2.88}
+{'loss': 168.5044, 'learning_rate': 2e-05, 'epoch': 2.89}
+{'loss': 169.9193, 'learning_rate': 2e-05, 'epoch': 2.9}
+{'loss': 163.9978, 'learning_rate': 2e-05, 'epoch': 2.91}
+{'loss': 160.5624, 'learning_rate': 2e-05, 'epoch': 2.92}
+{'loss': 168.6609, 'learning_rate': 2e-05, 'epoch': 2.93}
+{'loss': 176.1334, 'learning_rate': 2e-05, 'epoch': 2.94}
+{'loss': 183.0596, 'learning_rate': 2e-05, 'epoch': 2.95}
+{'loss': 137.6645, 'learning_rate': 2e-05, 'epoch': 2.96}
+{'loss': 207.355, 'learning_rate': 2e-05, 'epoch': 2.97}
+{'loss': 142.0311, 'learning_rate': 2e-05, 'epoch': 2.98}
+{'loss': 220.0689, 'learning_rate': 2e-05, 'epoch': 2.99}
+{'loss': 188.8789, 'learning_rate': 2e-05, 'epoch': 3.0}
+{'loss': 125.5, 'learning_rate': 2e-05, 'epoch': 3.01}
+{'loss': 135.3729, 'learning_rate': 2e-05, 'epoch': 3.02}
+{'loss': 138.8647, 'learning_rate': 2e-05, 'epoch': 3.03}
+{'loss': 179.7612, 'learning_rate': 2e-05, 'epoch': 3.04}
+{'loss': 155.7191, 'learning_rate': 2e-05, 'epoch': 3.05}
+{'loss': 142.4713, 'learning_rate': 2e-05, 'epoch': 3.06}
+{'loss': 153.5572, 'learning_rate': 2e-05, 'epoch': 3.07}
+{'loss': 158.0588, 'learning_rate': 2e-05, 'epoch': 3.08}
+{'loss': 188.1761, 'learning_rate': 2e-05, 'epoch': 3.09}
+{'loss': 149.4881, 'learning_rate': 2e-05, 'epoch': 3.1}
+{'loss': 143.047, 'learning_rate': 2e-05, 'epoch': 3.11}
+{'loss': 149.3641, 'learning_rate': 2e-05, 'epoch': 3.12}
+{'loss': 157.5219, 'learning_rate': 2e-05, 'epoch': 3.13}
+{'loss': 160.7546, 'learning_rate': 2e-05, 'epoch': 3.14}
+{'loss': 204.2928, 'learning_rate': 2e-05, 'epoch': 3.15}
+{'loss': 162.4894, 'learning_rate': 2e-05, 'epoch': 3.16}
+{'loss': 142.8077, 'learning_rate': 2e-05, 'epoch': 3.17}
+{'loss': 189.9061, 'learning_rate': 2e-05, 'epoch': 3.18}
+{'loss': 160.648, 'learning_rate': 2e-05, 'epoch': 3.19}
+{'loss': 195.4514, 'learning_rate': 2e-05, 'epoch': 3.2}
+  return fn(*args, **kwargs)                                                                              
+{'eval_loss': 214.12860107421875, 'eval_runtime': 91.6917, 'eval_samples_per_second': 17.45, 'eval_steps_per_second': 1.091, 'epoch': 3.2}
+                                                                                                          
+{'loss': 165.5777, 'learning_rate': 2e-05, 'epoch': 3.21}
+{'loss': 157.4871, 'learning_rate': 2e-05, 'epoch': 3.22}
+{'loss': 156.9734, 'learning_rate': 2e-05, 'epoch': 3.23}
+{'loss': 171.0179, 'learning_rate': 2e-05, 'epoch': 3.24}
+{'loss': 158.758, 'learning_rate': 2e-05, 'epoch': 3.25}
+{'loss': 180.9356, 'learning_rate': 2e-05, 'epoch': 3.26}
+{'loss': 132.6712, 'learning_rate': 2e-05, 'epoch': 3.27}
+{'loss': 142.0085, 'learning_rate': 2e-05, 'epoch': 3.28}
+{'loss': 122.8798, 'learning_rate': 2e-05, 'epoch': 3.29}
+{'loss': 166.3639, 'learning_rate': 2e-05, 'epoch': 3.3}
+{'loss': 126.7612, 'learning_rate': 2e-05, 'epoch': 3.31}
+{'loss': 188.9171, 'learning_rate': 2e-05, 'epoch': 3.32}
+{'loss': 151.0345, 'learning_rate': 2e-05, 'epoch': 3.33}
+{'loss': 159.7177, 'learning_rate': 2e-05, 'epoch': 3.34}
+{'loss': 150.0818, 'learning_rate': 2e-05, 'epoch': 3.35}
+{'loss': 162.671, 'learning_rate': 2e-05, 'epoch': 3.36}
+{'loss': 129.0101, 'learning_rate': 2e-05, 'epoch': 3.37}
+{'loss': 187.8155, 'learning_rate': 2e-05, 'epoch': 3.38}
+{'loss': 136.3083, 'learning_rate': 2e-05, 'epoch': 3.39}
+{'loss': 139.6308, 'learning_rate': 2e-05, 'epoch': 3.4}
+{'loss': 158.1506, 'learning_rate': 2e-05, 'epoch': 3.41}
+{'loss': 152.7077, 'learning_rate': 2e-05, 'epoch': 3.42}
+{'loss': 173.8158, 'learning_rate': 2e-05, 'epoch': 3.43}
+{'loss': 128.805, 'learning_rate': 2e-05, 'epoch': 3.44}
+{'loss': 152.3222, 'learning_rate': 2e-05, 'epoch': 3.45}
+{'loss': 163.3487, 'learning_rate': 2e-05, 'epoch': 3.46}
+{'loss': 169.0825, 'learning_rate': 2e-05, 'epoch': 3.47}
+{'loss': 156.5232, 'learning_rate': 2e-05, 'epoch': 3.48}
+{'loss': 188.6721, 'learning_rate': 2e-05, 'epoch': 3.49}
+{'loss': 201.6223, 'learning_rate': 2e-05, 'epoch': 3.5}
+{'loss': 294.7936, 'learning_rate': 2e-05, 'epoch': 3.51}
+{'loss': 155.2639, 'learning_rate': 2e-05, 'epoch': 3.52}
+{'loss': 148.6182, 'learning_rate': 2e-05, 'epoch': 3.53}
+{'loss': 207.8028, 'learning_rate': 2e-05, 'epoch': 3.54}
+{'loss': 163.1711, 'learning_rate': 2e-05, 'epoch': 3.55}
+{'loss': 162.5552, 'learning_rate': 2e-05, 'epoch': 3.56}
+{'loss': 167.8712, 'learning_rate': 2e-05, 'epoch': 3.57}
+{'loss': 155.6208, 'learning_rate': 2e-05, 'epoch': 3.58}
+{'loss': 178.2028, 'learning_rate': 2e-05, 'epoch': 3.59}
+{'loss': 174.9905, 'learning_rate': 2e-05, 'epoch': 3.6}
+  return fn(*args, **kwargs)                                                                              
+{'eval_loss': 211.39349365234375, 'eval_runtime': 91.4666, 'eval_samples_per_second': 17.493, 'eval_steps_per_second': 1.093, 'epoch': 3.6}
+                                                                                                          
+{'loss': 149.9229, 'learning_rate': 2e-05, 'epoch': 3.61}
+{'loss': 166.3071, 'learning_rate': 2e-05, 'epoch': 3.62}
+{'loss': 124.036, 'learning_rate': 2e-05, 'epoch': 3.63}
+{'loss': 172.7505, 'learning_rate': 2e-05, 'epoch': 3.64}
+{'loss': 148.0122, 'learning_rate': 2e-05, 'epoch': 3.65}
+{'loss': 185.0846, 'learning_rate': 2e-05, 'epoch': 3.66}
+{'loss': 153.4166, 'learning_rate': 2e-05, 'epoch': 3.67}
+{'loss': 144.2338, 'learning_rate': 2e-05, 'epoch': 3.68}
+{'loss': 158.5771, 'learning_rate': 2e-05, 'epoch': 3.69}
+{'loss': 163.8886, 'learning_rate': 2e-05, 'epoch': 3.7}
+{'loss': 151.2742, 'learning_rate': 2e-05, 'epoch': 3.71}
+{'loss': 169.2691, 'learning_rate': 2e-05, 'epoch': 3.72}
+{'loss': 125.0493, 'learning_rate': 2e-05, 'epoch': 3.73}
+{'loss': 144.3527, 'learning_rate': 2e-05, 'epoch': 3.74}
+{'loss': 210.2006, 'learning_rate': 2e-05, 'epoch': 3.75}
+{'loss': 162.8882, 'learning_rate': 2e-05, 'epoch': 3.76}
+{'loss': 163.0425, 'learning_rate': 2e-05, 'epoch': 3.77}
+{'loss': 144.6404, 'learning_rate': 2e-05, 'epoch': 3.78}
+{'loss': 169.7259, 'learning_rate': 2e-05, 'epoch': 3.79}
+{'loss': 117.309, 'learning_rate': 2e-05, 'epoch': 3.8}
+{'loss': 179.2435, 'learning_rate': 2e-05, 'epoch': 3.81}
+{'loss': 156.4202, 'learning_rate': 2e-05, 'epoch': 3.82}
+{'loss': 212.8055, 'learning_rate': 2e-05, 'epoch': 3.83}
+{'loss': 130.1424, 'learning_rate': 2e-05, 'epoch': 3.84}
+{'loss': 143.6542, 'learning_rate': 2e-05, 'epoch': 3.85}
+{'loss': 193.6444, 'learning_rate': 2e-05, 'epoch': 3.86}
+{'loss': 176.6723, 'learning_rate': 2e-05, 'epoch': 3.87}
+{'loss': 150.6032, 'learning_rate': 2e-05, 'epoch': 3.88}
+{'loss': 146.1843, 'learning_rate': 2e-05, 'epoch': 3.89}
+{'loss': 152.6586, 'learning_rate': 2e-05, 'epoch': 3.9}
+{'loss': 162.0343, 'learning_rate': 2e-05, 'epoch': 3.91}
+{'loss': 157.9043, 'learning_rate': 2e-05, 'epoch': 3.92}
+{'loss': 140.6674, 'learning_rate': 2e-05, 'epoch': 3.93}
+{'loss': 186.9754, 'learning_rate': 2e-05, 'epoch': 3.94}
+{'loss': 157.4324, 'learning_rate': 2e-05, 'epoch': 3.95}
+{'loss': 151.3968, 'learning_rate': 2e-05, 'epoch': 3.96}
+{'loss': 182.1434, 'learning_rate': 2e-05, 'epoch': 3.97}
+{'loss': 164.4491, 'learning_rate': 2e-05, 'epoch': 3.98}
+{'loss': 151.253, 'learning_rate': 2e-05, 'epoch': 3.99}
+{'loss': 136.0101, 'learning_rate': 2e-05, 'epoch': 4.0}
+                                                                                                          
+{'eval_loss': 206.38494873046875, 'eval_runtime': 91.5142, 'eval_samples_per_second': 17.484, 'eval_steps_per_second': 1.093, 'epoch': 4.0}
+{'train_runtime': 5141.9779, 'train_samples_per_second': 4.979, 'train_steps_per_second': 0.078, 'train_loss': 280.0906865310669, 'epoch': 4.0}

wandb/run-20250408_142458-wlfced8t/files/requirements.txt

@@ -0,0 +1,142 @@
+setuptools==58.1.0
+pip==23.0.1
+wcwidth==0.2.13
+triton==3.2.0
+sqlitedict==2.1.0
+sentencepiece==0.2.0
+pytz==2025.2
+py-cpuinfo==9.0.0
+pure_eval==0.2.3
+ptyprocess==0.7.0
+nvidia-cusparselt-cu12==0.6.2
+mpmath==1.3.0
+hjson==3.1.0
+Fraction==2.2.0
+antlr4-python3-runtime==4.9.3
+zstandard==0.23.0
+zipp==3.21.0
+xxhash==3.5.0
+urllib3==2.3.0
+tzdata==2025.2
+typing_extensions==4.13.1
+traitlets==5.14.3
+tqdm==4.67.1
+tornado==6.4.2
+threadpoolctl==3.6.0
+tcolorpy==0.1.7
+tabulate==0.9.0
+sympy==1.13.1
+smmap==5.0.2
+six==1.17.0
+setproctitle==1.3.5
+safetensors==0.5.3
+regex==2024.11.6
+pyzmq==26.4.0
+PyYAML==6.0.2
+Pygments==2.19.1
+pycountry==24.6.1
+pyarrow==19.0.1
+psutil==7.0.0
+protobuf==5.29.4
+propcache==0.3.1
+prompt_toolkit==3.0.50
+portalocker==3.1.1
+platformdirs==4.3.7
+pexpect==4.9.0
+pathvalidate==3.2.3
+parso==0.8.4
+packaging==24.2
+nvidia-nvtx-cu12==12.4.127
+nvidia-nvjitlink-cu12==12.4.127
+nvidia-nccl-cu12==2.21.5
+nvidia-curand-cu12==10.3.5.147
+nvidia-cufft-cu12==11.2.1.3
+nvidia-cuda-runtime-cu12==12.4.127
+nvidia-cuda-nvrtc-cu12==12.4.127
+nvidia-cuda-cupti-cu12==12.4.127
+nvidia-cublas-cu12==12.4.5.8
+numpy==2.0.2
+ninja==1.11.1.4
+networkx==3.2.1
+nest-asyncio==1.6.0
+msgpack==1.1.0
+MarkupSafe==3.0.2
+lxml==5.3.2
+joblib==1.4.2
+idna==3.10
+fsspec==2024.12.0
+frozenlist==1.5.0
+filelock==3.18.0
+executing==2.2.0
+exceptiongroup==1.2.2
+eval_type_backport==0.2.2
+einops==0.8.1
+dill==0.3.8
+decorator==5.2.1
+debugpy==1.8.13
+colorama==0.4.6
+click==8.1.8
+charset-normalizer==3.4.1
+chardet==5.2.0
+certifi==2025.1.31
+attrs==25.3.0
+async-timeout==5.0.1
+asttokens==3.0.0
+annotated-types==0.7.0
+aiohappyeyeballs==2.6.1
+absl-py==2.2.2
+typing-inspection==0.4.0
+tqdm-multiprocess==0.0.11
+tensorboardX==2.6.2.2
+stack-data==0.6.3
+sentry-sdk==2.25.1
+scipy==1.13.1
+sacrebleu==2.5.1
+requests==2.32.3
+python-dateutil==2.9.0.post0
+pydantic_core==2.33.1
+omegaconf==2.3.0
+nvidia-cusparse-cu12==12.3.1.170
+nvidia-cudnn-cu12==9.1.0.70
+numexpr==2.10.2
+nltk==3.9.1
+multiprocess==0.70.16
+multidict==6.3.2
+mbstrdecoder==1.1.4
+matplotlib-inline==0.1.7
+jupyter_core==5.7.2
+jsonlines==4.0.0
+Jinja2==3.1.6
+jedi==0.19.2
+importlib_resources==6.5.2
+importlib_metadata==8.6.1
+gitdb==4.0.12
+docker-pycreds==0.4.0
+comm==0.2.2
+aiosignal==1.3.2
+yarl==1.19.0
+typepy==1.3.4
+scikit-learn==1.6.1
+rouge-score==0.1.2
+pydantic==2.11.3
+pandas==2.2.3
+nvidia-cusolver-cu12==11.6.1.9
+jupyter_client==8.6.3
+ipython==8.18.1
+huggingface-hub==0.30.2
+GitPython==3.1.44
+torch==2.6.0
+tokenizers==0.15.2
+ipykernel==6.29.5
+aiohttp==3.11.16
+transformers==4.37.0
+deepspeed==0.16.5
+DataProperty==1.1.0
+bitsandbytes==0.45.5
+accelerate==0.28.0
+tabledata==1.3.4
+peft==0.8.0
+datasets==3.5.0
+pytablewriter==1.2.1
+evaluate==0.4.3
+wandb==0.19.9

wandb/run-20250408_142458-wlfced8t/files/wandb-metadata.json

@@ -0,0 +1,108 @@
+{
+  "os": "Linux-5.15.0-135-generic-x86_64-with-glibc2.35",
+  "python": "CPython 3.9.21",
+  "startedAt": "2025-04-08T14:24:58.120164Z",
+  "args": [
+    "--local_rank=0",
+    "--model_name_or_path",
+    "../models/TinyLlama_v1.1/",
+    "--data_path",
+    "../data/datasets/Llama-2-7b-hf/mix_wiki_alpaca_8000.json",
+    "--model_max_length",
+    "1024",
+    "--output_dir",
+    "./ckpts/tinyllama_v1.1/int2-g128/",
+    "--logging_dir",
+    "./ckpts/tinyllama_v1.1/int2-g128/runs/",
+    "--num_train_epochs",
+    "4",
+    "--bf16",
+    "True",
+    "--seed",
+    "42",
+    "--per_device_train_batch_size",
+    "16",
+    "--per_device_eval_batch_size",
+    "16",
+    "--gradient_accumulation_steps",
+    "4",
+    "--gradient_checkpointing",
+    "True",
+    "--evaluation_strategy",
+    "steps",
+    "--eval_steps",
+    "40",
+    "--load_best_model_at_end",
+    "True",
+    "--save_strategy",
+    "steps",
+    "--save_steps",
+    "40",
+    "--save_total_limit",
+    "2",
+    "--learning_rate",
+    "2e-5",
+    "--lr_scheduler_type",
+    "constant",
+    "--weight_decay",
+    "0.",
+    "--logging_steps",
+    "1",
+    "--report_to",
+    "tensorboard",
+    "wandb",
+    "--deepspeed",
+    "config/zero.json",
+    "--bits",
+    "2",
+    "--quant_type",
+    "int2-asym",
+    "--q_group_size",
+    "128",
+    "--train_kd",
+    "True",
+    "--kd_loss_type",
+    "cakld",
+    "--max_train_samples",
+    "999999",
+    "--clip",
+    "../quantization/clip_cache/TinyLlama_v1.1/int2-g128.pt"
+  ],
+  "program": "/workspace/BitDistiller/train/train.py",
+  "codePath": "train/train.py",
+  "git": {
+    "remote": "[email protected]:BrownianNotion/BitDistiller.git",
+    "commit": "17dbabd8bb62295551e4244147374632ce7a2287"
+  },
+  "email": "[email protected]",
+  "root": "./ckpts/tinyllama_v1.1/int2-g128/",
+  "host": "816cf5acb821",
+  "executable": "/workspace/BitDistiller/BitDistillerVenv/bin/python3.9",
+  "codePathLocal": "train.py",
+  "cpu_count": 32,
+  "cpu_count_logical": 64,
+  "gpu": "NVIDIA A100-PCIE-40GB",
+  "gpu_count": 1,
+  "disk": {
+    "/": {
+      "total": "213674622976",
+      "used": "15038046208"
+    }
+  },
+  "memory": {
+    "total": "270095581184"
+  },
+  "cpu": {
+    "count": 32,
+    "countLogical": 64
+  },
+  "gpu_nvidia": [
+    {
+      "name": "NVIDIA A100-PCIE-40GB",
+      "memoryTotal": "42949672960",
+      "cudaCores": 6912,
+      "architecture": "Ampere"
+    }
+  ],
+  "cudaVersion": "12.8"
+}

wandb/run-20250408_142458-wlfced8t/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_wandb":{"runtime":5187},"train/train_loss":280.0906865310669,"eval/steps_per_second":1.093,"eval/runtime":91.5142,"train/total_flos":1.627150507573248e+17,"eval/samples_per_second":17.484,"train/train_steps_per_second":0.078,"train/epoch":4,"train/global_step":400,"_runtime":5183.891265556,"train/learning_rate":2e-05,"_timestamp":1.7441274820110373e+09,"train/loss":136.0101,"eval/loss":206.38494873046875,"train/train_samples_per_second":4.979,"_step":410,"train/train_runtime":5141.9779}

wandb/run-20250408_142458-wlfced8t/logs/debug-core.log

@@ -0,0 +1,15 @@
+{"time":"2025-04-08T14:24:57.609759289Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmp7r_i2fib/port-9420.txt","pid":9420,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false}
+{"time":"2025-04-08T14:24:57.626067579Z","level":"INFO","msg":"Will exit if parent process dies.","ppid":9420}
+{"time":"2025-04-08T14:24:57.626060178Z","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":33329,"Zone":""}}
+{"time":"2025-04-08T14:24:57.799461315Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:40764"}
+{"time":"2025-04-08T14:24:58.122235867Z","level":"INFO","msg":"handleInformInit: received","streamId":"wlfced8t","id":"127.0.0.1:40764"}
+{"time":"2025-04-08T14:24:58.380862144Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"wlfced8t","id":"127.0.0.1:40764"}
+{"time":"2025-04-08T15:51:25.956066962Z","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"wlfced8t","id":"127.0.0.1:40764"}
+{"time":"2025-04-08T15:51:25.956245656Z","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"wlfced8t","id":"127.0.0.1:40764"}
+{"time":"2025-04-08T15:51:26.954886538Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"127.0.0.1:40764"}
+{"time":"2025-04-08T15:51:26.954913569Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"127.0.0.1:40764"}
+{"time":"2025-04-08T15:51:26.954921999Z","level":"INFO","msg":"server is shutting down"}
+{"time":"2025-04-08T15:51:26.95495002Z","level":"INFO","msg":"connection: closing","id":"127.0.0.1:40764"}
+{"time":"2025-04-08T15:51:26.955030402Z","level":"INFO","msg":"connection: closed successfully","id":"127.0.0.1:40764"}
+{"time":"2025-04-08T15:51:26.955048092Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"127.0.0.1:40764"}
+{"time":"2025-04-08T15:51:26.955060042Z","level":"INFO","msg":"server is closed"}

wandb/run-20250408_142458-wlfced8t/logs/debug-internal.log

@@ -0,0 +1,16 @@
+{"time":"2025-04-08T14:24:58.122459682Z","level":"INFO","msg":"stream: starting","core version":"0.19.9","symlink path":"ckpts/tinyllama_v1.1/int2-g128/wandb/run-20250408_142458-wlfced8t/logs/debug-core.log"}
+{"time":"2025-04-08T14:24:58.380784982Z","level":"INFO","msg":"created new stream","id":"wlfced8t"}
+{"time":"2025-04-08T14:24:58.380854043Z","level":"INFO","msg":"stream: started","id":"wlfced8t"}
+{"time":"2025-04-08T14:24:58.380871954Z","level":"INFO","msg":"writer: Do: started","stream_id":"wlfced8t"}
+{"time":"2025-04-08T14:24:58.380921165Z","level":"INFO","msg":"sender: started","stream_id":"wlfced8t"}
+{"time":"2025-04-08T14:24:58.380954676Z","level":"INFO","msg":"handler: started","stream_id":"wlfced8t"}
+{"time":"2025-04-08T14:24:58.59144642Z","level":"INFO","msg":"Starting system monitor"}
+{"time":"2025-04-08T15:51:25.126053606Z","level":"INFO","msg":"Stopping system monitor"}
+{"time":"2025-04-08T15:51:25.126119508Z","level":"INFO","msg":"Stopped system monitor"}
+{"time":"2025-04-08T15:51:25.715478158Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2025-04-08T15:51:25.950965444Z","level":"INFO","msg":"handler: operation stats","stats":{}}
+{"time":"2025-04-08T15:51:25.956097933Z","level":"INFO","msg":"stream: closing","id":"wlfced8t"}
+{"time":"2025-04-08T15:51:25.956113153Z","level":"INFO","msg":"handler: closed","stream_id":"wlfced8t"}
+{"time":"2025-04-08T15:51:25.956122893Z","level":"INFO","msg":"writer: Close: closed","stream_id":"wlfced8t"}
+{"time":"2025-04-08T15:51:25.956130383Z","level":"INFO","msg":"sender: closed","stream_id":"wlfced8t"}
+{"time":"2025-04-08T15:51:25.956237656Z","level":"INFO","msg":"stream: closed","id":"wlfced8t"}

wandb/run-20250408_142458-wlfced8t/logs/debug.log

@@ -0,0 +1,30 @@
+2025-04-08 14:24:58,116 INFO    MainThread:9420 [wandb_setup.py:_flush():67] Current SDK version is 0.19.9
+2025-04-08 14:24:58,116 INFO    MainThread:9420 [wandb_setup.py:_flush():67] Configure stats pid to 9420
+2025-04-08 14:24:58,116 INFO    MainThread:9420 [wandb_setup.py:_flush():67] Loading settings from /root/.config/wandb/settings
+2025-04-08 14:24:58,116 INFO    MainThread:9420 [wandb_setup.py:_flush():67] Loading settings from /workspace/BitDistiller/train/wandb/settings
+2025-04-08 14:24:58,116 INFO    MainThread:9420 [wandb_setup.py:_flush():67] Loading settings from environment variables
+2025-04-08 14:24:58,116 INFO    MainThread:9420 [wandb_init.py:setup_run_log_directory():662] Logging user logs to ./ckpts/tinyllama_v1.1/int2-g128/wandb/run-20250408_142458-wlfced8t/logs/debug.log
+2025-04-08 14:24:58,117 INFO    MainThread:9420 [wandb_init.py:setup_run_log_directory():663] Logging internal logs to ./ckpts/tinyllama_v1.1/int2-g128/wandb/run-20250408_142458-wlfced8t/logs/debug-internal.log
+2025-04-08 14:24:58,117 INFO    MainThread:9420 [wandb_init.py:init():781] calling init triggers
+2025-04-08 14:24:58,117 INFO    MainThread:9420 [wandb_init.py:init():786] wandb.init called with sweep_config: {}
+config: {'_wandb': {}}
+2025-04-08 14:24:58,117 INFO    MainThread:9420 [wandb_init.py:init():809] starting backend
+2025-04-08 14:24:58,117 INFO    MainThread:9420 [wandb_init.py:init():813] sending inform_init request
+2025-04-08 14:24:58,119 INFO    MainThread:9420 [backend.py:_multiprocessing_setup():101] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2025-04-08 14:24:58,119 INFO    MainThread:9420 [wandb_init.py:init():823] backend started and connected
+2025-04-08 14:24:58,124 INFO    MainThread:9420 [wandb_init.py:init():915] updated telemetry
+2025-04-08 14:24:58,304 INFO    MainThread:9420 [wandb_init.py:init():939] communicating run to backend with 90.0 second timeout
+2025-04-08 14:24:58,589 INFO    MainThread:9420 [wandb_init.py:init():1014] starting run threads in backend
+2025-04-08 14:24:58,680 INFO    MainThread:9420 [wandb_run.py:_console_start():2454] atexit reg
+2025-04-08 14:24:58,681 INFO    MainThread:9420 [wandb_run.py:_redirect():2306] redirect: wrap_raw
+2025-04-08 14:24:58,681 INFO    MainThread:9420 [wandb_run.py:_redirect():2371] Wrapping output streams.
+2025-04-08 14:24:58,681 INFO    MainThread:9420 [wandb_run.py:_redirect():2394] Redirects installed.
+2025-04-08 14:24:58,682 INFO    MainThread:9420 [wandb_init.py:init():1056] run started, returning control to user process
+2025-04-08 14:25:40,038 INFO    MainThread:9420 [wandb_run.py:_config_callback():1327] config_cb None None {'vocab_size': 32001, 'max_position_embeddings': 2048, 'hidden_size': 2048, 'intermediate_size': 5632, 'num_hidden_layers': 22, 'num_attention_heads': 32, 'num_key_value_heads': 4, 'hidden_act': 'silu', 'initializer_range': 0.02, 'rms_norm_eps': 1e-05, 'pretraining_tp': 1, 'use_cache': True, 'rope_theta': 10000.0, 'rope_scaling': None, 'attention_bias': False, 'attention_dropout': 0.0, 'return_dict': True, 'output_hidden_states': False, 'output_attentions': False, 'torchscript': False, 'torch_dtype': 'bfloat16', 'use_bfloat16': False, 'tf_legacy_loss': False, 'pruned_heads': {}, 'tie_word_embeddings': False, 'chunk_size_feed_forward': 0, 'is_encoder_decoder': False, 'is_decoder': False, 'cross_attention_hidden_size': None, 'add_cross_attention': False, 'tie_encoder_decoder': False, 'max_length': 20, 'min_length': 0, 'do_sample': False, 'early_stopping': False, 'num_beams': 1, 'num_beam_groups': 1, 'diversity_penalty': 0.0, 'temperature': 1.0, 'top_k': 50, 'top_p': 1.0, 'typical_p': 1.0, 'repetition_penalty': 1.0, 'length_penalty': 1.0, 'no_repeat_ngram_size': 0, 'encoder_no_repeat_ngram_size': 0, 'bad_words_ids': None, 'num_return_sequences': 1, 'output_scores': False, 'return_dict_in_generate': False, 'forced_bos_token_id': None, 'forced_eos_token_id': None, 'remove_invalid_values': False, 'exponential_decay_length_penalty': None, 'suppress_tokens': None, 'begin_suppress_tokens': None, 'architectures': ['LlamaForCausalLM'], 'finetuning_task': None, 'id2label': {0: 'LABEL_0', 1: 'LABEL_1'}, 'label2id': {'LABEL_0': 0, 'LABEL_1': 1}, 'tokenizer_class': None, 'prefix': None, 'bos_token_id': 1, 'pad_token_id': None, 'eos_token_id': 2, 'sep_token_id': None, 'decoder_start_token_id': None, 'task_specific_params': None, 'problem_type': None, '_name_or_path': '../models/TinyLlama_v1.1/', 'transformers_version': '4.37.0', 'model_type': 'llama', 'output_dir': './ckpts/tinyllama_v1.1/int2-g128/', 'overwrite_output_dir': False, 'do_train': False, 'do_eval': True, 'do_predict': False, 'evaluation_strategy': 'steps', 'prediction_loss_only': False, 'per_device_train_batch_size': 16, 'per_device_eval_batch_size': 16, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 4, 'eval_accumulation_steps': None, 'eval_delay': 0, 'learning_rate': 2e-05, 'weight_decay': 0.0, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 4.0, 'max_steps': -1, 'lr_scheduler_type': 'constant', 'lr_scheduler_kwargs': {}, 'warmup_ratio': 0.0, 'warmup_steps': 0, 'log_level': 'passive', 'log_level_replica': 'warning', 'log_on_each_node': True, 'logging_dir': './ckpts/tinyllama_v1.1/int2-g128/runs/', 'logging_strategy': 'steps', 'logging_first_step': False, 'logging_steps': 1.0, 'logging_nan_inf_filter': True, 'save_strategy': 'steps', 'save_steps': 40, 'save_total_limit': 2, 'save_safetensors': True, 'save_on_each_node': False, 'save_only_model': False, 'no_cuda': False, 'use_cpu': False, 'use_mps_device': False, 'seed': 42, 'data_seed': None, 'jit_mode_eval': False, 'use_ipex': False, 'bf16': True, 'fp16': False, 'fp16_opt_level': 'O1', 'half_precision_backend': 'auto', 'bf16_full_eval': False, 'fp16_full_eval': False, 'tf32': None, 'local_rank': 0, 'ddp_backend': None, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': 40, 'dataloader_num_workers': 0, 'past_index': -1, 'run_name': './ckpts/tinyllama_v1.1/int2-g128/', 'disable_tqdm': False, 'remove_unused_columns': True, 'label_names': None, 'load_best_model_at_end': True, 'metric_for_best_model': 'loss', 'greater_is_better': False, 'ignore_data_skip': False, 'fsdp': [], 'fsdp_min_num_params': 0, 'fsdp_config': {'min_num_params': 0, 'xla': False, 'xla_fsdp_grad_ckpt': False}, 'fsdp_transformer_layer_cls_to_wrap': None, 'deepspeed': 'config/zero.json', 'label_smoothing_factor': 0.0, 'optim': 'adamw_torch', 'optim_args': None, 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['tensorboard', 'wandb'], 'ddp_find_unused_parameters': None, 'ddp_bucket_cap_mb': None, 'ddp_broadcast_buffers': None, 'dataloader_pin_memory': True, 'dataloader_persistent_workers': False, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': False, 'resume_from_checkpoint': None, 'hub_model_id': None, 'hub_strategy': 'every_save', 'hub_token': '<HUB_TOKEN>', 'hub_private_repo': False, 'hub_always_push': False, 'gradient_checkpointing': True, 'gradient_checkpointing_kwargs': None, 'include_inputs_for_metrics': False, 'fp16_backend': 'auto', 'push_to_hub_model_id': None, 'push_to_hub_organization': None, 'push_to_hub_token': '<PUSH_TO_HUB_TOKEN>', 'mp_parameters': '', 'auto_find_batch_size': False, 'full_determinism': False, 'torchdynamo': None, 'ray_scope': 'last', 'ddp_timeout': 1800, 'torch_compile': False, 'torch_compile_backend': None, 'torch_compile_mode': None, 'dispatch_batches': None, 'split_batches': False, 'include_tokens_per_second': False, 'include_num_input_tokens_seen': False, 'neftune_noise_alpha': None, 'cache_dir': None, 'model_max_length': 1024, 'bits': 2, 'q_group_size': 128, 'quant_type': 'int2-asym', 'clip': '../quantization/clip_cache/TinyLlama_v1.1/int2-g128.pt', 'train_kd': True, 'kd_tmp': 1, 'kd_loss_type': 'cakld', 'cakld_steps': 10}
+2025-04-08 15:51:25,124 INFO    MainThread:9420 [wandb_run.py:_finish():2189] finishing run DeepFriedNLP/SNLP_BitDistiller/wlfced8t
+2025-04-08 15:51:25,125 INFO    MainThread:9420 [wandb_run.py:_atexit_cleanup():2419] got exitcode: 0
+2025-04-08 15:51:25,125 INFO    MainThread:9420 [wandb_run.py:_restore():2401] restore
+2025-04-08 15:51:25,125 INFO    MainThread:9420 [wandb_run.py:_restore():2407] restore done
+2025-04-08 15:51:25,953 INFO    MainThread:9420 [wandb_run.py:_footer_history_summary_info():4064] rendering history
+2025-04-08 15:51:25,954 INFO    MainThread:9420 [wandb_run.py:_footer_history_summary_info():4096] rendering summary
+2025-04-08 15:51:25,955 INFO    MainThread:9420 [wandb_run.py:_footer_sync_info():4025] logging synced files

wandb/run-20250408_142458-wlfced8t/run-wlfced8t.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:89d1ddb4b228042072d2a447f925cc0f73ee98db6218e72b836880c8ee4381d4
+size 1263539