diff --git a/run.log b/run.log deleted file mode 100644 index 73b964a808bd77c67b3c069230e6e0c6aea10307..0000000000000000000000000000000000000000 --- a/run.log +++ /dev/null @@ -1,497 +0,0 @@ -2021-07-10 11:28:58.039466: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory -[11:28:59] - INFO - __main__ - Training/evaluation parameters TrainingArguments( -_n_gpu=-1, -adafactor=False, -adam_beta1=0.9, -adam_beta2=0.999, -adam_epsilon=1e-08, -dataloader_drop_last=False, -dataloader_num_workers=0, -dataloader_pin_memory=True, -ddp_find_unused_parameters=None, -debug=[], -deepspeed=None, -disable_tqdm=False, -do_eval=False, -do_predict=False, -do_train=False, -eval_accumulation_steps=None, -eval_steps=500, -evaluation_strategy=IntervalStrategy.NO, -fp16=False, -fp16_backend=auto, -fp16_full_eval=False, -fp16_opt_level=O1, -gradient_accumulation_steps=1, -greater_is_better=None, -group_by_length=False, -ignore_data_skip=False, -label_names=None, -label_smoothing_factor=0.0, -learning_rate=0.0003, -length_column_name=length, -load_best_model_at_end=False, -local_rank=-1, -log_level=-1, -log_level_replica=-1, -log_on_each_node=True, -logging_dir=./runs/Jul10_11-28-59_t1v-n-112df4a9-w-0, -logging_first_step=False, -logging_steps=500, -logging_strategy=IntervalStrategy.STEPS, -lr_scheduler_type=SchedulerType.LINEAR, -max_grad_norm=1.0, -max_steps=-1, -metric_for_best_model=None, -mp_parameters=, -no_cuda=False, -num_train_epochs=8.0, -output_dir=./, -overwrite_output_dir=True, -past_index=-1, -per_device_eval_batch_size=4, -per_device_train_batch_size=4, -prediction_loss_only=False, -push_to_hub=True, -push_to_hub_model_id=flax-community/roberta-base-mr, -push_to_hub_organization=None, -push_to_hub_token=vdIAyRvCACJNslYtyLHufmNDnUIyknPzUgVDMFiXqJoulvMqjoubonLJzXOJQJczWfRMJumVaMFjGSFVnQAMdswvZkzNIthKrxBeARBXfqnIwjABkKpCbjGEgnkjpjKi, -remove_unused_columns=True, -report_to=['wandb'], -resume_from_checkpoint=None, -run_name=hf-flax-robert-base-mr, -save_on_each_node=False, -save_steps=500, -save_strategy=IntervalStrategy.STEPS, -save_total_limit=None, -seed=42, -sharded_ddp=[], -skip_memory_metrics=True, -tpu_metrics_debug=False, -tpu_num_cores=None, -use_legacy_prediction_loop=False, -warmup_ratio=0.0, -warmup_steps=1000, -weight_decay=0.0, -) -[11:28:59] - DEBUG - urllib3.connectionpool - Starting new HTTPS connection (1): s3.amazonaws.com:443 -[11:28:59] - DEBUG - urllib3.connectionpool - https://s3.amazonaws.com:443 "HEAD /datasets.huggingface.co/datasets/datasets/csv/csv.py HTTP/1.1" 200 0 -[11:28:59] - WARNING - datasets.builder - Using custom data configuration default-81af252916854639 -Downloading and preparing dataset csv/default (download: Unknown size, generated: Unknown size, post-processed: Unknown size, total: Unknown size) to /home/nipunsadvilkar/.cache/huggingface/datasets/csv/default-81af252916854639/0.0.0/e138af468cb14e747fb46a19c787ffcfa5170c821476d20d5304287ce12bbc23... - 0 tables [00:00, ? tables/s] 0 tables [00:00, ? tables/s] Dataset csv downloaded and prepared to /home/nipunsadvilkar/.cache/huggingface/datasets/csv/default-81af252916854639/0.0.0/e138af468cb14e747fb46a19c787ffcfa5170c821476d20d5304287ce12bbc23. Subsequent calls will reuse this data. - 0%| | 0/5 [00:00>, args: ('https://api.wandb.ai/files/nipunsadvilkar/hf-flax-robert-base-mr/2pc90wcd/file_stream',), kwargs: {'json': {'files': {'wandb-events.jsonl': {'offset': 1, 'content': ['{"system.cpu": 4.69, "system.memory": 2.63, "system.disk": 29.2, "system.proc.memory.availableMB": 333586.39, "system.proc.memory.rssMB": 72.37, "system.proc.memory.percent": 0.02, "system.proc.cpu.threads": 12.0, "system.network.sent": 2873699, "system.network.recv": 4435580, "_wandb": true, "_timestamp": 1625833687, "_runtime": 58}']}}}} -2021-07-09 12:28:10,019 DEBUG HandlerThread:33395 [handler.py:handle_request():124] handle_request: stop_status -2021-07-09 12:28:10,019 DEBUG SenderThread:33395 [sender.py:send_request():193] send_request: stop_status -2021-07-09 12:28:25,052 DEBUG HandlerThread:33395 [handler.py:handle_request():124] handle_request: stop_status -2021-07-09 12:28:25,052 DEBUG SenderThread:33395 [sender.py:send_request():193] send_request: stop_status -2021-07-09 12:28:28,818 INFO Thread-8 :33395 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_122709-2pc90wcd/files/output.log -2021-07-09 12:28:30,819 INFO Thread-8 :33395 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_122709-2pc90wcd/files/output.log -2021-07-09 12:28:32,820 INFO Thread-8 :33395 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_122709-2pc90wcd/files/output.log -2021-07-09 12:28:34,821 INFO Thread-8 :33395 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_122709-2pc90wcd/files/output.log -2021-07-09 12:28:36,822 INFO Thread-8 :33395 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_122709-2pc90wcd/files/output.log -2021-07-09 12:28:38,069 DEBUG SenderThread:33395 [sender.py:send():179] send: stats -2021-07-09 12:28:38,823 INFO Thread-8 :33395 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_122709-2pc90wcd/files/output.log -2021-07-09 12:28:40,086 DEBUG HandlerThread:33395 [handler.py:handle_request():124] handle_request: stop_status -2021-07-09 12:28:40,086 DEBUG SenderThread:33395 [sender.py:send_request():193] send_request: stop_status -2021-07-09 12:28:40,824 INFO Thread-8 :33395 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_122709-2pc90wcd/files/output.log -2021-07-09 12:28:42,825 INFO Thread-8 :33395 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_122709-2pc90wcd/files/output.log -2021-07-09 12:28:43,425 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:43,425 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:43,426 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:43,826 INFO Thread-8 :33395 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_122709-2pc90wcd/files/wandb-summary.json -2021-07-09 12:28:43,934 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:43,936 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:43,949 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:43,949 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:43,949 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:43,955 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:43,955 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:43,955 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:43,963 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:43,983 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:43,983 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:43,986 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:43,986 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:43,987 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:43,989 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:43,990 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:43,990 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,011 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,013 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,013 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,034 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,034 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,034 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,042 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,044 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,051 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,072 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,072 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,072 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,072 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,072 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,072 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,073 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,073 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,073 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,074 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,074 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,074 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,074 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,074 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,074 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,075 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,075 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,075 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,075 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,076 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,076 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,076 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,076 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,076 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,077 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,077 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,077 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,077 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,077 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,077 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,078 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,078 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,078 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,079 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,079 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,079 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,092 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,092 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,092 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,093 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,093 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,093 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,093 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,093 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,094 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,094 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,094 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,094 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,094 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,095 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,095 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,095 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,095 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,095 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,096 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,096 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,096 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,096 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,096 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,096 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,097 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,097 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,097 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,097 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,098 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,098 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,098 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,098 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,098 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,099 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,099 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,099 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,099 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,099 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,099 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,100 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,100 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,100 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,100 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,100 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,100 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,101 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,101 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,101 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,101 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,101 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,101 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,102 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,102 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,102 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,102 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,102 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,102 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,103 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,103 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,103 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,103 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,103 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,103 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,104 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,104 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,104 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,104 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,104 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,105 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,105 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,105 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,105 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,105 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,106 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,106 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,106 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,106 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,106 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,107 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,107 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,107 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,107 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,107 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,108 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,108 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,108 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,108 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,109 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,109 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,109 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,109 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,109 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,109 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,110 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,110 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,110 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,110 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,110 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,110 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,111 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,111 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,111 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,111 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,111 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,111 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,112 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,112 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,112 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,112 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,112 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,112 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,113 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,113 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,113 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,113 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,113 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,114 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,114 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,114 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,114 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,114 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,114 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,115 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,115 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,115 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,115 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,115 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,116 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,116 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,116 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,116 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,116 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,117 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,117 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,117 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,117 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,118 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,118 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,118 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,118 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,118 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,119 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,119 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,119 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,119 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,119 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,119 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,120 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,120 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,120 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,120 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,120 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,120 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,121 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,121 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,121 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,121 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,122 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,122 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,122 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,122 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,122 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,122 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,123 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,123 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,123 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,123 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,123 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,123 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,124 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,124 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,124 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,124 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,124 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,124 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,124 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,125 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,125 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,125 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,125 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,125 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,126 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,126 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,126 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,126 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,126 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,127 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,127 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,127 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,127 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,127 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,127 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,128 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,128 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,128 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,128 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,128 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,129 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,129 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,129 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,129 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,129 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,129 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,130 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,130 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,130 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,130 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,130 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,130 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,130 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,131 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,131 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,131 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,131 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,131 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,132 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,132 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,132 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,132 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,132 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,133 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,133 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,133 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,133 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,133 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,134 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,134 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,134 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,134 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,134 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,134 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,135 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,135 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,135 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,136 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,136 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,136 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,136 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,136 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,136 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,137 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,137 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,137 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,137 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,138 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,138 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,138 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,138 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,138 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,139 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,139 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,139 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,139 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,139 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,139 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,140 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,140 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,140 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,140 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,140 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,141 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,141 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,141 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,141 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,141 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,142 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,142 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,142 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,142 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,142 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,143 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,143 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,143 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,143 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,143 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,143 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,144 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,144 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,144 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,144 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,144 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,144 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,145 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,145 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,145 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,145 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,145 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,145 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,146 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,146 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,146 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,146 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,146 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,146 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,147 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,147 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,147 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,147 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,147 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,147 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,148 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,148 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,148 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,148 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,148 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,149 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,149 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,149 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,149 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,149 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,150 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,150 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,150 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,150 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,150 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,150 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,150 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,151 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,151 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,151 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,151 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,151 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,152 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,152 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,152 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,152 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,152 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,152 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,153 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,153 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,153 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,153 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,153 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,153 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,154 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,154 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,154 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,154 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,154 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,154 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,155 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,155 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,155 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,155 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,155 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,155 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,156 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,156 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,156 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,156 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,156 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,157 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,157 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,157 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,157 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,157 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,158 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,158 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,158 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,158 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,159 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,159 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,159 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,159 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,159 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,159 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,160 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,160 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,160 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,160 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,160 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,160 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,161 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,161 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,161 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,161 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,161 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,161 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,162 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,162 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,162 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,162 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,162 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,162 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,163 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,163 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,163 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,163 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,163 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,164 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,164 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,164 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,164 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,164 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,165 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,165 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,165 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,165 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,165 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,165 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,166 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,166 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,166 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,166 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,166 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,167 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,167 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,167 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,167 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,167 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,167 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,168 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,168 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,168 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,168 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,168 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,169 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,169 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,169 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,169 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,169 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,170 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,170 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,170 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,170 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,170 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,170 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,171 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,171 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,171 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,171 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,171 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,172 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,172 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,172 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,172 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,172 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,172 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,173 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,173 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,173 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,173 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,173 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,173 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,174 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,174 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,174 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,174 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,174 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,174 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,175 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,175 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,175 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,175 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,175 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,176 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,176 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,176 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,176 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,176 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,177 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,177 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,177 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,177 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,177 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,178 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,178 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,178 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,179 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,179 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,179 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,179 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,179 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,179 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,180 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,180 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,180 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,180 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,180 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,180 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,181 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,181 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,181 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,181 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,181 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,182 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,182 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,182 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,182 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,182 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,183 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,183 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,183 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,183 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,183 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,183 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,184 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,184 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,184 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,184 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,184 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,184 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,185 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,185 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,185 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,185 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,185 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,186 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,186 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,186 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,186 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,186 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,186 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,187 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,187 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,187 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,187 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,187 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,188 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,188 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,188 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,188 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,188 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,189 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,189 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,189 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,189 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,189 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,190 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,190 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,190 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,190 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,190 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,190 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,191 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,191 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,191 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,191 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,191 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,192 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,192 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,192 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,192 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,192 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,192 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,193 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,193 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,193 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,193 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,193 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,194 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,194 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,194 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,194 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,194 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,195 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,195 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,195 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,195 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,195 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,195 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,196 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,196 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,196 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,196 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,196 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,196 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,197 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,197 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,197 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,197 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,197 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,198 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,198 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,198 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,198 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,198 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,199 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,199 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,199 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,199 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,199 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,200 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,200 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,200 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,200 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,200 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,200 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,201 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,201 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,201 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,201 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,201 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,201 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,202 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,202 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,202 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,202 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,203 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,203 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,203 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,203 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,203 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,204 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,204 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,204 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,204 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,204 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,204 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,205 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,205 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,205 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,205 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,205 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,205 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,206 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,206 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,206 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,207 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,207 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,207 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,207 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,207 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,208 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,208 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,208 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,208 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,209 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,209 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,209 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,209 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,209 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,209 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,210 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,210 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,210 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,210 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,210 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,210 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,211 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,211 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,211 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,211 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,211 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,211 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,212 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,212 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,212 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,212 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,212 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,213 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,213 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,213 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,213 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,213 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,214 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,214 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,214 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,214 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,214 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,214 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,215 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,215 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,215 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,215 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,215 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,216 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,216 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,216 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,216 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,216 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,216 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,217 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,217 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,217 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,217 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,217 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,217 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,218 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,218 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,218 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,218 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,218 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,219 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,219 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,219 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,219 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,219 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,220 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,220 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,220 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,220 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,220 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,221 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,221 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,221 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,221 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,221 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,221 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,222 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,222 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,222 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,222 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,222 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,222 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,223 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,223 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,223 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,223 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,223 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,223 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,224 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,224 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,224 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,224 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,224 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,225 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,225 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,225 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,225 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,225 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,226 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,226 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,226 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,226 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,226 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,226 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,227 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,227 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,227 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,227 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,227 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,228 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,228 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,228 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,228 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,228 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,228 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,229 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,229 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,229 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,229 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,229 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,229 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,230 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,230 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,230 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,230 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,230 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,231 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,231 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,231 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,231 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,231 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,232 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,232 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,232 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,232 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,232 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,233 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,233 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,233 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,233 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,233 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,233 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,234 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,234 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,234 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,234 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,234 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,234 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,235 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,235 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,235 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,235 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,235 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,235 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,236 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,236 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,236 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,236 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,237 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,237 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,237 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,237 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,237 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,238 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,238 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,238 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,238 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,238 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,238 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,239 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,239 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,239 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,239 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,239 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,240 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,240 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,240 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,240 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,240 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,241 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,241 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,241 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,241 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,241 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,242 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,242 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,242 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,242 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,242 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,242 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,243 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,243 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,243 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,243 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,243 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,243 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,244 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,244 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,244 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,244 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,244 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,245 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,245 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,245 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,245 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,245 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,246 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,246 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,246 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,246 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,246 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,247 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,247 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,247 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,247 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,247 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,247 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,248 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,248 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:44,248 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:44,248 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:44,832 INFO Thread-8 :33395 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_122709-2pc90wcd/files/wandb-summary.json -2021-07-09 12:28:44,832 INFO Thread-8 :33395 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_122709-2pc90wcd/files/output.log -2021-07-09 12:28:46,827 INFO Thread-8 :33395 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_122709-2pc90wcd/files/output.log -2021-07-09 12:28:48,205 DEBUG SenderThread:33395 [sender.py:send():179] send: history -2021-07-09 12:28:48,206 DEBUG SenderThread:33395 [sender.py:send():179] send: summary -2021-07-09 12:28:48,208 INFO SenderThread:33395 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:28:48,828 INFO Thread-8 :33395 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_122709-2pc90wcd/files/wandb-summary.json -2021-07-09 12:28:48,829 INFO Thread-8 :33395 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_122709-2pc90wcd/files/output.log -2021-07-09 12:28:50,492 DEBUG HandlerThread:33395 [handler.py:handle_request():124] handle_request: shutdown -2021-07-09 12:28:50,492 INFO HandlerThread:33395 [handler.py:finish():638] shutting down handler -2021-07-09 12:28:50,829 INFO Thread-8 :33395 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_122709-2pc90wcd/files/output.log -2021-07-09 12:28:51,491 INFO SenderThread:33395 [sender.py:finish():945] shutting down sender -2021-07-09 12:28:51,492 INFO WriterThread:33395 [datastore.py:close():288] close: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_122709-2pc90wcd/run-2pc90wcd.wandb -2021-07-09 12:28:51,492 INFO SenderThread:33395 [dir_watcher.py:finish():282] shutting down directory watcher -2021-07-09 12:28:51,830 INFO SenderThread:33395 [dir_watcher.py:finish():312] scan: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_122709-2pc90wcd/files -2021-07-09 12:28:51,830 INFO SenderThread:33395 [dir_watcher.py:finish():318] scan save: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_122709-2pc90wcd/files/requirements.txt requirements.txt -2021-07-09 12:28:51,830 INFO SenderThread:33395 [dir_watcher.py:finish():318] scan save: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_122709-2pc90wcd/files/output.log output.log -2021-07-09 12:28:51,831 INFO SenderThread:33395 [dir_watcher.py:finish():318] scan save: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_122709-2pc90wcd/files/wandb-metadata.json wandb-metadata.json -2021-07-09 12:28:51,831 INFO SenderThread:33395 [dir_watcher.py:finish():318] scan save: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_122709-2pc90wcd/files/config.yaml config.yaml -2021-07-09 12:28:51,831 INFO SenderThread:33395 [dir_watcher.py:finish():318] scan save: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_122709-2pc90wcd/files/wandb-summary.json wandb-summary.json -2021-07-09 12:28:51,831 INFO SenderThread:33395 [file_pusher.py:finish():177] shutting down file pusher -2021-07-09 12:28:51,831 INFO SenderThread:33395 [file_pusher.py:join():182] waiting for file pusher -2021-07-09 12:28:52,029 INFO Thread-14 :33395 [upload_job.py:push():137] Uploaded file /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_122709-2pc90wcd/files/config.yaml -2021-07-09 12:28:52,038 INFO Thread-15 :33395 [upload_job.py:push():137] Uploaded file /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_122709-2pc90wcd/files/wandb-summary.json -2021-07-09 12:28:52,039 INFO Thread-12 :33395 [upload_job.py:push():137] Uploaded file /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_122709-2pc90wcd/files/requirements.txt -2021-07-09 12:28:52,064 INFO Thread-13 :33395 [upload_job.py:push():137] Uploaded file /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_122709-2pc90wcd/files/output.log -2021-07-09 12:28:52,398 INFO MainThread:33395 [internal.py:handle_exit():78] Internal process exited diff --git a/wandb/run-20210709_122709-2pc90wcd/logs/debug.log b/wandb/run-20210709_122709-2pc90wcd/logs/debug.log deleted file mode 100644 index 0e7c0d4633f788492ef1369797462bf162f7d09c..0000000000000000000000000000000000000000 --- a/wandb/run-20210709_122709-2pc90wcd/logs/debug.log +++ /dev/null @@ -1,53 +0,0 @@ -2021-07-09 12:27:09,053 INFO MainThread:32339 [wandb_setup.py:_flush():69] Unhandled environment var: WANDB_WATCH -2021-07-09 12:27:09,053 INFO MainThread:32339 [wandb_setup.py:_flush():69] setting env: {'project': 'hf-flax-robert-base-mr', 'entity': 'nipunsadvilkar'} -2021-07-09 12:27:09,053 INFO MainThread:32339 [wandb_setup.py:_flush():69] setting login settings: {} -2021-07-09 12:27:09,053 INFO MainThread:32339 [wandb_init.py:_log_setup():337] Logging user logs to /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_122709-2pc90wcd/logs/debug.log -2021-07-09 12:27:09,053 INFO MainThread:32339 [wandb_init.py:_log_setup():338] Logging internal logs to /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_122709-2pc90wcd/logs/debug-internal.log -2021-07-09 12:27:09,053 INFO MainThread:32339 [wandb_init.py:init():370] calling init triggers -2021-07-09 12:27:09,053 INFO MainThread:32339 [wandb_init.py:init():375] wandb.init called with sweep_config: {} -config: {} -2021-07-09 12:27:09,053 INFO MainThread:32339 [wandb_init.py:init():419] starting backend -2021-07-09 12:27:09,053 INFO MainThread:32339 [backend.py:_multiprocessing_setup():70] multiprocessing start_methods=fork,spawn,forkserver, using: spawn -2021-07-09 12:27:09,087 INFO MainThread:32339 [backend.py:ensure_launched():135] starting backend process... -2021-07-09 12:27:09,119 INFO MainThread:32339 [backend.py:ensure_launched():139] started backend process with pid: 33395 -2021-07-09 12:27:09,122 INFO MainThread:32339 [wandb_init.py:init():424] backend started and connected -2021-07-09 12:27:09,124 INFO MainThread:32339 [wandb_init.py:init():472] updated telemetry -2021-07-09 12:27:09,125 INFO MainThread:32339 [wandb_init.py:init():491] communicating current version -2021-07-09 12:27:09,709 INFO MainThread:32339 [wandb_init.py:init():496] got version response -2021-07-09 12:27:09,709 INFO MainThread:32339 [wandb_init.py:init():504] communicating run to backend with 30 second timeout -2021-07-09 12:27:09,783 INFO MainThread:32339 [wandb_init.py:init():529] starting run threads in backend -2021-07-09 12:27:09,838 INFO MainThread:32339 [wandb_run.py:_console_start():1623] atexit reg -2021-07-09 12:27:09,839 INFO MainThread:32339 [wandb_run.py:_redirect():1497] redirect: SettingsConsole.REDIRECT -2021-07-09 12:27:09,839 INFO MainThread:32339 [wandb_run.py:_redirect():1502] Redirecting console. -2021-07-09 12:27:09,841 INFO MainThread:32339 [wandb_run.py:_redirect():1558] Redirects installed. -2021-07-09 12:27:09,841 INFO MainThread:32339 [wandb_init.py:init():554] run started, returning control to user process -2021-07-09 12:27:09,842 INFO MainThread:32339 [wandb_run.py:_config_callback():872] config_cb None None {'output_dir': './', 'overwrite_output_dir': True, 'do_train': False, 'do_eval': False, 'do_predict': False, 'evaluation_strategy': 'IntervalStrategy.NO', 'prediction_loss_only': False, 'per_device_train_batch_size': 4, 'per_device_eval_batch_size': 4, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 1, 'eval_accumulation_steps': None, 'learning_rate': 0.0003, 'weight_decay': 0.0, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 8.0, 'max_steps': -1, 'lr_scheduler_type': 'SchedulerType.LINEAR', 'warmup_ratio': 0.0, 'warmup_steps': 1000, 'log_level': -1, 'log_level_replica': -1, 'log_on_each_node': True, 'logging_dir': './runs/Jul09_12-27-03_t1v-n-112df4a9-w-0', 'logging_strategy': 'IntervalStrategy.STEPS', 'logging_first_step': False, 'logging_steps': 500, 'save_strategy': 'IntervalStrategy.STEPS', 'save_steps': 500, 'save_total_limit': None, 'save_on_each_node': False, 'no_cuda': False, 'seed': 42, 'fp16': False, 'fp16_opt_level': 'O1', 'fp16_backend': 'auto', 'fp16_full_eval': False, 'local_rank': -1, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': 500, 'dataloader_num_workers': 0, 'past_index': -1, 'run_name': './', 'disable_tqdm': False, 'remove_unused_columns': True, 'label_names': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'sharded_ddp': [], 'deepspeed': None, 'label_smoothing_factor': 0.0, 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['tensorboard', 'wandb'], 'ddp_find_unused_parameters': None, 'dataloader_pin_memory': True, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': False, 'resume_from_checkpoint': None, 'push_to_hub_model_id': '', 'push_to_hub_organization': None, 'push_to_hub_token': None, 'mp_parameters': ''} -2021-07-09 12:27:09,843 INFO MainThread:32339 [wandb_run.py:_config_callback():872] config_cb None None {'model_name_or_path': None, 'model_type': 'roberta', 'config_name': './', 'tokenizer_name': './', 'cache_dir': None, 'use_fast_tokenizer': True, 'dtype': 'float32'} -2021-07-09 12:27:09,844 INFO MainThread:32339 [wandb_run.py:_config_callback():872] config_cb None None {'dataset_name': 'oscar', 'dataset_config_name': 'unshuffled_deduplicated_als', 'train_file': None, 'validation_file': None, 'train_ref_file': None, 'validation_ref_file': None, 'overwrite_cache': False, 'validation_split_percentage': 5, 'max_seq_length': 128, 'preprocessing_num_workers': None, 'mlm_probability': 0.15, 'pad_to_max_length': False, 'line_by_line': False} -2021-07-09 12:28:48,239 INFO MainThread:32339 [wandb_run.py:_atexit_cleanup():1593] got exitcode: 1 -2021-07-09 12:28:48,240 INFO MainThread:32339 [wandb_run.py:_restore():1565] restore -2021-07-09 12:28:52,518 ERROR MainThread:32339 [wandb_run.py:_atexit_cleanup():1611] Problem finishing run -Traceback (most recent call last): - File "/home/nipunsadvilkar/roberta_mr_env/lib/python3.8/site-packages/wandb/sdk/wandb_run.py", line 1601, in _atexit_cleanup - self._on_finish() - File "/home/nipunsadvilkar/roberta_mr_env/lib/python3.8/site-packages/wandb/sdk/wandb_run.py", line 1737, in _on_finish - self.history._flush() - File "/home/nipunsadvilkar/roberta_mr_env/lib/python3.8/site-packages/wandb/sdk/wandb_history.py", line 59, in _flush - self._callback(row=self._data, step=self._step) - File "/home/nipunsadvilkar/roberta_mr_env/lib/python3.8/site-packages/wandb/sdk/wandb_run.py", line 931, in _history_callback - self._backend.interface.publish_history( - File "/home/nipunsadvilkar/roberta_mr_env/lib/python3.8/site-packages/wandb/sdk/interface/interface.py", line 231, in publish_history - item.value_json = json_dumps_safer_history(v) # type: ignore - File "/home/nipunsadvilkar/roberta_mr_env/lib/python3.8/site-packages/wandb/util.py", line 667, in json_dumps_safer_history - return json.dumps(obj, cls=WandBHistoryJSONEncoder, **kwargs) - File "/usr/lib/python3.8/json/__init__.py", line 234, in dumps - return cls( - File "/usr/lib/python3.8/json/encoder.py", line 199, in encode - chunks = self.iterencode(o, _one_shot=True) - File "/usr/lib/python3.8/json/encoder.py", line 257, in iterencode - return _iterencode(o, 0) - File "/home/nipunsadvilkar/roberta_mr_env/lib/python3.8/site-packages/wandb/util.py", line 634, in default - return json.JSONEncoder.default(self, obj) - File "/usr/lib/python3.8/json/encoder.py", line 179, in default - raise TypeError(f'Object of type {o.__class__.__name__} ' -TypeError: Object of type DeviceArray is not JSON serializable diff --git a/wandb/run-20210709_122709-2pc90wcd/run-2pc90wcd.wandb b/wandb/run-20210709_122709-2pc90wcd/run-2pc90wcd.wandb deleted file mode 100644 index b6fd6dbce5ac3f1398c13f8358db8df85cf51a35..0000000000000000000000000000000000000000 Binary files a/wandb/run-20210709_122709-2pc90wcd/run-2pc90wcd.wandb and /dev/null differ diff --git a/wandb/run-20210709_124353-3pjvcule/files/config.yaml b/wandb/run-20210709_124353-3pjvcule/files/config.yaml deleted file mode 100644 index ab3d37a013838fc169594a1929bc84ac6663fbb1..0000000000000000000000000000000000000000 --- a/wandb/run-20210709_124353-3pjvcule/files/config.yaml +++ /dev/null @@ -1,19 +0,0 @@ -wandb_version: 1 - -_wandb: - desc: null - value: - cli_version: 0.10.33 - framework: huggingface - huggingface_version: 4.9.0.dev0 - is_jupyter_run: false - is_kaggle_kernel: false - python_version: 3.8.10 - t: - 1: - - 11 - 4: 3.8.10 - 5: 0.10.33 - 6: 4.9.0.dev0 - 8: - - 5 diff --git a/wandb/run-20210709_124353-3pjvcule/files/output.log b/wandb/run-20210709_124353-3pjvcule/files/output.log deleted file mode 100644 index 67fb84fd3ef63d3314f3818ae5b2ed6874545251..0000000000000000000000000000000000000000 --- a/wandb/run-20210709_124353-3pjvcule/files/output.log +++ /dev/null @@ -1,5 +0,0 @@ -/home/nipunsadvilkar/roberta_mr_env/lib/python3.8/site-packages/jax/lib/xla_bridge.py:382: UserWarning: jax.host_count has been renamed to jax.process_count. This alias will eventually be removed; please update your code. - warnings.warn( -/home/nipunsadvilkar/roberta_mr_env/lib/python3.8/site-packages/jax/lib/xla_bridge.py:369: UserWarning: jax.host_id has been renamed to jax.process_index. This alias will eventually be removed; please update your code. - warnings.warn( -Epoch ... (1/8): 0%| | 0/8 [00:00 /home/nipunsadvilkar/sample/roberta-base-mr/run_mlm_flax.py(647)() - 646 import ipdb; ipdb.set_trace() ---> 647 train_metrics = [] - 648 - - -ipdb> train_metrics -[{'learning_rate': ShardedDeviceArray([0.0001278, 0.0001278, 0.0001278, 0.0001278, 0.0001278, - 0.0001278, 0.0001278, 0.0001278], dtype=float32), 'loss': ShardedDeviceArray([7.9980755, 7.9980755, 7.9980755, 7.9980755, 7.9980755, - 7.9980755, 7.9980755, 7.9980755], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001281, 0.0001281, 0.0001281, 0.0001281, 0.0001281, - 0.0001281, 0.0001281, 0.0001281], dtype=float32), 'loss': ShardedDeviceArray([7.7923646, 7.7923646, 7.7923646, 7.7923646, 7.7923646, - 7.7923646, 7.7923646, 7.7923646], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001284, 0.0001284, 0.0001284, 0.0001284, 0.0001284, - 0.0001284, 0.0001284, 0.0001284], dtype=float32), 'loss': ShardedDeviceArray([7.851894, 7.851894, 7.851894, 7.851894, 7.851894, - 7.851894, 7.851894, 7.851894], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001287, 0.0001287, 0.0001287, 0.0001287, 0.0001287, - 0.0001287, 0.0001287, 0.0001287], dtype=float32), 'loss': ShardedDeviceArray([7.9031653, 7.9031653, 7.9031653, 7.9031653, 7.9031653, - 7.9031653, 7.9031653, 7.9031653], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.000129, 0.000129, 0.000129, 0.000129, 0.000129, - 0.000129, 0.000129, 0.000129], dtype=float32), 'loss': ShardedDeviceArray([7.9961405, 7.9961405, 7.9961405, 7.9961405, 7.9961405, - 7.9961405, 7.9961405, 7.9961405], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001293, 0.0001293, 0.0001293, 0.0001293, 0.0001293, - 0.0001293, 0.0001293, 0.0001293], dtype=float32), 'loss': ShardedDeviceArray([7.9092264, 7.9092264, 7.9092264, 7.9092264, 7.9092264, - 7.9092264, 7.9092264, 7.9092264], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001296, 0.0001296, 0.0001296, 0.0001296, 0.0001296, - 0.0001296, 0.0001296, 0.0001296], dtype=float32), 'loss': ShardedDeviceArray([7.845132, 7.845132, 7.845132, 7.845132, 7.845132, - 7.845132, 7.845132, 7.845132], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001299, 0.0001299, 0.0001299, 0.0001299, 0.0001299, - 0.0001299, 0.0001299, 0.0001299], dtype=float32), 'loss': ShardedDeviceArray([7.8732557, 7.8732557, 7.8732557, 7.8732557, 7.8732557, - 7.8732557, 7.8732557, 7.8732557], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001302, 0.0001302, 0.0001302, 0.0001302, 0.0001302, - 0.0001302, 0.0001302, 0.0001302], dtype=float32), 'loss': ShardedDeviceArray([7.875514, 7.875514, 7.875514, 7.875514, 7.875514, - 7.875514, 7.875514, 7.875514], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001305, 0.0001305, 0.0001305, 0.0001305, 0.0001305, - 0.0001305, 0.0001305, 0.0001305], dtype=float32), 'loss': ShardedDeviceArray([7.7459865, 7.7459865, 7.7459865, 7.7459865, 7.7459865, - 7.7459865, 7.7459865, 7.7459865], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001308, 0.0001308, 0.0001308, 0.0001308, 0.0001308, - 0.0001308, 0.0001308, 0.0001308], dtype=float32), 'loss': ShardedDeviceArray([8.150806, 8.150806, 8.150806, 8.150806, 8.150806, - 8.150806, 8.150806, 8.150806], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001311, 0.0001311, 0.0001311, 0.0001311, 0.0001311, - 0.0001311, 0.0001311, 0.0001311], dtype=float32), 'loss': ShardedDeviceArray([7.893339, 7.893339, 7.893339, 7.893339, 7.893339, - 7.893339, 7.893339, 7.893339], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001314, 0.0001314, 0.0001314, 0.0001314, 0.0001314, - 0.0001314, 0.0001314, 0.0001314], dtype=float32), 'loss': ShardedDeviceArray([8.129753, 8.129753, 8.129753, 8.129753, 8.129753, - 8.129753, 8.129753, 8.129753], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001317, 0.0001317, 0.0001317, 0.0001317, 0.0001317, - 0.0001317, 0.0001317, 0.0001317], dtype=float32), 'loss': ShardedDeviceArray([7.964561, 7.964561, 7.964561, 7.964561, 7.964561, - 7.964561, 7.964561, 7.964561], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.000132, 0.000132, 0.000132, 0.000132, 0.000132, - 0.000132, 0.000132, 0.000132], dtype=float32), 'loss': ShardedDeviceArray([7.915361, 7.915361, 7.915361, 7.915361, 7.915361, - 7.915361, 7.915361, 7.915361], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001323, 0.0001323, 0.0001323, 0.0001323, 0.0001323, - 0.0001323, 0.0001323, 0.0001323], dtype=float32), 'loss': ShardedDeviceArray([8.008739, 8.008739, 8.008739, 8.008739, 8.008739, - 8.008739, 8.008739, 8.008739], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001326, 0.0001326, 0.0001326, 0.0001326, 0.0001326, - 0.0001326, 0.0001326, 0.0001326], dtype=float32), 'loss': ShardedDeviceArray([7.92212, 7.92212, 7.92212, 7.92212, 7.92212, 7.92212, - 7.92212, 7.92212], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001329, 0.0001329, 0.0001329, 0.0001329, 0.0001329, - 0.0001329, 0.0001329, 0.0001329], dtype=float32), 'loss': ShardedDeviceArray([7.9835634, 7.9835634, 7.9835634, 7.9835634, 7.9835634, - 7.9835634, 7.9835634, 7.9835634], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001332, 0.0001332, 0.0001332, 0.0001332, 0.0001332, - 0.0001332, 0.0001332, 0.0001332], dtype=float32), 'loss': ShardedDeviceArray([7.877103, 7.877103, 7.877103, 7.877103, 7.877103, - 7.877103, 7.877103, 7.877103], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001335, 0.0001335, 0.0001335, 0.0001335, 0.0001335, - 0.0001335, 0.0001335, 0.0001335], dtype=float32), 'loss': ShardedDeviceArray([8.125843, 8.125843, 8.125843, 8.125843, 8.125843, - 8.125843, 8.125843, 8.125843], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001338, 0.0001338, 0.0001338, 0.0001338, 0.0001338, - 0.0001338, 0.0001338, 0.0001338], dtype=float32), 'loss': ShardedDeviceArray([7.8731213, 7.8731213, 7.8731213, 7.8731213, 7.8731213, - 7.8731213, 7.8731213, 7.8731213], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001341, 0.0001341, 0.0001341, 0.0001341, 0.0001341, - 0.0001341, 0.0001341, 0.0001341], dtype=float32), 'loss': ShardedDeviceArray([7.9708652, 7.9708652, 7.9708652, 7.9708652, 7.9708652, - 7.9708652, 7.9708652, 7.9708652], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001344, 0.0001344, 0.0001344, 0.0001344, 0.0001344, - 0.0001344, 0.0001344, 0.0001344], dtype=float32), 'loss': ShardedDeviceArray([7.783667, 7.783667, 7.783667, 7.783667, 7.783667, - 7.783667, 7.783667, 7.783667], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001347, 0.0001347, 0.0001347, 0.0001347, 0.0001347, - 0.0001347, 0.0001347, 0.0001347], dtype=float32), 'loss': ShardedDeviceArray([7.688542, 7.688542, 7.688542, 7.688542, 7.688542, - 7.688542, 7.688542, 7.688542], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.000135, 0.000135, 0.000135, 0.000135, 0.000135, - 0.000135, 0.000135, 0.000135], dtype=float32), 'loss': ShardedDeviceArray([7.810261, 7.810261, 7.810261, 7.810261, 7.810261, - 7.810261, 7.810261, 7.810261], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001353, 0.0001353, 0.0001353, 0.0001353, 0.0001353, - 0.0001353, 0.0001353, 0.0001353], dtype=float32), 'loss': ShardedDeviceArray([7.789603, 7.789603, 7.789603, 7.789603, 7.789603, - 7.789603, 7.789603, 7.789603], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001356, 0.0001356, 0.0001356, 0.0001356, 0.0001356, - 0.0001356, 0.0001356, 0.0001356], dtype=float32), 'loss': ShardedDeviceArray([7.976757, 7.976757, 7.976757, 7.976757, 7.976757, - 7.976757, 7.976757, 7.976757], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001359, 0.0001359, 0.0001359, 0.0001359, 0.0001359, - 0.0001359, 0.0001359, 0.0001359], dtype=float32), 'loss': ShardedDeviceArray([8.04048, 8.04048, 8.04048, 8.04048, 8.04048, 8.04048, - 8.04048, 8.04048], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001362, 0.0001362, 0.0001362, 0.0001362, 0.0001362, - 0.0001362, 0.0001362, 0.0001362], dtype=float32), 'loss': ShardedDeviceArray([7.758219, 7.758219, 7.758219, 7.758219, 7.758219, - 7.758219, 7.758219, 7.758219], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001365, 0.0001365, 0.0001365, 0.0001365, 0.0001365, - 0.0001365, 0.0001365, 0.0001365], dtype=float32), 'loss': ShardedDeviceArray([8.146248, 8.146248, 8.146248, 8.146248, 8.146248, - 8.146248, 8.146248, 8.146248], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001368, 0.0001368, 0.0001368, 0.0001368, 0.0001368, - 0.0001368, 0.0001368, 0.0001368], dtype=float32), 'loss': ShardedDeviceArray([7.768535, 7.768535, 7.768535, 7.768535, 7.768535, - 7.768535, 7.768535, 7.768535], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001371, 0.0001371, 0.0001371, 0.0001371, 0.0001371, - 0.0001371, 0.0001371, 0.0001371], dtype=float32), 'loss': ShardedDeviceArray([7.689828, 7.689828, 7.689828, 7.689828, 7.689828, - 7.689828, 7.689828, 7.689828], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001374, 0.0001374, 0.0001374, 0.0001374, 0.0001374, - 0.0001374, 0.0001374, 0.0001374], dtype=float32), 'loss': ShardedDeviceArray([7.796749, 7.796749, 7.796749, 7.796749, 7.796749, - 7.796749, 7.796749, 7.796749], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001377, 0.0001377, 0.0001377, 0.0001377, 0.0001377, - 0.0001377, 0.0001377, 0.0001377], dtype=float32), 'loss': ShardedDeviceArray([8.086693, 8.086693, 8.086693, 8.086693, 8.086693, - 8.086693, 8.086693, 8.086693], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.000138, 0.000138, 0.000138, 0.000138, 0.000138, - 0.000138, 0.000138, 0.000138], dtype=float32), 'loss': ShardedDeviceArray([8.044639, 8.044639, 8.044639, 8.044639, 8.044639, - 8.044639, 8.044639, 8.044639], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001383, 0.0001383, 0.0001383, 0.0001383, 0.0001383, - 0.0001383, 0.0001383, 0.0001383], dtype=float32), 'loss': ShardedDeviceArray([7.971658, 7.971658, 7.971658, 7.971658, 7.971658, - 7.971658, 7.971658, 7.971658], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001386, 0.0001386, 0.0001386, 0.0001386, 0.0001386, - 0.0001386, 0.0001386, 0.0001386], dtype=float32), 'loss': ShardedDeviceArray([7.994483, 7.994483, 7.994483, 7.994483, 7.994483, - 7.994483, 7.994483, 7.994483], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001389, 0.0001389, 0.0001389, 0.0001389, 0.0001389, - 0.0001389, 0.0001389, 0.0001389], dtype=float32), 'loss': ShardedDeviceArray([8.119253, 8.119253, 8.119253, 8.119253, 8.119253, - 8.119253, 8.119253, 8.119253], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001392, 0.0001392, 0.0001392, 0.0001392, 0.0001392, - 0.0001392, 0.0001392, 0.0001392], dtype=float32), 'loss': ShardedDeviceArray([7.9448185, 7.9448185, 7.9448185, 7.9448185, 7.9448185, - 7.9448185, 7.9448185, 7.9448185], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001395, 0.0001395, 0.0001395, 0.0001395, 0.0001395, - 0.0001395, 0.0001395, 0.0001395], dtype=float32), 'loss': ShardedDeviceArray([8.125984, 8.125984, 8.125984, 8.125984, 8.125984, - 8.125984, 8.125984, 8.125984], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001398, 0.0001398, 0.0001398, 0.0001398, 0.0001398, - 0.0001398, 0.0001398, 0.0001398], dtype=float32), 'loss': ShardedDeviceArray([7.8050995, 7.8050995, 7.8050995, 7.8050995, 7.8050995, - 7.8050995, 7.8050995, 7.8050995], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001401, 0.0001401, 0.0001401, 0.0001401, 0.0001401, - 0.0001401, 0.0001401, 0.0001401], dtype=float32), 'loss': ShardedDeviceArray([7.962894, 7.962894, 7.962894, 7.962894, 7.962894, - 7.962894, 7.962894, 7.962894], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001404, 0.0001404, 0.0001404, 0.0001404, 0.0001404, - 0.0001404, 0.0001404, 0.0001404], dtype=float32), 'loss': ShardedDeviceArray([7.7137384, 7.7137384, 7.7137384, 7.7137384, 7.7137384, - 7.7137384, 7.7137384, 7.7137384], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001407, 0.0001407, 0.0001407, 0.0001407, 0.0001407, - 0.0001407, 0.0001407, 0.0001407], dtype=float32), 'loss': ShardedDeviceArray([7.7208614, 7.7208614, 7.7208614, 7.7208614, 7.7208614, - 7.7208614, 7.7208614, 7.7208614], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.000141, 0.000141, 0.000141, 0.000141, 0.000141, - 0.000141, 0.000141, 0.000141], dtype=float32), 'loss': ShardedDeviceArray([7.9821005, 7.9821005, 7.9821005, 7.9821005, 7.9821005, - 7.9821005, 7.9821005, 7.9821005], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001413, 0.0001413, 0.0001413, 0.0001413, 0.0001413, - 0.0001413, 0.0001413, 0.0001413], dtype=float32), 'loss': ShardedDeviceArray([7.8411956, 7.8411956, 7.8411956, 7.8411956, 7.8411956, - 7.8411956, 7.8411956, 7.8411956], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001416, 0.0001416, 0.0001416, 0.0001416, 0.0001416, - 0.0001416, 0.0001416, 0.0001416], dtype=float32), 'loss': ShardedDeviceArray([8.111786, 8.111786, 8.111786, 8.111786, 8.111786, - 8.111786, 8.111786, 8.111786], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001419, 0.0001419, 0.0001419, 0.0001419, 0.0001419, - 0.0001419, 0.0001419, 0.0001419], dtype=float32), 'loss': ShardedDeviceArray([7.899773, 7.899773, 7.899773, 7.899773, 7.899773, - 7.899773, 7.899773, 7.899773], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001422, 0.0001422, 0.0001422, 0.0001422, 0.0001422, - 0.0001422, 0.0001422, 0.0001422], dtype=float32), 'loss': ShardedDeviceArray([7.9648595, 7.9648595, 7.9648595, 7.9648595, 7.9648595, - 7.9648595, 7.9648595, 7.9648595], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001425, 0.0001425, 0.0001425, 0.0001425, 0.0001425, - 0.0001425, 0.0001425, 0.0001425], dtype=float32), 'loss': ShardedDeviceArray([7.791326, 7.791326, 7.791326, 7.791326, 7.791326, - 7.791326, 7.791326, 7.791326], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001428, 0.0001428, 0.0001428, 0.0001428, 0.0001428, - 0.0001428, 0.0001428, 0.0001428], dtype=float32), 'loss': ShardedDeviceArray([7.6367683, 7.6367683, 7.6367683, 7.6367683, 7.6367683, - 7.6367683, 7.6367683, 7.6367683], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001431, 0.0001431, 0.0001431, 0.0001431, 0.0001431, - 0.0001431, 0.0001431, 0.0001431], dtype=float32), 'loss': ShardedDeviceArray([8.109274, 8.109274, 8.109274, 8.109274, 8.109274, - 8.109274, 8.109274, 8.109274], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001434, 0.0001434, 0.0001434, 0.0001434, 0.0001434, - 0.0001434, 0.0001434, 0.0001434], dtype=float32), 'loss': ShardedDeviceArray([7.957328, 7.957328, 7.957328, 7.957328, 7.957328, - 7.957328, 7.957328, 7.957328], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001437, 0.0001437, 0.0001437, 0.0001437, 0.0001437, - 0.0001437, 0.0001437, 0.0001437], dtype=float32), 'loss': ShardedDeviceArray([8.124847, 8.124847, 8.124847, 8.124847, 8.124847, - 8.124847, 8.124847, 8.124847], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.000144, 0.000144, 0.000144, 0.000144, 0.000144, - 0.000144, 0.000144, 0.000144], dtype=float32), 'loss': ShardedDeviceArray([8.072102, 8.072102, 8.072102, 8.072102, 8.072102, - 8.072102, 8.072102, 8.072102], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001443, 0.0001443, 0.0001443, 0.0001443, 0.0001443, - 0.0001443, 0.0001443, 0.0001443], dtype=float32), 'loss': ShardedDeviceArray([8.054686, 8.054686, 8.054686, 8.054686, 8.054686, - 8.054686, 8.054686, 8.054686], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001446, 0.0001446, 0.0001446, 0.0001446, 0.0001446, - 0.0001446, 0.0001446, 0.0001446], dtype=float32), 'loss': ShardedDeviceArray([7.875123, 7.875123, 7.875123, 7.875123, 7.875123, - 7.875123, 7.875123, 7.875123], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001449, 0.0001449, 0.0001449, 0.0001449, 0.0001449, - 0.0001449, 0.0001449, 0.0001449], dtype=float32), 'loss': ShardedDeviceArray([7.6778483, 7.6778483, 7.6778483, 7.6778483, 7.6778483, - 7.6778483, 7.6778483, 7.6778483], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001452, 0.0001452, 0.0001452, 0.0001452, 0.0001452, - 0.0001452, 0.0001452, 0.0001452], dtype=float32), 'loss': ShardedDeviceArray([7.8097315, 7.8097315, 7.8097315, 7.8097315, 7.8097315, - 7.8097315, 7.8097315, 7.8097315], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001455, 0.0001455, 0.0001455, 0.0001455, 0.0001455, - 0.0001455, 0.0001455, 0.0001455], dtype=float32), 'loss': ShardedDeviceArray([7.7626286, 7.7626286, 7.7626286, 7.7626286, 7.7626286, - 7.7626286, 7.7626286, 7.7626286], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001458, 0.0001458, 0.0001458, 0.0001458, 0.0001458, - 0.0001458, 0.0001458, 0.0001458], dtype=float32), 'loss': ShardedDeviceArray([7.825801, 7.825801, 7.825801, 7.825801, 7.825801, - 7.825801, 7.825801, 7.825801], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001461, 0.0001461, 0.0001461, 0.0001461, 0.0001461, - 0.0001461, 0.0001461, 0.0001461], dtype=float32), 'loss': ShardedDeviceArray([7.911273, 7.911273, 7.911273, 7.911273, 7.911273, - 7.911273, 7.911273, 7.911273], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001464, 0.0001464, 0.0001464, 0.0001464, 0.0001464, - 0.0001464, 0.0001464, 0.0001464], dtype=float32), 'loss': ShardedDeviceArray([7.7781672, 7.7781672, 7.7781672, 7.7781672, 7.7781672, - 7.7781672, 7.7781672, 7.7781672], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001467, 0.0001467, 0.0001467, 0.0001467, 0.0001467, - 0.0001467, 0.0001467, 0.0001467], dtype=float32), 'loss': ShardedDeviceArray([7.740696, 7.740696, 7.740696, 7.740696, 7.740696, - 7.740696, 7.740696, 7.740696], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.000147, 0.000147, 0.000147, 0.000147, 0.000147, - 0.000147, 0.000147, 0.000147], dtype=float32), 'loss': ShardedDeviceArray([7.7913523, 7.7913523, 7.7913523, 7.7913523, 7.7913523, - 7.7913523, 7.7913523, 7.7913523], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001473, 0.0001473, 0.0001473, 0.0001473, 0.0001473, - 0.0001473, 0.0001473, 0.0001473], dtype=float32), 'loss': ShardedDeviceArray([8.044626, 8.044626, 8.044626, 8.044626, 8.044626, - 8.044626, 8.044626, 8.044626], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001476, 0.0001476, 0.0001476, 0.0001476, 0.0001476, - 0.0001476, 0.0001476, 0.0001476], dtype=float32), 'loss': ShardedDeviceArray([8.033394, 8.033394, 8.033394, 8.033394, 8.033394, - 8.033394, 8.033394, 8.033394], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001479, 0.0001479, 0.0001479, 0.0001479, 0.0001479, - 0.0001479, 0.0001479, 0.0001479], dtype=float32), 'loss': ShardedDeviceArray([7.95217, 7.95217, 7.95217, 7.95217, 7.95217, 7.95217, - 7.95217, 7.95217], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001482, 0.0001482, 0.0001482, 0.0001482, 0.0001482, - 0.0001482, 0.0001482, 0.0001482], dtype=float32), 'loss': ShardedDeviceArray([7.8844614, 7.8844614, 7.8844614, 7.8844614, 7.8844614, - 7.8844614, 7.8844614, 7.8844614], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001485, 0.0001485, 0.0001485, 0.0001485, 0.0001485, - 0.0001485, 0.0001485, 0.0001485], dtype=float32), 'loss': ShardedDeviceArray([8.034703, 8.034703, 8.034703, 8.034703, 8.034703, - 8.034703, 8.034703, 8.034703], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001488, 0.0001488, 0.0001488, 0.0001488, 0.0001488, - 0.0001488, 0.0001488, 0.0001488], dtype=float32), 'loss': ShardedDeviceArray([7.847809, 7.847809, 7.847809, 7.847809, 7.847809, - 7.847809, 7.847809, 7.847809], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001491, 0.0001491, 0.0001491, 0.0001491, 0.0001491, - 0.0001491, 0.0001491, 0.0001491], dtype=float32), 'loss': ShardedDeviceArray([7.815194, 7.815194, 7.815194, 7.815194, 7.815194, - 7.815194, 7.815194, 7.815194], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001494, 0.0001494, 0.0001494, 0.0001494, 0.0001494, - 0.0001494, 0.0001494, 0.0001494], dtype=float32), 'loss': ShardedDeviceArray([7.7018757, 7.7018757, 7.7018757, 7.7018757, 7.7018757, - 7.7018757, 7.7018757, 7.7018757], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.0001497, 0.0001497, 0.0001497, 0.0001497, 0.0001497, - 0.0001497, 0.0001497, 0.0001497], dtype=float32), 'loss': ShardedDeviceArray([7.874889, 7.874889, 7.874889, 7.874889, 7.874889, - 7.874889, 7.874889, 7.874889], dtype=float32)}, {'learning_rate': ShardedDeviceArray([0.00015, 0.00015, 0.00015, 0.00015, 0.00015, 0.00015, - 0.00015, 0.00015], dtype=float32), 'loss': ShardedDeviceArray([8.018753, 8.018753, 8.018753, 8.018753, 8.018753, - 8.018753, 8.018753, 8.018753], dtype=float32)}] -ipdb> train_metrics -ipdb> tytrain_metrics -ipdb> tytrain_metrics -ipdb> tytrain_metrics -ipdb> tytrain_metrics -ipdb> tytrain_metrics -ipdb> tytrain_metrics -ipdb> tytrain_metrics -ipdb> tytrain_metrics -ipdb> tytrain_metrics -ipdb> tytrain_metrics -ipdb> tytrain_metrics -ipdb> tytrain_metrics -ipdb> tytrain_metrics -ipdb> tytrain_metrics -ipdb> tytrain_metrics -ipdb> tytrain_metrics -ipdb> tytrain_metrics -ipdb> tytrain_metrics -ipdb> tytrain_metrics -ipdb> tytrain_metrics -ipdb> tytrain_metrics -ipdb> tytrain_metrics -ipdb> tytrain_metrics -ipdb> tytrain_metrics -ipdb> tytrain_metrics -ipdb> tytrain_metrics -ipdb> tytrain_metrics -ipdb> tytrain_metrics -ipdb> tytrain_metrics -ipdb> tytrain_metrics -ipdb> tytrain_metrics -ipdb> tytrain_metrics -ipdb> tytrain_metrics -ipdb> tytrain_metrics -Evaluating ...: 0%| | 0/10 [00:00 tytrain_metrics -ipdb> tytrain_metrics -ipdb> tytrain_metrics -ipdb> tytrain_metrics -ipdb> tytrain_metrics -ipdb> tytrain_metrics -ipdb> tytrain_metrics -ipdb> tytrain_metrics - -Step... (500 | Loss: 8.205772399902344, Acc: 0.0773010179400444): 38%|██████████████████████████████████████▎ | 3/8 [13:50<23:03, 276.78s/it] \ No newline at end of file diff --git a/wandb/run-20210709_125049-89spqu1d/files/requirements.txt b/wandb/run-20210709_125049-89spqu1d/files/requirements.txt deleted file mode 100644 index 58a6f2b0a62dcb843c1f379c734d6bbe7a841939..0000000000000000000000000000000000000000 --- a/wandb/run-20210709_125049-89spqu1d/files/requirements.txt +++ /dev/null @@ -1,98 +0,0 @@ -absl-py==0.13.0 -aiohttp==3.7.4.post0 -async-timeout==3.0.1 -attrs==21.2.0 -backcall==0.2.0 -cachetools==4.2.2 -certifi==2021.5.30 -chardet==4.0.0 -chex==0.0.8 -click==8.0.1 -configparser==5.0.2 -cycler==0.10.0 -datasets==1.9.1.dev0 -decorator==5.0.9 -dill==0.3.4 -dm-tree==0.1.6 -docker-pycreds==0.4.0 -filelock==3.0.12 -flatbuffers==2.0 -flax==0.3.4 -fsspec==2021.6.1 -gitdb==4.0.7 -gitpython==3.1.18 -google-auth-oauthlib==0.4.4 -google-auth==1.32.1 -grpcio==1.38.1 -huggingface-hub==0.0.12 -idna==2.10 -ipdb==0.13.9 -ipython-genutils==0.2.0 -ipython==7.25.0 -jax==0.2.16 -jaxlib==0.1.68 -jedi==0.18.0 -joblib==1.0.1 -kiwisolver==1.3.1 -libtpu-nightly==0.1.dev20210615 -markdown==3.3.4 -matplotlib-inline==0.1.2 -matplotlib==3.4.2 -msgpack==1.0.2 -multidict==5.1.0 -multiprocess==0.70.12.2 -numpy==1.21.0 -oauthlib==3.1.1 -opt-einsum==3.3.0 -optax==0.0.9 -packaging==21.0 -pandas==1.3.0 -parso==0.8.2 -pathtools==0.1.2 -pexpect==4.8.0 -pickleshare==0.7.5 -pillow==8.3.1 -pip==20.0.2 -pkg-resources==0.0.0 -promise==2.3 -prompt-toolkit==3.0.19 -protobuf==3.17.3 -psutil==5.8.0 -ptyprocess==0.7.0 -pyarrow==4.0.1 -pyasn1-modules==0.2.8 -pyasn1==0.4.8 -pygments==2.9.0 -pyparsing==2.4.7 -python-dateutil==2.8.1 -pytz==2021.1 -pyyaml==5.4.1 -regex==2021.7.6 -requests-oauthlib==1.3.0 -requests==2.25.1 -rsa==4.7.2 -sacremoses==0.0.45 -scipy==1.7.0 -sentry-sdk==1.3.0 -setuptools==44.0.0 -shortuuid==1.0.1 -six==1.16.0 -smmap==4.0.0 -subprocess32==3.5.4 -tensorboard-data-server==0.6.1 -tensorboard-plugin-wit==1.8.0 -tensorboard==2.5.0 -tokenizers==0.10.3 -toml==0.10.2 -toolz==0.11.1 -tqdm==4.61.2 -traitlets==5.0.5 -transformers==4.9.0.dev0 -typing-extensions==3.10.0.0 -urllib3==1.26.6 -wandb==0.10.33 -wcwidth==0.2.5 -werkzeug==2.0.1 -wheel==0.36.2 -xxhash==2.0.2 -yarl==1.6.3 \ No newline at end of file diff --git a/wandb/run-20210709_125049-89spqu1d/files/wandb-metadata.json b/wandb/run-20210709_125049-89spqu1d/files/wandb-metadata.json deleted file mode 100644 index 343e746eb949e4a003723a56039d89462a2413d7..0000000000000000000000000000000000000000 --- a/wandb/run-20210709_125049-89spqu1d/files/wandb-metadata.json +++ /dev/null @@ -1,36 +0,0 @@ -{ - "os": "Linux-5.4.0-1043-gcp-x86_64-with-glibc2.29", - "python": "3.8.10", - "heartbeatAt": "2021-07-09T12:50:50.129806", - "startedAt": "2021-07-09T12:50:49.337388", - "docker": null, - "cpu_count": 96, - "cuda": null, - "args": [ - "--output_dir=./", - "--model_type=roberta", - "--config_name=./", - "--tokenizer_name=./", - "--dataset_name=oscar", - "--dataset_config_name=unshuffled_deduplicated_als", - "--max_seq_length=128", - "--per_device_train_batch_size=4", - "--per_device_eval_batch_size=4", - "--learning_rate=3e-4", - "--warmup_steps=1000", - "--overwrite_output_dir", - "--num_train_epochs=8" - ], - "state": "running", - "program": "run_mlm_flax.py", - "codePath": "run_mlm_flax.py", - "git": { - "remote": "https://huggingface.co/flax-community/roberta-base-mr", - "commit": "faa64e8a79301832a56c78b2b293675039070bb0" - }, - "email": "nipunsadvilkar@gmail.com", - "root": "/home/nipunsadvilkar/sample/roberta-base-mr", - "host": "t1v-n-112df4a9-w-0", - "username": "nipunsadvilkar", - "executable": "/home/nipunsadvilkar/roberta_mr_env/bin/python" -} diff --git a/wandb/run-20210709_125049-89spqu1d/files/wandb-summary.json b/wandb/run-20210709_125049-89spqu1d/files/wandb-summary.json deleted file mode 100644 index de417d3d096fef51acb6d8e33bdf6ebbe65a223f..0000000000000000000000000000000000000000 --- a/wandb/run-20210709_125049-89spqu1d/files/wandb-summary.json +++ /dev/null @@ -1 +0,0 @@ -{"train_time": 2.8953051567077637, "_runtime": 96, "_timestamp": 1625835145, "_step": 302, "training_learning_rate": {"_type": "histogram", "values": [3, 2, 2, 3, 2, 2, 3, 2, 2, 3, 2, 2, 3, 2, 2, 2, 3, 2, 2, 3, 2, 2, 3, 2, 2, 3, 2, 2, 3, 2, 2, 3], "bins": [0.00012780001270584762, 0.00012849376071244478, 0.00012918750871904194, 0.0001298812567256391, 0.00013057500473223627, 0.00013126876729074866, 0.00013196251529734582, 0.00013265626330394298, 0.00013335001131054014, 0.0001340437593171373, 0.00013473750732373446, 0.00013543125533033162, 0.000136125017888844, 0.00013681876589544117, 0.00013751251390203834, 0.0001382062619086355, 0.00013890000991523266, 0.00013959375792182982, 0.00014028750592842698, 0.00014098125393502414, 0.0001416750019416213, 0.0001423687645001337, 0.00014306251250673085, 0.00014375626051332802, 0.00014445000851992518, 0.00014514375652652234, 0.0001458375045331195, 0.00014653125253971666, 0.00014722501509822905, 0.0001479187631048262, 0.00014861251111142337, 0.00014930625911802053, 0.0001500000071246177]}, "train_step": 500, "train_learning_rate": 0.0001500000071246177, "training_loss": {"_type": "histogram", "values": [1, 0, 1, 2, 2, 1, 2, 2, 2, 6, 3, 2, 2, 2, 6, 2, 3, 3, 0, 3, 5, 3, 3, 2, 2, 3, 1, 1, 1, 2, 5, 2], "bins": [7.636768341064453, 7.65283203125, 7.668895721435547, 7.684959411621094, 7.701023101806641, 7.7170867919921875, 7.733150482177734, 7.749214172363281, 7.765277862548828, 7.781341552734375, 7.797405242919922, 7.813468933105469, 7.829532623291016, 7.8455963134765625, 7.861660003662109, 7.877723693847656, 7.893787384033203, 7.90985107421875, 7.925914764404297, 7.941978454589844, 7.958042144775391, 7.9741058349609375, 7.990169525146484, 8.006233215332031, 8.022296905517578, 8.038360595703125, 8.054424285888672, 8.070487976074219, 8.086551666259766, 8.102615356445312, 8.11867904663086, 8.134742736816406, 8.150806427001953]}, "train_loss": 8.018753051757812} \ No newline at end of file diff --git a/wandb/run-20210709_125049-89spqu1d/logs/debug-internal.log b/wandb/run-20210709_125049-89spqu1d/logs/debug-internal.log deleted file mode 100644 index fd36e5980d25ac99cbdc8bf1ad372c41f372b929..0000000000000000000000000000000000000000 --- a/wandb/run-20210709_125049-89spqu1d/logs/debug-internal.log +++ /dev/null @@ -1,1240 +0,0 @@ -2021-07-09 12:50:49,995 INFO MainThread:37232 [internal.py:wandb_internal():88] W&B internal server running at pid: 37232, started at: 2021-07-09 12:50:49.995101 -2021-07-09 12:50:49,997 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: check_version -2021-07-09 12:50:49,997 INFO WriterThread:37232 [datastore.py:open_for_write():80] open: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/run-89spqu1d.wandb -2021-07-09 12:50:49,999 DEBUG SenderThread:37232 [sender.py:send():179] send: header -2021-07-09 12:50:49,999 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: check_version -2021-07-09 12:50:50,064 DEBUG SenderThread:37232 [sender.py:send():179] send: run -2021-07-09 12:50:50,126 INFO SenderThread:37232 [dir_watcher.py:__init__():168] watching files in: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files -2021-07-09 12:50:50,126 INFO SenderThread:37232 [sender.py:_start_run_threads():716] run started: 89spqu1d with start time 1625835049 -2021-07-09 12:50:50,126 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:50:50,126 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:50:50,128 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: run_start -2021-07-09 12:50:50,129 DEBUG HandlerThread:37232 [meta.py:__init__():39] meta init -2021-07-09 12:50:50,129 DEBUG HandlerThread:37232 [meta.py:__init__():53] meta init done -2021-07-09 12:50:50,129 DEBUG HandlerThread:37232 [meta.py:probe():210] probe -2021-07-09 12:50:50,131 DEBUG HandlerThread:37232 [meta.py:_setup_git():200] setup git -2021-07-09 12:50:50,151 DEBUG HandlerThread:37232 [meta.py:_setup_git():207] setup git done -2021-07-09 12:50:50,151 DEBUG HandlerThread:37232 [meta.py:_save_pip():57] save pip -2021-07-09 12:50:50,152 DEBUG HandlerThread:37232 [meta.py:_save_pip():71] save pip done -2021-07-09 12:50:50,152 DEBUG HandlerThread:37232 [meta.py:probe():252] probe done -2021-07-09 12:50:50,183 DEBUG SenderThread:37232 [sender.py:send():179] send: files -2021-07-09 12:50:50,184 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-metadata.json with policy now -2021-07-09 12:50:50,188 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: stop_status -2021-07-09 12:50:50,188 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: stop_status -2021-07-09 12:50:50,216 DEBUG SenderThread:37232 [sender.py:send():179] send: config -2021-07-09 12:50:50,217 DEBUG SenderThread:37232 [sender.py:send():179] send: config -2021-07-09 12:50:50,217 DEBUG SenderThread:37232 [sender.py:send():179] send: config -2021-07-09 12:50:50,392 INFO Thread-11 :37232 [upload_job.py:push():137] Uploaded file /tmp/tmpicd15h30wandb/9d69te0o-wandb-metadata.json -2021-07-09 12:50:51,127 INFO Thread-8 :37232 [dir_watcher.py:_on_file_created():216] file/dir created: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/wandb-summary.json -2021-07-09 12:50:51,127 INFO Thread-8 :37232 [dir_watcher.py:_on_file_created():216] file/dir created: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/output.log -2021-07-09 12:50:51,128 INFO Thread-8 :37232 [dir_watcher.py:_on_file_created():216] file/dir created: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/requirements.txt -2021-07-09 12:50:51,128 INFO Thread-8 :37232 [dir_watcher.py:_on_file_created():216] file/dir created: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/wandb-metadata.json -2021-07-09 12:50:57,128 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/output.log -2021-07-09 12:50:59,129 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/output.log -2021-07-09 12:51:05,235 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: stop_status -2021-07-09 12:51:05,235 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: stop_status -2021-07-09 12:51:18,211 DEBUG SenderThread:37232 [sender.py:send():179] send: stats -2021-07-09 12:51:20,275 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: stop_status -2021-07-09 12:51:20,285 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: stop_status -2021-07-09 12:51:21,139 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/config.yaml -2021-07-09 12:51:35,312 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: stop_status -2021-07-09 12:51:35,313 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: stop_status -2021-07-09 12:51:48,297 DEBUG SenderThread:37232 [sender.py:send():179] send: stats -2021-07-09 12:51:50,344 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: stop_status -2021-07-09 12:51:50,344 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: stop_status -2021-07-09 12:52:05,424 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: stop_status -2021-07-09 12:52:05,424 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: stop_status -2021-07-09 12:52:11,160 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/output.log -2021-07-09 12:52:13,161 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/output.log -2021-07-09 12:52:15,161 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/output.log -2021-07-09 12:52:17,162 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/output.log -2021-07-09 12:52:18,380 DEBUG SenderThread:37232 [sender.py:send():179] send: stats -2021-07-09 12:52:19,163 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/output.log -2021-07-09 12:52:20,457 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: stop_status -2021-07-09 12:52:20,457 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: stop_status -2021-07-09 12:52:21,164 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/output.log -2021-07-09 12:52:23,164 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/output.log -2021-07-09 12:52:25,165 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/output.log -2021-07-09 12:52:25,427 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:25,428 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:25,428 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:25,863 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:25,864 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:25,868 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:25,869 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:25,869 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:25,875 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:25,875 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:25,875 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:25,880 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:25,880 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:25,880 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:25,886 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:25,887 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:25,887 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:25,889 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:25,889 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:25,889 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:25,893 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:25,893 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:25,893 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:25,896 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:25,896 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:25,896 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:25,901 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:25,901 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:25,901 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:25,904 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:25,904 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:25,904 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:25,910 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:25,910 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:25,910 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:25,914 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:25,915 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:25,915 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:25,916 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:25,916 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:25,917 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:25,920 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:25,920 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:25,920 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:25,925 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:25,925 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:25,925 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:25,927 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:25,927 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:25,927 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:25,930 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:25,930 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:25,930 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:25,933 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:25,934 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:25,934 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:25,935 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:25,935 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:25,936 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:25,938 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:25,938 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:25,938 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:25,941 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:25,942 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:25,942 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:25,947 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:25,948 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:25,948 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:25,951 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:25,951 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:25,951 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:25,956 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:25,956 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:25,957 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:25,961 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:25,962 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:25,962 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:25,965 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:25,966 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:25,966 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:25,968 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:25,968 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:25,968 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:25,977 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:25,979 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:25,979 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:25,988 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:25,989 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:25,989 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:25,991 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:25,991 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:25,991 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:25,997 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:25,997 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:25,997 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,007 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,011 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,011 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,012 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,012 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,012 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,013 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,013 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,013 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,013 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,013 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,013 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,014 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,014 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,014 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,015 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,015 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,015 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,015 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,015 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,016 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,016 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,016 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,016 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,017 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,017 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,017 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,017 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,017 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,018 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,019 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,019 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,019 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,020 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,020 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,020 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,021 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,021 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,021 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,021 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,022 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,022 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,022 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,022 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,022 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,023 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,023 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,023 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,023 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,023 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,023 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,024 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,024 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,024 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,024 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,024 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,025 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,025 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,025 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,025 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,026 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,026 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,026 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,026 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,027 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,027 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,027 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,027 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,027 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,028 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,028 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,028 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,028 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,029 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,029 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,029 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,029 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,029 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,029 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,030 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,030 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,030 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,030 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,030 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,031 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,031 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,031 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,031 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,031 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,031 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,032 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,032 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,032 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,032 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,032 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,032 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,033 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,033 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,033 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,034 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,034 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,034 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,034 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,034 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,034 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,035 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,035 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,035 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,035 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,036 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,036 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,036 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,036 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,036 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,037 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,037 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,037 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,037 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,037 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,037 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,038 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,038 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,038 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,038 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,038 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,039 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,039 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,039 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,039 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,039 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,040 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,040 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,040 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,040 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,040 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,041 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,041 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,041 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,041 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,041 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,041 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,042 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,042 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,042 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,042 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,042 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,042 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,043 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,043 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,043 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,043 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,043 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,043 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,044 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,044 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,044 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,045 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,045 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,045 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,045 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,045 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,045 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,046 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,046 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,046 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,046 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,047 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,047 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,047 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,047 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,047 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,048 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,048 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,048 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,048 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,048 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,048 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,049 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,049 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,049 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,049 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,049 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,049 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,050 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,050 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,050 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,050 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,050 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,050 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,051 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,051 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,051 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,051 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,052 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,052 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,052 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,052 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,052 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,052 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,053 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,053 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,053 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,053 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,053 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,053 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,054 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,054 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,054 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,054 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,054 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,055 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,055 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,055 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,055 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,055 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,055 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,056 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,056 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,056 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,056 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,056 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,056 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,057 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,057 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,057 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,058 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,058 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,058 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,058 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,058 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,058 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,059 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,059 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,059 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,059 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,060 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,060 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,060 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,060 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,060 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,060 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,061 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,061 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,061 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,061 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,061 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,062 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,062 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,062 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,062 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,062 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,062 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,063 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,063 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,063 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,063 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,063 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,063 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,064 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,064 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,064 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,064 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,065 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,065 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,065 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,065 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,065 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,066 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,066 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,066 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,066 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,066 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,066 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,067 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,067 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,067 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,067 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,067 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,067 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,068 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,068 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,068 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,068 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,069 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,069 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,069 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,069 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,069 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,069 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,070 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,070 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,070 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,070 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,070 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,071 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,071 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,071 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,071 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,071 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,071 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,072 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,072 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,072 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,072 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,072 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,073 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,073 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,073 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,073 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,074 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,074 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,074 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,074 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,074 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,074 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,075 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,075 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,075 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,075 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,075 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,076 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,076 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,076 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,076 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,076 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,077 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,077 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,077 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,077 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,077 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,078 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,078 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,078 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,078 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,078 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,078 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,079 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,079 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,079 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,079 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,079 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,079 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,080 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,080 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,080 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,080 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,080 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,081 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,081 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,081 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,081 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,081 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,082 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,082 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,082 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,082 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,082 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,082 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,083 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,083 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,083 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,083 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,083 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,084 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,084 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,084 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,084 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,084 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,084 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,085 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,085 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,085 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,085 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,085 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,085 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,086 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,086 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,086 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,086 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,087 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,087 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,087 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,087 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,087 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,088 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,088 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,088 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,088 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,088 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,088 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,089 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,089 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,089 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,089 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,089 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,090 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,090 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,090 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,090 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,091 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,091 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,091 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,091 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,091 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,091 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,092 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,092 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,092 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,092 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,092 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,092 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,093 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,093 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,093 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,093 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,093 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,093 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,094 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,094 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,094 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,094 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,094 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,095 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,095 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,095 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,095 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,095 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,096 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,096 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,096 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,096 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,096 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,097 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,097 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,097 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,097 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,097 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,097 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,098 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,098 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,098 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,098 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,098 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,099 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,099 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,099 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,099 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,099 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,100 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,100 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,100 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,100 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,100 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,101 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,101 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,101 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,101 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,101 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,101 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,102 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,102 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,102 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,102 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,102 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,102 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,103 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,103 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,103 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,103 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,104 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,104 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,104 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,104 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,104 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,105 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,105 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,105 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,105 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,105 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,105 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,106 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,106 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,106 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,106 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,106 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,107 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,107 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,107 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,107 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,107 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,108 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,108 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,108 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,108 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,108 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,109 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,109 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,109 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,109 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,109 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,109 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,110 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,110 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,110 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,110 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,110 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,110 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,111 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,111 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,111 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,111 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,112 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,112 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,112 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,112 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,112 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,113 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,113 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,113 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,113 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,113 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,113 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,114 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,114 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,114 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,114 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,114 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,114 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,115 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,115 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,115 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,115 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,115 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,116 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,116 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,116 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,116 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,117 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,117 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,117 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,117 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,117 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,117 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,118 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,118 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,118 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,118 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,119 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,119 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,119 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,119 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,119 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,120 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,120 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,120 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,120 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,120 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,120 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,121 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,121 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,121 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,121 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,121 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,122 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,122 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,122 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,122 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,122 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,122 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,123 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,123 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,123 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,123 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,124 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,124 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,124 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,124 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,124 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,124 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,125 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,125 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,125 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,125 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,125 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,125 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,126 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,126 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,126 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,126 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,126 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,127 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,127 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,127 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,127 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,127 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,128 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,128 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,128 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,128 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,128 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,129 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,129 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,129 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,129 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,129 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,129 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,130 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,130 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,130 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,130 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,130 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,131 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,131 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,131 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,131 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,131 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,132 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,132 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,132 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,132 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,132 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,133 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,133 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,133 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,133 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,133 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,133 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,134 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,134 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,134 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,134 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,134 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,134 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,135 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,135 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,135 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,135 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,135 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,136 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,136 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,136 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,136 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,137 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,137 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,137 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,137 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,137 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,137 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,138 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,138 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,138 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,138 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,139 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,139 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,139 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,139 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,139 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,140 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,140 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,140 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,140 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,140 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,140 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,141 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,141 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,141 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,141 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,141 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,141 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,142 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,142 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,142 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,142 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,142 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,143 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,143 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,143 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,143 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,143 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,144 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,144 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,144 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,144 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,144 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,145 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,145 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,145 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,145 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,145 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,145 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,146 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,146 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,146 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,146 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,146 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,147 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,147 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,147 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,147 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,147 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,148 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,148 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,148 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,148 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,148 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,149 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,149 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,149 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,149 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,149 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,149 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,150 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,150 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,150 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,150 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,150 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,150 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,151 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,151 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,151 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,151 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,151 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,151 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,152 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,152 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,152 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,152 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,152 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,153 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,153 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,153 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,153 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,153 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,154 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,154 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,154 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,154 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,154 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,155 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,155 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,155 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,155 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,155 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,155 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,156 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,156 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,156 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,156 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,156 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,157 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,157 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,157 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,157 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,158 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,158 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,158 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,158 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,158 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,158 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,159 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,159 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,159 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,159 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,159 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,160 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,160 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,160 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,160 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,160 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,161 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,161 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,161 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,161 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,161 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,162 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,162 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,162 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,162 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,162 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,162 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,163 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,163 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,163 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,163 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,163 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,164 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,164 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,164 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,164 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,164 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,165 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,165 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,165 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,165 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,165 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,166 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,166 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,166 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,167 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/wandb-summary.json -2021-07-09 12:52:26,167 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,167 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,167 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,167 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,168 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,168 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,168 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,168 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,168 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,169 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,169 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,169 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,169 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:26,169 DEBUG SenderThread:37232 [sender.py:send():179] send: history -2021-07-09 12:52:26,169 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 12:52:26,170 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 12:52:27,167 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/wandb-summary.json -2021-07-09 12:52:27,167 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/output.log -2021-07-09 12:52:35,492 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: stop_status -2021-07-09 12:52:35,493 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: stop_status -2021-07-09 12:52:47,174 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/output.log -2021-07-09 12:52:48,463 DEBUG SenderThread:37232 [sender.py:send():179] send: stats -2021-07-09 12:52:50,524 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: stop_status -2021-07-09 12:52:50,524 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: stop_status -2021-07-09 12:53:05,552 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: stop_status -2021-07-09 12:53:05,552 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: stop_status -2021-07-09 12:53:18,538 DEBUG SenderThread:37232 [sender.py:send():179] send: stats -2021-07-09 12:53:20,604 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: stop_status -2021-07-09 12:53:20,604 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: stop_status -2021-07-09 12:53:35,634 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: stop_status -2021-07-09 12:53:35,634 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: stop_status -2021-07-09 12:53:48,616 DEBUG SenderThread:37232 [sender.py:send():179] send: stats -2021-07-09 12:53:50,666 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: stop_status -2021-07-09 12:53:50,666 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: stop_status -2021-07-09 12:54:05,699 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: stop_status -2021-07-09 12:54:05,699 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: stop_status -2021-07-09 12:54:18,691 DEBUG SenderThread:37232 [sender.py:send():179] send: stats -2021-07-09 12:54:20,751 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: stop_status -2021-07-09 12:54:20,751 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: stop_status -2021-07-09 12:54:35,779 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: stop_status -2021-07-09 12:54:35,780 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: stop_status -2021-07-09 12:54:48,764 DEBUG SenderThread:37232 [sender.py:send():179] send: stats -2021-07-09 12:54:50,813 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: stop_status -2021-07-09 12:54:50,814 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: stop_status -2021-07-09 12:55:05,851 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: stop_status -2021-07-09 12:55:05,851 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: stop_status -2021-07-09 12:55:18,841 DEBUG SenderThread:37232 [sender.py:send():179] send: stats -2021-07-09 12:55:20,914 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: stop_status -2021-07-09 12:55:20,915 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: stop_status -2021-07-09 12:55:35,953 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: stop_status -2021-07-09 12:55:35,953 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: stop_status -2021-07-09 12:55:48,918 DEBUG SenderThread:37232 [sender.py:send():179] send: stats -2021-07-09 12:55:50,986 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: stop_status -2021-07-09 12:55:50,986 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: stop_status -2021-07-09 12:56:06,018 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: stop_status -2021-07-09 12:56:06,018 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: stop_status -2021-07-09 12:56:18,994 DEBUG SenderThread:37232 [sender.py:send():179] send: stats -2021-07-09 12:56:21,056 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: stop_status -2021-07-09 12:56:21,056 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: stop_status -2021-07-09 12:56:36,088 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: stop_status -2021-07-09 12:56:36,088 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: stop_status -2021-07-09 12:56:49,071 DEBUG SenderThread:37232 [sender.py:send():179] send: stats -2021-07-09 12:56:51,132 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: stop_status -2021-07-09 12:56:51,132 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: stop_status -2021-07-09 12:57:06,165 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: stop_status -2021-07-09 12:57:06,165 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: stop_status -2021-07-09 12:57:19,148 DEBUG SenderThread:37232 [sender.py:send():179] send: stats -2021-07-09 12:57:21,197 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: stop_status -2021-07-09 12:57:21,197 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: stop_status -2021-07-09 12:57:36,225 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: stop_status -2021-07-09 12:57:36,225 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: stop_status -2021-07-09 12:57:49,225 DEBUG SenderThread:37232 [sender.py:send():179] send: stats -2021-07-09 12:57:51,255 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: stop_status -2021-07-09 12:57:51,255 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: stop_status -2021-07-09 12:58:06,285 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: stop_status -2021-07-09 12:58:06,285 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: stop_status -2021-07-09 12:58:19,306 DEBUG SenderThread:37232 [sender.py:send():179] send: stats -2021-07-09 12:58:21,317 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: stop_status -2021-07-09 12:58:21,318 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: stop_status -2021-07-09 12:58:36,347 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: stop_status -2021-07-09 12:58:36,347 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: stop_status -2021-07-09 12:58:49,390 DEBUG SenderThread:37232 [sender.py:send():179] send: stats -2021-07-09 12:58:51,377 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: stop_status -2021-07-09 12:58:51,377 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: stop_status -2021-07-09 12:59:06,621 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: stop_status -2021-07-09 12:59:06,621 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: stop_status -2021-07-09 12:59:19,470 DEBUG SenderThread:37232 [sender.py:send():179] send: stats -2021-07-09 12:59:21,650 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: stop_status -2021-07-09 12:59:21,650 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: stop_status -2021-07-09 12:59:36,678 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: stop_status -2021-07-09 12:59:36,679 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: stop_status -2021-07-09 12:59:39,320 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/output.log -2021-07-09 12:59:47,323 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/output.log -2021-07-09 12:59:49,324 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/output.log -2021-07-09 12:59:49,552 DEBUG SenderThread:37232 [sender.py:send():179] send: stats -2021-07-09 12:59:51,325 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/output.log -2021-07-09 12:59:51,706 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: stop_status -2021-07-09 12:59:51,706 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: stop_status -2021-07-09 12:59:53,326 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/output.log -2021-07-09 12:59:55,326 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/output.log -2021-07-09 12:59:57,327 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/output.log -2021-07-09 12:59:59,328 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/output.log -2021-07-09 13:00:01,329 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/output.log -2021-07-09 13:00:06,735 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: stop_status -2021-07-09 13:00:06,735 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: stop_status -2021-07-09 13:00:19,335 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/output.log -2021-07-09 13:00:19,632 DEBUG SenderThread:37232 [sender.py:send():179] send: stats -2021-07-09 13:00:21,336 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/output.log -2021-07-09 13:00:21,767 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: stop_status -2021-07-09 13:00:21,767 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: stop_status -2021-07-09 13:00:25,337 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/output.log -2021-07-09 13:00:27,338 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/output.log -2021-07-09 13:00:35,340 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/output.log -2021-07-09 13:00:36,800 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: stop_status -2021-07-09 13:00:36,800 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: stop_status -2021-07-09 13:00:37,341 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/output.log -2021-07-09 13:00:39,342 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/output.log -2021-07-09 13:00:41,342 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/output.log -2021-07-09 13:00:43,343 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/output.log -2021-07-09 13:00:45,344 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/output.log -2021-07-09 13:00:49,345 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/output.log -2021-07-09 13:00:49,713 DEBUG SenderThread:37232 [sender.py:send():179] send: stats -2021-07-09 13:00:51,346 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/output.log -2021-07-09 13:00:51,828 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: stop_status -2021-07-09 13:00:51,829 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: stop_status -2021-07-09 13:00:53,347 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/output.log -2021-07-09 13:00:55,347 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/output.log -2021-07-09 13:00:57,348 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/output.log -2021-07-09 13:01:01,349 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/output.log -2021-07-09 13:01:03,350 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/output.log -2021-07-09 13:01:05,351 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/output.log -2021-07-09 13:01:06,861 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: stop_status -2021-07-09 13:01:06,862 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: stop_status -2021-07-09 13:01:13,353 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/output.log -2021-07-09 13:01:17,354 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/output.log -2021-07-09 13:01:19,355 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/output.log -2021-07-09 13:01:19,798 DEBUG SenderThread:37232 [sender.py:send():179] send: stats -2021-07-09 13:01:21,356 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/output.log -2021-07-09 13:01:21,894 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: stop_status -2021-07-09 13:01:21,894 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: stop_status -2021-07-09 13:01:25,357 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/output.log -2021-07-09 13:01:33,360 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/output.log -2021-07-09 13:01:35,360 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/output.log -2021-07-09 13:01:36,923 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: stop_status -2021-07-09 13:01:36,924 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: stop_status -2021-07-09 13:01:49,879 DEBUG SenderThread:37232 [sender.py:send():179] send: stats -2021-07-09 13:01:51,961 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: stop_status -2021-07-09 13:01:51,962 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: stop_status -2021-07-09 13:01:57,368 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/output.log -2021-07-09 13:02:01,369 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/output.log -2021-07-09 13:02:06,997 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: stop_status -2021-07-09 13:02:06,997 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: stop_status -2021-07-09 13:02:19,963 DEBUG SenderThread:37232 [sender.py:send():179] send: stats -2021-07-09 13:02:22,026 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: stop_status -2021-07-09 13:02:22,027 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: stop_status -2021-07-09 13:02:37,057 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: stop_status -2021-07-09 13:02:37,057 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: stop_status -2021-07-09 13:02:50,044 DEBUG SenderThread:37232 [sender.py:send():179] send: stats -2021-07-09 13:02:52,086 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: stop_status -2021-07-09 13:02:52,086 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: stop_status -2021-07-09 13:03:07,119 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: stop_status -2021-07-09 13:03:07,119 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: stop_status -2021-07-09 13:03:20,125 DEBUG SenderThread:37232 [sender.py:send():179] send: stats -2021-07-09 13:03:22,227 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: stop_status -2021-07-09 13:03:22,228 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: stop_status -2021-07-09 13:03:37,258 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: stop_status -2021-07-09 13:03:37,258 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: stop_status -2021-07-09 13:03:50,206 DEBUG SenderThread:37232 [sender.py:send():179] send: stats -2021-07-09 13:03:52,293 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: stop_status -2021-07-09 13:03:52,293 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: stop_status -2021-07-09 13:04:03,410 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/output.log -2021-07-09 13:04:07,321 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: stop_status -2021-07-09 13:04:07,321 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: stop_status -2021-07-09 13:04:07,412 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/output.log -2021-07-09 13:04:11,413 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/output.log -2021-07-09 13:04:13,414 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/output.log -2021-07-09 13:04:15,415 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/output.log -2021-07-09 13:04:17,416 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/output.log -2021-07-09 13:04:19,416 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/output.log -2021-07-09 13:04:20,288 DEBUG SenderThread:37232 [sender.py:send():179] send: stats -2021-07-09 13:04:21,417 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/output.log -2021-07-09 13:04:22,355 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: stop_status -2021-07-09 13:04:22,355 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: stop_status -2021-07-09 13:04:37,385 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: stop_status -2021-07-09 13:04:37,385 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: stop_status -2021-07-09 13:04:47,426 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/output.log -2021-07-09 13:04:48,276 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: poll_exit -2021-07-09 13:04:48,277 DEBUG SenderThread:37232 [sender.py:send():179] send: telemetry -2021-07-09 13:04:48,277 DEBUG SenderThread:37232 [sender.py:send():179] send: exit -2021-07-09 13:04:48,277 INFO SenderThread:37232 [sender.py:send_exit():287] handling exit code: 0 -2021-07-09 13:04:48,277 INFO SenderThread:37232 [sender.py:send_exit():295] send defer -2021-07-09 13:04:48,277 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: poll_exit -2021-07-09 13:04:48,278 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: defer -2021-07-09 13:04:48,278 INFO HandlerThread:37232 [handler.py:handle_request_defer():141] handle defer: 0 -2021-07-09 13:04:48,278 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: defer -2021-07-09 13:04:48,278 INFO SenderThread:37232 [sender.py:send_request_defer():304] handle sender defer: 0 -2021-07-09 13:04:48,278 INFO SenderThread:37232 [sender.py:transition_state():308] send defer: 1 -2021-07-09 13:04:48,278 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: defer -2021-07-09 13:04:48,278 INFO HandlerThread:37232 [handler.py:handle_request_defer():141] handle defer: 1 -2021-07-09 13:04:48,366 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: defer -2021-07-09 13:04:48,366 INFO SenderThread:37232 [sender.py:send_request_defer():304] handle sender defer: 1 -2021-07-09 13:04:48,367 INFO SenderThread:37232 [sender.py:transition_state():308] send defer: 2 -2021-07-09 13:04:48,367 DEBUG SenderThread:37232 [sender.py:send():179] send: stats -2021-07-09 13:04:48,367 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: defer -2021-07-09 13:04:48,367 INFO HandlerThread:37232 [handler.py:handle_request_defer():141] handle defer: 2 -2021-07-09 13:04:48,367 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: defer -2021-07-09 13:04:48,368 INFO SenderThread:37232 [sender.py:send_request_defer():304] handle sender defer: 2 -2021-07-09 13:04:48,368 INFO SenderThread:37232 [sender.py:transition_state():308] send defer: 3 -2021-07-09 13:04:48,368 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: defer -2021-07-09 13:04:48,368 INFO HandlerThread:37232 [handler.py:handle_request_defer():141] handle defer: 3 -2021-07-09 13:04:48,368 DEBUG SenderThread:37232 [sender.py:send():179] send: summary -2021-07-09 13:04:48,369 INFO SenderThread:37232 [sender.py:_save_file():841] saving file wandb-summary.json with policy end -2021-07-09 13:04:48,369 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: defer -2021-07-09 13:04:48,369 INFO SenderThread:37232 [sender.py:send_request_defer():304] handle sender defer: 3 -2021-07-09 13:04:48,369 INFO SenderThread:37232 [sender.py:transition_state():308] send defer: 4 -2021-07-09 13:04:48,369 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: defer -2021-07-09 13:04:48,370 INFO HandlerThread:37232 [handler.py:handle_request_defer():141] handle defer: 4 -2021-07-09 13:04:48,370 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: defer -2021-07-09 13:04:48,370 INFO SenderThread:37232 [sender.py:send_request_defer():304] handle sender defer: 4 -2021-07-09 13:04:48,379 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: poll_exit -2021-07-09 13:04:48,438 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/wandb-summary.json -2021-07-09 13:04:48,439 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/output.log -2021-07-09 13:04:48,439 INFO SenderThread:37232 [sender.py:transition_state():308] send defer: 5 -2021-07-09 13:04:48,439 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: poll_exit -2021-07-09 13:04:48,440 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: defer -2021-07-09 13:04:48,440 INFO HandlerThread:37232 [handler.py:handle_request_defer():141] handle defer: 5 -2021-07-09 13:04:48,440 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: defer -2021-07-09 13:04:48,440 INFO SenderThread:37232 [sender.py:send_request_defer():304] handle sender defer: 5 -2021-07-09 13:04:48,440 INFO SenderThread:37232 [dir_watcher.py:finish():282] shutting down directory watcher -2021-07-09 13:04:48,541 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: poll_exit -2021-07-09 13:04:49,439 INFO Thread-8 :37232 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/config.yaml -2021-07-09 13:04:49,439 INFO SenderThread:37232 [dir_watcher.py:finish():312] scan: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files -2021-07-09 13:04:49,440 INFO SenderThread:37232 [dir_watcher.py:finish():318] scan save: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/requirements.txt requirements.txt -2021-07-09 13:04:49,440 INFO SenderThread:37232 [dir_watcher.py:finish():318] scan save: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/output.log output.log -2021-07-09 13:04:49,440 INFO SenderThread:37232 [dir_watcher.py:finish():318] scan save: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/wandb-metadata.json wandb-metadata.json -2021-07-09 13:04:49,440 INFO SenderThread:37232 [dir_watcher.py:finish():318] scan save: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/config.yaml config.yaml -2021-07-09 13:04:49,440 INFO SenderThread:37232 [dir_watcher.py:finish():318] scan save: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/wandb-summary.json wandb-summary.json -2021-07-09 13:04:49,440 INFO SenderThread:37232 [sender.py:transition_state():308] send defer: 6 -2021-07-09 13:04:49,441 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: poll_exit -2021-07-09 13:04:49,444 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: defer -2021-07-09 13:04:49,444 INFO HandlerThread:37232 [handler.py:handle_request_defer():141] handle defer: 6 -2021-07-09 13:04:49,446 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: defer -2021-07-09 13:04:49,446 INFO SenderThread:37232 [sender.py:send_request_defer():304] handle sender defer: 6 -2021-07-09 13:04:49,446 INFO SenderThread:37232 [file_pusher.py:finish():177] shutting down file pusher -2021-07-09 13:04:49,542 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: poll_exit -2021-07-09 13:04:49,542 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: poll_exit -2021-07-09 13:04:49,643 INFO Thread-14 :37232 [upload_job.py:push():137] Uploaded file /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/config.yaml -2021-07-09 13:04:49,644 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: poll_exit -2021-07-09 13:04:49,644 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: poll_exit -2021-07-09 13:04:49,685 INFO Thread-12 :37232 [upload_job.py:push():137] Uploaded file /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/requirements.txt -2021-07-09 13:04:49,687 INFO Thread-15 :37232 [upload_job.py:push():137] Uploaded file /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/wandb-summary.json -2021-07-09 13:04:49,745 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: poll_exit -2021-07-09 13:04:49,745 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: poll_exit -2021-07-09 13:04:49,822 INFO Thread-13 :37232 [upload_job.py:push():137] Uploaded file /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/files/output.log -2021-07-09 13:04:49,846 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: poll_exit -2021-07-09 13:04:49,847 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: poll_exit -2021-07-09 13:04:49,948 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: poll_exit -2021-07-09 13:04:49,948 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: poll_exit -2021-07-09 13:04:50,022 INFO Thread-7 :37232 [sender.py:transition_state():308] send defer: 7 -2021-07-09 13:04:50,023 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: defer -2021-07-09 13:04:50,023 INFO HandlerThread:37232 [handler.py:handle_request_defer():141] handle defer: 7 -2021-07-09 13:04:50,023 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: defer -2021-07-09 13:04:50,023 INFO SenderThread:37232 [sender.py:send_request_defer():304] handle sender defer: 7 -2021-07-09 13:04:50,050 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: poll_exit -2021-07-09 13:04:50,227 INFO SenderThread:37232 [sender.py:transition_state():308] send defer: 8 -2021-07-09 13:04:50,227 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: poll_exit -2021-07-09 13:04:50,228 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: defer -2021-07-09 13:04:50,228 INFO HandlerThread:37232 [handler.py:handle_request_defer():141] handle defer: 8 -2021-07-09 13:04:50,228 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: defer -2021-07-09 13:04:50,228 INFO SenderThread:37232 [sender.py:send_request_defer():304] handle sender defer: 8 -2021-07-09 13:04:50,228 INFO SenderThread:37232 [sender.py:transition_state():308] send defer: 9 -2021-07-09 13:04:50,229 DEBUG SenderThread:37232 [sender.py:send():179] send: final -2021-07-09 13:04:50,229 DEBUG SenderThread:37232 [sender.py:send():179] send: footer -2021-07-09 13:04:50,229 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: defer -2021-07-09 13:04:50,229 INFO HandlerThread:37232 [handler.py:handle_request_defer():141] handle defer: 9 -2021-07-09 13:04:50,229 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: defer -2021-07-09 13:04:50,229 INFO SenderThread:37232 [sender.py:send_request_defer():304] handle sender defer: 9 -2021-07-09 13:04:50,329 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: poll_exit -2021-07-09 13:04:50,329 DEBUG SenderThread:37232 [sender.py:send_request():193] send_request: poll_exit -2021-07-09 13:04:50,329 INFO SenderThread:37232 [file_pusher.py:join():182] waiting for file pusher -2021-07-09 13:04:50,330 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: get_summary -2021-07-09 13:04:50,331 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: sampled_history -2021-07-09 13:04:50,332 DEBUG HandlerThread:37232 [handler.py:handle_request():124] handle_request: shutdown -2021-07-09 13:04:50,332 INFO HandlerThread:37232 [handler.py:finish():638] shutting down handler -2021-07-09 13:04:51,229 INFO WriterThread:37232 [datastore.py:close():288] close: /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/run-89spqu1d.wandb -2021-07-09 13:04:51,330 INFO SenderThread:37232 [sender.py:finish():945] shutting down sender -2021-07-09 13:04:51,330 INFO SenderThread:37232 [file_pusher.py:finish():177] shutting down file pusher -2021-07-09 13:04:51,330 INFO SenderThread:37232 [file_pusher.py:join():182] waiting for file pusher -2021-07-09 13:04:51,332 INFO MainThread:37232 [internal.py:handle_exit():78] Internal process exited diff --git a/wandb/run-20210709_125049-89spqu1d/logs/debug.log b/wandb/run-20210709_125049-89spqu1d/logs/debug.log deleted file mode 100644 index f16e46ce17a910c434a714b562ac800ef11bccd2..0000000000000000000000000000000000000000 --- a/wandb/run-20210709_125049-89spqu1d/logs/debug.log +++ /dev/null @@ -1,114 +0,0 @@ -2021-07-09 12:50:49,339 INFO MainThread:36178 [wandb_setup.py:_flush():69] Unhandled environment var: WANDB_WATCH -2021-07-09 12:50:49,339 INFO MainThread:36178 [wandb_setup.py:_flush():69] setting env: {'project': 'hf-flax-robert-base-mr', 'entity': 'nipunsadvilkar'} -2021-07-09 12:50:49,339 INFO MainThread:36178 [wandb_setup.py:_flush():69] setting login settings: {} -2021-07-09 12:50:49,339 INFO MainThread:36178 [wandb_init.py:_log_setup():337] Logging user logs to /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/logs/debug.log -2021-07-09 12:50:49,339 INFO MainThread:36178 [wandb_init.py:_log_setup():338] Logging internal logs to /home/nipunsadvilkar/sample/roberta-base-mr/wandb/run-20210709_125049-89spqu1d/logs/debug-internal.log -2021-07-09 12:50:49,339 INFO MainThread:36178 [wandb_init.py:init():370] calling init triggers -2021-07-09 12:50:49,339 INFO MainThread:36178 [wandb_init.py:init():375] wandb.init called with sweep_config: {} -config: {} -2021-07-09 12:50:49,339 INFO MainThread:36178 [wandb_init.py:init():419] starting backend -2021-07-09 12:50:49,339 INFO MainThread:36178 [backend.py:_multiprocessing_setup():70] multiprocessing start_methods=fork,spawn,forkserver, using: spawn -2021-07-09 12:50:49,373 INFO MainThread:36178 [backend.py:ensure_launched():135] starting backend process... -2021-07-09 12:50:49,407 INFO MainThread:36178 [backend.py:ensure_launched():139] started backend process with pid: 37232 -2021-07-09 12:50:49,409 INFO MainThread:36178 [wandb_init.py:init():424] backend started and connected -2021-07-09 12:50:49,412 INFO MainThread:36178 [wandb_init.py:init():472] updated telemetry -2021-07-09 12:50:49,413 INFO MainThread:36178 [wandb_init.py:init():491] communicating current version -2021-07-09 12:50:50,062 INFO MainThread:36178 [wandb_init.py:init():496] got version response -2021-07-09 12:50:50,062 INFO MainThread:36178 [wandb_init.py:init():504] communicating run to backend with 30 second timeout -2021-07-09 12:50:50,127 INFO MainThread:36178 [wandb_init.py:init():529] starting run threads in backend -2021-07-09 12:50:50,186 INFO MainThread:36178 [wandb_run.py:_console_start():1623] atexit reg -2021-07-09 12:50:50,187 INFO MainThread:36178 [wandb_run.py:_redirect():1497] redirect: SettingsConsole.REDIRECT -2021-07-09 12:50:50,187 INFO MainThread:36178 [wandb_run.py:_redirect():1502] Redirecting console. -2021-07-09 12:50:50,189 INFO MainThread:36178 [wandb_run.py:_redirect():1558] Redirects installed. -2021-07-09 12:50:50,189 INFO MainThread:36178 [wandb_init.py:init():554] run started, returning control to user process -2021-07-09 12:50:50,190 INFO MainThread:36178 [wandb_run.py:_config_callback():872] config_cb None None {'output_dir': './', 'overwrite_output_dir': True, 'do_train': False, 'do_eval': False, 'do_predict': False, 'evaluation_strategy': 'IntervalStrategy.NO', 'prediction_loss_only': False, 'per_device_train_batch_size': 4, 'per_device_eval_batch_size': 4, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 1, 'eval_accumulation_steps': None, 'learning_rate': 0.0003, 'weight_decay': 0.0, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 8.0, 'max_steps': -1, 'lr_scheduler_type': 'SchedulerType.LINEAR', 'warmup_ratio': 0.0, 'warmup_steps': 1000, 'log_level': -1, 'log_level_replica': -1, 'log_on_each_node': True, 'logging_dir': './runs/Jul09_12-50-44_t1v-n-112df4a9-w-0', 'logging_strategy': 'IntervalStrategy.STEPS', 'logging_first_step': False, 'logging_steps': 500, 'save_strategy': 'IntervalStrategy.STEPS', 'save_steps': 500, 'save_total_limit': None, 'save_on_each_node': False, 'no_cuda': False, 'seed': 42, 'fp16': False, 'fp16_opt_level': 'O1', 'fp16_backend': 'auto', 'fp16_full_eval': False, 'local_rank': -1, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': 500, 'dataloader_num_workers': 0, 'past_index': -1, 'run_name': './', 'disable_tqdm': False, 'remove_unused_columns': True, 'label_names': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'sharded_ddp': [], 'deepspeed': None, 'label_smoothing_factor': 0.0, 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['tensorboard', 'wandb'], 'ddp_find_unused_parameters': None, 'dataloader_pin_memory': True, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': False, 'resume_from_checkpoint': None, 'push_to_hub_model_id': '', 'push_to_hub_organization': None, 'push_to_hub_token': None, 'mp_parameters': ''} -2021-07-09 12:50:50,191 INFO MainThread:36178 [wandb_run.py:_config_callback():872] config_cb None None {'model_name_or_path': None, 'model_type': 'roberta', 'config_name': './', 'tokenizer_name': './', 'cache_dir': None, 'use_fast_tokenizer': True, 'dtype': 'float32'} -2021-07-09 12:50:50,192 INFO MainThread:36178 [wandb_run.py:_config_callback():872] config_cb None None {'dataset_name': 'oscar', 'dataset_config_name': 'unshuffled_deduplicated_als', 'train_file': None, 'validation_file': None, 'train_ref_file': None, 'validation_ref_file': None, 'overwrite_cache': False, 'validation_split_percentage': 5, 'max_seq_length': 128, 'preprocessing_num_workers': None, 'mlm_probability': 0.15, 'pad_to_max_length': False, 'line_by_line': False} -2021-07-09 13:04:45,688 INFO MainThread:36178 [wandb_run.py:_atexit_cleanup():1593] got exitcode: 0 -2021-07-09 13:04:45,689 INFO MainThread:36178 [wandb_run.py:_restore():1565] restore -2021-07-09 13:04:48,278 INFO MainThread:36178 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { - wandb_count: 1 -} -pusher_stats { - uploaded_bytes: 1200 - total_bytes: 1200 -} - -2021-07-09 13:04:48,440 INFO MainThread:36178 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { - wandb_count: 1 -} -pusher_stats { - uploaded_bytes: 1200 - total_bytes: 1200 -} - -2021-07-09 13:04:49,441 INFO MainThread:36178 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { - wandb_count: 3 -} -pusher_stats { - uploaded_bytes: 1200 - total_bytes: 30159 -} - -2021-07-09 13:04:49,543 INFO MainThread:36178 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { - wandb_count: 5 -} -pusher_stats { - uploaded_bytes: 36687 - total_bytes: 36687 -} - -2021-07-09 13:04:49,644 INFO MainThread:36178 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { - wandb_count: 5 -} -pusher_stats { - uploaded_bytes: 36687 - total_bytes: 36687 -} - -2021-07-09 13:04:49,746 INFO MainThread:36178 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { - wandb_count: 5 -} -pusher_stats { - uploaded_bytes: 36687 - total_bytes: 36687 -} - -2021-07-09 13:04:49,847 INFO MainThread:36178 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { - wandb_count: 5 -} -pusher_stats { - uploaded_bytes: 36687 - total_bytes: 36687 -} - -2021-07-09 13:04:49,949 INFO MainThread:36178 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { - wandb_count: 5 -} -pusher_stats { - uploaded_bytes: 36687 - total_bytes: 36687 -} - -2021-07-09 13:04:50,228 INFO MainThread:36178 [wandb_run.py:_wait_for_finish():1715] got exit ret: file_counts { - wandb_count: 5 -} -pusher_stats { - uploaded_bytes: 36687 - total_bytes: 36687 -} - -2021-07-09 13:04:50,330 INFO MainThread:36178 [wandb_run.py:_wait_for_finish():1715] got exit ret: done: true -exit_result { -} -file_counts { - wandb_count: 5 -} -pusher_stats { - uploaded_bytes: 36687 - total_bytes: 36687 -} - -2021-07-09 13:04:51,404 INFO MainThread:36178 [wandb_run.py:_show_summary():1870] rendering summary -2021-07-09 13:04:51,404 INFO MainThread:36178 [wandb_run.py:_show_history():1908] rendering history -2021-07-09 13:04:51,405 INFO MainThread:36178 [wandb_run.py:_show_files():1937] logging synced files diff --git a/wandb/run-20210709_125049-89spqu1d/run-89spqu1d.wandb b/wandb/run-20210709_125049-89spqu1d/run-89spqu1d.wandb deleted file mode 100644 index ac90648e0c3543667d562c273ff4b3a5eb496bff..0000000000000000000000000000000000000000 Binary files a/wandb/run-20210709_125049-89spqu1d/run-89spqu1d.wandb and /dev/null differ diff --git a/wandb/run-20210709_132941-xrzp5klf/files/config.yaml b/wandb/run-20210709_132941-xrzp5klf/files/config.yaml deleted file mode 100644 index 13edf918e4c23d301ffe12993aacfb2a12634334..0000000000000000000000000000000000000000 --- a/wandb/run-20210709_132941-xrzp5klf/files/config.yaml +++ /dev/null @@ -1,23 +0,0 @@ -wandb_version: 1 - -_wandb: - desc: null - value: - cli_version: 0.10.33 - framework: huggingface - huggingface_version: 4.9.0.dev0 - is_jupyter_run: false - is_kaggle_kernel: false - python_version: 3.8.10 - t: - 1: - - 3 - - 11 - 2: - - 3 - - 11 - 4: 3.8.10 - 5: 0.10.33 - 6: 4.9.0.dev0 - 8: - - 5 diff --git a/wandb/run-20210709_132941-xrzp5klf/files/events.out.tfevents.1625837383.t1v-n-112df4a9-w-0.40707.3.v2 b/wandb/run-20210709_132941-xrzp5klf/files/events.out.tfevents.1625837383.t1v-n-112df4a9-w-0.40707.3.v2 deleted file mode 120000 index 02e41f944bffd22256d3fcd23ecd4de8ddd86bb9..0000000000000000000000000000000000000000 --- a/wandb/run-20210709_132941-xrzp5klf/files/events.out.tfevents.1625837383.t1v-n-112df4a9-w-0.40707.3.v2 +++ /dev/null @@ -1 +0,0 @@ -/home/nipunsadvilkar/sample/roberta-base-mr/events.out.tfevents.1625837383.t1v-n-112df4a9-w-0.40707.3.v2 \ No newline at end of file diff --git a/wandb/run-20210709_132941-xrzp5klf/files/output.log b/wandb/run-20210709_132941-xrzp5klf/files/output.log deleted file mode 100644 index 15010660537ae805f6e9739ab98c552e057d1347..0000000000000000000000000000000000000000 --- a/wandb/run-20210709_132941-xrzp5klf/files/output.log +++ /dev/null @@ -1,38 +0,0 @@ -2021-07-09 13:29:43.266078: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcuda.so.1'; dlerror: libcuda.so.1: cannot open shared object file: No such file or directory -2021-07-09 13:29:43.266109: W tensorflow/stream_executor/cuda/cuda_driver.cc:326] failed call to cuInit: UNKNOWN ERROR (303) -/home/nipunsadvilkar/roberta_mr_env/lib/python3.8/site-packages/jax/lib/xla_bridge.py:382: UserWarning: jax.host_count has been renamed to jax.process_count. This alias will eventually be removed; please update your code. - warnings.warn( -/home/nipunsadvilkar/roberta_mr_env/lib/python3.8/site-packages/jax/lib/xla_bridge.py:369: UserWarning: jax.host_id has been renamed to jax.process_index. This alias will eventually be removed; please update your code. - warnings.warn( -Epoch ... (1/8): 0%| | 0/8 [00:00