Upload folder using huggingface_hub

model_index.json

@@ -0,0 +1,37 @@
+{
+  "_class_name": "StableDiffusionPipeline",
+  "_diffusers_version": "0.33.1",
+  "feature_extractor": [
+    null,
+    null
+  ],
+  "image_encoder": [
+    null,
+    null
+  ],
+  "requires_safety_checker": null,
+  "safety_checker": [
+    null,
+    null
+  ],
+  "scheduler": [
+    "diffusers",
+    "DDIMScheduler"
+  ],
+  "text_encoder": [
+    "transformers",
+    "CLIPTextModel"
+  ],
+  "tokenizer": [
+    "transformers",
+    "CLIPTokenizer"
+  ],
+  "unet": [
+    "diffusers",
+    "UNet2DConditionModel"
+  ],
+  "vae": [
+    "diffusers",
+    "AutoencoderKL"
+  ]
+}

scheduler/scheduler_config.json

@@ -0,0 +1,20 @@
+{
+  "_class_name": "DDIMScheduler",
+  "_diffusers_version": "0.33.1",
+  "beta_end": 0.012,
+  "beta_schedule": "scaled_linear",
+  "beta_start": 0.00085,
+  "clip_sample": false,
+  "clip_sample_range": 1.0,
+  "dynamic_thresholding_ratio": 0.995,
+  "num_train_timesteps": 1000,
+  "prediction_type": "v_prediction",
+  "rescale_betas_zero_snr": false,
+  "sample_max_value": 1.0,
+  "set_alpha_to_one": false,
+  "skip_prk_steps": true,
+  "steps_offset": 1,
+  "thresholding": false,
+  "timestep_spacing": "leading",
+  "trained_betas": null
+}

sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-20250723-085809.log

@@ -0,0 +1,301 @@
+07/23/2025 08:58:09 AM  Finding last checkpoint in logdir: ../logs
+07/23/2025 08:58:09 AM  Seed: 469552848
+07/23/2025 08:58:09 AM  unet attention_head_dim: [5, 10, 20, 20]
+07/23/2025 08:58:09 AM Inferred yaml: v2-inference-v.yaml, attn: sd2, prediction_type: v_prediction
+07/23/2025 08:58:10 AM  * Using FlowMatching noise scheduler for training: flow-matching
+07/23/2025 08:58:10 AM * Using SDP attention *
+07/23/2025 08:58:13 AM * DLMA resolution 768, buckets: [[768, 768], [832, 704], [704, 832], [896, 640], [640, 896], [960, 576], [576, 960], [1024, 576], [576, 1024], [1152, 512], [512, 1152], [1280, 448], [448, 1280], [1408, 384], [384, 1408], [1472, 320], [320, 1472]]
+07/23/2025 08:58:13 AM  Preloading images...
+07/23/2025 08:58:21 AM  *** Error opening /workspace/flowmatch/data/fm-2/captions_shotdecksf_gemma3-221x3-clusters/clusters/cluster_13_items/IJFFD793.jpg to get metadata. File may be corrupt and will be skipped.
+07/23/2025 08:58:21 AM  *** exception: [Errno 2] No such file or directory: '/workspace/flowmatch/data/fm-2/captions_shotdecksf_gemma3-221x3-clusters/clusters/cluster_13_items/IJFFD793.jpg'
+07/23/2025 08:58:21 AM  *** Error opening /workspace/flowmatch/data/fm-2/captions_shotdecksf_gemma3-221x3-clusters/clusters/cluster_98_items/YJZSP29K.jpg to get metadata. File may be corrupt and will be skipped.
+07/23/2025 08:58:21 AM  *** exception: [Errno 2] No such file or directory: '/workspace/flowmatch/data/fm-2/captions_shotdecksf_gemma3-221x3-clusters/clusters/cluster_98_items/YJZSP29K.jpg'
+07/23/2025 08:58:21 AM  *** Error opening /workspace/flowmatch/data/fm-2/captions_shotdecksf_gemma3-221x3-clusters/clusters/cluster_197_items/BOVL26EH.jpg to get metadata. File may be corrupt and will be skipped.
+07/23/2025 08:58:21 AM  *** exception: [Errno 2] No such file or directory: '/workspace/flowmatch/data/fm-2/captions_shotdecksf_gemma3-221x3-clusters/clusters/cluster_197_items/BOVL26EH.jpg'
+07/23/2025 08:58:21 AM  *** Error opening /workspace/flowmatch/data/fm-2/captions_shotdecksf_gemma3-221x3-clusters/clusters/cluster_197_items/FST8WHY8.jpg to get metadata. File may be corrupt and will be skipped.
+07/23/2025 08:58:21 AM  *** exception: [Errno 2] No such file or directory: '/workspace/flowmatch/data/fm-2/captions_shotdecksf_gemma3-221x3-clusters/clusters/cluster_197_items/FST8WHY8.jpg'
+07/23/2025 08:58:21 AM  *** Error opening /workspace/flowmatch/data/fm-2/captions_shotdecksf_gemma3-221x3-clusters/clusters/cluster_195_items/QM3NAFD6.jpg to get metadata. File may be corrupt and will be skipped.
+07/23/2025 08:58:21 AM  *** exception: [Errno 2] No such file or directory: '/workspace/flowmatch/data/fm-2/captions_shotdecksf_gemma3-221x3-clusters/clusters/cluster_195_items/QM3NAFD6.jpg'
+07/23/2025 08:58:21 AM  *** Error opening /workspace/flowmatch/data/fm-2/captions_shotdecksf_gemma3-221x3-clusters/clusters/cluster_195_items/MRZ4MUQH.jpg to get metadata. File may be corrupt and will be skipped.
+07/23/2025 08:58:21 AM  *** exception: [Errno 2] No such file or directory: '/workspace/flowmatch/data/fm-2/captions_shotdecksf_gemma3-221x3-clusters/clusters/cluster_195_items/MRZ4MUQH.jpg'
+07/23/2025 08:58:21 AM  *** Error opening /workspace/flowmatch/data/fm-2/captions_shotdecksf_gemma3-221x3-clusters/clusters/cluster_195_items/QWDPXX5Y.jpg to get metadata. File may be corrupt and will be skipped.
+07/23/2025 08:58:21 AM  *** exception: [Errno 2] No such file or directory: '/workspace/flowmatch/data/fm-2/captions_shotdecksf_gemma3-221x3-clusters/clusters/cluster_195_items/QWDPXX5Y.jpg'
+07/23/2025 08:58:21 AM  *** Error opening /workspace/flowmatch/data/fm-2/captions_shotdecksf_gemma3-221x3-clusters/clusters/cluster_195_items/DLICNCMF.jpg to get metadata. File may be corrupt and will be skipped.
+07/23/2025 08:58:21 AM  *** exception: [Errno 2] No such file or directory: '/workspace/flowmatch/data/fm-2/captions_shotdecksf_gemma3-221x3-clusters/clusters/cluster_195_items/DLICNCMF.jpg'
+07/23/2025 08:58:21 AM  *** Error opening /workspace/flowmatch/data/fm-2/captions_shotdecksf_gemma3-221x3-clusters/clusters/cluster_483_items/5Y2TCV1F.jpg to get metadata. File may be corrupt and will be skipped.
+07/23/2025 08:58:21 AM  *** exception: [Errno 2] No such file or directory: '/workspace/flowmatch/data/fm-2/captions_shotdecksf_gemma3-221x3-clusters/clusters/cluster_483_items/5Y2TCV1F.jpg'
+07/23/2025 08:58:21 AM  *** Error opening /workspace/flowmatch/data/fm-2/captions_shotdecksf_gemma3-221x3-clusters/clusters/cluster_83_items/BGSZ410F.jpg to get metadata. File may be corrupt and will be skipped.
+07/23/2025 08:58:21 AM  *** exception: [Errno 2] No such file or directory: '/workspace/flowmatch/data/fm-2/captions_shotdecksf_gemma3-221x3-clusters/clusters/cluster_83_items/BGSZ410F.jpg'
+07/23/2025 08:58:21 AM  *** Error opening /workspace/flowmatch/data/fm-2/captions_shotdecksf_gemma3-221x3-clusters/clusters/cluster_308_items/MWG7NC5L.jpg to get metadata. File may be corrupt and will be skipped.
+07/23/2025 08:58:21 AM  *** exception: [Errno 2] No such file or directory: '/workspace/flowmatch/data/fm-2/captions_shotdecksf_gemma3-221x3-clusters/clusters/cluster_308_items/MWG7NC5L.jpg'
+07/23/2025 08:58:21 AM  *** Error opening /workspace/flowmatch/data/fm-2/captions_shotdecksf_gemma3-221x3-clusters/clusters/cluster_461_items/7ASEME05.jpg to get metadata. File may be corrupt and will be skipped.
+07/23/2025 08:58:21 AM  *** exception: [Errno 2] No such file or directory: '/workspace/flowmatch/data/fm-2/captions_shotdecksf_gemma3-221x3-clusters/clusters/cluster_461_items/7ASEME05.jpg'
+07/23/2025 08:58:21 AM  *** Error opening /workspace/flowmatch/data/fm-2/captions_shotdecksf_gemma3-221x3-clusters/clusters/cluster_157_items/8SF59L1A.jpg to get metadata. File may be corrupt and will be skipped.
+07/23/2025 08:58:21 AM  *** exception: [Errno 2] No such file or directory: '/workspace/flowmatch/data/fm-2/captions_shotdecksf_gemma3-221x3-clusters/clusters/cluster_157_items/8SF59L1A.jpg'
+07/23/2025 08:58:21 AM  *** Error opening /workspace/flowmatch/data/fm-2/captions_shotdecksf_gemma3-221x3-clusters/clusters/cluster_428_items/A1V0BWV5.jpg to get metadata. File may be corrupt and will be skipped.
+07/23/2025 08:58:21 AM  *** exception: [Errno 2] No such file or directory: '/workspace/flowmatch/data/fm-2/captions_shotdecksf_gemma3-221x3-clusters/clusters/cluster_428_items/A1V0BWV5.jpg'
+07/23/2025 08:58:21 AM  *** Error opening /workspace/flowmatch/data/fm-2/captions_shotdecksf_gemma3-221x3-clusters/base/7TFLW33Z.jpg to get metadata. File may be corrupt and will be skipped.
+07/23/2025 08:58:21 AM  *** exception: [Errno 2] No such file or directory: '/workspace/flowmatch/data/fm-2/captions_shotdecksf_gemma3-221x3-clusters/base/7TFLW33Z.jpg'
+07/23/2025 08:58:21 AM  * From 40165 images, dropped 328 undersized images (39837 remaining). Remove --skip_undersized_images to log which ones.
+07/23/2025 08:58:21 AM * DLMA resolution 896, buckets: [[896, 896], [1024, 768], [768, 1024], [1088, 704], [704, 1088], [1216, 640], [640, 1216], [1344, 576], [576, 1344], [1536, 512], [512, 1536], [1792, 448], [448, 1792]]
+07/23/2025 08:58:21 AM  Preloading images...
+07/23/2025 08:58:29 AM  *** Error opening /workspace/flowmatch/data/fm-2/captions_shotdecksf_gemma3-221x3-clusters/clusters/cluster_13_items/IJFFD793.jpg to get metadata. File may be corrupt and will be skipped.
+07/23/2025 08:58:29 AM  *** exception: [Errno 2] No such file or directory: '/workspace/flowmatch/data/fm-2/captions_shotdecksf_gemma3-221x3-clusters/clusters/cluster_13_items/IJFFD793.jpg'
+07/23/2025 08:58:29 AM  *** Error opening /workspace/flowmatch/data/fm-2/captions_shotdecksf_gemma3-221x3-clusters/clusters/cluster_98_items/YJZSP29K.jpg to get metadata. File may be corrupt and will be skipped.
+07/23/2025 08:58:29 AM  *** exception: [Errno 2] No such file or directory: '/workspace/flowmatch/data/fm-2/captions_shotdecksf_gemma3-221x3-clusters/clusters/cluster_98_items/YJZSP29K.jpg'
+07/23/2025 08:58:29 AM  *** Error opening /workspace/flowmatch/data/fm-2/captions_shotdecksf_gemma3-221x3-clusters/clusters/cluster_197_items/BOVL26EH.jpg to get metadata. File may be corrupt and will be skipped.
+07/23/2025 08:58:29 AM  *** exception: [Errno 2] No such file or directory: '/workspace/flowmatch/data/fm-2/captions_shotdecksf_gemma3-221x3-clusters/clusters/cluster_197_items/BOVL26EH.jpg'
+07/23/2025 08:58:29 AM  *** Error opening /workspace/flowmatch/data/fm-2/captions_shotdecksf_gemma3-221x3-clusters/clusters/cluster_197_items/FST8WHY8.jpg to get metadata. File may be corrupt and will be skipped.
+07/23/2025 08:58:29 AM  *** exception: [Errno 2] No such file or directory: '/workspace/flowmatch/data/fm-2/captions_shotdecksf_gemma3-221x3-clusters/clusters/cluster_197_items/FST8WHY8.jpg'
+07/23/2025 08:58:29 AM  *** Error opening /workspace/flowmatch/data/fm-2/captions_shotdecksf_gemma3-221x3-clusters/clusters/cluster_195_items/QM3NAFD6.jpg to get metadata. File may be corrupt and will be skipped.
+07/23/2025 08:58:29 AM  *** exception: [Errno 2] No such file or directory: '/workspace/flowmatch/data/fm-2/captions_shotdecksf_gemma3-221x3-clusters/clusters/cluster_195_items/QM3NAFD6.jpg'
+07/23/2025 08:58:29 AM  *** Error opening /workspace/flowmatch/data/fm-2/captions_shotdecksf_gemma3-221x3-clusters/clusters/cluster_195_items/MRZ4MUQH.jpg to get metadata. File may be corrupt and will be skipped.
+07/23/2025 08:58:29 AM  *** exception: [Errno 2] No such file or directory: '/workspace/flowmatch/data/fm-2/captions_shotdecksf_gemma3-221x3-clusters/clusters/cluster_195_items/MRZ4MUQH.jpg'
+07/23/2025 08:58:29 AM  *** Error opening /workspace/flowmatch/data/fm-2/captions_shotdecksf_gemma3-221x3-clusters/clusters/cluster_195_items/QWDPXX5Y.jpg to get metadata. File may be corrupt and will be skipped.
+07/23/2025 08:58:29 AM  *** exception: [Errno 2] No such file or directory: '/workspace/flowmatch/data/fm-2/captions_shotdecksf_gemma3-221x3-clusters/clusters/cluster_195_items/QWDPXX5Y.jpg'
+07/23/2025 08:58:29 AM  *** Error opening /workspace/flowmatch/data/fm-2/captions_shotdecksf_gemma3-221x3-clusters/clusters/cluster_195_items/DLICNCMF.jpg to get metadata. File may be corrupt and will be skipped.
+07/23/2025 08:58:29 AM  *** exception: [Errno 2] No such file or directory: '/workspace/flowmatch/data/fm-2/captions_shotdecksf_gemma3-221x3-clusters/clusters/cluster_195_items/DLICNCMF.jpg'
+07/23/2025 08:58:29 AM  *** Error opening /workspace/flowmatch/data/fm-2/captions_shotdecksf_gemma3-221x3-clusters/clusters/cluster_483_items/5Y2TCV1F.jpg to get metadata. File may be corrupt and will be skipped.
+07/23/2025 08:58:29 AM  *** exception: [Errno 2] No such file or directory: '/workspace/flowmatch/data/fm-2/captions_shotdecksf_gemma3-221x3-clusters/clusters/cluster_483_items/5Y2TCV1F.jpg'
+07/23/2025 08:58:29 AM  *** Error opening /workspace/flowmatch/data/fm-2/captions_shotdecksf_gemma3-221x3-clusters/clusters/cluster_83_items/BGSZ410F.jpg to get metadata. File may be corrupt and will be skipped.
+07/23/2025 08:58:29 AM  *** exception: [Errno 2] No such file or directory: '/workspace/flowmatch/data/fm-2/captions_shotdecksf_gemma3-221x3-clusters/clusters/cluster_83_items/BGSZ410F.jpg'
+07/23/2025 08:58:29 AM  *** Error opening /workspace/flowmatch/data/fm-2/captions_shotdecksf_gemma3-221x3-clusters/clusters/cluster_308_items/MWG7NC5L.jpg to get metadata. File may be corrupt and will be skipped.
+07/23/2025 08:58:29 AM  *** exception: [Errno 2] No such file or directory: '/workspace/flowmatch/data/fm-2/captions_shotdecksf_gemma3-221x3-clusters/clusters/cluster_308_items/MWG7NC5L.jpg'
+07/23/2025 08:58:29 AM  *** Error opening /workspace/flowmatch/data/fm-2/captions_shotdecksf_gemma3-221x3-clusters/clusters/cluster_461_items/7ASEME05.jpg to get metadata. File may be corrupt and will be skipped.
+07/23/2025 08:58:29 AM  *** exception: [Errno 2] No such file or directory: '/workspace/flowmatch/data/fm-2/captions_shotdecksf_gemma3-221x3-clusters/clusters/cluster_461_items/7ASEME05.jpg'
+07/23/2025 08:58:29 AM  *** Error opening /workspace/flowmatch/data/fm-2/captions_shotdecksf_gemma3-221x3-clusters/clusters/cluster_157_items/8SF59L1A.jpg to get metadata. File may be corrupt and will be skipped.
+07/23/2025 08:58:29 AM  *** exception: [Errno 2] No such file or directory: '/workspace/flowmatch/data/fm-2/captions_shotdecksf_gemma3-221x3-clusters/clusters/cluster_157_items/8SF59L1A.jpg'
+07/23/2025 08:58:29 AM  *** Error opening /workspace/flowmatch/data/fm-2/captions_shotdecksf_gemma3-221x3-clusters/clusters/cluster_428_items/A1V0BWV5.jpg to get metadata. File may be corrupt and will be skipped.
+07/23/2025 08:58:29 AM  *** exception: [Errno 2] No such file or directory: '/workspace/flowmatch/data/fm-2/captions_shotdecksf_gemma3-221x3-clusters/clusters/cluster_428_items/A1V0BWV5.jpg'
+07/23/2025 08:58:29 AM  *** Error opening /workspace/flowmatch/data/fm-2/captions_shotdecksf_gemma3-221x3-clusters/base/7TFLW33Z.jpg to get metadata. File may be corrupt and will be skipped.
+07/23/2025 08:58:29 AM  *** exception: [Errno 2] No such file or directory: '/workspace/flowmatch/data/fm-2/captions_shotdecksf_gemma3-221x3-clusters/base/7TFLW33Z.jpg'
+07/23/2025 08:58:29 AM  * From 40165 images, dropped 12879 undersized images (27286 remaining). Remove --skip_undersized_images to log which ones.
+07/23/2025 08:58:29 AM * DLMA resolution 1024, buckets: [[1024, 1024], [1152, 896], [896, 1152], [1216, 832], [832, 1216], [1344, 768], [768, 1344], [1472, 704], [704, 1472], [1600, 640], [640, 1600], [1792, 576], [576, 1792], [2048, 512], [512, 2048]]
+07/23/2025 08:58:29 AM  Preloading images...
+07/23/2025 08:58:37 AM  *** Error opening /workspace/flowmatch/data/fm-2/captions_shotdecksf_gemma3-221x3-clusters/clusters/cluster_13_items/IJFFD793.jpg to get metadata. File may be corrupt and will be skipped.
+07/23/2025 08:58:37 AM  *** exception: [Errno 2] No such file or directory: '/workspace/flowmatch/data/fm-2/captions_shotdecksf_gemma3-221x3-clusters/clusters/cluster_13_items/IJFFD793.jpg'
+07/23/2025 08:58:37 AM  *** Error opening /workspace/flowmatch/data/fm-2/captions_shotdecksf_gemma3-221x3-clusters/clusters/cluster_98_items/YJZSP29K.jpg to get metadata. File may be corrupt and will be skipped.
+07/23/2025 08:58:37 AM  *** exception: [Errno 2] No such file or directory: '/workspace/flowmatch/data/fm-2/captions_shotdecksf_gemma3-221x3-clusters/clusters/cluster_98_items/YJZSP29K.jpg'
+07/23/2025 08:58:37 AM  *** Error opening /workspace/flowmatch/data/fm-2/captions_shotdecksf_gemma3-221x3-clusters/clusters/cluster_197_items/BOVL26EH.jpg to get metadata. File may be corrupt and will be skipped.
+07/23/2025 08:58:37 AM  *** exception: [Errno 2] No such file or directory: '/workspace/flowmatch/data/fm-2/captions_shotdecksf_gemma3-221x3-clusters/clusters/cluster_197_items/BOVL26EH.jpg'
+07/23/2025 08:58:37 AM  *** Error opening /workspace/flowmatch/data/fm-2/captions_shotdecksf_gemma3-221x3-clusters/clusters/cluster_197_items/FST8WHY8.jpg to get metadata. File may be corrupt and will be skipped.
+07/23/2025 08:58:37 AM  *** exception: [Errno 2] No such file or directory: '/workspace/flowmatch/data/fm-2/captions_shotdecksf_gemma3-221x3-clusters/clusters/cluster_197_items/FST8WHY8.jpg'
+07/23/2025 08:58:37 AM  *** Error opening /workspace/flowmatch/data/fm-2/captions_shotdecksf_gemma3-221x3-clusters/clusters/cluster_195_items/QM3NAFD6.jpg to get metadata. File may be corrupt and will be skipped.
+07/23/2025 08:58:37 AM  *** exception: [Errno 2] No such file or directory: '/workspace/flowmatch/data/fm-2/captions_shotdecksf_gemma3-221x3-clusters/clusters/cluster_195_items/QM3NAFD6.jpg'
+07/23/2025 08:58:37 AM  *** Error opening /workspace/flowmatch/data/fm-2/captions_shotdecksf_gemma3-221x3-clusters/clusters/cluster_195_items/MRZ4MUQH.jpg to get metadata. File may be corrupt and will be skipped.
+07/23/2025 08:58:37 AM  *** exception: [Errno 2] No such file or directory: '/workspace/flowmatch/data/fm-2/captions_shotdecksf_gemma3-221x3-clusters/clusters/cluster_195_items/MRZ4MUQH.jpg'
+07/23/2025 08:58:37 AM  *** Error opening /workspace/flowmatch/data/fm-2/captions_shotdecksf_gemma3-221x3-clusters/clusters/cluster_195_items/QWDPXX5Y.jpg to get metadata. File may be corrupt and will be skipped.
+07/23/2025 08:58:37 AM  *** exception: [Errno 2] No such file or directory: '/workspace/flowmatch/data/fm-2/captions_shotdecksf_gemma3-221x3-clusters/clusters/cluster_195_items/QWDPXX5Y.jpg'
+07/23/2025 08:58:37 AM  *** Error opening /workspace/flowmatch/data/fm-2/captions_shotdecksf_gemma3-221x3-clusters/clusters/cluster_195_items/DLICNCMF.jpg to get metadata. File may be corrupt and will be skipped.
+07/23/2025 08:58:37 AM  *** exception: [Errno 2] No such file or directory: '/workspace/flowmatch/data/fm-2/captions_shotdecksf_gemma3-221x3-clusters/clusters/cluster_195_items/DLICNCMF.jpg'
+07/23/2025 08:58:37 AM  *** Error opening /workspace/flowmatch/data/fm-2/captions_shotdecksf_gemma3-221x3-clusters/clusters/cluster_483_items/5Y2TCV1F.jpg to get metadata. File may be corrupt and will be skipped.
+07/23/2025 08:58:37 AM  *** exception: [Errno 2] No such file or directory: '/workspace/flowmatch/data/fm-2/captions_shotdecksf_gemma3-221x3-clusters/clusters/cluster_483_items/5Y2TCV1F.jpg'
+07/23/2025 08:58:37 AM  *** Error opening /workspace/flowmatch/data/fm-2/captions_shotdecksf_gemma3-221x3-clusters/clusters/cluster_83_items/BGSZ410F.jpg to get metadata. File may be corrupt and will be skipped.
+07/23/2025 08:58:37 AM  *** exception: [Errno 2] No such file or directory: '/workspace/flowmatch/data/fm-2/captions_shotdecksf_gemma3-221x3-clusters/clusters/cluster_83_items/BGSZ410F.jpg'
+07/23/2025 08:58:37 AM  *** Error opening /workspace/flowmatch/data/fm-2/captions_shotdecksf_gemma3-221x3-clusters/clusters/cluster_308_items/MWG7NC5L.jpg to get metadata. File may be corrupt and will be skipped.
+07/23/2025 08:58:37 AM  *** exception: [Errno 2] No such file or directory: '/workspace/flowmatch/data/fm-2/captions_shotdecksf_gemma3-221x3-clusters/clusters/cluster_308_items/MWG7NC5L.jpg'
+07/23/2025 08:58:37 AM  *** Error opening /workspace/flowmatch/data/fm-2/captions_shotdecksf_gemma3-221x3-clusters/clusters/cluster_461_items/7ASEME05.jpg to get metadata. File may be corrupt and will be skipped.
+07/23/2025 08:58:37 AM  *** exception: [Errno 2] No such file or directory: '/workspace/flowmatch/data/fm-2/captions_shotdecksf_gemma3-221x3-clusters/clusters/cluster_461_items/7ASEME05.jpg'
+07/23/2025 08:58:37 AM  *** Error opening /workspace/flowmatch/data/fm-2/captions_shotdecksf_gemma3-221x3-clusters/clusters/cluster_157_items/8SF59L1A.jpg to get metadata. File may be corrupt and will be skipped.
+07/23/2025 08:58:37 AM  *** exception: [Errno 2] No such file or directory: '/workspace/flowmatch/data/fm-2/captions_shotdecksf_gemma3-221x3-clusters/clusters/cluster_157_items/8SF59L1A.jpg'
+07/23/2025 08:58:37 AM  *** Error opening /workspace/flowmatch/data/fm-2/captions_shotdecksf_gemma3-221x3-clusters/clusters/cluster_428_items/A1V0BWV5.jpg to get metadata. File may be corrupt and will be skipped.
+07/23/2025 08:58:37 AM  *** exception: [Errno 2] No such file or directory: '/workspace/flowmatch/data/fm-2/captions_shotdecksf_gemma3-221x3-clusters/clusters/cluster_428_items/A1V0BWV5.jpg'
+07/23/2025 08:58:37 AM  *** Error opening /workspace/flowmatch/data/fm-2/captions_shotdecksf_gemma3-221x3-clusters/base/7TFLW33Z.jpg to get metadata. File may be corrupt and will be skipped.
+07/23/2025 08:58:37 AM  *** exception: [Errno 2] No such file or directory: '/workspace/flowmatch/data/fm-2/captions_shotdecksf_gemma3-221x3-clusters/base/7TFLW33Z.jpg'
+07/23/2025 08:58:37 AM  * From 40165 images, dropped 19580 undersized images (20585 remaining). Remove --skip_undersized_images to log which ones.
+07/23/2025 08:58:37 AM  * Removed 293 images from the training set to use for validation
+07/23/2025 08:58:37 AM  * DLMA initialized with 293 images.
+07/23/2025 08:58:37 AM  ** Dataset 'val': 154 batches, num_images: 308, batch_size: 2
+07/23/2025 08:58:37 AM  * Aspect ratio bucket ('default_batch', 1472, 320) has only 3 images. At batch size 6 this makes for an effective multiplier of 2.0, which may cause problems. Consider adding 3 or more images with aspect ratio 23:5, or reducing your batch_size.
+07/23/2025 08:58:37 AM  * Aspect ratio bucket ('default_batch', 2048, 512) has only 2 images. At batch size 6 this makes for an effective multiplier of 3.0, which may cause problems. Consider adding 4 or more images with aspect ratio 4:1, or reducing your batch_size.
+07/23/2025 08:58:37 AM  * Aspect ratio bucket ('default_batch', 576, 1792) has only 6 images. At batch size 6 this makes for an effective multiplier of 2.0, which may cause problems. Consider adding 6 or more images with aspect ratio 9:28, or reducing your batch_size.
+07/23/2025 08:58:38 AM  * Aspect ratio bucket ('default_batch', 384, 1408) has only 3 images. At batch size 6 this makes for an effective multiplier of 2.0, which may cause problems. Consider adding 3 or more images with aspect ratio 3:11, or reducing your batch_size.
+07/23/2025 08:58:38 AM  * Aspect ratio bucket ('default_batch', 320, 1472) has only 6 images. At batch size 6 this makes for an effective multiplier of 2.0, which may cause problems. Consider adding 6 or more images with aspect ratio 5:23, or reducing your batch_size.
+07/23/2025 08:58:38 AM  - Plugin plugins.caption_munger.CaptionMungerPlugin loaded to <class 'plugins.caption_munger.CaptionMungerPlugin'>
+07/23/2025 08:58:38 AM  - Plugin plugins.interruptible.InterruptiblePlugin loaded to <class 'plugins.interruptible.InterruptiblePlugin'>
+07/23/2025 08:58:38 AM  * DLMA initialized with 87415 source images. After applying multipliers, each epoch will train on at least 4824 images.
+07/23/2025 08:58:48 AM  ** Dataset 'train': 821 batches, num_images: 4926, batch_size: 6
+07/23/2025 08:58:48 AM  * unet optimizer: AdamW (686 parameters) *
+07/23/2025 08:58:48 AM     lr: 2e-06, betas: [0.9, 0.999], epsilon: 1e-08, weight_decay: 0.0001 *
+07/23/2025 08:58:48 AM  Grad scaler enabled: False (amp mode)
+07/23/2025 08:58:48 AM  Pretraining GPU Memory: 4942 / 24564 MB
+07/23/2025 08:58:48 AM  saving ckpts every 1000000000.0 minutes
+07/23/2025 08:58:48 AM  saving ckpts every 100 epochs
+07/23/2025 08:58:48 AM  unet device: cuda:0, precision: torch.float32, training: True
+07/23/2025 08:58:48 AM  text_encoder device: cuda:0, precision: torch.float16, training: False
+07/23/2025 08:58:48 AM  vae device: cuda:0, precision: torch.float16, training: False
+07/23/2025 08:58:48 AM  scheduler: <class 'flow_match_model.TrainFlowMatchScheduler'>
+07/23/2025 08:58:48 AM  Project name: sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k
+07/23/2025 08:58:48 AM  grad_accum: 1
+07/23/2025 08:58:48 AM  batch_size: 6
+07/23/2025 08:58:48 AM  epoch_len: 821
+07/23/2025 09:11:19 AM  * Saving diffusers model to ../logs/sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-20250723-085809/ckpts/rolling-sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-ep00-gs00410
+07/23/2025 09:11:27 AM  Saving optimizer state to ../logs/sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-20250723-085809/ckpts/rolling-sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-ep00-gs00410
+07/23/2025 09:11:34 AM Execution of InterruptiblePlugin took 14.701716184616089 seconds which is longer than the limit of 0.5 seconds
+07/23/2025 09:24:37 AM  * Saving diffusers model to ../logs/sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-20250723-085809/ckpts/rolling-sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-ep00-gs00820
+07/23/2025 09:24:42 AM  Saving optimizer state to ../logs/sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-20250723-085809/ckpts/rolling-sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-ep00-gs00820
+07/23/2025 09:24:49 AM Execution of InterruptiblePlugin took 12.271726608276367 seconds which is longer than the limit of 0.5 seconds
+07/23/2025 09:26:35 AM Execution of plugin.transform_caption took 3.471446990966797 seconds which is longer than the limit of 0.5 seconds
+07/23/2025 09:29:11 AM Execution of plugin.transform_caption took 3.235290050506592 seconds which is longer than the limit of 0.5 seconds
+07/23/2025 09:35:29 AM  * Saving diffusers model to ../logs/sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-20250723-085809/ckpts/rolling-sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-ep01-gs01230
+07/23/2025 09:35:34 AM  Saving optimizer state to ../logs/sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-20250723-085809/ckpts/rolling-sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-ep01-gs01230
+07/23/2025 09:35:43 AM Execution of InterruptiblePlugin took 14.041560411453247 seconds which is longer than the limit of 0.5 seconds
+07/23/2025 09:48:34 AM  * Saving diffusers model to ../logs/sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-20250723-085809/ckpts/rolling-sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-ep01-gs01639
+07/23/2025 09:48:39 AM  Saving optimizer state to ../logs/sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-20250723-085809/ckpts/rolling-sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-ep01-gs01639
+07/23/2025 09:48:46 AM Execution of InterruptiblePlugin took 11.795836687088013 seconds which is longer than the limit of 0.5 seconds
+07/23/2025 09:59:35 AM  * Saving diffusers model to ../logs/sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-20250723-085809/ckpts/rolling-sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-ep02-gs02049
+07/23/2025 09:59:41 AM  Saving optimizer state to ../logs/sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-20250723-085809/ckpts/rolling-sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-ep02-gs02049
+07/23/2025 09:59:47 AM Execution of InterruptiblePlugin took 12.317729234695435 seconds which is longer than the limit of 0.5 seconds
+07/23/2025 10:04:09 AM Execution of plugin.transform_caption took 7.317157983779907 seconds which is longer than the limit of 0.5 seconds
+07/23/2025 10:12:48 AM  * Saving diffusers model to ../logs/sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-20250723-085809/ckpts/rolling-sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-ep02-gs02458
+07/23/2025 10:12:54 AM  Saving optimizer state to ../logs/sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-20250723-085809/ckpts/rolling-sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-ep02-gs02458
+07/23/2025 10:13:02 AM Execution of InterruptiblePlugin took 13.927639245986938 seconds which is longer than the limit of 0.5 seconds
+07/23/2025 10:23:34 AM  * Saving diffusers model to ../logs/sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-20250723-085809/ckpts/rolling-sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-ep03-gs02869
+07/23/2025 10:23:39 AM  Saving optimizer state to ../logs/sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-20250723-085809/ckpts/rolling-sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-ep03-gs02869
+07/23/2025 10:23:46 AM Execution of InterruptiblePlugin took 12.102501392364502 seconds which is longer than the limit of 0.5 seconds
+07/23/2025 10:37:00 AM  * Saving diffusers model to ../logs/sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-20250723-085809/ckpts/rolling-sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-ep03-gs03279
+07/23/2025 10:37:05 AM  Saving optimizer state to ../logs/sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-20250723-085809/ckpts/rolling-sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-ep03-gs03279
+07/23/2025 10:37:11 AM Execution of InterruptiblePlugin took 11.69479751586914 seconds which is longer than the limit of 0.5 seconds
+07/23/2025 10:47:38 AM  * Saving diffusers model to ../logs/sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-20250723-085809/ckpts/rolling-sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-ep04-gs03688
+07/23/2025 10:47:44 AM  Saving optimizer state to ../logs/sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-20250723-085809/ckpts/rolling-sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-ep04-gs03688
+07/23/2025 10:47:51 AM Execution of InterruptiblePlugin took 12.988811731338501 seconds which is longer than the limit of 0.5 seconds
+07/23/2025 11:01:34 AM  * Saving diffusers model to ../logs/sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-20250723-085809/ckpts/rolling-sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-ep04-gs04097
+07/23/2025 11:01:41 AM  Saving optimizer state to ../logs/sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-20250723-085809/ckpts/rolling-sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-ep04-gs04097
+07/23/2025 11:01:47 AM Execution of InterruptiblePlugin took 12.719388246536255 seconds which is longer than the limit of 0.5 seconds
+07/23/2025 11:12:33 AM  * Saving diffusers model to ../logs/sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-20250723-085809/ckpts/rolling-sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-ep05-gs04506
+07/23/2025 11:12:39 AM  Saving optimizer state to ../logs/sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-20250723-085809/ckpts/rolling-sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-ep05-gs04506
+07/23/2025 11:12:45 AM Execution of InterruptiblePlugin took 12.09020709991455 seconds which is longer than the limit of 0.5 seconds
+07/23/2025 11:25:21 AM  * Saving diffusers model to ../logs/sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-20250723-085809/ckpts/rolling-sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-ep05-gs04915
+07/23/2025 11:25:26 AM  Saving optimizer state to ../logs/sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-20250723-085809/ckpts/rolling-sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-ep05-gs04915
+07/23/2025 11:25:33 AM Execution of InterruptiblePlugin took 12.17073917388916 seconds which is longer than the limit of 0.5 seconds
+07/23/2025 11:35:56 AM  * Saving diffusers model to ../logs/sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-20250723-085809/ckpts/rolling-sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-ep06-gs05326
+07/23/2025 11:36:02 AM  Saving optimizer state to ../logs/sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-20250723-085809/ckpts/rolling-sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-ep06-gs05326
+07/23/2025 11:36:11 AM Execution of InterruptiblePlugin took 14.330196142196655 seconds which is longer than the limit of 0.5 seconds
+07/23/2025 11:49:13 AM  * Saving diffusers model to ../logs/sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-20250723-085809/ckpts/rolling-sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-ep06-gs05736
+07/23/2025 11:49:19 AM  Saving optimizer state to ../logs/sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-20250723-085809/ckpts/rolling-sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-ep06-gs05736
+07/23/2025 11:49:25 AM Execution of InterruptiblePlugin took 12.604533672332764 seconds which is longer than the limit of 0.5 seconds
+07/23/2025 11:54:42 AM Execution of plugin.transform_caption took 3.3861029148101807 seconds which is longer than the limit of 0.5 seconds
+07/23/2025 11:59:57 AM  * Saving diffusers model to ../logs/sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-20250723-085809/ckpts/rolling-sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-ep07-gs06146
+07/23/2025 12:00:03 PM  Saving optimizer state to ../logs/sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-20250723-085809/ckpts/rolling-sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-ep07-gs06146
+07/23/2025 12:00:09 PM Execution of InterruptiblePlugin took 12.159730911254883 seconds which is longer than the limit of 0.5 seconds
+07/23/2025 12:13:06 PM  * Saving diffusers model to ../logs/sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-20250723-085809/ckpts/rolling-sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-ep07-gs06555
+07/23/2025 12:13:12 PM  Saving optimizer state to ../logs/sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-20250723-085809/ckpts/rolling-sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-ep07-gs06555
+07/23/2025 12:13:18 PM Execution of InterruptiblePlugin took 12.365245819091797 seconds which is longer than the limit of 0.5 seconds
+07/23/2025 12:23:57 PM  * Saving diffusers model to ../logs/sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-20250723-085809/ckpts/rolling-sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-ep08-gs06965
+07/23/2025 12:24:03 PM  Saving optimizer state to ../logs/sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-20250723-085809/ckpts/rolling-sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-ep08-gs06965
+07/23/2025 12:24:10 PM Execution of InterruptiblePlugin took 13.04531741142273 seconds which is longer than the limit of 0.5 seconds
+07/23/2025 12:37:00 PM  * Saving diffusers model to ../logs/sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-20250723-085809/ckpts/rolling-sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-ep08-gs07374
+07/23/2025 12:37:06 PM  Saving optimizer state to ../logs/sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-20250723-085809/ckpts/rolling-sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-ep08-gs07374
+07/23/2025 12:37:12 PM Execution of InterruptiblePlugin took 12.612109899520874 seconds which is longer than the limit of 0.5 seconds
+07/23/2025 12:47:56 PM  * Saving diffusers model to ../logs/sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-20250723-085809/ckpts/rolling-sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-ep09-gs07785
+07/23/2025 12:48:02 PM  Saving optimizer state to ../logs/sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-20250723-085809/ckpts/rolling-sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-ep09-gs07785
+07/23/2025 12:48:08 PM Execution of InterruptiblePlugin took 11.76731824874878 seconds which is longer than the limit of 0.5 seconds
+07/23/2025 12:54:42 PM Execution of plugin.transform_caption took 3.2821173667907715 seconds which is longer than the limit of 0.5 seconds
+07/23/2025 12:54:51 PM Execution of plugin.transform_caption took 3.2314810752868652 seconds which is longer than the limit of 0.5 seconds
+07/23/2025 12:55:12 PM Execution of plugin.transform_caption took 4.170757055282593 seconds which is longer than the limit of 0.5 seconds
+07/23/2025 01:02:01 PM  * Saving diffusers model to ../logs/sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-20250723-085809/ckpts/rolling-sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-ep09-gs08195
+07/23/2025 01:02:14 PM  Saving optimizer state to ../logs/sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-20250723-085809/ckpts/rolling-sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-ep09-gs08195
+07/23/2025 01:02:26 PM Execution of InterruptiblePlugin took 24.66141438484192 seconds which is longer than the limit of 0.5 seconds
+07/23/2025 01:13:32 PM  * Saving diffusers model to ../logs/sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-20250723-085809/ckpts/rolling-sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-ep10-gs08605
+07/23/2025 01:13:38 PM  Saving optimizer state to ../logs/sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-20250723-085809/ckpts/rolling-sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-ep10-gs08605
+07/23/2025 01:13:46 PM Execution of InterruptiblePlugin took 13.334459066390991 seconds which is longer than the limit of 0.5 seconds
+07/23/2025 01:26:58 PM  * Saving diffusers model to ../logs/sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-20250723-085809/ckpts/rolling-sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-ep10-gs09014
+07/23/2025 01:27:04 PM  Saving optimizer state to ../logs/sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-20250723-085809/ckpts/rolling-sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-ep10-gs09014
+07/23/2025 01:27:11 PM Execution of InterruptiblePlugin took 12.350433111190796 seconds which is longer than the limit of 0.5 seconds
+07/23/2025 01:38:00 PM  * Saving diffusers model to ../logs/sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-20250723-085809/ckpts/rolling-sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-ep11-gs09424
+07/23/2025 01:38:07 PM  Saving optimizer state to ../logs/sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-20250723-085809/ckpts/rolling-sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-ep11-gs09424
+07/23/2025 01:38:13 PM Execution of InterruptiblePlugin took 12.203232049942017 seconds which is longer than the limit of 0.5 seconds
+07/23/2025 01:44:49 PM Execution of plugin.transform_caption took 3.773069381713867 seconds which is longer than the limit of 0.5 seconds
+07/23/2025 01:51:21 PM  * Saving diffusers model to ../logs/sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-20250723-085809/ckpts/rolling-sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-ep11-gs09833
+07/23/2025 01:51:27 PM  Saving optimizer state to ../logs/sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-20250723-085809/ckpts/rolling-sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-ep11-gs09833
+07/23/2025 01:51:33 PM Execution of InterruptiblePlugin took 11.658567190170288 seconds which is longer than the limit of 0.5 seconds
+07/23/2025 02:02:22 PM  * Saving diffusers model to ../logs/sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-20250723-085809/ckpts/rolling-sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-ep12-gs10243
+07/23/2025 02:02:28 PM  Saving optimizer state to ../logs/sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-20250723-085809/ckpts/rolling-sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-ep12-gs10243
+07/23/2025 02:02:35 PM Execution of InterruptiblePlugin took 12.632521629333496 seconds which is longer than the limit of 0.5 seconds
+07/23/2025 02:04:27 PM Execution of plugin.transform_caption took 10.16742491722107 seconds which is longer than the limit of 0.5 seconds
+07/23/2025 02:11:21 PM Execution of plugin.transform_caption took 3.3150854110717773 seconds which is longer than the limit of 0.5 seconds
+07/23/2025 02:15:45 PM  * Saving diffusers model to ../logs/sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-20250723-085809/ckpts/rolling-sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-ep12-gs10652
+07/23/2025 02:15:50 PM  Saving optimizer state to ../logs/sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-20250723-085809/ckpts/rolling-sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-ep12-gs10652
+07/23/2025 02:15:56 PM Execution of InterruptiblePlugin took 11.62908411026001 seconds which is longer than the limit of 0.5 seconds
+07/23/2025 02:26:45 PM  * Saving diffusers model to ../logs/sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-20250723-085809/ckpts/rolling-sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-ep13-gs11063
+07/23/2025 02:26:51 PM  Saving optimizer state to ../logs/sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-20250723-085809/ckpts/rolling-sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-ep13-gs11063
+07/23/2025 02:26:57 PM Execution of InterruptiblePlugin took 12.121954441070557 seconds which is longer than the limit of 0.5 seconds
+07/23/2025 02:40:17 PM  * Saving diffusers model to ../logs/sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-20250723-085809/ckpts/rolling-sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-ep13-gs11473
+07/23/2025 02:40:23 PM  Saving optimizer state to ../logs/sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-20250723-085809/ckpts/rolling-sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-ep13-gs11473
+07/23/2025 02:40:30 PM Execution of InterruptiblePlugin took 13.193635940551758 seconds which is longer than the limit of 0.5 seconds
+07/23/2025 02:51:29 PM  * Saving diffusers model to ../logs/sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-20250723-085809/ckpts/rolling-sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-ep14-gs11883
+07/23/2025 02:51:34 PM  Saving optimizer state to ../logs/sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-20250723-085809/ckpts/rolling-sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-ep14-gs11883
+07/23/2025 02:51:41 PM Execution of InterruptiblePlugin took 12.194420099258423 seconds which is longer than the limit of 0.5 seconds
+07/23/2025 02:56:50 PM Execution of plugin.transform_caption took 3.5864827632904053 seconds which is longer than the limit of 0.5 seconds
+07/23/2025 03:05:21 PM  * Saving diffusers model to ../logs/sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-20250723-085809/ckpts/rolling-sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-ep14-gs12293
+07/23/2025 03:05:27 PM  Saving optimizer state to ../logs/sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-20250723-085809/ckpts/rolling-sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-ep14-gs12293
+07/23/2025 03:05:34 PM Execution of InterruptiblePlugin took 12.754526376724243 seconds which is longer than the limit of 0.5 seconds
+07/23/2025 03:16:45 PM  * Saving diffusers model to ../logs/sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-20250723-085809/ckpts/rolling-sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-ep15-gs12704
+07/23/2025 03:16:52 PM  Saving optimizer state to ../logs/sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-20250723-085809/ckpts/rolling-sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-ep15-gs12704
+07/23/2025 03:16:58 PM Execution of InterruptiblePlugin took 13.077876329421997 seconds which is longer than the limit of 0.5 seconds
+07/23/2025 03:30:07 PM  * Saving diffusers model to ../logs/sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-20250723-085809/ckpts/rolling-sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-ep15-gs13114
+07/23/2025 03:30:13 PM  Saving optimizer state to ../logs/sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-20250723-085809/ckpts/rolling-sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-ep15-gs13114
+07/23/2025 03:30:20 PM Execution of InterruptiblePlugin took 13.731325626373291 seconds which is longer than the limit of 0.5 seconds
+07/23/2025 03:41:16 PM  * Saving diffusers model to ../logs/sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-20250723-085809/ckpts/rolling-sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-ep16-gs13524
+07/23/2025 03:41:22 PM  Saving optimizer state to ../logs/sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-20250723-085809/ckpts/rolling-sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-ep16-gs13524
+07/23/2025 03:41:30 PM Execution of InterruptiblePlugin took 14.183850288391113 seconds which is longer than the limit of 0.5 seconds
+07/23/2025 03:41:41 PM Execution of plugin.transform_caption took 6.991438627243042 seconds which is longer than the limit of 0.5 seconds
+07/23/2025 03:54:10 PM  * Saving diffusers model to ../logs/sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-20250723-085809/ckpts/rolling-sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-ep16-gs13933
+07/23/2025 03:54:16 PM  Saving optimizer state to ../logs/sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-20250723-085809/ckpts/rolling-sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-ep16-gs13933
+07/23/2025 03:54:22 PM Execution of InterruptiblePlugin took 12.073235988616943 seconds which is longer than the limit of 0.5 seconds
+07/23/2025 04:05:12 PM  * Saving diffusers model to ../logs/sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-20250723-085809/ckpts/rolling-sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-ep17-gs14342
+07/23/2025 04:05:18 PM  Saving optimizer state to ../logs/sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-20250723-085809/ckpts/rolling-sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-ep17-gs14342
+07/23/2025 04:05:26 PM Execution of InterruptiblePlugin took 13.525968551635742 seconds which is longer than the limit of 0.5 seconds
+07/23/2025 04:18:41 PM  * Saving diffusers model to ../logs/sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-20250723-085809/ckpts/rolling-sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-ep17-gs14750
+07/23/2025 04:18:46 PM  Saving optimizer state to ../logs/sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-20250723-085809/ckpts/rolling-sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-ep17-gs14750
+07/23/2025 04:18:53 PM Execution of InterruptiblePlugin took 11.94441556930542 seconds which is longer than the limit of 0.5 seconds
+07/23/2025 04:29:40 PM  * Saving diffusers model to ../logs/sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-20250723-085809/ckpts/rolling-sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-ep18-gs15160
+07/23/2025 04:29:46 PM  Saving optimizer state to ../logs/sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-20250723-085809/ckpts/rolling-sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-ep18-gs15160
+07/23/2025 04:29:53 PM Execution of InterruptiblePlugin took 12.795083284378052 seconds which is longer than the limit of 0.5 seconds
+07/23/2025 04:42:42 PM  * Saving diffusers model to ../logs/sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-20250723-085809/ckpts/rolling-sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-ep18-gs15569
+07/23/2025 04:42:51 PM  Saving optimizer state to ../logs/sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-20250723-085809/ckpts/rolling-sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-ep18-gs15569
+07/23/2025 04:42:58 PM Execution of InterruptiblePlugin took 15.606266260147095 seconds which is longer than the limit of 0.5 seconds
+07/23/2025 04:53:47 PM  * Saving diffusers model to ../logs/sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-20250723-085809/ckpts/rolling-sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-ep19-gs15980
+07/23/2025 04:53:53 PM  Saving optimizer state to ../logs/sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-20250723-085809/ckpts/rolling-sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-ep19-gs15980
+07/23/2025 04:54:00 PM Execution of InterruptiblePlugin took 13.108078718185425 seconds which is longer than the limit of 0.5 seconds
+07/23/2025 04:55:47 PM Execution of plugin.transform_caption took 4.184926748275757 seconds which is longer than the limit of 0.5 seconds
+07/23/2025 05:07:52 PM  * Saving diffusers model to ../logs/sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-20250723-085809/ckpts/rolling-sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-ep19-gs16390
+07/23/2025 05:07:58 PM  Saving optimizer state to ../logs/sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-20250723-085809/ckpts/rolling-sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-ep19-gs16390
+07/23/2025 05:08:04 PM Execution of InterruptiblePlugin took 12.541584014892578 seconds which is longer than the limit of 0.5 seconds
+07/23/2025 05:19:02 PM  * Saving diffusers model to ../logs/sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-20250723-085809/ckpts/rolling-sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-ep20-gs16799
+07/23/2025 05:19:09 PM  Saving optimizer state to ../logs/sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-20250723-085809/ckpts/rolling-sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-ep20-gs16799
+07/23/2025 05:19:16 PM Execution of InterruptiblePlugin took 13.99230170249939 seconds which is longer than the limit of 0.5 seconds
+07/23/2025 05:32:25 PM  * Saving diffusers model to ../logs/sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-20250723-085809/ckpts/rolling-sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-ep20-gs17207
+07/23/2025 05:32:30 PM  Saving optimizer state to ../logs/sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-20250723-085809/ckpts/rolling-sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-ep20-gs17207
+07/23/2025 05:32:36 PM Execution of InterruptiblePlugin took 11.880703449249268 seconds which is longer than the limit of 0.5 seconds
+07/23/2025 05:43:29 PM  * Saving diffusers model to ../logs/sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-20250723-085809/ckpts/rolling-sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-ep21-gs17616
+07/23/2025 05:43:35 PM  Saving optimizer state to ../logs/sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-20250723-085809/ckpts/rolling-sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-ep21-gs17616
+07/23/2025 05:43:42 PM Execution of InterruptiblePlugin took 12.984358549118042 seconds which is longer than the limit of 0.5 seconds
+07/23/2025 05:56:58 PM  * Saving diffusers model to ../logs/sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-20250723-085809/ckpts/rolling-sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-ep21-gs18025
+07/23/2025 05:57:04 PM  Saving optimizer state to ../logs/sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-20250723-085809/ckpts/rolling-sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-ep21-gs18025
+07/23/2025 05:57:11 PM Execution of InterruptiblePlugin took 12.752715110778809 seconds which is longer than the limit of 0.5 seconds
+07/23/2025 06:08:08 PM  * Saving diffusers model to ../logs/sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-20250723-085809/ckpts/rolling-sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-ep22-gs18435
+07/23/2025 06:08:15 PM  Saving optimizer state to ../logs/sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-20250723-085809/ckpts/rolling-sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-ep22-gs18435
+07/23/2025 06:08:21 PM Execution of InterruptiblePlugin took 12.125662088394165 seconds which is longer than the limit of 0.5 seconds
+07/23/2025 06:21:47 PM  * Saving diffusers model to ../logs/sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-20250723-085809/ckpts/rolling-sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-ep22-gs18845
+07/23/2025 06:21:52 PM  Saving optimizer state to ../logs/sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-20250723-085809/ckpts/rolling-sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-ep22-gs18845
+07/23/2025 06:21:59 PM Execution of InterruptiblePlugin took 12.46973729133606 seconds which is longer than the limit of 0.5 seconds
+07/23/2025 06:33:03 PM  * Saving diffusers model to ../logs/sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-20250723-085809/ckpts/rolling-sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-ep23-gs19257
+07/23/2025 06:33:09 PM  Saving optimizer state to ../logs/sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-20250723-085809/ckpts/rolling-sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-ep23-gs19257
+07/23/2025 06:33:17 PM Execution of InterruptiblePlugin took 14.044326782226562 seconds which is longer than the limit of 0.5 seconds
+07/23/2025 06:46:42 PM  * Saving diffusers model to ../logs/sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-20250723-085809/ckpts/rolling-sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-ep23-gs19668
+07/23/2025 06:46:48 PM  Saving optimizer state to ../logs/sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-20250723-085809/ckpts/rolling-sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-ep23-gs19668
+07/23/2025 06:46:56 PM Execution of InterruptiblePlugin took 14.455474376678467 seconds which is longer than the limit of 0.5 seconds
+07/23/2025 06:57:47 PM  * Saving diffusers model to ../logs/sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-20250723-085809/ckpts/rolling-sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-ep24-gs20077
+07/23/2025 06:57:53 PM  Saving optimizer state to ../logs/sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-20250723-085809/ckpts/rolling-sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-ep24-gs20077
+07/23/2025 06:57:59 PM Execution of InterruptiblePlugin took 12.370712041854858 seconds which is longer than the limit of 0.5 seconds
+07/23/2025 07:11:42 PM  * Saving diffusers model to ../logs/sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-20250723-085809/ckpts/rolling-sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-ep24-gs20486
+07/23/2025 07:11:48 PM  Saving optimizer state to ../logs/sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-20250723-085809/ckpts/rolling-sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-ep24-gs20486
+07/23/2025 07:11:57 PM Execution of InterruptiblePlugin took 14.517040491104126 seconds which is longer than the limit of 0.5 seconds
+07/23/2025 07:11:59 PM  * Saving diffusers model to ../logs/sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-20250723-085809/ckpts/sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-ep25-gs20487
+07/23/2025 07:12:05 PM Training complete
+07/23/2025 07:12:05 PM Total training time took 613.27 minutes, total steps: 20487
+07/23/2025 07:12:05 PM Average epoch time: 24.29 minutes
+07/23/2025 07:12:05 PM  ***************************
+07/23/2025 07:12:05 PM  **** Finished training ****
+07/23/2025 07:12:05 PM  ***************************

sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-20250723-085809_main.json

@@ -0,0 +1,148 @@
+{
+  "config": "sdsf-flowmatch-2.json",
+  "batch_size": 6,
+  "clip_skip": 0,
+  "clip_grad_norm": 10.0,
+  "contrastive_learning_batch_ids": [],
+  "contrastive_learning_max_negative_loss": 1.0,
+  "contrastive_learning_negative_loss_scale": 0.1,
+  "cond_dropout": 0.1,
+  "data_root": "../data/fm-2",
+  "disable_amp": false,
+  "disable_textenc_training": true,
+  "disable_unet_training": false,
+  "disable_xformers": false,
+  "flip_p": 0.0,
+  "gpuid": 0,
+  "gradient_checkpointing": true,
+  "grad_accum": 1,
+  "logdir": "../logs",
+  "log_step": 100,
+  "lowvram": false,
+  "max_epochs": 25,
+  "notebook": false,
+  "optimizer_config": "optimizer-unet-2e6sqrt-adamw.json",
+  "_plugins_disabled": [
+    "plugins.textual_inversion.TextualInversionPlugin"
+  ],
+  "plugins": [
+    "plugins.caption_munger.CaptionMungerPlugin",
+    "plugins.interruptible.InterruptiblePlugin"
+  ],
+  "project_name": "sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k",
+  "resolution": [
+    768,
+    896,
+    1024
+  ],
+  "run_name": null,
+  "sample_prompts": "sample_prompts.json",
+  "sample_steps": 1000,
+  "save_ckpt_dir": null,
+  "save_ckpts_from_n_epochs": 0,
+  "save_every_n_epochs": 100,
+  "save_optimizer": false,
+  "scale_lr": false,
+  "shuffle_tags": false,
+  "validation_config": "validation.json",
+  "wandb": false,
+  "write_schedule": false,
+  "rated_dataset": false,
+  "rated_dataset_target_dropout_percent": 50,
+  "zero_frequency_noise_ratio": 0.0,
+  "enable_zero_terminal_snr": false,
+  "mix_zero_terminal_snr": false,
+  "load_settings_every_epoch": false,
+  "min_snr_gamma": null,
+  "loss_type": "sd3-cosmap",
+  "ema_decay_rate": null,
+  "ema_strength_target": null,
+  "ema_update_interval": null,
+  "ema_device": null,
+  "ema_sample_nonema_model": false,
+  "ema_sample_ema_model": false,
+  "ema_resume_model": null,
+  "no_save_ckpt": true,
+  "no_prepend_last": true,
+  "amp": true,
+  "attn_type": "sdp",
+  "seed": 469552848,
+  "embedding_perturbation": 0.0,
+  "latents_perturbation": 0.0,
+  "log_huber": false,
+  "save_full_precision": false,
+  "timestep_start": 0,
+  "timestep_end": 1000,
+  "train_sampler": "flow-matching",
+  "pyramid_noise_discount": null,
+  "batch_share_noise": false,
+  "batch_share_timesteps": false,
+  "use_grokfast": false,
+  "lr": null,
+  "max_steps": null,
+  "no_initial_validation": false,
+  "match_zero_terminal_snr": false,
+  "debug_invert_min_snr_gamma": false,
+  "teacher": null,
+  "teacher_is_sdxl": false,
+  "teacher_loss_scale": 1,
+  "contrastive_learning_delta_loss_method": true,
+  "contrastive_learning_delta_timestep_start": 0,
+  "everything_contrastive_learning_p": 0.0,
+  "batch_id_dropout_p": 0,
+  "cond_dropout_noise_p": 0,
+  "jacobian_descent": false,
+  "use_masks": false,
+  "lora": false,
+  "lora_resume": null,
+  "lora_rank": 16,
+  "lora_alpha": 8,
+  "init_grad_scale": null,
+  "forward_slice_size": 2,
+  "auto_decay_steps_multiplier": 1.0,
+  "resume_ckpt": "../logs/sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20250722-173322/ckpts/sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-ep25-gs20491",
+  "keep_tags": 0,
+  "contrastive_learning_use_l1_loss": false,
+  "contrastive_learning_no_average_negatives": false,
+  "contrastive_learning_save_on_cpu": false,
+  "contrastive_learning_dropout_p": 0,
+  "contrastive_learning_curriculum_alpha": 0,
+  "all_caption_variants": false,
+  "ckpt_every_n_minutes": 1000000000.0,
+  "skip_undersized_images": true,
+  "test_images": false,
+  "batch_size_curriculum_alpha": 0.25,
+  "initial_batch_size": 20,
+  "final_batch_size": 20,
+  "cond_dropout_curriculum_alpha": 0,
+  "cond_dropout_curriculum_source": "timestep",
+  "final_cond_dropout": 0.1,
+  "negative_loss_margin": 1,
+  "timestep_curriculum_alpha": 0,
+  "timestep_initial_start": 800,
+  "timestep_initial_end": 1000,
+  "everything_contrastive_learning_curriculum_alpha": 0,
+  "max_backward_slice_size": 4,
+  "caption_variants": [
+    "short",
+    "natural_language"
+  ],
+  "freeze_unet_balanced": false,
+  "cond_dropout_loss_scale": 1,
+  "contrastive_learning_info_nce_sample_count": 0,
+  "contrastive_learning_info_nce_temperature": 1,
+  "interleave_batch_size_1": false,
+  "interleave_batch_size_1_alpha": 0,
+  "log_named_parameters_magnitudes": true,
+  "lr_decay_steps": 20525,
+  "lr_scheduler": "constant",
+  "lr_warmup_steps": 410,
+  "lr_advance_steps": 0,
+  "keep_same_sample_at_different_resolutions_together": false,
+  "timesteps_multirank_stratified": false,
+  "timesteps_multirank_stratified_alpha": 1.5,
+  "timesteps_multirank_stratified_beta": 2,
+  "min_snr_alpha": 1,
+  "teacher_p": 0.5,
+  "use_compel": false
+}

sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20k-20250723-085809_opt.json

@@ -0,0 +1,38 @@
+{
+  "base": {
+    "optimizer": "adamw",
+    "lr": 2e-06,
+    "lr_scheduler": "polynomial",
+    "lr_end": 1e-08,
+    "power": 2,
+    "lr_decay_steps": 20525,
+    "lr_warmup_steps": 410,
+    "betas": [
+      0.9,
+      0.999
+    ],
+    "epsilon": 1e-08,
+    "weight_decay": 0.0001,
+    "lr_advance_steps": 0
+  },
+  "text_encoder_overrides": {
+    "optimizer": "adamw",
+    "lr": 1e-06,
+    "lr_scheduler": "cosine",
+    "lr_decay_steps": 20525,
+    "lr_warmup_steps": 410,
+    "betas": [
+      0.9,
+      0.999
+    ],
+    "epsilon": 1e-08,
+    "weight_decay": 0.0001,
+    "lr_advance_steps": 0
+  },
+  "text_encoder_freezing": {
+    "unfreeze_last_n_layers": null
+  },
+  "apply_grad_scaler_step_tweaks": false,
+  "log_grad_norm": true,
+  "use_grad_scaler": false
+}

text_encoder/config.json

@@ -0,0 +1,25 @@
+{
+  "_name_or_path": "../logs/sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20250722-173322/ckpts/sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-ep25-gs20491/text_encoder",
+  "architectures": [
+    "CLIPTextModel"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 0,
+  "dropout": 0.0,
+  "eos_token_id": 2,
+  "hidden_act": "gelu",
+  "hidden_size": 1024,
+  "initializer_factor": 1.0,
+  "initializer_range": 0.02,
+  "intermediate_size": 4096,
+  "layer_norm_eps": 1e-05,
+  "max_position_embeddings": 77,
+  "model_type": "clip_text_model",
+  "num_attention_heads": 16,
+  "num_hidden_layers": 23,
+  "pad_token_id": 1,
+  "projection_dim": 512,
+  "torch_dtype": "float16",
+  "transformers_version": "4.38.2",
+  "vocab_size": 49408
+}

text_encoder/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bc1827c465450322616f06dea41596eac7d493f4e95904dcb51f0fc745c4e13f
+size 680820392

tokenizer/special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "<|startoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "!",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer/tokenizer_config.json

@@ -0,0 +1,39 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "!",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49406": {
+      "content": "<|startoftext|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49407": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|startoftext|>",
+  "clean_up_tokenization_spaces": true,
+  "do_lower_case": true,
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "model_max_length": 77,
+  "pad_token": "!",
+  "tokenizer_class": "CLIPTokenizer",
+  "unk_token": "<|endoftext|>",
+  "use_fast": false
+}

unet/config.json

@@ -0,0 +1,73 @@
+{
+  "_class_name": "UNet2DConditionModel",
+  "_diffusers_version": "0.33.1",
+  "_name_or_path": "../logs/sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20250722-173322/ckpts/sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-ep25-gs20491/unet",
+  "act_fn": "silu",
+  "addition_embed_type": null,
+  "addition_embed_type_num_heads": 64,
+  "addition_time_embed_dim": null,
+  "attention_head_dim": [
+    5,
+    10,
+    20,
+    20
+  ],
+  "attention_type": "default",
+  "block_out_channels": [
+    320,
+    640,
+    1280,
+    1280
+  ],
+  "center_input_sample": false,
+  "class_embed_type": null,
+  "class_embeddings_concat": false,
+  "conv_in_kernel": 3,
+  "conv_out_kernel": 3,
+  "cross_attention_dim": 1024,
+  "cross_attention_norm": null,
+  "down_block_types": [
+    "CrossAttnDownBlock2D",
+    "CrossAttnDownBlock2D",
+    "CrossAttnDownBlock2D",
+    "DownBlock2D"
+  ],
+  "downsample_padding": 1,
+  "dropout": 0.0,
+  "dual_cross_attention": false,
+  "encoder_hid_dim": null,
+  "encoder_hid_dim_type": null,
+  "flip_sin_to_cos": true,
+  "freq_shift": 0,
+  "in_channels": 4,
+  "layers_per_block": 2,
+  "mid_block_only_cross_attention": null,
+  "mid_block_scale_factor": 1,
+  "mid_block_type": "UNetMidBlock2DCrossAttn",
+  "norm_eps": 1e-05,
+  "norm_num_groups": 32,
+  "num_attention_heads": null,
+  "num_class_embeds": null,
+  "only_cross_attention": false,
+  "out_channels": 4,
+  "projection_class_embeddings_input_dim": null,
+  "resnet_out_scale_factor": 1.0,
+  "resnet_skip_time_act": false,
+  "resnet_time_scale_shift": "default",
+  "reverse_transformer_layers_per_block": null,
+  "sample_size": 96,
+  "time_cond_proj_dim": null,
+  "time_embedding_act_fn": null,
+  "time_embedding_dim": null,
+  "time_embedding_type": "positional",
+  "timestep_post_act": null,
+  "transformer_layers_per_block": 1,
+  "up_block_types": [
+    "UpBlock2D",
+    "CrossAttnUpBlock2D",
+    "CrossAttnUpBlock2D",
+    "CrossAttnUpBlock2D"
+  ],
+  "upcast_attention": true,
+  "use_linear_projection": true
+}

unet/diffusion_pytorch_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b9e59ec4d5073d44db2cdd94a0366b9606677755a19c47794b503ad13a1599d5
+size 3463726504

vae/config.json

@@ -0,0 +1,38 @@
+{
+  "_class_name": "AutoencoderKL",
+  "_diffusers_version": "0.33.1",
+  "_name_or_path": "../logs/sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-20250722-173322/ckpts/sdsf-flowmatch-512-kk-768-5e7k-3k-clip10-3k-fixloss-1k-4e6-9k-largeres-ep25-gs20491/vae",
+  "act_fn": "silu",
+  "block_out_channels": [
+    128,
+    256,
+    512,
+    512
+  ],
+  "down_block_types": [
+    "DownEncoderBlock2D",
+    "DownEncoderBlock2D",
+    "DownEncoderBlock2D",
+    "DownEncoderBlock2D"
+  ],
+  "force_upcast": true,
+  "in_channels": 3,
+  "latent_channels": 4,
+  "latents_mean": null,
+  "latents_std": null,
+  "layers_per_block": 2,
+  "mid_block_add_attention": true,
+  "norm_num_groups": 32,
+  "out_channels": 3,
+  "sample_size": 768,
+  "scaling_factor": 0.18215,
+  "shift_factor": null,
+  "up_block_types": [
+    "UpDecoderBlock2D",
+    "UpDecoderBlock2D",
+    "UpDecoderBlock2D",
+    "UpDecoderBlock2D"
+  ],
+  "use_post_quant_conv": true,
+  "use_quant_conv": true
+}

vae/diffusion_pytorch_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3e4c08995484ee61270175e9e7a072b66a6e4eeb5f0c266667fe1f45b90daf9a
+size 167335342