Add files using upload-large-folder tool

CausVid.egg-info/PKG-INFO

@@ -0,0 +1,5 @@
+Metadata-Version: 2.4
+Name: CausVid
+Version: 0.0.1
+License-File: LICENSE.md
+Dynamic: license-file

CausVid.egg-info/SOURCES.txt

@@ -0,0 +1,7 @@
+LICENSE.md
+README.md
+setup.py
+CausVid.egg-info/PKG-INFO
+CausVid.egg-info/SOURCES.txt
+CausVid.egg-info/dependency_links.txt
+CausVid.egg-info/top_level.txt

CausVid.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

CausVid.egg-info/top_level.txt

@@ -0,0 +1 @@
+

configs/sdxl_8node_dmd_config.yaml

@@ -0,0 +1,42 @@
+model_name: sdxl
+generator_grad:
+  model: true
+real_score_grad:
+  model: false
+fake_score_grad:
+  model: true
+denoising_step_list:
+- 999
+- 749
+- 499
+- 249
+num_train_timestep: 1000
+real_guidance_scale: 8.0
+generator_task: image
+denoising_loss_type: noise
+mixed_precision: true
+seed: 0
+wandb_host: WANDB_HOST
+wandb_key: WANDB_KEY
+wandb_entity: tyin
+wandb_project: causvid
+wandb_name: causvid_sdxl_test_run_1node
+sharding_strategy: full
+lr: 5.0e-07
+beta1: 0.9
+beta2: 0.999
+data_path: captions_laion6.25.txt
+batch_size: 2
+log_iters: 1000
+negative_prompt: ''
+dfake_gen_update_ratio: 5
+image_or_video_shape:
+- 2
+- 1
+- 4
+- 128
+- 128
+output_path: /mnt/localssd/sdxl_logs
+distillation_loss: dmd
+gradient_checkpointing: false
+warp_denoising_step: false

configs/wan_bidirectional_dmd.yaml

@@ -0,0 +1,48 @@
+model_name: wan
+generator_fsdp_wrap_strategy: size
+real_score_fsdp_wrap_strategy: size
+fake_score_fsdp_wrap_strategy: size
+text_encoder_fsdp_wrap_strategy: size
+generator_grad:
+  model: true
+real_score_grad:
+  model: false
+fake_score_grad:
+  model: true
+denoising_step_list:
+- 1000
+- 757
+- 522
+num_train_timestep: 1000
+timestep_shift: 8.0
+real_guidance_scale: 3.5
+generator_task: bidirectional_video
+real_task_type: bidirectional_video
+fake_task_type: bidirectional_video
+denoising_loss_type: flow
+mixed_precision: true
+seed: 0
+wandb_host: WANDB_HOST
+wandb_key: WANDB_KEY
+wandb_entity: tyin
+wandb_project: causvid
+wandb_name: wan_bidirectional_dmd
+sharding_strategy: hybrid_full
+lr: 2.0e-06
+beta1: 0.9
+beta2: 0.999
+data_path: sample_dataset/mixkit_prompts.txt
+batch_size: 1
+log_iters: 200
+negative_prompt: '色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走'
+dfake_gen_update_ratio: 5
+image_or_video_shape:
+- 1
+- 21
+- 16
+- 60
+- 104
+output_path: /mnt/localssd/wan_bidirectional_dmd
+distillation_loss: dmd
+gradient_checkpointing: true
+warp_denoising_step: false

configs/wan_bidirectional_dmd_from_scratch.yaml

@@ -0,0 +1,49 @@
+model_name: wan
+generator_fsdp_wrap_strategy: size
+real_score_fsdp_wrap_strategy: size
+fake_score_fsdp_wrap_strategy: size
+text_encoder_fsdp_wrap_strategy: size
+generator_grad:
+  model: true
+real_score_grad:
+  model: false
+fake_score_grad:
+  model: true
+denoising_step_list:
+- 1000
+- 757
+- 522
+num_train_timestep: 1000
+timestep_shift: 8.0
+real_guidance_scale: 3.5
+generator_task: bidirectional_video
+real_task_type: bidirectional_video
+fake_task_type: bidirectional_video
+denoising_loss_type: flow
+mixed_precision: true
+seed: 0
+wandb_host: WANDB_HOST
+wandb_key: WANDB_KEY
+wandb_entity: tyin
+wandb_project: causvid
+wandb_name: wan_bidirectional_dmd_from_scratch
+sharding_strategy: hybrid_full
+lr: 2.0e-06
+beta1: 0.9
+beta2: 0.999
+data_path: mixkit_latents_lmdb
+batch_size: 1
+log_iters: 200
+negative_prompt: '色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走'
+dfake_gen_update_ratio: 5
+image_or_video_shape:
+- 1
+- 21
+- 16
+- 60
+- 104
+output_path: /mnt/localssd/wan_bidirectional_dmd_from_scratch
+distillation_loss: dmd
+gradient_checkpointing: true
+backward_simulation: false
+warp_denoising_step: false

configs/wan_causal_dmd.yaml

@@ -0,0 +1,53 @@
+model_name: wan
+generator_name: causal_wan
+generator_ckpt: "wan_causal_ode_checkpoint_model_003000/model.pt"
+generator_fsdp_wrap_strategy: size
+real_score_fsdp_wrap_strategy: size
+fake_score_fsdp_wrap_strategy: size
+text_encoder_fsdp_wrap_strategy: size
+generator_grad:
+  model: true
+real_score_grad:
+  model: false
+fake_score_grad:
+  model: true
+denoising_step_list:
+- 1000
+- 757
+- 522
+- 0
+num_train_timestep: 1000
+timestep_shift: 8.0
+real_guidance_scale: 3.5
+generator_task: causal_video
+real_task_type: bidirectional_video
+fake_task_type: bidirectional_video
+denoising_loss_type: flow
+mixed_precision: true
+seed: 0
+wandb_host: WANDB_HOST
+wandb_key: WANDB_KEY
+wandb_entity: tyin
+wandb_project: causvid
+wandb_name: wan_causal_dmd
+sharding_strategy: hybrid_full
+lr: 2.0e-06
+beta1: 0.9
+beta2: 0.999
+data_path: mixkit_latents_lmdb
+batch_size: 1
+log_iters: 200
+negative_prompt: '色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走'
+dfake_gen_update_ratio: 5
+image_or_video_shape:
+- 1
+- 21
+- 16
+- 60
+- 104
+output_path: /mnt/localssd/wan_causal_dmd
+distillation_loss: dmd
+gradient_checkpointing: true
+backward_simulation: false
+num_frame_per_block: 3
+warp_denoising_step: false

configs/wan_causal_dmd_warp_4step_cfg2.yaml

@@ -0,0 +1,54 @@
+model_name: wan
+generator_name: causal_wan
+generator_ckpt: "wan_causal_ode_checkpoint_model_003000/model.pt"
+generator_fsdp_wrap_strategy: size
+real_score_fsdp_wrap_strategy: size
+fake_score_fsdp_wrap_strategy: size
+text_encoder_fsdp_wrap_strategy: size
+generator_grad:
+  model: true
+real_score_grad:
+  model: false
+fake_score_grad:
+  model: true
+denoising_step_list:
+- 1000
+- 750
+- 500
+- 250
+- 0
+num_train_timestep: 1000
+timestep_shift: 8.0
+real_guidance_scale: 2.0
+generator_task: causal_video
+real_task_type: bidirectional_video
+fake_task_type: bidirectional_video
+denoising_loss_type: flow
+mixed_precision: true
+seed: 0
+wandb_host: WANDB_HOST
+wandb_key: WANDB_KEY
+wandb_entity: tyin
+wandb_project: causvid
+wandb_name: wan_causal_dmd
+sharding_strategy: hybrid_full
+lr: 2.0e-06
+beta1: 0.9
+beta2: 0.999
+data_path: mixkit_latents_lmdb
+batch_size: 1
+log_iters: 200
+negative_prompt: '色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走'
+dfake_gen_update_ratio: 5
+image_or_video_shape:
+- 1
+- 21
+- 16
+- 60
+- 104
+output_path: /mnt/localssd/wan_causal_dmd_warp_4step
+distillation_loss: dmd
+gradient_checkpointing: true
+backward_simulation: false
+num_frame_per_block: 3
+warp_denoising_step: true

configs/wan_causal_ode.yaml

@@ -0,0 +1,31 @@
+model_name: causal_wan
+generator_ckpt: "pretrained_ode.pt"
+generator_grad:
+  model: true
+denoising_step_list:
+- 1000
+- 757
+- 522
+- 0
+generator_task: causal_video
+generator_fsdp_wrap_strategy: size
+text_encoder_fsdp_wrap_strategy: size
+mixed_precision: true
+seed: 0
+wandb_host: https://api.wandb.ai
+wandb_key: 5409d3b960b01b25cec0f6abb5361b4022f0cc41
+wandb_entity: liyitong-Tsinghua University
+wandb_project: self-forcing
+wandb_name: wan_causal_ode
+sharding_strategy: hybrid_full
+lr: 2.0e-06
+beta1: 0.9
+beta2: 0.999
+data_path: ../mixkit_ode_lmdb
+batch_size: 2
+log_iters: 200
+output_path: ./output
+distillation_loss: ode
+gradient_checkpointing: true
+num_frame_per_block: 3
+warp_denoising_step: false

output/wandb/run-20250908_063808-yfn0incq/files/output.log

@@ -0,0 +1,26 @@
+run dir: ./output/wandb/run-20250908_063808-yfn0incq/files
+Start training step 0...
+ cache a block wise causal mask with block size of 3 frames
+BlockMask(shape=(1, 1, 32768, 32768), sparsity=42.52%,
+(0, 0)
+████░░
+████░░
+████░░░░░░░░
+██████████░░
+██████████░░
+██████████░░░░░░░░
+████████████████░░
+████████████████░░
+████████████████░░░░░░░░
+██████████████████████░░
+██████████████████████░░
+██████████████████████░░░░░░░░
+████████████████████████████░░
+████████████████████████████░░
+████████████████████████████████░░
+████████████████████████████████░░
+████████████████████████████████░░░░░░░░
+████████████████████████████████████████
+████████████████████████████████████████
+████████████████████████████████████████
+)

output/wandb/run-20250908_063808-yfn0incq/files/requirements.txt

@@ -0,0 +1,145 @@
+nvidia-cuda-nvrtc-cu12==12.8.93
+filelock==3.19.1
+aiofiles==24.1.0
+PyYAML==6.0.2
+httpx==0.28.1
+multidict==6.6.4
+tifffile==2025.5.10
+tzdata==2025.2
+urllib3==2.5.0
+decord==0.6.0
+certifi==2025.8.3
+setuptools==78.1.1
+websocket-client==1.8.0
+gitdb==4.0.12
+yarl==1.20.1
+tqdm==4.67.1
+open_clip_torch==3.1.0
+pyparsing==3.2.3
+fastapi==0.116.1
+nvidia-curand-cu12==10.3.9.90
+mdurl==0.1.2
+torchvision==0.23.0
+h11==0.16.0
+pytz==2025.2
+six==1.17.0
+nvidia-cufile-cu12==1.13.1.3
+aiohappyeyeballs==2.6.1
+wandb==0.21.3
+nvidia-nvjitlink-cu12==12.8.93
+cycler==0.12.1
+anyio==4.10.0
+scikit-image==0.25.2
+nvidia-cuda-runtime-cu12==12.8.90
+sentencepiece==0.2.1
+rich==14.1.0
+pydantic==2.11.7
+GitPython==3.1.45
+python-dateutil==2.9.0.post0
+accelerate==1.10.1
+proglog==0.1.12
+sentry-sdk==2.37.0
+nvidia-cusparselt-cu12==0.7.1
+dashscope==1.24.3
+platformdirs==4.4.0
+safehttpx==0.1.6
+fsspec==2025.9.0
+lazy_loader==0.4
+typing_extensions==4.15.0
+semantic-version==2.10.0
+numpy==1.24.4
+nvidia-cufft-cu12==11.3.3.83
+ruff==0.12.12
+nvidia-cudnn-cu12==9.10.2.21
+annotated-types==0.7.0
+decorator==4.4.2
+antlr4-python3-runtime==4.9.3
+psutil==7.0.0
+Brotli==1.1.0
+tomlkit==0.13.3
+httpcore==1.0.9
+kiwisolver==1.4.9
+nvidia-cusparse-cu12==12.5.8.93
+groovy==0.1.2
+protobuf==6.32.0
+orjson==3.11.3
+scipy==1.15.3
+regex==2025.9.1
+MarkupSafe==3.0.2
+av==13.1.0
+timm==1.0.19
+uvicorn==0.35.0
+wheel==0.45.1
+Pygments==2.19.2
+websockets==15.0.1
+lmdb==1.7.3
+sympy==1.14.0
+einops==0.8.1
+idna==3.10
+triton==3.4.0
+torch==2.8.0
+moviepy==1.0.3
+nvidia-nvtx-cu12==12.8.90
+matplotlib==3.10.6
+pillow==11.3.0
+charset-normalizer==3.4.3
+attrs==25.3.0
+aiosignal==1.4.0
+markdown-it-py==4.0.0
+requests==2.32.5
+typer==0.17.4
+huggingface-hub==0.34.4
+nvidia-nccl-cu12==2.27.3
+propcache==0.3.2
+opencv-python==4.11.0.86
+ffmpy==0.6.1
+jmespath==1.0.1
+botocore==1.40.25
+pydantic_core==2.33.2
+fonttools==4.59.2
+omegaconf==2.3.0
+pycparser==2.22
+mpmath==1.3.0
+flash_attn==2.8.3
+smmap==5.0.2
+gradio_client==1.12.1
+exceptiongroup==1.3.0
+cffi==1.17.1
+diffusers==0.31.0
+nvidia-cusolver-cu12==11.7.3.90
+contourpy==1.3.2
+cryptography==45.0.7
+nvidia-cuda-cupti-cu12==12.8.90
+wcwidth==0.2.13
+zipp==3.23.0
+safetensors==0.6.2
+gradio==5.44.1
+click==8.2.1
+frozenlist==1.7.0
+networkx==3.4.2
+s3transfer==0.13.1
+shellingham==1.5.4
+starlette==0.47.3
+packaging==25.0
+ftfy==6.3.1
+importlib_metadata==8.7.0
+transformers==4.56.1
+nvidia-cublas-cu12==12.8.4.1
+Jinja2==3.1.6
+pycocotools==2.0.10
+sniffio==1.3.1
+hf-xet==1.1.9
+typing-inspection==0.4.1
+pandas==2.3.2
+python-multipart==0.0.20
+aiohttp==3.12.15
+clip==1.0
+pydub==0.25.1
+easydict==1.13
+pip==25.2
+tokenizers==0.22.0
+imageio==2.37.0
+async-timeout==5.0.1
+boto3==1.40.25
+imageio-ffmpeg==0.6.0
+CausVid==0.0.1

output/wandb/run-20250908_063808-yfn0incq/files/wandb-metadata.json

@@ -0,0 +1,93 @@
+{
+  "os": "Linux-6.8.0-1028-aws-x86_64-with-glibc2.39",
+  "python": "CPython 3.10.18",
+  "startedAt": "2025-09-08T06:38:08.309033Z",
+  "args": [
+    "--config_path",
+    "configs/wan_causal_ode.yaml"
+  ],
+  "program": "/home/yitongli/CausVid/causvid/train_ode.py",
+  "codePath": "causvid/train_ode.py",
+  "codePathLocal": "causvid/train_ode.py",
+  "git": {
+    "remote": "https://github.com/tianweiy/CausVid.git",
+    "commit": "adb6a5ecd07666b4d0290042915c8406e6d5ce22"
+  },
+  "email": "[email protected]",
+  "root": "./output",
+  "host": "ip-172-31-3-169",
+  "executable": "/home/yitongli/miniconda3/envs/causvid/bin/python3.10",
+  "cpu_count": 48,
+  "cpu_count_logical": 96,
+  "gpu": "NVIDIA A100-SXM4-80GB",
+  "gpu_count": 8,
+  "disk": {
+    "/": {
+      "total": "2079114358784",
+      "used": "1367494828032"
+    }
+  },
+  "memory": {
+    "total": "1204521443328"
+  },
+  "gpu_nvidia": [
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-88ce35f2-86d5-a394-f90d-a52e2cd45949"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-1039594d-55ae-3eb3-339e-6d4d62c81d42"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-1f47056a-9c2c-8af1-8232-636203622830"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-38de1dff-a680-ec66-625b-49f2a31e3b42"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-712ffb29-5da2-92e5-99b7-9f148e8c1204"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-194ed489-5bfa-defc-85b1-812953e7d84a"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-21b1f0be-4c00-4af7-0649-9942ae8fd0e4"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-195cca5b-c8ee-790b-4d30-ae4d7aa93057"
+    }
+  ],
+  "cudaVersion": "12.8",
+  "writerId": "051fauhxsqz43onnqgc6jbdjqxushh74"
+}

output/wandb/run-20250908_063808-yfn0incq/logs/debug-core.log

@@ -0,0 +1,7 @@
+{"time":"2025-09-08T06:38:08.329996016Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpmfwk91u6/port-3438882.txt","pid":3438882,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2025-09-08T06:38:08.330648777Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":3438882}
+{"time":"2025-09-08T06:38:08.330657813Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-3438882-3439217-2707863955/socket","Net":"unix"}}
+{"time":"2025-09-08T06:38:08.516356689Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2025-09-08T06:38:08.52538569Z","level":"INFO","msg":"handleInformInit: received","streamId":"yfn0incq","id":"1(@)"}
+{"time":"2025-09-08T06:38:08.725905857Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"yfn0incq","id":"1(@)"}
+{"time":"2025-09-08T06:46:21.545953845Z","level":"INFO","msg":"server: parent process exited, terminating service process"}

output/wandb/run-20250908_063808-yfn0incq/logs/debug-internal.log

@@ -0,0 +1,6 @@
+{"time":"2025-09-08T06:38:08.52564227Z","level":"INFO","msg":"stream: starting","core version":"0.21.3"}
+{"time":"2025-09-08T06:38:08.725733203Z","level":"INFO","msg":"stream: created new stream","id":"yfn0incq"}
+{"time":"2025-09-08T06:38:08.725892734Z","level":"INFO","msg":"stream: started","id":"yfn0incq"}
+{"time":"2025-09-08T06:38:08.725914161Z","level":"INFO","msg":"writer: started","stream_id":"yfn0incq"}
+{"time":"2025-09-08T06:38:08.725929156Z","level":"INFO","msg":"sender: started","stream_id":"yfn0incq"}
+{"time":"2025-09-08T06:38:08.725920349Z","level":"INFO","msg":"handler: started","stream_id":"yfn0incq"}

output/wandb/run-20250908_063808-yfn0incq/logs/debug.log

@@ -0,0 +1,22 @@
+2025-09-08 06:38:08,310 INFO    MainThread:3438882 [wandb_setup.py:_flush():81] Current SDK version is 0.21.3
+2025-09-08 06:38:08,310 INFO    MainThread:3438882 [wandb_setup.py:_flush():81] Configure stats pid to 3438882
+2025-09-08 06:38:08,310 INFO    MainThread:3438882 [wandb_setup.py:_flush():81] Loading settings from /home/yitongli/.config/wandb/settings
+2025-09-08 06:38:08,310 INFO    MainThread:3438882 [wandb_setup.py:_flush():81] Loading settings from /home/yitongli/CausVid/wandb/settings
+2025-09-08 06:38:08,310 INFO    MainThread:3438882 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2025-09-08 06:38:08,310 INFO    MainThread:3438882 [wandb_init.py:setup_run_log_directory():686] Logging user logs to ./output/wandb/run-20250908_063808-yfn0incq/logs/debug.log
+2025-09-08 06:38:08,310 INFO    MainThread:3438882 [wandb_init.py:setup_run_log_directory():687] Logging internal logs to ./output/wandb/run-20250908_063808-yfn0incq/logs/debug-internal.log
+2025-09-08 06:38:08,310 INFO    MainThread:3438882 [wandb_init.py:init():813] calling init triggers
+2025-09-08 06:38:08,310 INFO    MainThread:3438882 [wandb_init.py:init():818] wandb.init called with sweep_config: {}
+config: {'model_name': 'causal_wan', 'generator_grad': {'model': True}, 'denoising_step_list': [1000, 757, 522, 0], 'generator_task': 'causal_video', 'generator_fsdp_wrap_strategy': 'size', 'text_encoder_fsdp_wrap_strategy': 'size', 'mixed_precision': True, 'seed': 9828283, 'wandb_host': 'https://api.wandb.ai', 'wandb_key': '5409d3b960b01b25cec0f6abb5361b4022f0cc41', 'wandb_entity': 'liyitong-Tsinghua University', 'wandb_project': 'self-forcing', 'wandb_name': 'wan_causal_ode', 'sharding_strategy': 'hybrid_full', 'lr': 2e-06, 'beta1': 0.9, 'beta2': 0.999, 'data_path': '../mixkit_ode_lmdb', 'batch_size': 2, 'log_iters': 200, 'output_path': './output', 'distillation_loss': 'ode', 'gradient_checkpointing': True, 'num_frame_per_block': 3, 'warp_denoising_step': False, 'no_save': False, '_wandb': {}}
+2025-09-08 06:38:08,310 INFO    MainThread:3438882 [wandb_init.py:init():854] starting backend
+2025-09-08 06:38:08,516 INFO    MainThread:3438882 [wandb_init.py:init():857] sending inform_init request
+2025-09-08 06:38:08,520 INFO    MainThread:3438882 [wandb_init.py:init():865] backend started and connected
+2025-09-08 06:38:08,523 INFO    MainThread:3438882 [wandb_init.py:init():936] updated telemetry
+2025-09-08 06:38:08,528 INFO    MainThread:3438882 [wandb_init.py:init():960] communicating run to backend with 90.0 second timeout
+2025-09-08 06:38:08,916 INFO    MainThread:3438882 [wandb_init.py:init():1011] starting run threads in backend
+2025-09-08 06:38:09,029 INFO    MainThread:3438882 [wandb_run.py:_console_start():2494] atexit reg
+2025-09-08 06:38:09,029 INFO    MainThread:3438882 [wandb_run.py:_redirect():2342] redirect: wrap_raw
+2025-09-08 06:38:09,029 INFO    MainThread:3438882 [wandb_run.py:_redirect():2411] Wrapping output streams.
+2025-09-08 06:38:09,030 INFO    MainThread:3438882 [wandb_run.py:_redirect():2434] Redirects installed.
+2025-09-08 06:38:09,032 INFO    MainThread:3438882 [wandb_init.py:init():1057] run started, returning control to user process
+2025-09-08 06:38:09,410 INFO    MainThread:3438882 [wandb_run.py:_config_callback():1380] config_cb None None {'_wandb': {'code_path': 'source-self-forcing-causvid_train_ode.py'}}

output/wandb/run-20250908_064634-xd44venm/files/wandb-metadata.json

@@ -0,0 +1,93 @@
+{
+  "os": "Linux-6.8.0-1028-aws-x86_64-with-glibc2.39",
+  "python": "CPython 3.10.18",
+  "startedAt": "2025-09-08T06:46:34.797035Z",
+  "args": [
+    "--config_path",
+    "configs/wan_causal_ode.yaml"
+  ],
+  "program": "/home/yitongli/CausVid/causvid/train_ode.py",
+  "codePath": "causvid/train_ode.py",
+  "codePathLocal": "causvid/train_ode.py",
+  "git": {
+    "remote": "https://github.com/tianweiy/CausVid.git",
+    "commit": "adb6a5ecd07666b4d0290042915c8406e6d5ce22"
+  },
+  "email": "[email protected]",
+  "root": "./output",
+  "host": "ip-172-31-3-169",
+  "executable": "/home/yitongli/miniconda3/envs/causvid/bin/python3.10",
+  "cpu_count": 48,
+  "cpu_count_logical": 96,
+  "gpu": "NVIDIA A100-SXM4-80GB",
+  "gpu_count": 8,
+  "disk": {
+    "/": {
+      "total": "2079114358784",
+      "used": "1367495385088"
+    }
+  },
+  "memory": {
+    "total": "1204521443328"
+  },
+  "gpu_nvidia": [
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-88ce35f2-86d5-a394-f90d-a52e2cd45949"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-1039594d-55ae-3eb3-339e-6d4d62c81d42"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-1f47056a-9c2c-8af1-8232-636203622830"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-38de1dff-a680-ec66-625b-49f2a31e3b42"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-712ffb29-5da2-92e5-99b7-9f148e8c1204"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-194ed489-5bfa-defc-85b1-812953e7d84a"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-21b1f0be-4c00-4af7-0649-9942ae8fd0e4"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-195cca5b-c8ee-790b-4d30-ae4d7aa93057"
+    }
+  ],
+  "cudaVersion": "12.8",
+  "writerId": "u3mfnczl6vj98a25sgfj3dvcuv00y0jd"
+}

output/wandb/run-20250908_064634-xd44venm/logs/debug-core.log

@@ -0,0 +1,7 @@
+{"time":"2025-09-08T06:46:34.819015681Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmp9b1_vyfw/port-3454108.txt","pid":3454108,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2025-09-08T06:46:34.819705153Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":3454108}
+{"time":"2025-09-08T06:46:34.819700396Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-3454108-3454427-2811185260/socket","Net":"unix"}}
+{"time":"2025-09-08T06:46:35.005856196Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2025-09-08T06:46:35.017139949Z","level":"INFO","msg":"handleInformInit: received","streamId":"xd44venm","id":"1(@)"}
+{"time":"2025-09-08T06:46:35.219626779Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"xd44venm","id":"1(@)"}
+{"time":"2025-09-08T06:56:07.684243507Z","level":"INFO","msg":"server: parent process exited, terminating service process"}

output/wandb/run-20250908_065643-lyib7g2q/files/output.log

@@ -0,0 +1,25 @@
+run dir: ./output/wandb/run-20250908_065643-lyib7g2q/files
+ cache a block wise causal mask with block size of 3 frames
+BlockMask(shape=(1, 1, 32768, 32768), sparsity=42.52%,
+(0, 0)
+████░░
+████░░
+████░░░░░░░░
+██████████░░
+██████████░░
+██████████░░░░░░░░
+████████████████░░
+████████████████░░
+████████████████░░░░░░░░
+██████████████████████░░
+██████████████████████░░
+██████████████████████░░░░░░░░
+████████████████████████████░░
+████████████████████████████░░
+████████████████████████████████░░
+████████████████████████████████░░
+████████████████████████████████░░░░░░░░
+████████████████████████████████████████
+████████████████████████████████████████
+████████████████████████████████████████
+)

output/wandb/run-20250908_065643-lyib7g2q/files/requirements.txt

@@ -0,0 +1,145 @@
+nvidia-cuda-nvrtc-cu12==12.8.93
+filelock==3.19.1
+aiofiles==24.1.0
+PyYAML==6.0.2
+httpx==0.28.1
+multidict==6.6.4
+tifffile==2025.5.10
+tzdata==2025.2
+urllib3==2.5.0
+decord==0.6.0
+certifi==2025.8.3
+setuptools==78.1.1
+websocket-client==1.8.0
+gitdb==4.0.12
+yarl==1.20.1
+tqdm==4.67.1
+open_clip_torch==3.1.0
+pyparsing==3.2.3
+fastapi==0.116.1
+nvidia-curand-cu12==10.3.9.90
+mdurl==0.1.2
+torchvision==0.23.0
+h11==0.16.0
+pytz==2025.2
+six==1.17.0
+nvidia-cufile-cu12==1.13.1.3
+aiohappyeyeballs==2.6.1
+wandb==0.21.3
+nvidia-nvjitlink-cu12==12.8.93
+cycler==0.12.1
+anyio==4.10.0
+scikit-image==0.25.2
+nvidia-cuda-runtime-cu12==12.8.90
+sentencepiece==0.2.1
+rich==14.1.0
+pydantic==2.11.7
+GitPython==3.1.45
+python-dateutil==2.9.0.post0
+accelerate==1.10.1
+proglog==0.1.12
+sentry-sdk==2.37.0
+nvidia-cusparselt-cu12==0.7.1
+dashscope==1.24.3
+platformdirs==4.4.0
+safehttpx==0.1.6
+fsspec==2025.9.0
+lazy_loader==0.4
+typing_extensions==4.15.0
+semantic-version==2.10.0
+numpy==1.24.4
+nvidia-cufft-cu12==11.3.3.83
+ruff==0.12.12
+nvidia-cudnn-cu12==9.10.2.21
+annotated-types==0.7.0
+decorator==4.4.2
+antlr4-python3-runtime==4.9.3
+psutil==7.0.0
+Brotli==1.1.0
+tomlkit==0.13.3
+httpcore==1.0.9
+kiwisolver==1.4.9
+nvidia-cusparse-cu12==12.5.8.93
+groovy==0.1.2
+protobuf==6.32.0
+orjson==3.11.3
+scipy==1.15.3
+regex==2025.9.1
+MarkupSafe==3.0.2
+av==13.1.0
+timm==1.0.19
+uvicorn==0.35.0
+wheel==0.45.1
+Pygments==2.19.2
+websockets==15.0.1
+lmdb==1.7.3
+sympy==1.14.0
+einops==0.8.1
+idna==3.10
+triton==3.4.0
+torch==2.8.0
+moviepy==1.0.3
+nvidia-nvtx-cu12==12.8.90
+matplotlib==3.10.6
+pillow==11.3.0
+charset-normalizer==3.4.3
+attrs==25.3.0
+aiosignal==1.4.0
+markdown-it-py==4.0.0
+requests==2.32.5
+typer==0.17.4
+huggingface-hub==0.34.4
+nvidia-nccl-cu12==2.27.3
+propcache==0.3.2
+opencv-python==4.11.0.86
+ffmpy==0.6.1
+jmespath==1.0.1
+botocore==1.40.25
+pydantic_core==2.33.2
+fonttools==4.59.2
+omegaconf==2.3.0
+pycparser==2.22
+mpmath==1.3.0
+flash_attn==2.8.3
+smmap==5.0.2
+gradio_client==1.12.1
+exceptiongroup==1.3.0
+cffi==1.17.1
+diffusers==0.31.0
+nvidia-cusolver-cu12==11.7.3.90
+contourpy==1.3.2
+cryptography==45.0.7
+nvidia-cuda-cupti-cu12==12.8.90
+wcwidth==0.2.13
+zipp==3.23.0
+safetensors==0.6.2
+gradio==5.44.1
+click==8.2.1
+frozenlist==1.7.0
+networkx==3.4.2
+s3transfer==0.13.1
+shellingham==1.5.4
+starlette==0.47.3
+packaging==25.0
+ftfy==6.3.1
+importlib_metadata==8.7.0
+transformers==4.56.1
+nvidia-cublas-cu12==12.8.4.1
+Jinja2==3.1.6
+pycocotools==2.0.10
+sniffio==1.3.1
+hf-xet==1.1.9
+typing-inspection==0.4.1
+pandas==2.3.2
+python-multipart==0.0.20
+aiohttp==3.12.15
+clip==1.0
+pydub==0.25.1
+easydict==1.13
+pip==25.2
+tokenizers==0.22.0
+imageio==2.37.0
+async-timeout==5.0.1
+boto3==1.40.25
+imageio-ffmpeg==0.6.0
+CausVid==0.0.1

output/wandb/run-20250908_065643-lyib7g2q/files/wandb-metadata.json

@@ -0,0 +1,93 @@
+{
+  "os": "Linux-6.8.0-1028-aws-x86_64-with-glibc2.39",
+  "python": "CPython 3.10.18",
+  "startedAt": "2025-09-08T06:56:43.722768Z",
+  "args": [
+    "--config_path",
+    "configs/wan_causal_ode.yaml"
+  ],
+  "program": "/home/yitongli/CausVid/causvid/train_ode.py",
+  "codePath": "causvid/train_ode.py",
+  "codePathLocal": "causvid/train_ode.py",
+  "git": {
+    "remote": "https://github.com/tianweiy/CausVid.git",
+    "commit": "adb6a5ecd07666b4d0290042915c8406e6d5ce22"
+  },
+  "email": "[email protected]",
+  "root": "./output",
+  "host": "ip-172-31-3-169",
+  "executable": "/home/yitongli/miniconda3/envs/causvid/bin/python3.10",
+  "cpu_count": 48,
+  "cpu_count_logical": 96,
+  "gpu": "NVIDIA A100-SXM4-80GB",
+  "gpu_count": 8,
+  "disk": {
+    "/": {
+      "total": "2079114358784",
+      "used": "1367495954432"
+    }
+  },
+  "memory": {
+    "total": "1204521443328"
+  },
+  "gpu_nvidia": [
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-88ce35f2-86d5-a394-f90d-a52e2cd45949"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-1039594d-55ae-3eb3-339e-6d4d62c81d42"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-1f47056a-9c2c-8af1-8232-636203622830"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-38de1dff-a680-ec66-625b-49f2a31e3b42"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-712ffb29-5da2-92e5-99b7-9f148e8c1204"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-194ed489-5bfa-defc-85b1-812953e7d84a"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-21b1f0be-4c00-4af7-0649-9942ae8fd0e4"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-195cca5b-c8ee-790b-4d30-ae4d7aa93057"
+    }
+  ],
+  "cudaVersion": "12.8",
+  "writerId": "nrbwifb91qzia1ndhn1iu2g7eodid6yk"
+}

output/wandb/run-20250908_065643-lyib7g2q/logs/debug-core.log

@@ -0,0 +1,7 @@
+{"time":"2025-09-08T06:56:43.750357486Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmp4mhsv4lj/port-3470051.txt","pid":3470051,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2025-09-08T06:56:43.750987576Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":3470051}
+{"time":"2025-09-08T06:56:43.750983708Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-3470051-3470349-2506732261/socket","Net":"unix"}}
+{"time":"2025-09-08T06:56:43.936283119Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2025-09-08T06:56:43.944508948Z","level":"INFO","msg":"handleInformInit: received","streamId":"lyib7g2q","id":"1(@)"}
+{"time":"2025-09-08T06:56:44.157850577Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"lyib7g2q","id":"1(@)"}
+{"time":"2025-09-08T07:06:27.322661473Z","level":"INFO","msg":"server: parent process exited, terminating service process"}

output/wandb/run-20250908_065643-lyib7g2q/logs/debug-internal.log

@@ -0,0 +1,6 @@
+{"time":"2025-09-08T06:56:43.94460419Z","level":"INFO","msg":"stream: starting","core version":"0.21.3"}
+{"time":"2025-09-08T06:56:44.157805156Z","level":"INFO","msg":"stream: created new stream","id":"lyib7g2q"}
+{"time":"2025-09-08T06:56:44.157846153Z","level":"INFO","msg":"stream: started","id":"lyib7g2q"}
+{"time":"2025-09-08T06:56:44.157856214Z","level":"INFO","msg":"handler: started","stream_id":"lyib7g2q"}
+{"time":"2025-09-08T06:56:44.15785768Z","level":"INFO","msg":"sender: started","stream_id":"lyib7g2q"}
+{"time":"2025-09-08T06:56:44.157877833Z","level":"INFO","msg":"writer: started","stream_id":"lyib7g2q"}

output/wandb/run-20250908_065643-lyib7g2q/logs/debug.log

@@ -0,0 +1,22 @@
+2025-09-08 06:56:43,725 INFO    MainThread:3470051 [wandb_setup.py:_flush():81] Current SDK version is 0.21.3
+2025-09-08 06:56:43,725 INFO    MainThread:3470051 [wandb_setup.py:_flush():81] Configure stats pid to 3470051
+2025-09-08 06:56:43,725 INFO    MainThread:3470051 [wandb_setup.py:_flush():81] Loading settings from /home/yitongli/.config/wandb/settings
+2025-09-08 06:56:43,725 INFO    MainThread:3470051 [wandb_setup.py:_flush():81] Loading settings from /home/yitongli/CausVid/wandb/settings
+2025-09-08 06:56:43,725 INFO    MainThread:3470051 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2025-09-08 06:56:43,725 INFO    MainThread:3470051 [wandb_init.py:setup_run_log_directory():686] Logging user logs to ./output/wandb/run-20250908_065643-lyib7g2q/logs/debug.log
+2025-09-08 06:56:43,725 INFO    MainThread:3470051 [wandb_init.py:setup_run_log_directory():687] Logging internal logs to ./output/wandb/run-20250908_065643-lyib7g2q/logs/debug-internal.log
+2025-09-08 06:56:43,725 INFO    MainThread:3470051 [wandb_init.py:init():813] calling init triggers
+2025-09-08 06:56:43,725 INFO    MainThread:3470051 [wandb_init.py:init():818] wandb.init called with sweep_config: {}
+config: {'model_name': 'causal_wan', 'generator_grad': {'model': True}, 'denoising_step_list': [1000, 757, 522, 0], 'generator_task': 'causal_video', 'generator_fsdp_wrap_strategy': 'size', 'text_encoder_fsdp_wrap_strategy': 'size', 'mixed_precision': True, 'seed': 7956051, 'wandb_host': 'https://api.wandb.ai', 'wandb_key': '5409d3b960b01b25cec0f6abb5361b4022f0cc41', 'wandb_entity': 'liyitong-Tsinghua University', 'wandb_project': 'self-forcing', 'wandb_name': 'wan_causal_ode', 'sharding_strategy': 'hybrid_full', 'lr': 2e-06, 'beta1': 0.9, 'beta2': 0.999, 'data_path': '../mixkit_ode_lmdb', 'batch_size': 2, 'log_iters': 200, 'output_path': './output', 'distillation_loss': 'ode', 'gradient_checkpointing': True, 'num_frame_per_block': 3, 'warp_denoising_step': False, 'no_save': False, '_wandb': {}}
+2025-09-08 06:56:43,726 INFO    MainThread:3470051 [wandb_init.py:init():854] starting backend
+2025-09-08 06:56:43,936 INFO    MainThread:3470051 [wandb_init.py:init():857] sending inform_init request
+2025-09-08 06:56:43,942 INFO    MainThread:3470051 [wandb_init.py:init():865] backend started and connected
+2025-09-08 06:56:43,949 INFO    MainThread:3470051 [wandb_init.py:init():936] updated telemetry
+2025-09-08 06:56:43,956 INFO    MainThread:3470051 [wandb_init.py:init():960] communicating run to backend with 90.0 second timeout
+2025-09-08 06:56:44,314 INFO    MainThread:3470051 [wandb_init.py:init():1011] starting run threads in backend
+2025-09-08 06:56:44,487 INFO    MainThread:3470051 [wandb_run.py:_console_start():2494] atexit reg
+2025-09-08 06:56:44,487 INFO    MainThread:3470051 [wandb_run.py:_redirect():2342] redirect: wrap_raw
+2025-09-08 06:56:44,487 INFO    MainThread:3470051 [wandb_run.py:_redirect():2411] Wrapping output streams.
+2025-09-08 06:56:44,487 INFO    MainThread:3470051 [wandb_run.py:_redirect():2434] Redirects installed.
+2025-09-08 06:56:44,489 INFO    MainThread:3470051 [wandb_init.py:init():1057] run started, returning control to user process
+2025-09-08 06:56:44,862 INFO    MainThread:3470051 [wandb_run.py:_config_callback():1380] config_cb None None {'_wandb': {'code_path': 'source-self-forcing-causvid_train_ode.py'}}

output/wandb/run-20250908_081933-xtabuxq1/files/config.yaml

@@ -0,0 +1,159 @@
+_wandb:
+    value:
+        cli_version: 0.21.3
+        code_path: source-self-forcing-causvid_train_ode.py
+        e:
+            0fh2pmjwzgt2no6kkv0vzcqewqutpwmq:
+                args:
+                    - --config_path
+                    - configs/wan_causal_ode.yaml
+                codePath: causvid/train_ode.py
+                codePathLocal: causvid/train_ode.py
+                cpu_count: 48
+                cpu_count_logical: 96
+                cudaVersion: "12.8"
+                disk:
+                    /:
+                        total: "2079114358784"
+                        used: "1383118893056"
+                email: [email protected]
+                executable: /home/yitongli/miniconda3/envs/causvid/bin/python3.10
+                git:
+                    commit: adb6a5ecd07666b4d0290042915c8406e6d5ce22
+                    remote: https://github.com/tianweiy/CausVid.git
+                gpu: NVIDIA A100-SXM4-80GB
+                gpu_count: 8
+                gpu_nvidia:
+                    - architecture: Ampere
+                      cudaCores: 6912
+                      memoryTotal: "85899345920"
+                      name: NVIDIA A100-SXM4-80GB
+                      uuid: GPU-88ce35f2-86d5-a394-f90d-a52e2cd45949
+                    - architecture: Ampere
+                      cudaCores: 6912
+                      memoryTotal: "85899345920"
+                      name: NVIDIA A100-SXM4-80GB
+                      uuid: GPU-1039594d-55ae-3eb3-339e-6d4d62c81d42
+                    - architecture: Ampere
+                      cudaCores: 6912
+                      memoryTotal: "85899345920"
+                      name: NVIDIA A100-SXM4-80GB
+                      uuid: GPU-1f47056a-9c2c-8af1-8232-636203622830
+                    - architecture: Ampere
+                      cudaCores: 6912
+                      memoryTotal: "85899345920"
+                      name: NVIDIA A100-SXM4-80GB
+                      uuid: GPU-38de1dff-a680-ec66-625b-49f2a31e3b42
+                    - architecture: Ampere
+                      cudaCores: 6912
+                      memoryTotal: "85899345920"
+                      name: NVIDIA A100-SXM4-80GB
+                      uuid: GPU-712ffb29-5da2-92e5-99b7-9f148e8c1204
+                    - architecture: Ampere
+                      cudaCores: 6912
+                      memoryTotal: "85899345920"
+                      name: NVIDIA A100-SXM4-80GB
+                      uuid: GPU-194ed489-5bfa-defc-85b1-812953e7d84a
+                    - architecture: Ampere
+                      cudaCores: 6912
+                      memoryTotal: "85899345920"
+                      name: NVIDIA A100-SXM4-80GB
+                      uuid: GPU-21b1f0be-4c00-4af7-0649-9942ae8fd0e4
+                    - architecture: Ampere
+                      cudaCores: 6912
+                      memoryTotal: "85899345920"
+                      name: NVIDIA A100-SXM4-80GB
+                      uuid: GPU-195cca5b-c8ee-790b-4d30-ae4d7aa93057
+                host: ip-172-31-3-169
+                memory:
+                    total: "1204521443328"
+                os: Linux-6.8.0-1028-aws-x86_64-with-glibc2.39
+                program: /home/yitongli/CausVid/causvid/train_ode.py
+                python: CPython 3.10.18
+                root: ./output
+                startedAt: "2025-09-08T08:19:33.312470Z"
+                writerId: 0fh2pmjwzgt2no6kkv0vzcqewqutpwmq
+        m: []
+        python_version: 3.10.18
+        t:
+            "1":
+                - 1
+                - 11
+                - 41
+                - 49
+                - 71
+                - 83
+                - 105
+            "2":
+                - 1
+                - 11
+                - 41
+                - 49
+                - 71
+                - 83
+                - 105
+            "3":
+                - 16
+                - 17
+            "4": 3.10.18
+            "5": 0.21.3
+            "6": 4.56.1
+            "12": 0.21.3
+            "13": linux-x86_64
+batch_size:
+    value: 2
+beta1:
+    value: 0.9
+beta2:
+    value: 0.999
+data_path:
+    value: ../mixkit_ode_lmdb
+denoising_step_list:
+    value:
+        - 1000
+        - 757
+        - 522
+        - 0
+distillation_loss:
+    value: ode
+generator_fsdp_wrap_strategy:
+    value: size
+generator_grad:
+    value:
+        model: true
+generator_task:
+    value: causal_video
+gradient_checkpointing:
+    value: true
+log_iters:
+    value: 200
+lr:
+    value: 2e-06
+mixed_precision:
+    value: true
+model_name:
+    value: causal_wan
+no_save:
+    value: false
+num_frame_per_block:
+    value: 3
+output_path:
+    value: ./output
+seed:
+    value: 4894427
+sharding_strategy:
+    value: hybrid_full
+text_encoder_fsdp_wrap_strategy:
+    value: size
+wandb_entity:
+    value: liyitong-Tsinghua University
+wandb_host:
+    value: https://api.wandb.ai
+wandb_key:
+    value: 5409d3b960b01b25cec0f6abb5361b4022f0cc41
+wandb_name:
+    value: wan_causal_ode
+wandb_project:
+    value: self-forcing
+warp_denoising_step:
+    value: false

output/wandb/run-20250908_081933-xtabuxq1/files/wandb-metadata.json

@@ -0,0 +1,93 @@
+{
+  "os": "Linux-6.8.0-1028-aws-x86_64-with-glibc2.39",
+  "python": "CPython 3.10.18",
+  "startedAt": "2025-09-08T08:19:33.312470Z",
+  "args": [
+    "--config_path",
+    "configs/wan_causal_ode.yaml"
+  ],
+  "program": "/home/yitongli/CausVid/causvid/train_ode.py",
+  "codePath": "causvid/train_ode.py",
+  "codePathLocal": "causvid/train_ode.py",
+  "git": {
+    "remote": "https://github.com/tianweiy/CausVid.git",
+    "commit": "adb6a5ecd07666b4d0290042915c8406e6d5ce22"
+  },
+  "email": "[email protected]",
+  "root": "./output",
+  "host": "ip-172-31-3-169",
+  "executable": "/home/yitongli/miniconda3/envs/causvid/bin/python3.10",
+  "cpu_count": 48,
+  "cpu_count_logical": 96,
+  "gpu": "NVIDIA A100-SXM4-80GB",
+  "gpu_count": 8,
+  "disk": {
+    "/": {
+      "total": "2079114358784",
+      "used": "1383118893056"
+    }
+  },
+  "memory": {
+    "total": "1204521443328"
+  },
+  "gpu_nvidia": [
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-88ce35f2-86d5-a394-f90d-a52e2cd45949"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-1039594d-55ae-3eb3-339e-6d4d62c81d42"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-1f47056a-9c2c-8af1-8232-636203622830"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-38de1dff-a680-ec66-625b-49f2a31e3b42"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-712ffb29-5da2-92e5-99b7-9f148e8c1204"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-194ed489-5bfa-defc-85b1-812953e7d84a"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-21b1f0be-4c00-4af7-0649-9942ae8fd0e4"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-195cca5b-c8ee-790b-4d30-ae4d7aa93057"
+    }
+  ],
+  "cudaVersion": "12.8",
+  "writerId": "0fh2pmjwzgt2no6kkv0vzcqewqutpwmq"
+}

output/wandb/run-20250908_081933-xtabuxq1/logs/debug-core.log

@@ -0,0 +1,14 @@
+{"time":"2025-09-08T08:19:33.333709351Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpl2jclwy9/port-3826501.txt","pid":3826501,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2025-09-08T08:19:33.334301481Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":3826501}
+{"time":"2025-09-08T08:19:33.334288666Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-3826501-3826821-56449121/socket","Net":"unix"}}
+{"time":"2025-09-08T08:19:33.519765449Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2025-09-08T08:19:33.527245947Z","level":"INFO","msg":"handleInformInit: received","streamId":"xtabuxq1","id":"1(@)"}
+{"time":"2025-09-08T08:19:33.72624599Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"xtabuxq1","id":"1(@)"}
+{"time":"2025-09-08T08:21:14.249662492Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
+{"time":"2025-09-08T08:21:14.249744829Z","level":"INFO","msg":"server is shutting down"}
+{"time":"2025-09-08T08:21:14.249739693Z","level":"INFO","msg":"connection: closing","id":"1(@)"}
+{"time":"2025-09-08T08:21:14.249900377Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-3826501-3826821-56449121/socket","Net":"unix"}}
+{"time":"2025-09-08T08:21:14.249928173Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
+{"time":"2025-09-08T08:21:14.74398118Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
+{"time":"2025-09-08T08:21:14.744024642Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
+{"time":"2025-09-08T08:21:14.74403815Z","level":"INFO","msg":"server is closed"}

output/wandb/run-20250908_081933-xtabuxq1/logs/debug.log

@@ -0,0 +1,24 @@
+2025-09-08 08:19:33,313 INFO    MainThread:3826501 [wandb_setup.py:_flush():81] Current SDK version is 0.21.3
+2025-09-08 08:19:33,313 INFO    MainThread:3826501 [wandb_setup.py:_flush():81] Configure stats pid to 3826501
+2025-09-08 08:19:33,313 INFO    MainThread:3826501 [wandb_setup.py:_flush():81] Loading settings from /home/yitongli/.config/wandb/settings
+2025-09-08 08:19:33,313 INFO    MainThread:3826501 [wandb_setup.py:_flush():81] Loading settings from /home/yitongli/CausVid/wandb/settings
+2025-09-08 08:19:33,313 INFO    MainThread:3826501 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2025-09-08 08:19:33,313 INFO    MainThread:3826501 [wandb_init.py:setup_run_log_directory():686] Logging user logs to ./output/wandb/run-20250908_081933-xtabuxq1/logs/debug.log
+2025-09-08 08:19:33,313 INFO    MainThread:3826501 [wandb_init.py:setup_run_log_directory():687] Logging internal logs to ./output/wandb/run-20250908_081933-xtabuxq1/logs/debug-internal.log
+2025-09-08 08:19:33,313 INFO    MainThread:3826501 [wandb_init.py:init():813] calling init triggers
+2025-09-08 08:19:33,313 INFO    MainThread:3826501 [wandb_init.py:init():818] wandb.init called with sweep_config: {}
+config: {'model_name': 'causal_wan', 'generator_grad': {'model': True}, 'denoising_step_list': [1000, 757, 522, 0], 'generator_task': 'causal_video', 'generator_fsdp_wrap_strategy': 'size', 'text_encoder_fsdp_wrap_strategy': 'size', 'mixed_precision': True, 'seed': 4894427, 'wandb_host': 'https://api.wandb.ai', 'wandb_key': '5409d3b960b01b25cec0f6abb5361b4022f0cc41', 'wandb_entity': 'liyitong-Tsinghua University', 'wandb_project': 'self-forcing', 'wandb_name': 'wan_causal_ode', 'sharding_strategy': 'hybrid_full', 'lr': 2e-06, 'beta1': 0.9, 'beta2': 0.999, 'data_path': '../mixkit_ode_lmdb', 'batch_size': 2, 'log_iters': 200, 'output_path': './output', 'distillation_loss': 'ode', 'gradient_checkpointing': True, 'num_frame_per_block': 3, 'warp_denoising_step': False, 'no_save': False, '_wandb': {}}
+2025-09-08 08:19:33,314 INFO    MainThread:3826501 [wandb_init.py:init():854] starting backend
+2025-09-08 08:19:33,519 INFO    MainThread:3826501 [wandb_init.py:init():857] sending inform_init request
+2025-09-08 08:19:33,524 INFO    MainThread:3826501 [wandb_init.py:init():865] backend started and connected
+2025-09-08 08:19:33,527 INFO    MainThread:3826501 [wandb_init.py:init():936] updated telemetry
+2025-09-08 08:19:33,532 INFO    MainThread:3826501 [wandb_init.py:init():960] communicating run to backend with 90.0 second timeout
+2025-09-08 08:19:33,885 INFO    MainThread:3826501 [wandb_init.py:init():1011] starting run threads in backend
+2025-09-08 08:19:33,999 INFO    MainThread:3826501 [wandb_run.py:_console_start():2494] atexit reg
+2025-09-08 08:19:33,999 INFO    MainThread:3826501 [wandb_run.py:_redirect():2342] redirect: wrap_raw
+2025-09-08 08:19:33,999 INFO    MainThread:3826501 [wandb_run.py:_redirect():2411] Wrapping output streams.
+2025-09-08 08:19:33,999 INFO    MainThread:3826501 [wandb_run.py:_redirect():2434] Redirects installed.
+2025-09-08 08:19:34,001 INFO    MainThread:3826501 [wandb_init.py:init():1057] run started, returning control to user process
+2025-09-08 08:19:35,140 INFO    MainThread:3826501 [wandb_run.py:_config_callback():1380] config_cb None None {'_wandb': {'code_path': 'source-self-forcing-causvid_train_ode.py'}}
+2025-09-08 08:21:14,249 INFO    wandb-AsyncioManager-main:3826501 [service_client.py:_forward_responses():84] Reached EOF.
+2025-09-08 08:21:14,250 INFO    wandb-AsyncioManager-main:3826501 [mailbox.py:close():137] Closing mailbox, abandoning 1 handles.

output/wandb/run-20250908_084516-b1cckt0e/logs/debug-core.log

@@ -0,0 +1,7 @@
+{"time":"2025-09-08T08:45:16.764735165Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmp2ihrl71u/port-3877754.txt","pid":3877754,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2025-09-08T08:45:16.76537437Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":3877754}
+{"time":"2025-09-08T08:45:16.765346611Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-3877754-3878040-3730929431/socket","Net":"unix"}}
+{"time":"2025-09-08T08:45:16.950019952Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2025-09-08T08:45:16.958321194Z","level":"INFO","msg":"handleInformInit: received","streamId":"b1cckt0e","id":"1(@)"}
+{"time":"2025-09-08T08:45:17.166215832Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"b1cckt0e","id":"1(@)"}
+{"time":"2025-09-08T08:48:41.404482127Z","level":"INFO","msg":"server: parent process exited, terminating service process"}

output/wandb/run-20250908_084949-b9yvqkt5/files/config.yaml

@@ -0,0 +1,160 @@
+_wandb:
+    value:
+        cli_version: 0.21.3
+        code_path: source-self-forcing-causvid_train_ode.py
+        e:
+            ytk5y3nz1jsnyvdlk65guvab4uhpkdm4:
+                args:
+                    - --config_path
+                    - configs/wan_causal_ode.yaml
+                codePath: causvid/train_ode.py
+                codePathLocal: causvid/train_ode.py
+                cpu_count: 48
+                cpu_count_logical: 96
+                cudaVersion: "12.8"
+                disk:
+                    /:
+                        total: "2079114358784"
+                        used: "1386641809408"
+                email: [email protected]
+                executable: /home/yitongli/miniconda3/envs/causvid/bin/python3.10
+                git:
+                    commit: adb6a5ecd07666b4d0290042915c8406e6d5ce22
+                    remote: https://github.com/tianweiy/CausVid.git
+                gpu: NVIDIA A100-SXM4-80GB
+                gpu_count: 8
+                gpu_nvidia:
+                    - architecture: Ampere
+                      cudaCores: 6912
+                      memoryTotal: "85899345920"
+                      name: NVIDIA A100-SXM4-80GB
+                      uuid: GPU-88ce35f2-86d5-a394-f90d-a52e2cd45949
+                    - architecture: Ampere
+                      cudaCores: 6912
+                      memoryTotal: "85899345920"
+                      name: NVIDIA A100-SXM4-80GB
+                      uuid: GPU-1039594d-55ae-3eb3-339e-6d4d62c81d42
+                    - architecture: Ampere
+                      cudaCores: 6912
+                      memoryTotal: "85899345920"
+                      name: NVIDIA A100-SXM4-80GB
+                      uuid: GPU-1f47056a-9c2c-8af1-8232-636203622830
+                    - architecture: Ampere
+                      cudaCores: 6912
+                      memoryTotal: "85899345920"
+                      name: NVIDIA A100-SXM4-80GB
+                      uuid: GPU-38de1dff-a680-ec66-625b-49f2a31e3b42
+                    - architecture: Ampere
+                      cudaCores: 6912
+                      memoryTotal: "85899345920"
+                      name: NVIDIA A100-SXM4-80GB
+                      uuid: GPU-712ffb29-5da2-92e5-99b7-9f148e8c1204
+                    - architecture: Ampere
+                      cudaCores: 6912
+                      memoryTotal: "85899345920"
+                      name: NVIDIA A100-SXM4-80GB
+                      uuid: GPU-194ed489-5bfa-defc-85b1-812953e7d84a
+                    - architecture: Ampere
+                      cudaCores: 6912
+                      memoryTotal: "85899345920"
+                      name: NVIDIA A100-SXM4-80GB
+                      uuid: GPU-21b1f0be-4c00-4af7-0649-9942ae8fd0e4
+                    - architecture: Ampere
+                      cudaCores: 6912
+                      memoryTotal: "85899345920"
+                      name: NVIDIA A100-SXM4-80GB
+                      uuid: GPU-195cca5b-c8ee-790b-4d30-ae4d7aa93057
+                host: ip-172-31-3-169
+                memory:
+                    total: "1204521443328"
+                os: Linux-6.8.0-1028-aws-x86_64-with-glibc2.39
+                program: /home/yitongli/CausVid/causvid/train_ode.py
+                python: CPython 3.10.18
+                root: ./output
+                startedAt: "2025-09-08T08:49:49.541042Z"
+                writerId: ytk5y3nz1jsnyvdlk65guvab4uhpkdm4
+        m: []
+        python_version: 3.10.18
+        t:
+            "1":
+                - 1
+                - 11
+                - 41
+                - 49
+                - 71
+                - 83
+                - 105
+            "2":
+                - 1
+                - 11
+                - 41
+                - 49
+                - 71
+                - 83
+                - 105
+            "3":
+                - 16
+                - 17
+                - 61
+            "4": 3.10.18
+            "5": 0.21.3
+            "6": 4.56.1
+            "12": 0.21.3
+            "13": linux-x86_64
+batch_size:
+    value: 2
+beta1:
+    value: 0.9
+beta2:
+    value: 0.999
+data_path:
+    value: ../mixkit_ode_lmdb
+denoising_step_list:
+    value:
+        - 1000
+        - 757
+        - 522
+        - 0
+distillation_loss:
+    value: ode
+generator_fsdp_wrap_strategy:
+    value: size
+generator_grad:
+    value:
+        model: true
+generator_task:
+    value: causal_video
+gradient_checkpointing:
+    value: true
+log_iters:
+    value: 200
+lr:
+    value: 2e-06
+mixed_precision:
+    value: true
+model_name:
+    value: causal_wan
+no_save:
+    value: false
+num_frame_per_block:
+    value: 3
+output_path:
+    value: ./output
+seed:
+    value: 2913400
+sharding_strategy:
+    value: hybrid_full
+text_encoder_fsdp_wrap_strategy:
+    value: size
+wandb_entity:
+    value: liyitong-Tsinghua University
+wandb_host:
+    value: https://api.wandb.ai
+wandb_key:
+    value: 5409d3b960b01b25cec0f6abb5361b4022f0cc41
+wandb_name:
+    value: wan_causal_ode
+wandb_project:
+    value: self-forcing
+warp_denoising_step:
+    value: false

output/wandb/run-20250908_084949-b9yvqkt5/files/output.log

@@ -0,0 +1,94 @@
+run dir: ./output/wandb/run-20250908_084949-b9yvqkt5/files
+KV inference with 3 frames per block
+ODERegression initialized.
+ cache a block wise causal mask with block size of 3 frames
+BlockMask(shape=(1, 1, 32768, 32768), sparsity=42.52%,
+(0, 0)
+████░░
+████░░
+████░░░░░░░░
+██████████░░
+██████████░░
+██████████░░░░░░░░
+████████████████░░
+████████████████░░
+████████████████░░░░░░░░
+██████████████████████░░
+██████████████████████░░
+██████████████████████░░░░░░░░
+████████████████████████████░░
+████████████████████████████░░
+████████████████████████████████░░
+████████████████████████████████░░
+████████████████████████████████░░░░░░░░
+████████████████████████████████████████
+████████████████████████████████████████
+████████████████████████████████████████
+)
+arrive
+Start gathering distributed model states...
+/home/yitongli/miniconda3/envs/causvid/lib/python3.10/site-packages/torch/distributed/fsdp/fully_sharded_data_parallel.py:678: FutureWarning: FSDP.state_dict_type() and FSDP.set_state_dict_type() are being deprecated. Please use APIs, get_state_dict() and set_state_dict(), which can support different parallelisms, FSDP1, FSDP2, DDP. API doc: https://pytorch.org/docs/stable/distributed.checkpoint.html#torch.distributed.checkpoint.state_dict.get_state_dict .Tutorial: https://pytorch.org/tutorials/recipes/distributed_checkpoint_recipe.html .
+  warnings.warn(
+Model saved to ./output/2025-09-08-08-49-49.305637_seed2913400/checkpoint_model_000000/model.pt
+training step 0...
+###########  torch.Size([1, 21, 16, 60, 104])
+Traceback (most recent call last):
+  File "/home/yitongli/CausVid/causvid/train_ode.py", line 229, in <module>
+    main()
+  File "/home/yitongli/CausVid/causvid/train_ode.py", line 223, in main
+    trainer.train()
+  File "/home/yitongli/CausVid/causvid/train_ode.py", line 187, in train
+    self.generate_video()
+  File "/home/yitongli/CausVid/causvid/train_ode.py", line 181, in generate_video
+    self.distillation_model.generate_video(ode_latent, conditional_dict)
+  File "/home/yitongli/CausVid/causvid/ode_regression.py", line 199, in generate_video
+    video = self.inference_pipeline.inference(
+  File "/home/yitongli/CausVid/causvid/models/wan/causal_inference.py", line 222, in inference
+    video = self.vae.decode_to_pixel(output)
+  File "/home/yitongli/CausVid/causvid/models/wan/wan_wrapper.py", line 83, in decode_to_pixel
+    output = [
+  File "/home/yitongli/CausVid/causvid/models/wan/wan_wrapper.py", line 84, in <listcomp>
+    self.model.decode(u.unsqueeze(0),
+  File "/home/yitongli/CausVid/causvid/models/wan/wan_base/modules/vae.py", line 553, in decode
+    x = self.conv2(z)
+  File "/home/yitongli/miniconda3/envs/causvid/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1773, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+  File "/home/yitongli/miniconda3/envs/causvid/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1784, in _call_impl
+    return forward_call(*args, **kwargs)
+  File "/home/yitongli/CausVid/causvid/models/wan/wan_base/modules/vae.py", line 36, in forward
+    return super().forward(x)
+  File "/home/yitongli/miniconda3/envs/causvid/lib/python3.10/site-packages/torch/nn/modules/conv.py", line 717, in forward
+    return self._conv_forward(input, self.weight, self.bias)
+  File "/home/yitongli/miniconda3/envs/causvid/lib/python3.10/site-packages/torch/nn/modules/conv.py", line 712, in _conv_forward
+    return F.conv3d(
+RuntimeError: Input type (c10::BFloat16) and bias type (float) should be the same
+[rank0]: Traceback (most recent call last):
+[rank0]:   File "/home/yitongli/CausVid/causvid/train_ode.py", line 229, in <module>
+[rank0]:     main()
+[rank0]:   File "/home/yitongli/CausVid/causvid/train_ode.py", line 223, in main
+[rank0]:     trainer.train()
+[rank0]:   File "/home/yitongli/CausVid/causvid/train_ode.py", line 187, in train
+[rank0]:     self.generate_video()
+[rank0]:   File "/home/yitongli/CausVid/causvid/train_ode.py", line 181, in generate_video
+[rank0]:     self.distillation_model.generate_video(ode_latent, conditional_dict)
+[rank0]:   File "/home/yitongli/CausVid/causvid/ode_regression.py", line 199, in generate_video
+[rank0]:     video = self.inference_pipeline.inference(
+[rank0]:   File "/home/yitongli/CausVid/causvid/models/wan/causal_inference.py", line 222, in inference
+[rank0]:     video = self.vae.decode_to_pixel(output)
+[rank0]:   File "/home/yitongli/CausVid/causvid/models/wan/wan_wrapper.py", line 83, in decode_to_pixel
+[rank0]:     output = [
+[rank0]:   File "/home/yitongli/CausVid/causvid/models/wan/wan_wrapper.py", line 84, in <listcomp>
+[rank0]:     self.model.decode(u.unsqueeze(0),
+[rank0]:   File "/home/yitongli/CausVid/causvid/models/wan/wan_base/modules/vae.py", line 553, in decode
+[rank0]:     x = self.conv2(z)
+[rank0]:   File "/home/yitongli/miniconda3/envs/causvid/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1773, in _wrapped_call_impl
+[rank0]:     return self._call_impl(*args, **kwargs)
+[rank0]:   File "/home/yitongli/miniconda3/envs/causvid/lib/python3.10/site-packages/torch/nn/modules/module.py", line 1784, in _call_impl
+[rank0]:     return forward_call(*args, **kwargs)
+[rank0]:   File "/home/yitongli/CausVid/causvid/models/wan/wan_base/modules/vae.py", line 36, in forward
+[rank0]:     return super().forward(x)
+[rank0]:   File "/home/yitongli/miniconda3/envs/causvid/lib/python3.10/site-packages/torch/nn/modules/conv.py", line 717, in forward
+[rank0]:     return self._conv_forward(input, self.weight, self.bias)
+[rank0]:   File "/home/yitongli/miniconda3/envs/causvid/lib/python3.10/site-packages/torch/nn/modules/conv.py", line 712, in _conv_forward
+[rank0]:     return F.conv3d(
+[rank0]: RuntimeError: Input type (c10::BFloat16) and bias type (float) should be the same

output/wandb/run-20250908_084949-b9yvqkt5/files/requirements.txt

@@ -0,0 +1,145 @@
+nvidia-cuda-nvrtc-cu12==12.8.93
+filelock==3.19.1
+aiofiles==24.1.0
+PyYAML==6.0.2
+httpx==0.28.1
+multidict==6.6.4
+tifffile==2025.5.10
+tzdata==2025.2
+urllib3==2.5.0
+decord==0.6.0
+certifi==2025.8.3
+setuptools==78.1.1
+websocket-client==1.8.0
+gitdb==4.0.12
+yarl==1.20.1
+tqdm==4.67.1
+open_clip_torch==3.1.0
+pyparsing==3.2.3
+fastapi==0.116.1
+nvidia-curand-cu12==10.3.9.90
+mdurl==0.1.2
+torchvision==0.23.0
+h11==0.16.0
+pytz==2025.2
+six==1.17.0
+nvidia-cufile-cu12==1.13.1.3
+aiohappyeyeballs==2.6.1
+wandb==0.21.3
+nvidia-nvjitlink-cu12==12.8.93
+cycler==0.12.1
+anyio==4.10.0
+scikit-image==0.25.2
+nvidia-cuda-runtime-cu12==12.8.90
+sentencepiece==0.2.1
+rich==14.1.0
+pydantic==2.11.7
+GitPython==3.1.45
+python-dateutil==2.9.0.post0
+accelerate==1.10.1
+proglog==0.1.12
+sentry-sdk==2.37.0
+nvidia-cusparselt-cu12==0.7.1
+dashscope==1.24.3
+platformdirs==4.4.0
+safehttpx==0.1.6
+fsspec==2025.9.0
+lazy_loader==0.4
+typing_extensions==4.15.0
+semantic-version==2.10.0
+numpy==1.24.4
+nvidia-cufft-cu12==11.3.3.83
+ruff==0.12.12
+nvidia-cudnn-cu12==9.10.2.21
+annotated-types==0.7.0
+decorator==4.4.2
+antlr4-python3-runtime==4.9.3
+psutil==7.0.0
+Brotli==1.1.0
+tomlkit==0.13.3
+httpcore==1.0.9
+kiwisolver==1.4.9
+nvidia-cusparse-cu12==12.5.8.93
+groovy==0.1.2
+protobuf==6.32.0
+orjson==3.11.3
+scipy==1.15.3
+regex==2025.9.1
+MarkupSafe==3.0.2
+av==13.1.0
+timm==1.0.19
+uvicorn==0.35.0
+wheel==0.45.1
+Pygments==2.19.2
+websockets==15.0.1
+lmdb==1.7.3
+sympy==1.14.0
+einops==0.8.1
+idna==3.10
+triton==3.4.0
+torch==2.8.0
+moviepy==1.0.3
+nvidia-nvtx-cu12==12.8.90
+matplotlib==3.10.6
+pillow==11.3.0
+charset-normalizer==3.4.3
+attrs==25.3.0
+aiosignal==1.4.0
+markdown-it-py==4.0.0
+requests==2.32.5
+typer==0.17.4
+huggingface-hub==0.34.4
+nvidia-nccl-cu12==2.27.3
+propcache==0.3.2
+opencv-python==4.11.0.86
+ffmpy==0.6.1
+jmespath==1.0.1
+botocore==1.40.25
+pydantic_core==2.33.2
+fonttools==4.59.2
+omegaconf==2.3.0
+pycparser==2.22
+mpmath==1.3.0
+flash_attn==2.8.3
+smmap==5.0.2
+gradio_client==1.12.1
+exceptiongroup==1.3.0
+cffi==1.17.1
+diffusers==0.31.0
+nvidia-cusolver-cu12==11.7.3.90
+contourpy==1.3.2
+cryptography==45.0.7
+nvidia-cuda-cupti-cu12==12.8.90
+wcwidth==0.2.13
+zipp==3.23.0
+safetensors==0.6.2
+gradio==5.44.1
+click==8.2.1
+frozenlist==1.7.0
+networkx==3.4.2
+s3transfer==0.13.1
+shellingham==1.5.4
+starlette==0.47.3
+packaging==25.0
+ftfy==6.3.1
+importlib_metadata==8.7.0
+transformers==4.56.1
+nvidia-cublas-cu12==12.8.4.1
+Jinja2==3.1.6
+pycocotools==2.0.10
+sniffio==1.3.1
+hf-xet==1.1.9
+typing-inspection==0.4.1
+pandas==2.3.2
+python-multipart==0.0.20
+aiohttp==3.12.15
+clip==1.0
+pydub==0.25.1
+easydict==1.13
+pip==25.2
+tokenizers==0.22.0
+imageio==2.37.0
+async-timeout==5.0.1
+boto3==1.40.25
+imageio-ffmpeg==0.6.0
+CausVid==0.0.1

output/wandb/run-20250908_084949-b9yvqkt5/files/wandb-metadata.json

@@ -0,0 +1,93 @@
+{
+  "os": "Linux-6.8.0-1028-aws-x86_64-with-glibc2.39",
+  "python": "CPython 3.10.18",
+  "startedAt": "2025-09-08T08:49:49.541042Z",
+  "args": [
+    "--config_path",
+    "configs/wan_causal_ode.yaml"
+  ],
+  "program": "/home/yitongli/CausVid/causvid/train_ode.py",
+  "codePath": "causvid/train_ode.py",
+  "codePathLocal": "causvid/train_ode.py",
+  "git": {
+    "remote": "https://github.com/tianweiy/CausVid.git",
+    "commit": "adb6a5ecd07666b4d0290042915c8406e6d5ce22"
+  },
+  "email": "[email protected]",
+  "root": "./output",
+  "host": "ip-172-31-3-169",
+  "executable": "/home/yitongli/miniconda3/envs/causvid/bin/python3.10",
+  "cpu_count": 48,
+  "cpu_count_logical": 96,
+  "gpu": "NVIDIA A100-SXM4-80GB",
+  "gpu_count": 8,
+  "disk": {
+    "/": {
+      "total": "2079114358784",
+      "used": "1386641809408"
+    }
+  },
+  "memory": {
+    "total": "1204521443328"
+  },
+  "gpu_nvidia": [
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-88ce35f2-86d5-a394-f90d-a52e2cd45949"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-1039594d-55ae-3eb3-339e-6d4d62c81d42"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-1f47056a-9c2c-8af1-8232-636203622830"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-38de1dff-a680-ec66-625b-49f2a31e3b42"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-712ffb29-5da2-92e5-99b7-9f148e8c1204"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-194ed489-5bfa-defc-85b1-812953e7d84a"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-21b1f0be-4c00-4af7-0649-9942ae8fd0e4"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-195cca5b-c8ee-790b-4d30-ae4d7aa93057"
+    }
+  ],
+  "cudaVersion": "12.8",
+  "writerId": "ytk5y3nz1jsnyvdlk65guvab4uhpkdm4"
+}

output/wandb/run-20250908_084949-b9yvqkt5/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_wandb":{"runtime":134},"_runtime":134.067707153,"generator_loss":0.020751953125,"generator_grad_norm":5.278256416320801,"loss_at_time_250":0.016326904296875,"_timestamp":1.7573215036196253e+09,"_step":0}

output/wandb/run-20250908_084949-b9yvqkt5/logs/debug-core.log

@@ -0,0 +1,14 @@
+{"time":"2025-09-08T08:49:49.562614129Z","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpr7mbm8le/port-3899796.txt","pid":3899796,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2025-09-08T08:49:49.563183638Z","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-3899796-3900061-1880589413/socket","Net":"unix"}}
+{"time":"2025-09-08T08:49:49.563187036Z","level":"INFO","msg":"server: will exit if parent process dies","ppid":3899796}
+{"time":"2025-09-08T08:49:49.748718746Z","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2025-09-08T08:49:49.756209089Z","level":"INFO","msg":"handleInformInit: received","streamId":"b9yvqkt5","id":"1(@)"}
+{"time":"2025-09-08T08:49:49.964234191Z","level":"INFO","msg":"handleInformInit: stream started","streamId":"b9yvqkt5","id":"1(@)"}
+{"time":"2025-09-08T08:52:04.255668903Z","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
+{"time":"2025-09-08T08:52:04.255755182Z","level":"INFO","msg":"connection: closing","id":"1(@)"}
+{"time":"2025-09-08T08:52:04.255795637Z","level":"INFO","msg":"server is shutting down"}
+{"time":"2025-09-08T08:52:04.256573202Z","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
+{"time":"2025-09-08T08:52:04.263624933Z","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-3899796-3900061-1880589413/socket","Net":"unix"}}
+{"time":"2025-09-08T08:52:04.76114449Z","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
+{"time":"2025-09-08T08:52:04.761165019Z","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
+{"time":"2025-09-08T08:52:04.761174679Z","level":"INFO","msg":"server is closed"}

output/wandb/run-20250908_084949-b9yvqkt5/logs/debug-internal.log

@@ -0,0 +1,11 @@
+{"time":"2025-09-08T08:49:49.756354218Z","level":"INFO","msg":"stream: starting","core version":"0.21.3"}
+{"time":"2025-09-08T08:49:49.964190017Z","level":"INFO","msg":"stream: created new stream","id":"b9yvqkt5"}
+{"time":"2025-09-08T08:49:49.964229664Z","level":"INFO","msg":"stream: started","id":"b9yvqkt5"}
+{"time":"2025-09-08T08:49:49.964243758Z","level":"INFO","msg":"writer: started","stream_id":"b9yvqkt5"}
+{"time":"2025-09-08T08:49:49.964270263Z","level":"INFO","msg":"handler: started","stream_id":"b9yvqkt5"}
+{"time":"2025-09-08T08:49:49.964261355Z","level":"INFO","msg":"sender: started","stream_id":"b9yvqkt5"}
+{"time":"2025-09-08T08:52:04.25574272Z","level":"INFO","msg":"stream: closing","id":"b9yvqkt5"}
+{"time":"2025-09-08T08:52:04.575873408Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2025-09-08T08:52:04.755032763Z","level":"INFO","msg":"handler: closed","stream_id":"b9yvqkt5"}
+{"time":"2025-09-08T08:52:04.755112443Z","level":"INFO","msg":"sender: closed","stream_id":"b9yvqkt5"}
+{"time":"2025-09-08T08:52:04.755121083Z","level":"INFO","msg":"stream: closed","id":"b9yvqkt5"}

output/wandb/run-20250908_084949-b9yvqkt5/logs/debug.log

@@ -0,0 +1,24 @@
+2025-09-08 08:49:49,542 INFO    MainThread:3899796 [wandb_setup.py:_flush():81] Current SDK version is 0.21.3
+2025-09-08 08:49:49,542 INFO    MainThread:3899796 [wandb_setup.py:_flush():81] Configure stats pid to 3899796
+2025-09-08 08:49:49,542 INFO    MainThread:3899796 [wandb_setup.py:_flush():81] Loading settings from /home/yitongli/.config/wandb/settings
+2025-09-08 08:49:49,542 INFO    MainThread:3899796 [wandb_setup.py:_flush():81] Loading settings from /home/yitongli/CausVid/wandb/settings
+2025-09-08 08:49:49,542 INFO    MainThread:3899796 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2025-09-08 08:49:49,542 INFO    MainThread:3899796 [wandb_init.py:setup_run_log_directory():686] Logging user logs to ./output/wandb/run-20250908_084949-b9yvqkt5/logs/debug.log
+2025-09-08 08:49:49,542 INFO    MainThread:3899796 [wandb_init.py:setup_run_log_directory():687] Logging internal logs to ./output/wandb/run-20250908_084949-b9yvqkt5/logs/debug-internal.log
+2025-09-08 08:49:49,542 INFO    MainThread:3899796 [wandb_init.py:init():813] calling init triggers
+2025-09-08 08:49:49,542 INFO    MainThread:3899796 [wandb_init.py:init():818] wandb.init called with sweep_config: {}
+config: {'model_name': 'causal_wan', 'generator_grad': {'model': True}, 'denoising_step_list': [1000, 757, 522, 0], 'generator_task': 'causal_video', 'generator_fsdp_wrap_strategy': 'size', 'text_encoder_fsdp_wrap_strategy': 'size', 'mixed_precision': True, 'seed': 2913400, 'wandb_host': 'https://api.wandb.ai', 'wandb_key': '5409d3b960b01b25cec0f6abb5361b4022f0cc41', 'wandb_entity': 'liyitong-Tsinghua University', 'wandb_project': 'self-forcing', 'wandb_name': 'wan_causal_ode', 'sharding_strategy': 'hybrid_full', 'lr': 2e-06, 'beta1': 0.9, 'beta2': 0.999, 'data_path': '../mixkit_ode_lmdb', 'batch_size': 2, 'log_iters': 200, 'output_path': './output', 'distillation_loss': 'ode', 'gradient_checkpointing': True, 'num_frame_per_block': 3, 'warp_denoising_step': False, 'no_save': False, '_wandb': {}}
+2025-09-08 08:49:49,542 INFO    MainThread:3899796 [wandb_init.py:init():854] starting backend
+2025-09-08 08:49:49,748 INFO    MainThread:3899796 [wandb_init.py:init():857] sending inform_init request
+2025-09-08 08:49:49,752 INFO    MainThread:3899796 [wandb_init.py:init():865] backend started and connected
+2025-09-08 08:49:49,754 INFO    MainThread:3899796 [wandb_init.py:init():936] updated telemetry
+2025-09-08 08:49:49,759 INFO    MainThread:3899796 [wandb_init.py:init():960] communicating run to backend with 90.0 second timeout
+2025-09-08 08:49:50,186 INFO    MainThread:3899796 [wandb_init.py:init():1011] starting run threads in backend
+2025-09-08 08:49:50,300 INFO    MainThread:3899796 [wandb_run.py:_console_start():2494] atexit reg
+2025-09-08 08:49:50,300 INFO    MainThread:3899796 [wandb_run.py:_redirect():2342] redirect: wrap_raw
+2025-09-08 08:49:50,300 INFO    MainThread:3899796 [wandb_run.py:_redirect():2411] Wrapping output streams.
+2025-09-08 08:49:50,301 INFO    MainThread:3899796 [wandb_run.py:_redirect():2434] Redirects installed.
+2025-09-08 08:49:50,303 INFO    MainThread:3899796 [wandb_init.py:init():1057] run started, returning control to user process
+2025-09-08 08:49:50,698 INFO    MainThread:3899796 [wandb_run.py:_config_callback():1380] config_cb None None {'_wandb': {'code_path': 'source-self-forcing-causvid_train_ode.py'}}
+2025-09-08 08:52:04,255 INFO    wandb-AsyncioManager-main:3899796 [service_client.py:_forward_responses():84] Reached EOF.
+2025-09-08 08:52:04,255 INFO    wandb-AsyncioManager-main:3899796 [mailbox.py:close():137] Closing mailbox, abandoning 1 handles.

output/wandb/run-20250908_085656-y8q57cbx/files/output.log

@@ -0,0 +1 @@
+run dir: ./output/wandb/run-20250908_085656-y8q57cbx/files

output/wandb/run-20250908_085656-y8q57cbx/files/requirements.txt

@@ -0,0 +1,145 @@
+nvidia-cuda-nvrtc-cu12==12.8.93
+filelock==3.19.1
+aiofiles==24.1.0
+PyYAML==6.0.2
+httpx==0.28.1
+multidict==6.6.4
+tifffile==2025.5.10
+tzdata==2025.2
+urllib3==2.5.0
+decord==0.6.0
+certifi==2025.8.3
+setuptools==78.1.1
+websocket-client==1.8.0
+gitdb==4.0.12
+yarl==1.20.1
+tqdm==4.67.1
+open_clip_torch==3.1.0
+pyparsing==3.2.3
+fastapi==0.116.1
+nvidia-curand-cu12==10.3.9.90
+mdurl==0.1.2
+torchvision==0.23.0
+h11==0.16.0
+pytz==2025.2
+six==1.17.0
+nvidia-cufile-cu12==1.13.1.3
+aiohappyeyeballs==2.6.1
+wandb==0.21.3
+nvidia-nvjitlink-cu12==12.8.93
+cycler==0.12.1
+anyio==4.10.0
+scikit-image==0.25.2
+nvidia-cuda-runtime-cu12==12.8.90
+sentencepiece==0.2.1
+rich==14.1.0
+pydantic==2.11.7
+GitPython==3.1.45
+python-dateutil==2.9.0.post0
+accelerate==1.10.1
+proglog==0.1.12
+sentry-sdk==2.37.0
+nvidia-cusparselt-cu12==0.7.1
+dashscope==1.24.3
+platformdirs==4.4.0
+safehttpx==0.1.6
+fsspec==2025.9.0
+lazy_loader==0.4
+typing_extensions==4.15.0
+semantic-version==2.10.0
+numpy==1.24.4
+nvidia-cufft-cu12==11.3.3.83
+ruff==0.12.12
+nvidia-cudnn-cu12==9.10.2.21
+annotated-types==0.7.0
+decorator==4.4.2
+antlr4-python3-runtime==4.9.3
+psutil==7.0.0
+Brotli==1.1.0
+tomlkit==0.13.3
+httpcore==1.0.9
+kiwisolver==1.4.9
+nvidia-cusparse-cu12==12.5.8.93
+groovy==0.1.2
+protobuf==6.32.0
+orjson==3.11.3
+scipy==1.15.3
+regex==2025.9.1
+MarkupSafe==3.0.2
+av==13.1.0
+timm==1.0.19
+uvicorn==0.35.0
+wheel==0.45.1
+Pygments==2.19.2
+websockets==15.0.1
+lmdb==1.7.3
+sympy==1.14.0
+einops==0.8.1
+idna==3.10
+triton==3.4.0
+torch==2.8.0
+moviepy==1.0.3
+nvidia-nvtx-cu12==12.8.90
+matplotlib==3.10.6
+pillow==11.3.0
+charset-normalizer==3.4.3
+attrs==25.3.0
+aiosignal==1.4.0
+markdown-it-py==4.0.0
+requests==2.32.5
+typer==0.17.4
+huggingface-hub==0.34.4
+nvidia-nccl-cu12==2.27.3
+propcache==0.3.2
+opencv-python==4.11.0.86
+ffmpy==0.6.1
+jmespath==1.0.1
+botocore==1.40.25
+pydantic_core==2.33.2
+fonttools==4.59.2
+omegaconf==2.3.0
+pycparser==2.22
+mpmath==1.3.0
+flash_attn==2.8.3
+smmap==5.0.2
+gradio_client==1.12.1
+exceptiongroup==1.3.0
+cffi==1.17.1
+diffusers==0.31.0
+nvidia-cusolver-cu12==11.7.3.90
+contourpy==1.3.2
+cryptography==45.0.7
+nvidia-cuda-cupti-cu12==12.8.90
+wcwidth==0.2.13
+zipp==3.23.0
+safetensors==0.6.2
+gradio==5.44.1
+click==8.2.1
+frozenlist==1.7.0
+networkx==3.4.2
+s3transfer==0.13.1
+shellingham==1.5.4
+starlette==0.47.3
+packaging==25.0
+ftfy==6.3.1
+importlib_metadata==8.7.0
+transformers==4.56.1
+nvidia-cublas-cu12==12.8.4.1
+Jinja2==3.1.6
+pycocotools==2.0.10
+sniffio==1.3.1
+hf-xet==1.1.9
+typing-inspection==0.4.1
+pandas==2.3.2
+python-multipart==0.0.20
+aiohttp==3.12.15
+clip==1.0
+pydub==0.25.1
+easydict==1.13
+pip==25.2
+tokenizers==0.22.0
+imageio==2.37.0
+async-timeout==5.0.1
+boto3==1.40.25
+imageio-ffmpeg==0.6.0
+CausVid==0.0.1

output/wandb/run-20250908_085656-y8q57cbx/files/wandb-metadata.json

@@ -0,0 +1,93 @@
+{
+  "os": "Linux-6.8.0-1028-aws-x86_64-with-glibc2.39",
+  "python": "CPython 3.10.18",
+  "startedAt": "2025-09-08T08:56:56.977649Z",
+  "args": [
+    "--config_path",
+    "configs/wan_causal_ode.yaml"
+  ],
+  "program": "/home/yitongli/CausVid/causvid/train_ode.py",
+  "codePath": "causvid/train_ode.py",
+  "codePathLocal": "causvid/train_ode.py",
+  "git": {
+    "remote": "https://github.com/tianweiy/CausVid.git",
+    "commit": "adb6a5ecd07666b4d0290042915c8406e6d5ce22"
+  },
+  "email": "[email protected]",
+  "root": "./output",
+  "host": "ip-172-31-3-169",
+  "executable": "/home/yitongli/miniconda3/envs/causvid/bin/python3.10",
+  "cpu_count": 48,
+  "cpu_count_logical": 96,
+  "gpu": "NVIDIA A100-SXM4-80GB",
+  "gpu_count": 8,
+  "disk": {
+    "/": {
+      "total": "2079114358784",
+      "used": "1396408811520"
+    }
+  },
+  "memory": {
+    "total": "1204521443328"
+  },
+  "gpu_nvidia": [
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-88ce35f2-86d5-a394-f90d-a52e2cd45949"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-1039594d-55ae-3eb3-339e-6d4d62c81d42"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-1f47056a-9c2c-8af1-8232-636203622830"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-38de1dff-a680-ec66-625b-49f2a31e3b42"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-712ffb29-5da2-92e5-99b7-9f148e8c1204"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-194ed489-5bfa-defc-85b1-812953e7d84a"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-21b1f0be-4c00-4af7-0649-9942ae8fd0e4"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-195cca5b-c8ee-790b-4d30-ae4d7aa93057"
+    }
+  ],
+  "cudaVersion": "12.8",
+  "writerId": "j1na06z1wvv9hx6dftanb741hwvst9hr"
+}

output/wandb/run-20250908_091534-f394z0xa/files/config.yaml

@@ -0,0 +1,160 @@
+_wandb:
+    value:
+        cli_version: 0.21.3
+        code_path: source-self-forcing-causvid_train_ode.py
+        e:
+            wcpbsusleulz3a9y7yho5o0ck602eklv:
+                args:
+                    - --config_path
+                    - configs/wan_causal_ode.yaml
+                codePath: causvid/train_ode.py
+                codePathLocal: causvid/train_ode.py
+                cpu_count: 48
+                cpu_count_logical: 96
+                cudaVersion: "12.8"
+                disk:
+                    /:
+                        total: "2079114358784"
+                        used: "1396822892544"
+                email: [email protected]
+                executable: /home/yitongli/miniconda3/envs/causvid/bin/python3.10
+                git:
+                    commit: adb6a5ecd07666b4d0290042915c8406e6d5ce22
+                    remote: https://github.com/tianweiy/CausVid.git
+                gpu: NVIDIA A100-SXM4-80GB
+                gpu_count: 8
+                gpu_nvidia:
+                    - architecture: Ampere
+                      cudaCores: 6912
+                      memoryTotal: "85899345920"
+                      name: NVIDIA A100-SXM4-80GB
+                      uuid: GPU-88ce35f2-86d5-a394-f90d-a52e2cd45949
+                    - architecture: Ampere
+                      cudaCores: 6912
+                      memoryTotal: "85899345920"
+                      name: NVIDIA A100-SXM4-80GB
+                      uuid: GPU-1039594d-55ae-3eb3-339e-6d4d62c81d42
+                    - architecture: Ampere
+                      cudaCores: 6912
+                      memoryTotal: "85899345920"
+                      name: NVIDIA A100-SXM4-80GB
+                      uuid: GPU-1f47056a-9c2c-8af1-8232-636203622830
+                    - architecture: Ampere
+                      cudaCores: 6912
+                      memoryTotal: "85899345920"
+                      name: NVIDIA A100-SXM4-80GB
+                      uuid: GPU-38de1dff-a680-ec66-625b-49f2a31e3b42
+                    - architecture: Ampere
+                      cudaCores: 6912
+                      memoryTotal: "85899345920"
+                      name: NVIDIA A100-SXM4-80GB
+                      uuid: GPU-712ffb29-5da2-92e5-99b7-9f148e8c1204
+                    - architecture: Ampere
+                      cudaCores: 6912
+                      memoryTotal: "85899345920"
+                      name: NVIDIA A100-SXM4-80GB
+                      uuid: GPU-194ed489-5bfa-defc-85b1-812953e7d84a
+                    - architecture: Ampere
+                      cudaCores: 6912
+                      memoryTotal: "85899345920"
+                      name: NVIDIA A100-SXM4-80GB
+                      uuid: GPU-21b1f0be-4c00-4af7-0649-9942ae8fd0e4
+                    - architecture: Ampere
+                      cudaCores: 6912
+                      memoryTotal: "85899345920"
+                      name: NVIDIA A100-SXM4-80GB
+                      uuid: GPU-195cca5b-c8ee-790b-4d30-ae4d7aa93057
+                host: ip-172-31-3-169
+                memory:
+                    total: "1204521443328"
+                os: Linux-6.8.0-1028-aws-x86_64-with-glibc2.39
+                program: /home/yitongli/CausVid/causvid/train_ode.py
+                python: CPython 3.10.18
+                root: ./output
+                startedAt: "2025-09-08T09:15:34.924173Z"
+                writerId: wcpbsusleulz3a9y7yho5o0ck602eklv
+        m: []
+        python_version: 3.10.18
+        t:
+            "1":
+                - 1
+                - 11
+                - 41
+                - 49
+                - 71
+                - 83
+                - 105
+            "2":
+                - 1
+                - 11
+                - 41
+                - 49
+                - 71
+                - 83
+                - 105
+            "3":
+                - 16
+                - 17
+                - 61
+            "4": 3.10.18
+            "5": 0.21.3
+            "6": 4.56.1
+            "12": 0.21.3
+            "13": linux-x86_64
+batch_size:
+    value: 2
+beta1:
+    value: 0.9
+beta2:
+    value: 0.999
+data_path:
+    value: ../mixkit_ode_lmdb
+denoising_step_list:
+    value:
+        - 1000
+        - 757
+        - 522
+        - 0
+distillation_loss:
+    value: ode
+generator_fsdp_wrap_strategy:
+    value: size
+generator_grad:
+    value:
+        model: true
+generator_task:
+    value: causal_video
+gradient_checkpointing:
+    value: true
+log_iters:
+    value: 200
+lr:
+    value: 2e-06
+mixed_precision:
+    value: true
+model_name:
+    value: causal_wan
+no_save:
+    value: false
+num_frame_per_block:
+    value: 3
+output_path:
+    value: ./output
+seed:
+    value: 4288120
+sharding_strategy:
+    value: hybrid_full
+text_encoder_fsdp_wrap_strategy:
+    value: size
+wandb_entity:
+    value: liyitong-Tsinghua University
+wandb_host:
+    value: https://api.wandb.ai
+wandb_key:
+    value: 5409d3b960b01b25cec0f6abb5361b4022f0cc41
+wandb_name:
+    value: wan_causal_ode
+wandb_project:
+    value: self-forcing
+warp_denoising_step:
+    value: false

output/wandb/run-20250908_091534-f394z0xa/files/output.log

@@ -0,0 +1,66 @@
+run dir: ./output/wandb/run-20250908_091534-f394z0xa/files
+KV inference with 3 frames per block
+ODERegression initialized.
+ cache a block wise causal mask with block size of 3 frames
+BlockMask(shape=(1, 1, 32768, 32768), sparsity=42.52%,
+(0, 0)
+████░░
+████░░
+████░░░░░░░░
+██████████░░
+██████████░░
+██████████░░░░░░░░
+████████████████░░
+████████████████░░
+████████████████░░░░░░░░
+██████████████████████░░
+██████████████████████░░
+██████████████████████░░░░░░░░
+████████████████████████████░░
+████████████████████████████░░
+████████████████████████████████░░
+████████████████████████████████░░
+████████████████████████████████░░░░░░░░
+████████████████████████████████████████
+████████████████████████████████████████
+████████████████████████████████████████
+)
+arrive
+Start gathering distributed model states...
+/home/yitongli/miniconda3/envs/causvid/lib/python3.10/site-packages/torch/distributed/fsdp/fully_sharded_data_parallel.py:678: FutureWarning: FSDP.state_dict_type() and FSDP.set_state_dict_type() are being deprecated. Please use APIs, get_state_dict() and set_state_dict(), which can support different parallelisms, FSDP1, FSDP2, DDP. API doc: https://pytorch.org/docs/stable/distributed.checkpoint.html#torch.distributed.checkpoint.state_dict.get_state_dict .Tutorial: https://pytorch.org/tutorials/recipes/distributed_checkpoint_recipe.html .
+  warnings.warn(
+Model saved to ./output/2025-09-08-09-15-34.671833_seed4288120/checkpoint_model_000000/model.pt
+training step 0...
+Saving video:   0%|                                                                    | 0/3 [00:00<?, ?it/s]
+Traceback (most recent call last):
+  File "/home/yitongli/CausVid/causvid/train_ode.py", line 285, in <module>
+    main()
+  File "/home/yitongli/CausVid/causvid/train_ode.py", line 279, in main
+    trainer.train()
+  File "/home/yitongli/CausVid/causvid/train_ode.py", line 243, in train
+    self.generate_video()
+  File "/home/yitongli/CausVid/causvid/train_ode.py", line 217, in generate_video
+    save_video(video, output_path, fps=15, quality=5)
+  File "/home/yitongli/CausVid/causvid/train_ode.py", line 42, in save_video
+    writer.append_data(frame)
+  File "/home/yitongli/miniconda3/envs/causvid/lib/python3.10/site-packages/imageio/core/format.py", line 590, in append_data
+    return self._append_data(im, total_meta)
+  File "/home/yitongli/miniconda3/envs/causvid/lib/python3.10/site-packages/imageio/plugins/ffmpeg.py", line 584, in _append_data
+    raise ValueError("Image must have 1, 2, 3 or 4 channels")
+ValueError: Image must have 1, 2, 3 or 4 channels
+[rank0]: Traceback (most recent call last):
+[rank0]:   File "/home/yitongli/CausVid/causvid/train_ode.py", line 285, in <module>
+[rank0]:     main()
+[rank0]:   File "/home/yitongli/CausVid/causvid/train_ode.py", line 279, in main
+[rank0]:     trainer.train()
+[rank0]:   File "/home/yitongli/CausVid/causvid/train_ode.py", line 243, in train
+[rank0]:     self.generate_video()
+[rank0]:   File "/home/yitongli/CausVid/causvid/train_ode.py", line 217, in generate_video
+[rank0]:     save_video(video, output_path, fps=15, quality=5)
+[rank0]:   File "/home/yitongli/CausVid/causvid/train_ode.py", line 42, in save_video
+[rank0]:     writer.append_data(frame)
+[rank0]:   File "/home/yitongli/miniconda3/envs/causvid/lib/python3.10/site-packages/imageio/core/format.py", line 590, in append_data
+[rank0]:     return self._append_data(im, total_meta)
+[rank0]:   File "/home/yitongli/miniconda3/envs/causvid/lib/python3.10/site-packages/imageio/plugins/ffmpeg.py", line 584, in _append_data
+[rank0]:     raise ValueError("Image must have 1, 2, 3 or 4 channels")
+[rank0]: ValueError: Image must have 1, 2, 3 or 4 channels

output/wandb/run-20250908_091534-f394z0xa/files/requirements.txt

@@ -0,0 +1,145 @@
+nvidia-cuda-nvrtc-cu12==12.8.93
+filelock==3.19.1
+aiofiles==24.1.0
+PyYAML==6.0.2
+httpx==0.28.1
+multidict==6.6.4
+tifffile==2025.5.10
+tzdata==2025.2
+urllib3==2.5.0
+decord==0.6.0
+certifi==2025.8.3
+setuptools==78.1.1
+websocket-client==1.8.0
+gitdb==4.0.12
+yarl==1.20.1
+tqdm==4.67.1
+open_clip_torch==3.1.0
+pyparsing==3.2.3
+fastapi==0.116.1
+nvidia-curand-cu12==10.3.9.90
+mdurl==0.1.2
+torchvision==0.23.0
+h11==0.16.0
+pytz==2025.2
+six==1.17.0
+nvidia-cufile-cu12==1.13.1.3
+aiohappyeyeballs==2.6.1
+wandb==0.21.3
+nvidia-nvjitlink-cu12==12.8.93
+cycler==0.12.1
+anyio==4.10.0
+scikit-image==0.25.2
+nvidia-cuda-runtime-cu12==12.8.90
+sentencepiece==0.2.1
+rich==14.1.0
+pydantic==2.11.7
+GitPython==3.1.45
+python-dateutil==2.9.0.post0
+accelerate==1.10.1
+proglog==0.1.12
+sentry-sdk==2.37.0
+nvidia-cusparselt-cu12==0.7.1
+dashscope==1.24.3
+platformdirs==4.4.0
+safehttpx==0.1.6
+fsspec==2025.9.0
+lazy_loader==0.4
+typing_extensions==4.15.0
+semantic-version==2.10.0
+numpy==1.24.4
+nvidia-cufft-cu12==11.3.3.83
+ruff==0.12.12
+nvidia-cudnn-cu12==9.10.2.21
+annotated-types==0.7.0
+decorator==4.4.2
+antlr4-python3-runtime==4.9.3
+psutil==7.0.0
+Brotli==1.1.0
+tomlkit==0.13.3
+httpcore==1.0.9
+kiwisolver==1.4.9
+nvidia-cusparse-cu12==12.5.8.93
+groovy==0.1.2
+protobuf==6.32.0
+orjson==3.11.3
+scipy==1.15.3
+regex==2025.9.1
+MarkupSafe==3.0.2
+av==13.1.0
+timm==1.0.19
+uvicorn==0.35.0
+wheel==0.45.1
+Pygments==2.19.2
+websockets==15.0.1
+lmdb==1.7.3
+sympy==1.14.0
+einops==0.8.1
+idna==3.10
+triton==3.4.0
+torch==2.8.0
+moviepy==1.0.3
+nvidia-nvtx-cu12==12.8.90
+matplotlib==3.10.6
+pillow==11.3.0
+charset-normalizer==3.4.3
+attrs==25.3.0
+aiosignal==1.4.0
+markdown-it-py==4.0.0
+requests==2.32.5
+typer==0.17.4
+huggingface-hub==0.34.4
+nvidia-nccl-cu12==2.27.3
+propcache==0.3.2
+opencv-python==4.11.0.86
+ffmpy==0.6.1
+jmespath==1.0.1
+botocore==1.40.25
+pydantic_core==2.33.2
+fonttools==4.59.2
+omegaconf==2.3.0
+pycparser==2.22
+mpmath==1.3.0
+flash_attn==2.8.3
+smmap==5.0.2
+gradio_client==1.12.1
+exceptiongroup==1.3.0
+cffi==1.17.1
+diffusers==0.31.0
+nvidia-cusolver-cu12==11.7.3.90
+contourpy==1.3.2
+cryptography==45.0.7
+nvidia-cuda-cupti-cu12==12.8.90
+wcwidth==0.2.13
+zipp==3.23.0
+safetensors==0.6.2
+gradio==5.44.1
+click==8.2.1
+frozenlist==1.7.0
+networkx==3.4.2
+s3transfer==0.13.1
+shellingham==1.5.4
+starlette==0.47.3
+packaging==25.0
+ftfy==6.3.1
+importlib_metadata==8.7.0
+transformers==4.56.1
+nvidia-cublas-cu12==12.8.4.1
+Jinja2==3.1.6
+pycocotools==2.0.10
+sniffio==1.3.1
+hf-xet==1.1.9
+typing-inspection==0.4.1
+pandas==2.3.2
+python-multipart==0.0.20
+aiohttp==3.12.15
+clip==1.0
+pydub==0.25.1
+easydict==1.13
+pip==25.2
+tokenizers==0.22.0
+imageio==2.37.0
+async-timeout==5.0.1
+boto3==1.40.25
+imageio-ffmpeg==0.6.0
+CausVid==0.0.1

output/wandb/run-20250908_091534-f394z0xa/files/wandb-metadata.json

@@ -0,0 +1,93 @@
+{
+  "os": "Linux-6.8.0-1028-aws-x86_64-with-glibc2.39",
+  "python": "CPython 3.10.18",
+  "startedAt": "2025-09-08T09:15:34.924173Z",
+  "args": [
+    "--config_path",
+    "configs/wan_causal_ode.yaml"
+  ],
+  "program": "/home/yitongli/CausVid/causvid/train_ode.py",
+  "codePath": "causvid/train_ode.py",
+  "codePathLocal": "causvid/train_ode.py",
+  "git": {
+    "remote": "https://github.com/tianweiy/CausVid.git",
+    "commit": "adb6a5ecd07666b4d0290042915c8406e6d5ce22"
+  },
+  "email": "[email protected]",
+  "root": "./output",
+  "host": "ip-172-31-3-169",
+  "executable": "/home/yitongli/miniconda3/envs/causvid/bin/python3.10",
+  "cpu_count": 48,
+  "cpu_count_logical": 96,
+  "gpu": "NVIDIA A100-SXM4-80GB",
+  "gpu_count": 8,
+  "disk": {
+    "/": {
+      "total": "2079114358784",
+      "used": "1396822892544"
+    }
+  },
+  "memory": {
+    "total": "1204521443328"
+  },
+  "gpu_nvidia": [
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-88ce35f2-86d5-a394-f90d-a52e2cd45949"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-1039594d-55ae-3eb3-339e-6d4d62c81d42"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-1f47056a-9c2c-8af1-8232-636203622830"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-38de1dff-a680-ec66-625b-49f2a31e3b42"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-712ffb29-5da2-92e5-99b7-9f148e8c1204"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-194ed489-5bfa-defc-85b1-812953e7d84a"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-21b1f0be-4c00-4af7-0649-9942ae8fd0e4"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-195cca5b-c8ee-790b-4d30-ae4d7aa93057"
+    }
+  ],
+  "cudaVersion": "12.8",
+  "writerId": "wcpbsusleulz3a9y7yho5o0ck602eklv"
+}

output/wandb/run-20250908_091534-f394z0xa/files/wandb-summary.json

@@ -0,0 +1 @@
+{"generator_grad_norm":2.03125,"loss_at_time_500":0.148712158203125,"_timestamp":1.7573230600618684e+09,"_step":0,"_wandb":{"runtime":142},"_runtime":142.749615808,"generator_loss":0.16015625}

output/wandb/run-20250908_091534-f394z0xa/logs/debug-internal.log

@@ -0,0 +1,11 @@
+{"time":"2025-09-08T09:15:35.140001084Z","level":"INFO","msg":"stream: starting","core version":"0.21.3"}
+{"time":"2025-09-08T09:15:35.350074186Z","level":"INFO","msg":"stream: created new stream","id":"f394z0xa"}
+{"time":"2025-09-08T09:15:35.350117697Z","level":"INFO","msg":"stream: started","id":"f394z0xa"}
+{"time":"2025-09-08T09:15:35.350127873Z","level":"INFO","msg":"sender: started","stream_id":"f394z0xa"}
+{"time":"2025-09-08T09:15:35.350125845Z","level":"INFO","msg":"handler: started","stream_id":"f394z0xa"}
+{"time":"2025-09-08T09:15:35.350157006Z","level":"INFO","msg":"writer: started","stream_id":"f394z0xa"}
+{"time":"2025-09-08T09:17:58.289962144Z","level":"INFO","msg":"stream: closing","id":"f394z0xa"}
+{"time":"2025-09-08T09:17:58.601880656Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2025-09-08T09:17:58.768362935Z","level":"INFO","msg":"handler: closed","stream_id":"f394z0xa"}
+{"time":"2025-09-08T09:17:58.768430865Z","level":"INFO","msg":"sender: closed","stream_id":"f394z0xa"}
+{"time":"2025-09-08T09:17:58.768439365Z","level":"INFO","msg":"stream: closed","id":"f394z0xa"}

output/wandb/run-20250908_091534-f394z0xa/logs/debug.log

@@ -0,0 +1,24 @@
+2025-09-08 09:15:34,925 INFO    MainThread:4032801 [wandb_setup.py:_flush():81] Current SDK version is 0.21.3
+2025-09-08 09:15:34,925 INFO    MainThread:4032801 [wandb_setup.py:_flush():81] Configure stats pid to 4032801
+2025-09-08 09:15:34,925 INFO    MainThread:4032801 [wandb_setup.py:_flush():81] Loading settings from /home/yitongli/.config/wandb/settings
+2025-09-08 09:15:34,925 INFO    MainThread:4032801 [wandb_setup.py:_flush():81] Loading settings from /home/yitongli/CausVid/wandb/settings
+2025-09-08 09:15:34,925 INFO    MainThread:4032801 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2025-09-08 09:15:34,925 INFO    MainThread:4032801 [wandb_init.py:setup_run_log_directory():686] Logging user logs to ./output/wandb/run-20250908_091534-f394z0xa/logs/debug.log
+2025-09-08 09:15:34,925 INFO    MainThread:4032801 [wandb_init.py:setup_run_log_directory():687] Logging internal logs to ./output/wandb/run-20250908_091534-f394z0xa/logs/debug-internal.log
+2025-09-08 09:15:34,925 INFO    MainThread:4032801 [wandb_init.py:init():813] calling init triggers
+2025-09-08 09:15:34,925 INFO    MainThread:4032801 [wandb_init.py:init():818] wandb.init called with sweep_config: {}
+config: {'model_name': 'causal_wan', 'generator_grad': {'model': True}, 'denoising_step_list': [1000, 757, 522, 0], 'generator_task': 'causal_video', 'generator_fsdp_wrap_strategy': 'size', 'text_encoder_fsdp_wrap_strategy': 'size', 'mixed_precision': True, 'seed': 4288120, 'wandb_host': 'https://api.wandb.ai', 'wandb_key': '5409d3b960b01b25cec0f6abb5361b4022f0cc41', 'wandb_entity': 'liyitong-Tsinghua University', 'wandb_project': 'self-forcing', 'wandb_name': 'wan_causal_ode', 'sharding_strategy': 'hybrid_full', 'lr': 2e-06, 'beta1': 0.9, 'beta2': 0.999, 'data_path': '../mixkit_ode_lmdb', 'batch_size': 2, 'log_iters': 200, 'output_path': './output', 'distillation_loss': 'ode', 'gradient_checkpointing': True, 'num_frame_per_block': 3, 'warp_denoising_step': False, 'no_save': False, '_wandb': {}}
+2025-09-08 09:15:34,925 INFO    MainThread:4032801 [wandb_init.py:init():854] starting backend
+2025-09-08 09:15:35,132 INFO    MainThread:4032801 [wandb_init.py:init():857] sending inform_init request
+2025-09-08 09:15:35,137 INFO    MainThread:4032801 [wandb_init.py:init():865] backend started and connected
+2025-09-08 09:15:35,142 INFO    MainThread:4032801 [wandb_init.py:init():936] updated telemetry
+2025-09-08 09:15:35,147 INFO    MainThread:4032801 [wandb_init.py:init():960] communicating run to backend with 90.0 second timeout
+2025-09-08 09:15:35,538 INFO    MainThread:4032801 [wandb_init.py:init():1011] starting run threads in backend
+2025-09-08 09:15:35,661 INFO    MainThread:4032801 [wandb_run.py:_console_start():2494] atexit reg
+2025-09-08 09:15:35,661 INFO    MainThread:4032801 [wandb_run.py:_redirect():2342] redirect: wrap_raw
+2025-09-08 09:15:35,661 INFO    MainThread:4032801 [wandb_run.py:_redirect():2411] Wrapping output streams.
+2025-09-08 09:15:35,661 INFO    MainThread:4032801 [wandb_run.py:_redirect():2434] Redirects installed.
+2025-09-08 09:15:35,664 INFO    MainThread:4032801 [wandb_init.py:init():1057] run started, returning control to user process
+2025-09-08 09:15:36,097 INFO    MainThread:4032801 [wandb_run.py:_config_callback():1380] config_cb None None {'_wandb': {'code_path': 'source-self-forcing-causvid_train_ode.py'}}
+2025-09-08 09:17:58,290 INFO    wandb-AsyncioManager-main:4032801 [service_client.py:_forward_responses():84] Reached EOF.
+2025-09-08 09:17:58,290 INFO    wandb-AsyncioManager-main:4032801 [mailbox.py:close():137] Closing mailbox, abandoning 1 handles.