upload models

.gitattributes

@@ -33,3 +33,9 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+toyset/1.png filter=lfs diff=lfs merge=lfs -text
+toyset/2.png filter=lfs diff=lfs merge=lfs -text
+toyset/3.png filter=lfs diff=lfs merge=lfs -text
+toyset/4.png filter=lfs diff=lfs merge=lfs -text
+toyset/5.png filter=lfs diff=lfs merge=lfs -text
+toyset/good.png filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,2 @@
+key628.txt
+key628.txt.pub

app.py

@@ -0,0 +1,209 @@
+import gradio as gr
+import os
+import cv2
+import numpy as np
+import torch
+import torch.nn.functional as F
+from PIL import Image
+import tempfile
+import io
+
+from depth_anything.dpt import DepthAnything_AC
+
+
+def normalize_depth(disparity_tensor):
+    """Standard normalization method to convert disparity to depth"""
+    eps = 1e-6
+    disparity_min = disparity_tensor.min()
+    disparity_max = disparity_tensor.max()
+    normalized_disparity = (disparity_tensor - disparity_min) / (disparity_max - disparity_min + eps)
+    return normalized_disparity
+
+
+def load_model(model_path='checkpoints/depth_anything_AC_vits.pth', encoder='vits'):
+    """Load trained depth estimation model"""
+    model_configs = {
+        'vitl': {'encoder': 'vitl', 'features': 256, 'out_channels': [256, 512, 1024, 1024], 'version': 'v2'},
+        'vitb': {'encoder': 'vitb', 'features': 128, 'out_channels': [96, 192, 384, 768], 'version': 'v2'},
+        'vits': {'encoder': 'vits', 'features': 64, 'out_channels': [48, 96, 192, 384], 'version': 'v2'}
+    }
+    
+    model = DepthAnything_AC(model_configs[encoder])
+    
+    if os.path.exists(model_path):
+        checkpoint = torch.load(model_path, map_location='cpu')
+        model.load_state_dict(checkpoint, strict=False)
+    else:
+        print(f"Warning: Model file {model_path} not found")
+        
+    model.eval()
+    if torch.cuda.is_available():
+        model.cuda()
+    
+    return model
+
+
+def preprocess_image(image, target_size=518):
+    """Preprocess input image"""
+    if isinstance(image, Image.Image):
+        image = np.array(image)
+    
+    if len(image.shape) == 3 and image.shape[2] == 3:
+        pass
+    elif len(image.shape) == 3 and image.shape[2] == 4:
+        image = image[:, :, :3]
+    
+    image = image.astype(np.float32) / 255.0
+    h, w = image.shape[:2]
+    scale = target_size / min(h, w)
+    new_h, new_w = int(h * scale), int(w * scale)
+    
+    new_h = ((new_h + 13) // 14) * 14
+    new_w = ((new_w + 13) // 14) * 14
+    image = cv2.resize(image, (new_w, new_h), interpolation=cv2.INTER_CUBIC)
+
+    mean = np.array([0.485, 0.456, 0.406])
+    std = np.array([0.229, 0.224, 0.225])
+    image = (image - mean) / std
+
+    image = torch.from_numpy(image.transpose(2, 0, 1)).float()
+    image = image.unsqueeze(0)
+    
+    return image, (h, w)
+
+
+def postprocess_depth(depth_tensor, original_size):
+    """Post-process depth map"""
+    if depth_tensor.dim() == 3:
+        depth_tensor = depth_tensor.unsqueeze(1)
+    elif depth_tensor.dim() == 2:
+        depth_tensor = depth_tensor.unsqueeze(0).unsqueeze(1)
+    
+    h, w = original_size
+    depth = F.interpolate(depth_tensor, size=(h, w), mode='bilinear', align_corners=True)
+    depth = depth.squeeze().cpu().numpy()
+    
+    return depth
+
+
+def create_colored_depth_map(depth, colormap='spectral'):
+    """Create colored depth map"""
+    if colormap == 'inferno':
+        depth_colored = cv2.applyColorMap((depth * 255).astype(np.uint8), cv2.COLORMAP_INFERNO)
+        depth_colored = cv2.cvtColor(depth_colored, cv2.COLOR_BGR2RGB)
+    elif colormap == 'spectral':
+        from matplotlib import cm
+        spectral_cmap = cm.get_cmap('Spectral_r')
+        depth_colored = (spectral_cmap(depth) * 255).astype(np.uint8)
+        depth_colored = depth_colored[:, :, :3]
+    else:
+        depth_colored = (depth * 255).astype(np.uint8)
+        depth_colored = np.stack([depth_colored] * 3, axis=2)
+    
+    return depth_colored
+
+
+print("Loading model...")
+model = load_model()
+print("Model loaded successfully!")
+
+
+def predict_depth(input_image, colormap_choice):
+    """Main depth prediction function"""
+    try:
+        image_tensor, original_size = preprocess_image(input_image)
+        
+        if torch.cuda.is_available():
+            image_tensor = image_tensor.cuda()
+        
+        with torch.no_grad():
+            prediction = model(image_tensor)
+            disparity_tensor = prediction['out']
+            depth_tensor = normalize_depth(disparity_tensor)
+        
+        depth = postprocess_depth(depth_tensor, original_size)
+        
+        depth_colored = create_colored_depth_map(depth, colormap_choice.lower())
+        
+        return Image.fromarray(depth_colored)
+        
+    except Exception as e:
+        print(f"Error during inference: {str(e)}")
+        return None
+
+
+with gr.Blocks(title="Depth Anything AC - Depth Estimation Demo", theme=gr.themes.Soft()) as demo:
+    gr.Markdown("""
+    # 🌊 Depth Anything AC - Depth Estimation Demo
+    
+    Upload an image and AI will generate the corresponding depth map! Different colors in the depth map represent different distances, allowing you to see the three-dimensional structure of the image.
+    
+    ## How to Use
+    1. Click the upload area to select an image
+    2. Choose your preferred colormap style
+    3. Click the "Generate Depth Map" button
+    4. View the results and download
+    """)
+    
+    with gr.Row():
+        with gr.Column():
+            input_image = gr.Image(
+                label="Upload Image",
+                type="pil",
+                height=400
+            )
+            
+            colormap_choice = gr.Dropdown(
+                choices=["Spectral", "Inferno", "Gray"],
+                value="Spectral",
+                label="Colormap"
+            )
+            
+            submit_btn = gr.Button(
+                "🎯 Generate Depth Map",
+                variant="primary",
+                size="lg"
+            )
+            
+        with gr.Column():
+            output_image = gr.Image(
+                label="Depth Map Result",
+                type="pil",
+                height=400
+            )
+    
+    gr.Examples(
+        examples=[
+            ["toyset/1.png", "Spectral"],
+            ["toyset/2.png", "Spectral"],
+            ["toyset/good.png", "Spectral"],
+        ] if os.path.exists("toyset") else [],
+        inputs=[input_image, colormap_choice],
+        outputs=output_image,
+        fn=predict_depth,
+        cache_examples=False,
+        label="Try these example images"
+    )
+    
+    submit_btn.click(
+        fn=predict_depth,
+        inputs=[input_image, colormap_choice],
+        outputs=output_image,
+        show_progress=True
+    )
+    
+    gr.Markdown("""
+    ## 📝 Notes
+    - **Spectral**: Rainbow spectrum with distinct near-far contrast
+    - **Inferno**: Flame spectrum with warm tones
+    - **Gray**: Grayscale with classic effect
+    """)
+
+
+if __name__ == "__main__":
+    demo.launch(
+        server_name="0.0.0.0",
+        server_port=7860,
+        share=False,
+        show_error=True
+    ) 

depth_anything/dinov2.py

@@ -0,0 +1,415 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under the Apache License, Version 2.0
+# found in the LICENSE file in the root directory of this source tree.
+
+# References:
+#   https://github.com/facebookresearch/dino/blob/main/vision_transformer.py
+#   https://github.com/rwightman/pytorch-image-models/tree/master/timm/models/vision_transformer.py
+
+from functools import partial
+import math
+import logging
+from typing import Sequence, Tuple, Union, Callable
+
+import torch
+import torch.nn as nn
+import torch.utils.checkpoint
+from torch.nn.init import trunc_normal_
+
+from .dinov2_layers import Mlp, PatchEmbed, SwiGLUFFNFused, MemEffAttention, NestedTensorBlock as Block
+
+
+logger = logging.getLogger("dinov2")
+
+
+def named_apply(fn: Callable, module: nn.Module, name="", depth_first=True, include_root=False) -> nn.Module:
+    if not depth_first and include_root:
+        fn(module=module, name=name)
+    for child_name, child_module in module.named_children():
+        child_name = ".".join((name, child_name)) if name else child_name
+        named_apply(fn=fn, module=child_module, name=child_name, depth_first=depth_first, include_root=True)
+    if depth_first and include_root:
+        fn(module=module, name=name)
+    return module
+
+
+class BlockChunk(nn.ModuleList):
+    def forward(self, x):
+        for b in self:
+            x = b(x)
+        return x
+
+
+class DinoVisionTransformer(nn.Module):
+    def __init__(
+        self,
+        img_size=224,
+        patch_size=16,
+        in_chans=3,
+        embed_dim=768,
+        depth=12,
+        num_heads=12,
+        mlp_ratio=4.0,
+        qkv_bias=True,
+        ffn_bias=True,
+        proj_bias=True,
+        drop_path_rate=0.0,
+        drop_path_uniform=False,
+        init_values=None,  # for layerscale: None or 0 => no layerscale
+        embed_layer=PatchEmbed,
+        act_layer=nn.GELU,
+        block_fn=Block,
+        ffn_layer="mlp",
+        block_chunks=1,
+        num_register_tokens=0,
+        interpolate_antialias=False,
+        interpolate_offset=0.1,
+    ):
+        """
+        Args:
+            img_size (int, tuple): input image size
+            patch_size (int, tuple): patch size
+            in_chans (int): number of input channels
+            embed_dim (int): embedding dimension
+            depth (int): depth of transformer
+            num_heads (int): number of attention heads
+            mlp_ratio (int): ratio of mlp hidden dim to embedding dim
+            qkv_bias (bool): enable bias for qkv if True
+            proj_bias (bool): enable bias for proj in attn if True
+            ffn_bias (bool): enable bias for ffn if True
+            drop_path_rate (float): stochastic depth rate
+            drop_path_uniform (bool): apply uniform drop rate across blocks
+            weight_init (str): weight init scheme
+            init_values (float): layer-scale init values
+            embed_layer (nn.Module): patch embedding layer
+            act_layer (nn.Module): MLP activation layer
+            block_fn (nn.Module): transformer block class
+            ffn_layer (str): "mlp", "swiglu", "swiglufused" or "identity"
+            block_chunks: (int) split block sequence into block_chunks units for FSDP wrap
+            num_register_tokens: (int) number of extra cls tokens (so-called "registers")
+            interpolate_antialias: (str) flag to apply anti-aliasing when interpolating positional embeddings
+            interpolate_offset: (float) work-around offset to apply when interpolating positional embeddings
+        """
+        super().__init__()
+        norm_layer = partial(nn.LayerNorm, eps=1e-6)
+
+        self.num_features = self.embed_dim = embed_dim  # num_features for consistency with other models
+        self.num_tokens = 1
+        self.n_blocks = depth
+        self.num_heads = num_heads
+        self.patch_size = patch_size
+        self.num_register_tokens = num_register_tokens
+        self.interpolate_antialias = interpolate_antialias
+        self.interpolate_offset = interpolate_offset
+
+        self.patch_embed = embed_layer(img_size=img_size, patch_size=patch_size, in_chans=in_chans, embed_dim=embed_dim)
+        num_patches = self.patch_embed.num_patches
+
+        self.cls_token = nn.Parameter(torch.zeros(1, 1, embed_dim))
+        self.pos_embed = nn.Parameter(torch.zeros(1, num_patches + self.num_tokens, embed_dim))
+        assert num_register_tokens >= 0
+        self.register_tokens = (
+            nn.Parameter(torch.zeros(1, num_register_tokens, embed_dim)) if num_register_tokens else None
+        )
+
+        if drop_path_uniform is True:
+            dpr = [drop_path_rate] * depth
+        else:
+            dpr = [x.item() for x in torch.linspace(0, drop_path_rate, depth)]  # stochastic depth decay rule
+
+        if ffn_layer == "mlp":
+            logger.info("using MLP layer as FFN")
+            ffn_layer = Mlp
+        elif ffn_layer == "swiglufused" or ffn_layer == "swiglu":
+            logger.info("using SwiGLU layer as FFN")
+            ffn_layer = SwiGLUFFNFused
+        elif ffn_layer == "identity":
+            logger.info("using Identity layer as FFN")
+
+            def f(*args, **kwargs):
+                return nn.Identity()
+
+            ffn_layer = f
+        else:
+            raise NotImplementedError
+
+        blocks_list = [
+            block_fn(
+                dim=embed_dim,
+                num_heads=num_heads,
+                mlp_ratio=mlp_ratio,
+                qkv_bias=qkv_bias,
+                proj_bias=proj_bias,
+                ffn_bias=ffn_bias,
+                drop_path=dpr[i],
+                norm_layer=norm_layer,
+                act_layer=act_layer,
+                ffn_layer=ffn_layer,
+                init_values=init_values,
+            )
+            for i in range(depth)
+        ]
+        if block_chunks > 0:
+            self.chunked_blocks = True
+            chunked_blocks = []
+            chunksize = depth // block_chunks
+            for i in range(0, depth, chunksize):
+                # this is to keep the block index consistent if we chunk the block list
+                chunked_blocks.append([nn.Identity()] * i + blocks_list[i : i + chunksize])
+            self.blocks = nn.ModuleList([BlockChunk(p) for p in chunked_blocks])
+        else:
+            self.chunked_blocks = False
+            self.blocks = nn.ModuleList(blocks_list)
+
+        self.norm = norm_layer(embed_dim)
+        self.head = nn.Identity()
+
+        self.mask_token = nn.Parameter(torch.zeros(1, embed_dim))
+
+        self.init_weights()
+
+    def init_weights(self):
+        trunc_normal_(self.pos_embed, std=0.02)
+        nn.init.normal_(self.cls_token, std=1e-6)
+        if self.register_tokens is not None:
+            nn.init.normal_(self.register_tokens, std=1e-6)
+        named_apply(init_weights_vit_timm, self)
+
+    def interpolate_pos_encoding(self, x, w, h):
+        previous_dtype = x.dtype
+        npatch = x.shape[1] - 1
+        N = self.pos_embed.shape[1] - 1
+        if npatch == N and w == h:
+            return self.pos_embed
+        pos_embed = self.pos_embed.float()
+        class_pos_embed = pos_embed[:, 0]
+        patch_pos_embed = pos_embed[:, 1:]
+        dim = x.shape[-1]
+        w0 = w // self.patch_size
+        h0 = h // self.patch_size
+        # we add a small number to avoid floating point error in the interpolation
+        # see discussion at https://github.com/facebookresearch/dino/issues/8
+        # DINOv2 with register modify the interpolate_offset from 0.1 to 0.0
+        w0, h0 = w0 + self.interpolate_offset, h0 + self.interpolate_offset
+        # w0, h0 = w0 + 0.1, h0 + 0.1
+        
+        sqrt_N = math.sqrt(N)
+        sx, sy = float(w0) / sqrt_N, float(h0) / sqrt_N
+        patch_pos_embed = nn.functional.interpolate(
+            patch_pos_embed.reshape(1, int(sqrt_N), int(sqrt_N), dim).permute(0, 3, 1, 2),
+            scale_factor=(sx, sy),
+            # (int(w0), int(h0)), # to solve the upsampling shape issue
+            mode="bicubic",
+            antialias=self.interpolate_antialias
+        )
+        
+        assert int(w0) == patch_pos_embed.shape[-2]
+        assert int(h0) == patch_pos_embed.shape[-1]
+        patch_pos_embed = patch_pos_embed.permute(0, 2, 3, 1).view(1, -1, dim)
+        return torch.cat((class_pos_embed.unsqueeze(0), patch_pos_embed), dim=1).to(previous_dtype)
+
+    def prepare_tokens_with_masks(self, x, masks=None):
+        B, nc, w, h = x.shape
+        x = self.patch_embed(x)
+        if masks is not None:
+            x = torch.where(masks.unsqueeze(-1), self.mask_token.to(x.dtype).unsqueeze(0), x)
+
+        x = torch.cat((self.cls_token.expand(x.shape[0], -1, -1), x), dim=1)
+        x = x + self.interpolate_pos_encoding(x, w, h)
+
+        if self.register_tokens is not None:
+            x = torch.cat(
+                (
+                    x[:, :1],
+                    self.register_tokens.expand(x.shape[0], -1, -1),
+                    x[:, 1:],
+                ),
+                dim=1,
+            )
+
+        return x
+
+    def forward_features_list(self, x_list, masks_list):
+        x = [self.prepare_tokens_with_masks(x, masks) for x, masks in zip(x_list, masks_list)]
+        for blk in self.blocks:
+            x = blk(x)
+
+        all_x = x
+        output = []
+        for x, masks in zip(all_x, masks_list):
+            x_norm = self.norm(x)
+            output.append(
+                {
+                    "x_norm_clstoken": x_norm[:, 0],
+                    "x_norm_regtokens": x_norm[:, 1 : self.num_register_tokens + 1],
+                    "x_norm_patchtokens": x_norm[:, self.num_register_tokens + 1 :],
+                    "x_prenorm": x,
+                    "masks": masks,
+                }
+            )
+        return output
+
+    def forward_features(self, x, masks=None):
+        if isinstance(x, list):
+            return self.forward_features_list(x, masks)
+
+        x = self.prepare_tokens_with_masks(x, masks)
+
+        for blk in self.blocks:
+            x = blk(x)
+
+        x_norm = self.norm(x)
+        return {
+            "x_norm_clstoken": x_norm[:, 0],
+            "x_norm_regtokens": x_norm[:, 1 : self.num_register_tokens + 1],
+            "x_norm_patchtokens": x_norm[:, self.num_register_tokens + 1 :],
+            "x_prenorm": x,
+            "masks": masks,
+        }
+
+    def _get_intermediate_layers_not_chunked(self, x, n=1):
+        x = self.prepare_tokens_with_masks(x)
+        # If n is an int, take the n last blocks. If it's a list, take them
+        output, total_block_len = [], len(self.blocks)
+        blocks_to_take = range(total_block_len - n, total_block_len) if isinstance(n, int) else n
+        for i, blk in enumerate(self.blocks):
+            x = blk(x)
+            if i in blocks_to_take:
+                output.append(x)
+        assert len(output) == len(blocks_to_take), f"only {len(output)} / {len(blocks_to_take)} blocks found"
+        return output
+
+    def _get_intermediate_layers_chunked(self, x, n=1):
+        x = self.prepare_tokens_with_masks(x)
+        output, i, total_block_len = [], 0, len(self.blocks[-1])
+        # If n is an int, take the n last blocks. If it's a list, take them
+        blocks_to_take = range(total_block_len - n, total_block_len) if isinstance(n, int) else n
+        for block_chunk in self.blocks:
+            for blk in block_chunk[i:]:  # Passing the nn.Identity()
+                x = blk(x)
+                if i in blocks_to_take:
+                    output.append(x)
+                i += 1
+        assert len(output) == len(blocks_to_take), f"only {len(output)} / {len(blocks_to_take)} blocks found"
+        return output
+
+    def get_intermediate_layers(
+        self,
+        x: torch.Tensor,
+        n: Union[int, Sequence] = 1,  # Layers or n last layers to take
+        reshape: bool = False,
+        return_class_token: bool = False,
+        norm=True
+    ) -> Tuple[Union[torch.Tensor, Tuple[torch.Tensor]]]:
+        if self.chunked_blocks:
+            outputs = self._get_intermediate_layers_chunked(x, n)
+        else:
+            outputs = self._get_intermediate_layers_not_chunked(x, n)
+        if norm:
+            outputs = [self.norm(out) for out in outputs]
+        class_tokens = [out[:, 0] for out in outputs]
+        outputs = [out[:, 1 + self.num_register_tokens:] for out in outputs]
+        if reshape:
+            B, _, w, h = x.shape
+            outputs = [
+                out.reshape(B, w // self.patch_size, h // self.patch_size, -1).permute(0, 3, 1, 2).contiguous()
+                for out in outputs
+            ]
+        if return_class_token:
+            return tuple(zip(outputs, class_tokens))
+        return tuple(outputs)
+
+    def forward(self, *args, is_training=False, **kwargs):
+        ret = self.forward_features(*args, **kwargs)
+        if is_training:
+            return ret
+        else:
+            return self.head(ret["x_norm_clstoken"])
+
+
+def init_weights_vit_timm(module: nn.Module, name: str = ""):
+    """ViT weight initialization, original timm impl (for reproducibility)"""
+    if isinstance(module, nn.Linear):
+        trunc_normal_(module.weight, std=0.02)
+        if module.bias is not None:
+            nn.init.zeros_(module.bias)
+
+
+def vit_small(patch_size=16, num_register_tokens=0, **kwargs):
+    model = DinoVisionTransformer(
+        patch_size=patch_size,
+        embed_dim=384,
+        depth=12,
+        num_heads=6,
+        mlp_ratio=4,
+        block_fn=partial(Block, attn_class=MemEffAttention),
+        num_register_tokens=num_register_tokens,
+        **kwargs,
+    )
+    return model
+
+
+def vit_base(patch_size=16, num_register_tokens=0, **kwargs):
+    model = DinoVisionTransformer(
+        patch_size=patch_size,
+        embed_dim=768,
+        depth=12,
+        num_heads=12,
+        mlp_ratio=4,
+        block_fn=partial(Block, attn_class=MemEffAttention),
+        num_register_tokens=num_register_tokens,
+        **kwargs,
+    )
+    return model
+
+
+def vit_large(patch_size=16, num_register_tokens=0, **kwargs):
+    model = DinoVisionTransformer(
+        patch_size=patch_size,
+        embed_dim=1024,
+        depth=24,
+        num_heads=16,
+        mlp_ratio=4,
+        block_fn=partial(Block, attn_class=MemEffAttention),
+        num_register_tokens=num_register_tokens,
+        **kwargs,
+    )
+    return model
+
+
+def vit_giant2(patch_size=16, num_register_tokens=0, **kwargs):
+    """
+    Close to ViT-giant, with embed-dim 1536 and 24 heads => embed-dim per head 64
+    """
+    model = DinoVisionTransformer(
+        patch_size=patch_size,
+        embed_dim=1536,
+        depth=40,
+        num_heads=24,
+        mlp_ratio=4,
+        block_fn=partial(Block, attn_class=MemEffAttention),
+        num_register_tokens=num_register_tokens,
+        **kwargs,
+    )
+    return model
+
+
+def DINOv2(model_name):
+    model_zoo = {
+        "vits": vit_small, 
+        "vitb": vit_base, 
+        "vitl": vit_large, 
+        "vitg": vit_giant2
+    }
+    
+    return model_zoo[model_name](
+        img_size=518,
+        patch_size=14,
+        init_values=1.0,
+        ffn_layer="mlp" if model_name != "vitg" else "swiglufused",
+        block_chunks=0,
+        num_register_tokens=0,
+        interpolate_antialias=False,
+        interpolate_offset=0.1
+    )

depth_anything/dinov2_layers/__init__.py

@@ -0,0 +1,11 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+from .mlp import Mlp
+from .patch_embed import PatchEmbed
+from .swiglu_ffn import SwiGLUFFN, SwiGLUFFNFused
+from .block import NestedTensorBlock
+from .attention import MemEffAttention

depth_anything/dinov2_layers/attention.py

@@ -0,0 +1,83 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+# References:
+#   https://github.com/facebookresearch/dino/blob/master/vision_transformer.py
+#   https://github.com/rwightman/pytorch-image-models/tree/master/timm/models/vision_transformer.py
+
+import logging
+
+from torch import Tensor
+from torch import nn
+
+
+logger = logging.getLogger("dinov2")
+
+
+try:
+    from xformers.ops import memory_efficient_attention, unbind, fmha
+
+    XFORMERS_AVAILABLE = True
+except ImportError:
+    logger.warning("xFormers not available")
+    XFORMERS_AVAILABLE = False
+
+
+class Attention(nn.Module):
+    def __init__(
+        self,
+        dim: int,
+        num_heads: int = 8,
+        qkv_bias: bool = False,
+        proj_bias: bool = True,
+        attn_drop: float = 0.0,
+        proj_drop: float = 0.0,
+    ) -> None:
+        super().__init__()
+        self.num_heads = num_heads
+        head_dim = dim // num_heads
+        self.scale = head_dim**-0.5
+
+        self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias)
+        self.attn_drop = nn.Dropout(attn_drop)
+        self.proj = nn.Linear(dim, dim, bias=proj_bias)
+        self.proj_drop = nn.Dropout(proj_drop)
+
+    def forward(self, x: Tensor) -> Tensor:
+        B, N, C = x.shape
+        qkv = self.qkv(x).reshape(B, N, 3, self.num_heads, C // self.num_heads).permute(2, 0, 3, 1, 4)
+
+        q, k, v = qkv[0] * self.scale, qkv[1], qkv[2]
+        attn = q @ k.transpose(-2, -1)
+
+        attn = attn.softmax(dim=-1)
+        attn = self.attn_drop(attn)
+
+        x = (attn @ v).transpose(1, 2).reshape(B, N, C)
+        x = self.proj(x)
+        x = self.proj_drop(x)
+        return x
+
+
+class MemEffAttention(Attention):
+    def forward(self, x: Tensor, attn_bias=None) -> Tensor:
+        if not XFORMERS_AVAILABLE:
+            assert attn_bias is None, "xFormers is required for nested tensors usage"
+            return super().forward(x)
+
+        B, N, C = x.shape
+        qkv = self.qkv(x).reshape(B, N, 3, self.num_heads, C // self.num_heads)
+
+        q, k, v = unbind(qkv, 2)
+
+        x = memory_efficient_attention(q, k, v, attn_bias=attn_bias)
+        x = x.reshape([B, N, C])
+
+        x = self.proj(x)
+        x = self.proj_drop(x)
+        return x
+
+        

depth_anything/dinov2_layers/block.py

@@ -0,0 +1,252 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+# References:
+#   https://github.com/facebookresearch/dino/blob/master/vision_transformer.py
+#   https://github.com/rwightman/pytorch-image-models/tree/master/timm/layers/patch_embed.py
+
+import logging
+from typing import Callable, List, Any, Tuple, Dict
+
+import torch
+from torch import nn, Tensor
+
+from .attention import Attention, MemEffAttention
+from .drop_path import DropPath
+from .layer_scale import LayerScale
+from .mlp import Mlp
+
+
+logger = logging.getLogger("dinov2")
+
+
+try:
+    from xformers.ops import fmha
+    from xformers.ops import scaled_index_add, index_select_cat
+
+    XFORMERS_AVAILABLE = True
+except ImportError:
+    logger.warning("xFormers not available")
+    XFORMERS_AVAILABLE = False
+
+
+class Block(nn.Module):
+    def __init__(
+        self,
+        dim: int,
+        num_heads: int,
+        mlp_ratio: float = 4.0,
+        qkv_bias: bool = False,
+        proj_bias: bool = True,
+        ffn_bias: bool = True,
+        drop: float = 0.0,
+        attn_drop: float = 0.0,
+        init_values=None,
+        drop_path: float = 0.0,
+        act_layer: Callable[..., nn.Module] = nn.GELU,
+        norm_layer: Callable[..., nn.Module] = nn.LayerNorm,
+        attn_class: Callable[..., nn.Module] = Attention,
+        ffn_layer: Callable[..., nn.Module] = Mlp,
+    ) -> None:
+        super().__init__()
+        # print(f"biases: qkv: {qkv_bias}, proj: {proj_bias}, ffn: {ffn_bias}")
+        self.norm1 = norm_layer(dim)
+        self.attn = attn_class(
+            dim,
+            num_heads=num_heads,
+            qkv_bias=qkv_bias,
+            proj_bias=proj_bias,
+            attn_drop=attn_drop,
+            proj_drop=drop,
+        )
+        self.ls1 = LayerScale(dim, init_values=init_values) if init_values else nn.Identity()
+        self.drop_path1 = DropPath(drop_path) if drop_path > 0.0 else nn.Identity()
+
+        self.norm2 = norm_layer(dim)
+        mlp_hidden_dim = int(dim * mlp_ratio)
+        self.mlp = ffn_layer(
+            in_features=dim,
+            hidden_features=mlp_hidden_dim,
+            act_layer=act_layer,
+            drop=drop,
+            bias=ffn_bias,
+        )
+        self.ls2 = LayerScale(dim, init_values=init_values) if init_values else nn.Identity()
+        self.drop_path2 = DropPath(drop_path) if drop_path > 0.0 else nn.Identity()
+
+        self.sample_drop_ratio = drop_path
+
+    def forward(self, x: Tensor) -> Tensor:
+        def attn_residual_func(x: Tensor) -> Tensor:
+            return self.ls1(self.attn(self.norm1(x)))
+
+        def ffn_residual_func(x: Tensor) -> Tensor:
+            return self.ls2(self.mlp(self.norm2(x)))
+
+        if self.training and self.sample_drop_ratio > 0.1:
+            # the overhead is compensated only for a drop path rate larger than 0.1
+            x = drop_add_residual_stochastic_depth(
+                x,
+                residual_func=attn_residual_func,
+                sample_drop_ratio=self.sample_drop_ratio,
+            )
+            x = drop_add_residual_stochastic_depth(
+                x,
+                residual_func=ffn_residual_func,
+                sample_drop_ratio=self.sample_drop_ratio,
+            )
+        elif self.training and self.sample_drop_ratio > 0.0:
+            x = x + self.drop_path1(attn_residual_func(x))
+            x = x + self.drop_path1(ffn_residual_func(x))  # FIXME: drop_path2
+        else:
+            x = x + attn_residual_func(x)
+            x = x + ffn_residual_func(x)
+        return x
+
+
+def drop_add_residual_stochastic_depth(
+    x: Tensor,
+    residual_func: Callable[[Tensor], Tensor],
+    sample_drop_ratio: float = 0.0,
+) -> Tensor:
+    # 1) extract subset using permutation
+    b, n, d = x.shape
+    sample_subset_size = max(int(b * (1 - sample_drop_ratio)), 1)
+    brange = (torch.randperm(b, device=x.device))[:sample_subset_size]
+    x_subset = x[brange]
+
+    # 2) apply residual_func to get residual
+    residual = residual_func(x_subset)
+
+    x_flat = x.flatten(1)
+    residual = residual.flatten(1)
+
+    residual_scale_factor = b / sample_subset_size
+
+    # 3) add the residual
+    x_plus_residual = torch.index_add(x_flat, 0, brange, residual.to(dtype=x.dtype), alpha=residual_scale_factor)
+    return x_plus_residual.view_as(x)
+
+
+def get_branges_scales(x, sample_drop_ratio=0.0):
+    b, n, d = x.shape
+    sample_subset_size = max(int(b * (1 - sample_drop_ratio)), 1)
+    brange = (torch.randperm(b, device=x.device))[:sample_subset_size]
+    residual_scale_factor = b / sample_subset_size
+    return brange, residual_scale_factor
+
+
+def add_residual(x, brange, residual, residual_scale_factor, scaling_vector=None):
+    if scaling_vector is None:
+        x_flat = x.flatten(1)
+        residual = residual.flatten(1)
+        x_plus_residual = torch.index_add(x_flat, 0, brange, residual.to(dtype=x.dtype), alpha=residual_scale_factor)
+    else:
+        x_plus_residual = scaled_index_add(
+            x, brange, residual.to(dtype=x.dtype), scaling=scaling_vector, alpha=residual_scale_factor
+        )
+    return x_plus_residual
+
+
+attn_bias_cache: Dict[Tuple, Any] = {}
+
+
+def get_attn_bias_and_cat(x_list, branges=None):
+    """
+    this will perform the index select, cat the tensors, and provide the attn_bias from cache
+    """
+    batch_sizes = [b.shape[0] for b in branges] if branges is not None else [x.shape[0] for x in x_list]
+    all_shapes = tuple((b, x.shape[1]) for b, x in zip(batch_sizes, x_list))
+    if all_shapes not in attn_bias_cache.keys():
+        seqlens = []
+        for b, x in zip(batch_sizes, x_list):
+            for _ in range(b):
+                seqlens.append(x.shape[1])
+        attn_bias = fmha.BlockDiagonalMask.from_seqlens(seqlens)
+        attn_bias._batch_sizes = batch_sizes
+        attn_bias_cache[all_shapes] = attn_bias
+
+    if branges is not None:
+        cat_tensors = index_select_cat([x.flatten(1) for x in x_list], branges).view(1, -1, x_list[0].shape[-1])
+    else:
+        tensors_bs1 = tuple(x.reshape([1, -1, *x.shape[2:]]) for x in x_list)
+        cat_tensors = torch.cat(tensors_bs1, dim=1)
+
+    return attn_bias_cache[all_shapes], cat_tensors
+
+
+def drop_add_residual_stochastic_depth_list(
+    x_list: List[Tensor],
+    residual_func: Callable[[Tensor, Any], Tensor],
+    sample_drop_ratio: float = 0.0,
+    scaling_vector=None,
+) -> Tensor:
+    # 1) generate random set of indices for dropping samples in the batch
+    branges_scales = [get_branges_scales(x, sample_drop_ratio=sample_drop_ratio) for x in x_list]
+    branges = [s[0] for s in branges_scales]
+    residual_scale_factors = [s[1] for s in branges_scales]
+
+    # 2) get attention bias and index+concat the tensors
+    attn_bias, x_cat = get_attn_bias_and_cat(x_list, branges)
+
+    # 3) apply residual_func to get residual, and split the result
+    residual_list = attn_bias.split(residual_func(x_cat, attn_bias=attn_bias))  # type: ignore
+
+    outputs = []
+    for x, brange, residual, residual_scale_factor in zip(x_list, branges, residual_list, residual_scale_factors):
+        outputs.append(add_residual(x, brange, residual, residual_scale_factor, scaling_vector).view_as(x))
+    return outputs
+
+
+class NestedTensorBlock(Block):
+    def forward_nested(self, x_list: List[Tensor]) -> List[Tensor]:
+        """
+        x_list contains a list of tensors to nest together and run
+        """
+        assert isinstance(self.attn, MemEffAttention)
+
+        if self.training and self.sample_drop_ratio > 0.0:
+
+            def attn_residual_func(x: Tensor, attn_bias=None) -> Tensor:
+                return self.attn(self.norm1(x), attn_bias=attn_bias)
+
+            def ffn_residual_func(x: Tensor, attn_bias=None) -> Tensor:
+                return self.mlp(self.norm2(x))
+
+            x_list = drop_add_residual_stochastic_depth_list(
+                x_list,
+                residual_func=attn_residual_func,
+                sample_drop_ratio=self.sample_drop_ratio,
+                scaling_vector=self.ls1.gamma if isinstance(self.ls1, LayerScale) else None,
+            )
+            x_list = drop_add_residual_stochastic_depth_list(
+                x_list,
+                residual_func=ffn_residual_func,
+                sample_drop_ratio=self.sample_drop_ratio,
+                scaling_vector=self.ls2.gamma if isinstance(self.ls1, LayerScale) else None,
+            )
+            return x_list
+        else:
+
+            def attn_residual_func(x: Tensor, attn_bias=None) -> Tensor:
+                return self.ls1(self.attn(self.norm1(x), attn_bias=attn_bias))
+
+            def ffn_residual_func(x: Tensor, attn_bias=None) -> Tensor:
+                return self.ls2(self.mlp(self.norm2(x)))
+
+            attn_bias, x = get_attn_bias_and_cat(x_list)
+            x = x + attn_residual_func(x, attn_bias=attn_bias)
+            x = x + ffn_residual_func(x)
+            return attn_bias.split(x)
+
+    def forward(self, x_or_x_list):
+        if isinstance(x_or_x_list, Tensor):
+            return super().forward(x_or_x_list)
+        elif isinstance(x_or_x_list, list):
+            assert XFORMERS_AVAILABLE, "Please install xFormers for nested tensors usage"
+            return self.forward_nested(x_or_x_list)
+        else:
+            raise AssertionError

depth_anything/dinov2_layers/drop_path.py

@@ -0,0 +1,35 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+# References:
+#   https://github.com/facebookresearch/dino/blob/master/vision_transformer.py
+#   https://github.com/rwightman/pytorch-image-models/tree/master/timm/layers/drop.py
+
+
+from torch import nn
+
+
+def drop_path(x, drop_prob: float = 0.0, training: bool = False):
+    if drop_prob == 0.0 or not training:
+        return x
+    keep_prob = 1 - drop_prob
+    shape = (x.shape[0],) + (1,) * (x.ndim - 1)  # work with diff dim tensors, not just 2D ConvNets
+    random_tensor = x.new_empty(shape).bernoulli_(keep_prob)
+    if keep_prob > 0.0:
+        random_tensor.div_(keep_prob)
+    output = x * random_tensor
+    return output
+
+
+class DropPath(nn.Module):
+    """Drop paths (Stochastic Depth) per sample (when applied in main path of residual blocks)."""
+
+    def __init__(self, drop_prob=None):
+        super(DropPath, self).__init__()
+        self.drop_prob = drop_prob
+
+    def forward(self, x):
+        return drop_path(x, self.drop_prob, self.training)

depth_anything/dinov2_layers/layer_scale.py

@@ -0,0 +1,28 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+# Modified from: https://github.com/huggingface/pytorch-image-models/blob/main/timm/models/vision_transformer.py#L103-L110
+
+from typing import Union
+
+import torch
+from torch import Tensor
+from torch import nn
+
+
+class LayerScale(nn.Module):
+    def __init__(
+        self,
+        dim: int,
+        init_values: Union[float, Tensor] = 1e-5,
+        inplace: bool = False,
+    ) -> None:
+        super().__init__()
+        self.inplace = inplace
+        self.gamma = nn.Parameter(init_values * torch.ones(dim))
+
+    def forward(self, x: Tensor) -> Tensor:
+        return x.mul_(self.gamma) if self.inplace else x * self.gamma

depth_anything/dinov2_layers/mlp.py

@@ -0,0 +1,41 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+# References:
+#   https://github.com/facebookresearch/dino/blob/master/vision_transformer.py
+#   https://github.com/rwightman/pytorch-image-models/tree/master/timm/layers/mlp.py
+
+
+from typing import Callable, Optional
+
+from torch import Tensor, nn
+
+
+class Mlp(nn.Module):
+    def __init__(
+        self,
+        in_features: int,
+        hidden_features: Optional[int] = None,
+        out_features: Optional[int] = None,
+        act_layer: Callable[..., nn.Module] = nn.GELU,
+        drop: float = 0.0,
+        bias: bool = True,
+    ) -> None:
+        super().__init__()
+        out_features = out_features or in_features
+        hidden_features = hidden_features or in_features
+        self.fc1 = nn.Linear(in_features, hidden_features, bias=bias)
+        self.act = act_layer()
+        self.fc2 = nn.Linear(hidden_features, out_features, bias=bias)
+        self.drop = nn.Dropout(drop)
+
+    def forward(self, x: Tensor) -> Tensor:
+        x = self.fc1(x)
+        x = self.act(x)
+        x = self.drop(x)
+        x = self.fc2(x)
+        x = self.drop(x)
+        return x

depth_anything/dinov2_layers/patch_embed.py

@@ -0,0 +1,89 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+# References:
+#   https://github.com/facebookresearch/dino/blob/master/vision_transformer.py
+#   https://github.com/rwightman/pytorch-image-models/tree/master/timm/layers/patch_embed.py
+
+from typing import Callable, Optional, Tuple, Union
+
+from torch import Tensor
+import torch.nn as nn
+
+
+def make_2tuple(x):
+    if isinstance(x, tuple):
+        assert len(x) == 2
+        return x
+
+    assert isinstance(x, int)
+    return (x, x)
+
+
+class PatchEmbed(nn.Module):
+    """
+    2D image to patch embedding: (B,C,H,W) -> (B,N,D)
+
+    Args:
+        img_size: Image size.
+        patch_size: Patch token size.
+        in_chans: Number of input image channels.
+        embed_dim: Number of linear projection output channels.
+        norm_layer: Normalization layer.
+    """
+
+    def __init__(
+        self,
+        img_size: Union[int, Tuple[int, int]] = 224,
+        patch_size: Union[int, Tuple[int, int]] = 16,
+        in_chans: int = 3,
+        embed_dim: int = 768,
+        norm_layer: Optional[Callable] = None,
+        flatten_embedding: bool = True,
+    ) -> None:
+        super().__init__()
+
+        image_HW = make_2tuple(img_size)
+        patch_HW = make_2tuple(patch_size)
+        patch_grid_size = (
+            image_HW[0] // patch_HW[0],
+            image_HW[1] // patch_HW[1],
+        )
+
+        self.img_size = image_HW
+        self.patch_size = patch_HW
+        self.patches_resolution = patch_grid_size
+        self.num_patches = patch_grid_size[0] * patch_grid_size[1]
+
+        self.in_chans = in_chans
+        self.embed_dim = embed_dim
+
+        self.flatten_embedding = flatten_embedding
+
+        self.proj = nn.Conv2d(in_chans, embed_dim, kernel_size=patch_HW, stride=patch_HW)
+        self.norm = norm_layer(embed_dim) if norm_layer else nn.Identity()
+
+    def forward(self, x: Tensor) -> Tensor:
+        _, _, H, W = x.shape
+        patch_H, patch_W = self.patch_size
+
+        assert H % patch_H == 0, f"Input image height {H} is not a multiple of patch height {patch_H}"
+        assert W % patch_W == 0, f"Input image width {W} is not a multiple of patch width: {patch_W}"
+
+        x = self.proj(x)  # B C H W
+        H, W = x.size(2), x.size(3)
+        x = x.flatten(2).transpose(1, 2)  # B HW C
+        x = self.norm(x)
+        if not self.flatten_embedding:
+            x = x.reshape(-1, H, W, self.embed_dim)  # B H W C
+        return x
+
+    def flops(self) -> float:
+        Ho, Wo = self.patches_resolution
+        flops = Ho * Wo * self.embed_dim * self.in_chans * (self.patch_size[0] * self.patch_size[1])
+        if self.norm is not None:
+            flops += Ho * Wo * self.embed_dim
+        return flops

depth_anything/dinov2_layers/swiglu_ffn.py

@@ -0,0 +1,63 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+from typing import Callable, Optional
+
+from torch import Tensor, nn
+import torch.nn.functional as F
+
+
+class SwiGLUFFN(nn.Module):
+    def __init__(
+        self,
+        in_features: int,
+        hidden_features: Optional[int] = None,
+        out_features: Optional[int] = None,
+        act_layer: Callable[..., nn.Module] = None,
+        drop: float = 0.0,
+        bias: bool = True,
+    ) -> None:
+        super().__init__()
+        out_features = out_features or in_features
+        hidden_features = hidden_features or in_features
+        self.w12 = nn.Linear(in_features, 2 * hidden_features, bias=bias)
+        self.w3 = nn.Linear(hidden_features, out_features, bias=bias)
+
+    def forward(self, x: Tensor) -> Tensor:
+        x12 = self.w12(x)
+        x1, x2 = x12.chunk(2, dim=-1)
+        hidden = F.silu(x1) * x2
+        return self.w3(hidden)
+
+
+try:
+    from xformers.ops import SwiGLU
+
+    XFORMERS_AVAILABLE = True
+except ImportError:
+    SwiGLU = SwiGLUFFN
+    XFORMERS_AVAILABLE = False
+
+
+class SwiGLUFFNFused(SwiGLU):
+    def __init__(
+        self,
+        in_features: int,
+        hidden_features: Optional[int] = None,
+        out_features: Optional[int] = None,
+        act_layer: Callable[..., nn.Module] = None,
+        drop: float = 0.0,
+        bias: bool = True,
+    ) -> None:
+        out_features = out_features or in_features
+        hidden_features = hidden_features or in_features
+        hidden_features = (int(hidden_features * 2 / 3) + 7) // 8 * 8
+        super().__init__(
+            in_features=in_features,
+            hidden_features=hidden_features,
+            out_features=out_features,
+            bias=bias,
+        )

depth_anything/dpt.py

@@ -0,0 +1,268 @@
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+import torchhub.facebookresearch_dinov2_main.hubconf as dinov2
+
+from depth_anything.util.blocksv2 import FeatureFusionBlock, _make_scratch
+from torchvision.transforms import Compose
+from depth_anything.util.transform import Resize, NormalizeImage, PrepareForNet
+import cv2
+import numpy as np
+
+def _make_fusion_block(features, use_bn, size = None):
+    return FeatureFusionBlock(
+        features,
+        nn.ReLU(False),
+        deconv=False,
+        bn=use_bn,
+        expand=False,
+        align_corners=True,
+        size=size,
+    )
+
+
+class DPTHead(nn.Module):
+    def __init__(self, nclass, in_channels, features=256, use_bn=False, out_channels=[256, 512, 1024, 1024], use_clstoken=False):
+        super(DPTHead, self).__init__()
+        
+        self.nclass = nclass
+        self.use_clstoken = use_clstoken
+        
+        self.projects = nn.ModuleList([
+            nn.Conv2d(
+                in_channels=in_channels,
+                out_channels=out_channel,
+                kernel_size=1,
+                stride=1,
+                padding=0,
+            ) for out_channel in out_channels
+        ])
+        
+        self.resize_layers = nn.ModuleList([
+            nn.ConvTranspose2d(
+                in_channels=out_channels[0],
+                out_channels=out_channels[0],
+                kernel_size=4,
+                stride=4,
+                padding=0),
+            nn.ConvTranspose2d(
+                in_channels=out_channels[1],
+                out_channels=out_channels[1],
+                kernel_size=2,
+                stride=2,
+                padding=0),
+            nn.Identity(),
+            nn.Conv2d(
+                in_channels=out_channels[3],
+                out_channels=out_channels[3],
+                kernel_size=3,
+                stride=2,
+                padding=1)
+        ])
+        
+        if use_clstoken:
+            self.readout_projects = nn.ModuleList()
+            for _ in range(len(self.projects)):
+                self.readout_projects.append(
+                    nn.Sequential(
+                        nn.Linear(2 * in_channels, in_channels),
+                        nn.GELU()))
+        
+        self.scratch = _make_scratch(
+            out_channels,
+            features,
+            groups=1,
+            expand=False,
+        )
+
+        self.scratch.stem_transpose = nn.Identity()
+        
+        self.scratch.refinenet1 = _make_fusion_block(features, use_bn)
+        self.scratch.refinenet2 = _make_fusion_block(features, use_bn)
+        self.scratch.refinenet3 = _make_fusion_block(features, use_bn)
+        self.scratch.refinenet4 = _make_fusion_block(features, use_bn)
+
+        head_features_1 = features
+        head_features_2 = 32
+        
+        if nclass > 1:
+            self.scratch.output_conv = nn.Sequential(
+                nn.Conv2d(head_features_1, head_features_1, kernel_size=3, stride=1, padding=1),
+                nn.ReLU(True),
+                nn.Conv2d(head_features_1, nclass, kernel_size=1, stride=1, padding=0),
+            )
+        else:
+            self.scratch.output_conv1 = nn.Conv2d(head_features_1, head_features_1 // 2, kernel_size=3, stride=1, padding=1)
+            
+            self.scratch.output_conv2 = nn.Sequential(
+                nn.Conv2d(head_features_1 // 2, head_features_2, kernel_size=3, stride=1, padding=1),
+                nn.ReLU(True),
+                nn.Conv2d(head_features_2, 1, kernel_size=1, stride=1, padding=0),
+                nn.ReLU(True),
+                nn.Identity(),
+            )
+       
+    def forward(self, out_features, patch_h, patch_w,need_fp=False,need_prior=False,teacher_features=None,alpha=0.8):
+        depth_out={}
+        out = []
+        for i, x in enumerate(out_features):
+            if self.use_clstoken:
+                x, cls_token = x[0], x[1]
+                readout = cls_token.unsqueeze(1).expand_as(x)
+                x = self.readout_projects[i](torch.cat((x, readout), -1))
+            else:
+                x = x[0]
+            x = x.permute(0, 2, 1).reshape((x.shape[0], x.shape[-1], patch_h, patch_w)).contiguous()
+            
+            x = self.projects[i](x)
+            x = self.resize_layers[i](x)
+            
+            out.append(x)
+            
+        layer_1, layer_2, layer_3, layer_4 = out
+        
+        layer_1_rn = self.scratch.layer1_rn(layer_1)
+        layer_2_rn = self.scratch.layer2_rn(layer_2)
+        layer_3_rn = self.scratch.layer3_rn(layer_3)
+        layer_4_rn = self.scratch.layer4_rn(layer_4)
+    
+        path_4 = self.scratch.refinenet4(layer_4_rn, size=layer_3_rn.shape[2:])
+        path_3 = self.scratch.refinenet3(path_4, layer_3_rn, size=layer_2_rn.shape[2:])
+        path_2 = self.scratch.refinenet2(path_3, layer_2_rn, size=layer_1_rn.shape[2:])
+        path_1 = self.scratch.refinenet1(path_2, layer_1_rn)
+        
+        out = self.scratch.output_conv1(path_1)
+        out = F.interpolate(out, (int(patch_h * 14), int(patch_w * 14)), mode="bilinear", align_corners=True)
+        out = self.scratch.output_conv2(out)
+
+        depth_out['out']=out
+        
+        return depth_out
+        
+        
+class DPT_DINOv2(nn.Module):
+    def __init__(self, encoder='vitl', features=256, out_channels=[256, 512, 1024, 1024], use_bn=False, use_clstoken=False, localhub=True, version='v1'):
+        super(DPT_DINOv2, self).__init__()
+        
+        assert encoder in ['vits', 'vitb', 'vitl']
+        self.intermediate_layer_idx = {
+            'vits': [2, 5, 8, 11],
+            'vitb': [2, 5, 8, 11], 
+            'vitl': [4, 11, 17, 23], 
+            'vitg': [9, 19, 29, 39]
+        }
+        self.encoder = encoder
+        self.version = version
+        # in case the Internet connection is not stable, please load the DINOv2 locally
+        # if localhub:
+        #     self.pretrained = torch.hub.load('torchhub/facebookresearch_dinov2_main', 'dinov2_{:}14'.format(encoder), source='local', pretrained=True)
+        # else:
+        #     self.pretrained = torch.hub.load('facebookresearch/dinov2', 'dinov2_{:}14'.format(encoder))
+        self.pretrained = dinov2.__dict__['dinov2_{:}14'.format(encoder)](pretrained=True)
+        
+        dim = self.pretrained.blocks[0].attn.qkv.in_features
+        
+        self.depth_head = DPTHead(1, dim, features, use_bn, out_channels=out_channels, use_clstoken=use_clstoken)
+
+        
+    def forward(self, x,need_fp=False,teacher_features=None,alpha=0.8,prior_mode='teacher'):
+        depth_out={}
+        h, w = x.shape[-2:]
+        if self.version == 'v1':
+            features = self.pretrained.get_intermediate_layers(x, 4, return_class_token=True)
+        else:
+
+            features = self.pretrained.get_intermediate_layers(x, self.intermediate_layer_idx[self.encoder], return_class_token=True)
+        patch_h, patch_w = h // 14, w // 14
+        
+        depth_all = self.depth_head(features, patch_h, patch_w,need_fp,teacher_features,alpha)
+        depth=depth_all['out']
+        depth = F.interpolate(depth, size=(h, w), mode="bilinear", align_corners=True)
+        depth = F.relu(depth).squeeze(1)
+        depth_out['out']=depth
+
+        return depth_out
+
+
+class DepthAnything_AC(DPT_DINOv2):
+    def __init__(self, config):
+        super().__init__(**config)
+
+
+    def get_intermediate_features(self, x):
+        """
+        Extract intermediate features from the model
+        
+        Args:
+            x: Input tensor of shape (B, C, H, W)
+            
+        Returns:
+            dict: Dictionary containing intermediate features including:
+                - encoder_features: List of encoder feature maps
+                - decoder_features: List of decoder feature maps  
+                - decoder_features_path: List of decoder path features
+                - cls_token: List of classification tokens
+        """
+        features = {
+            'encoder_features': [],
+            'decoder_features': [],
+            'decoder_features_path': [],
+            'cls_token': []
+        }
+        
+        h, w = x.shape[-2:]
+        patch_h, patch_w = h // 14, w // 14
+      
+        all_features = []
+        for i in range(len(self.pretrained.blocks)):
+            feat = self.pretrained.get_intermediate_layers(x, [i], return_class_token=True)[0]
+            all_features.append(feat)
+            if i in [2, 5, 8, 11]: 
+                feat_map = feat[0]  
+                B, N, C = feat_map.shape
+                H = W = int(np.sqrt(N))
+                features['encoder_features'].append(feat_map.reshape(B, H, W, C).permute(0, 3, 1, 2))
+        
+        out_features = []
+        for layer_idx in self.intermediate_layer_idx[self.encoder]:
+            out_features.append(all_features[layer_idx])
+        out = []
+        for i, feat in enumerate(out_features):
+            if self.depth_head.use_clstoken:
+                feat_map, cls_token = feat[0], feat[1]
+                readout = cls_token.unsqueeze(1).expand_as(feat_map)
+                feat_map = self.depth_head.readout_projects[i](torch.cat((feat_map, readout), -1))
+                features['cls_token'].append(cls_token)
+            else:
+                feat_map = feat[0]
+            feat_map = feat_map.permute(0, 2, 1).reshape((feat_map.shape[0], feat_map.shape[-1], patch_h, patch_w)).contiguous()
+            
+            feat_map = self.depth_head.projects[i](feat_map)
+            feat_map = self.depth_head.resize_layers[i](feat_map)
+            
+            out.append(feat_map)
+        
+        layer_1, layer_2, layer_3, layer_4 = out
+        
+        layer_1_rn = self.depth_head.scratch.layer1_rn(layer_1)
+        layer_2_rn = self.depth_head.scratch.layer2_rn(layer_2)
+        layer_3_rn = self.depth_head.scratch.layer3_rn(layer_3)
+        layer_4_rn = self.depth_head.scratch.layer4_rn(layer_4)
+            
+        features['decoder_features'].append(layer_1_rn)
+        features['decoder_features'].append(layer_2_rn)
+        features['decoder_features'].append(layer_3_rn)
+        features['decoder_features'].append(layer_4_rn)
+
+        path_4 = self.depth_head.scratch.refinenet4(layer_4_rn, size=layer_3_rn.shape[2:])
+        path_3 = self.depth_head.scratch.refinenet3(path_4, layer_3_rn, size=layer_2_rn.shape[2:])
+        path_2 = self.depth_head.scratch.refinenet2(path_3, layer_2_rn, size=layer_1_rn.shape[2:])
+        path_1 = self.depth_head.scratch.refinenet1(path_2, layer_1_rn)
+
+        features['decoder_features_path'].append(path_1)
+        features['decoder_features_path'].append(path_2)
+        features['decoder_features_path'].append(path_3)
+        features['decoder_features_path'].append(path_4)
+        
+        return features
+    

depth_anything/dpt_teacher.py

@@ -0,0 +1,173 @@
+import argparse
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+from depth_anything.util.blocksv1 import FeatureFusionBlock, _make_scratch
+
+
+def _make_fusion_block(features, use_bn, size = None):
+    return FeatureFusionBlock(
+        features,
+        nn.ReLU(False),
+        deconv=False,
+        bn=use_bn,
+        expand=False,
+        align_corners=True,
+        size=size,
+    )
+
+
+class DPTHead(nn.Module):
+    def __init__(self, nclass, in_channels, features=256, use_bn=False, out_channels=[256, 512, 1024, 1024], use_clstoken=False):
+        super(DPTHead, self).__init__()
+        
+        self.nclass = nclass
+        self.use_clstoken = use_clstoken
+        
+        self.projects = nn.ModuleList([
+            nn.Conv2d(
+                in_channels=in_channels,
+                out_channels=out_channel,
+                kernel_size=1,
+                stride=1,
+                padding=0,
+            ) for out_channel in out_channels
+        ])
+        
+        self.resize_layers = nn.ModuleList([
+            nn.ConvTranspose2d(
+                in_channels=out_channels[0],
+                out_channels=out_channels[0],
+                kernel_size=4,
+                stride=4,
+                padding=0),
+            nn.ConvTranspose2d(
+                in_channels=out_channels[1],
+                out_channels=out_channels[1],
+                kernel_size=2,
+                stride=2,
+                padding=0),
+            nn.Identity(),
+            nn.Conv2d(
+                in_channels=out_channels[3],
+                out_channels=out_channels[3],
+                kernel_size=3,
+                stride=2,
+                padding=1)
+        ])
+        
+        if use_clstoken:
+            self.readout_projects = nn.ModuleList()
+            for _ in range(len(self.projects)):
+                self.readout_projects.append(
+                    nn.Sequential(
+                        nn.Linear(2 * in_channels, in_channels),
+                        nn.GELU()))
+        
+        self.scratch = _make_scratch(
+            out_channels,
+            features,
+            groups=1,
+            expand=False,
+        )
+
+        self.scratch.stem_transpose = nn.Identity()
+        
+        self.scratch.refinenet1 = _make_fusion_block(features, use_bn)
+        self.scratch.refinenet2 = _make_fusion_block(features, use_bn)
+        self.scratch.refinenet3 = _make_fusion_block(features, use_bn)
+        self.scratch.refinenet4 = _make_fusion_block(features, use_bn)
+
+        head_features_1 = features
+        head_features_2 = 32
+        
+        if nclass > 1:
+            self.scratch.output_conv = nn.Sequential(
+                nn.Conv2d(head_features_1, head_features_1, kernel_size=3, stride=1, padding=1),
+                nn.ReLU(True),
+                nn.Conv2d(head_features_1, nclass, kernel_size=1, stride=1, padding=0),
+            )
+        else:
+            self.scratch.output_conv1 = nn.Conv2d(head_features_1, head_features_1 // 2, kernel_size=3, stride=1, padding=1)
+            
+            self.scratch.output_conv2 = nn.Sequential(
+                nn.Conv2d(head_features_1 // 2, head_features_2, kernel_size=3, stride=1, padding=1),
+                nn.ReLU(True),
+                nn.Conv2d(head_features_2, 1, kernel_size=1, stride=1, padding=0),
+                nn.ReLU(True),
+                nn.Identity(),
+            )
+            
+    def forward(self, out_features, patch_h, patch_w):
+        out = []
+        for i, x in enumerate(out_features):
+            if self.use_clstoken:
+                x, cls_token = x[0], x[1]
+                readout = cls_token.unsqueeze(1).expand_as(x)
+                x = self.readout_projects[i](torch.cat((x, readout), -1))
+            else:
+                x = x[0]
+            x = x.permute(0, 2, 1).reshape((x.shape[0], x.shape[-1], patch_h, patch_w)).contiguous()
+            
+            x = self.projects[i](x)
+            x = self.resize_layers[i](x)
+            
+            out.append(x)
+            
+        
+        layer_1, layer_2, layer_3, layer_4 = out
+        
+        layer_1_rn = self.scratch.layer1_rn(layer_1)
+        layer_2_rn = self.scratch.layer2_rn(layer_2)
+        layer_3_rn = self.scratch.layer3_rn(layer_3)
+        layer_4_rn = self.scratch.layer4_rn(layer_4)
+        
+        path_4 = self.scratch.refinenet4(layer_4_rn, size=layer_3_rn.shape[2:])
+        path_3 = self.scratch.refinenet3(path_4, layer_3_rn, size=layer_2_rn.shape[2:])
+        path_2 = self.scratch.refinenet2(path_3, layer_2_rn, size=layer_1_rn.shape[2:])
+        path_1 = self.scratch.refinenet1(path_2, layer_1_rn)
+        
+        out = self.scratch.output_conv1(path_1)
+        out = F.interpolate(out, (int(patch_h * 14), int(patch_w * 14)), mode="bilinear", align_corners=True)
+        out = self.scratch.output_conv2(out)
+        
+        return out
+        
+        
+class DPT_DINOv2(nn.Module):
+    def __init__(self, encoder='vitl', features=256, out_channels=[256, 512, 1024, 1024], use_bn=False, use_clstoken=False, localhub=True):
+        super(DPT_DINOv2, self).__init__()
+        
+        assert encoder in ['vits', 'vitb', 'vitl']
+        
+        # in case the Internet connection is not stable, please load the DINOv2 locally
+        if localhub:
+            self.pretrained = torch.hub.load('torchhub/facebookresearch_dinov2_main', 'dinov2_{:}14'.format(encoder), source='local', pretrained=True)
+        else:
+            self.pretrained = torch.hub.load('facebookresearch/dinov2', 'dinov2_{:}14'.format(encoder))
+         
+        dim = self.pretrained.blocks[0].attn.qkv.in_features
+        
+        self.depth_head = DPTHead(1, dim, features, use_bn, out_channels=out_channels, use_clstoken=use_clstoken)
+        
+    def forward(self, x):
+        h, w = x.shape[-2:]
+        
+        features = self.pretrained.get_intermediate_layers(x, 4, return_class_token=True)
+        patch_h, patch_w = h // 14, w // 14
+
+        depth = self.depth_head(features, patch_h, patch_w)
+        depth = F.interpolate(depth, size=(h, w), mode="bilinear", align_corners=True)
+        depth = F.relu(depth)
+
+
+        return depth.squeeze(1)
+
+
+class DepthAnythingTeacher(DPT_DINOv2):
+    def __init__(self, config):
+        super().__init__(**config)
+
+
+    

depth_anything/util/blocksv1.py

@@ -0,0 +1,153 @@
+import torch.nn as nn
+
+
+def _make_scratch(in_shape, out_shape, groups=1, expand=False):
+    scratch = nn.Module()
+
+    out_shape1 = out_shape
+    out_shape2 = out_shape
+    out_shape3 = out_shape
+    if len(in_shape) >= 4:
+        out_shape4 = out_shape
+
+    if expand:
+        out_shape1 = out_shape
+        out_shape2 = out_shape*2
+        out_shape3 = out_shape*4
+        if len(in_shape) >= 4:
+            out_shape4 = out_shape*8
+
+    scratch.layer1_rn = nn.Conv2d(
+        in_shape[0], out_shape1, kernel_size=3, stride=1, padding=1, bias=False, groups=groups
+    )
+    scratch.layer2_rn = nn.Conv2d(
+        in_shape[1], out_shape2, kernel_size=3, stride=1, padding=1, bias=False, groups=groups
+    )
+    scratch.layer3_rn = nn.Conv2d(
+        in_shape[2], out_shape3, kernel_size=3, stride=1, padding=1, bias=False, groups=groups
+    )
+    if len(in_shape) >= 4:
+        scratch.layer4_rn = nn.Conv2d(
+            in_shape[3], out_shape4, kernel_size=3, stride=1, padding=1, bias=False, groups=groups
+        )
+
+    return scratch
+
+
+class ResidualConvUnit(nn.Module):
+    """Residual convolution module.
+    """
+
+    def __init__(self, features, activation, bn):
+        """Init.
+
+        Args:
+            features (int): number of features
+        """
+        super().__init__()
+
+        self.bn = bn
+
+        self.groups=1
+
+        self.conv1 = nn.Conv2d(
+            features, features, kernel_size=3, stride=1, padding=1, bias=True, groups=self.groups
+        )
+        
+        self.conv2 = nn.Conv2d(
+            features, features, kernel_size=3, stride=1, padding=1, bias=True, groups=self.groups
+        )
+
+        if self.bn==True:
+            self.bn1 = nn.BatchNorm2d(features)
+            self.bn2 = nn.BatchNorm2d(features)
+
+        self.activation = activation
+
+        self.skip_add = nn.quantized.FloatFunctional()
+
+    def forward(self, x):
+        """Forward pass.
+
+        Args:
+            x (tensor): input
+
+        Returns:
+            tensor: output
+        """
+        
+        out = self.activation(x)
+        out = self.conv1(out)
+        if self.bn==True:
+            out = self.bn1(out)
+       
+        out = self.activation(out)
+        out = self.conv2(out)
+        if self.bn==True:
+            out = self.bn2(out)
+
+        if self.groups > 1:
+            out = self.conv_merge(out)
+
+        return self.skip_add.add(out, x)
+
+
+class FeatureFusionBlock(nn.Module):
+    """Feature fusion block.
+    """
+
+    def __init__(self, features, activation, deconv=False, bn=False, expand=False, align_corners=True, size=None):
+        """Init.
+
+        Args:
+            features (int): number of features
+        """
+        super(FeatureFusionBlock, self).__init__()
+
+        self.deconv = deconv
+        self.align_corners = align_corners
+
+        self.groups=1
+
+        self.expand = expand
+        out_features = features
+        if self.expand==True:
+            out_features = features//2
+        
+        self.out_conv = nn.Conv2d(features, out_features, kernel_size=1, stride=1, padding=0, bias=True, groups=1)
+
+        self.resConfUnit1 = ResidualConvUnit(features, activation, bn)
+        self.resConfUnit2 = ResidualConvUnit(features, activation, bn)
+        
+        self.skip_add = nn.quantized.FloatFunctional()
+
+        self.size=size
+
+    def forward(self, *xs, size=None):
+        """Forward pass.
+
+        Returns:
+            tensor: output
+        """
+        output = xs[0]
+
+        if len(xs) == 2:
+            res = self.resConfUnit1(xs[1])
+            output = self.skip_add.add(output, res)
+
+        output = self.resConfUnit2(output)
+
+        if (size is None) and (self.size is None):
+            modifier = {"scale_factor": 2}
+        elif size is None:
+            modifier = {"size": self.size}
+        else:
+            modifier = {"size": size}
+
+        output = nn.functional.interpolate(
+            output, **modifier, mode="bilinear", align_corners=self.align_corners
+        )
+
+        output = self.out_conv(output)
+
+        return output

depth_anything/util/blocksv2.py

@@ -0,0 +1,148 @@
+import torch.nn as nn
+
+
+def _make_scratch(in_shape, out_shape, groups=1, expand=False):
+    scratch = nn.Module()
+
+    out_shape1 = out_shape
+    out_shape2 = out_shape
+    out_shape3 = out_shape
+    if len(in_shape) >= 4:
+        out_shape4 = out_shape
+
+    if expand:
+        out_shape1 = out_shape
+        out_shape2 = out_shape * 2
+        out_shape3 = out_shape * 4
+        if len(in_shape) >= 4:
+            out_shape4 = out_shape * 8
+
+    scratch.layer1_rn = nn.Conv2d(in_shape[0], out_shape1, kernel_size=3, stride=1, padding=1, bias=False, groups=groups)
+    scratch.layer2_rn = nn.Conv2d(in_shape[1], out_shape2, kernel_size=3, stride=1, padding=1, bias=False, groups=groups)
+    scratch.layer3_rn = nn.Conv2d(in_shape[2], out_shape3, kernel_size=3, stride=1, padding=1, bias=False, groups=groups)
+    if len(in_shape) >= 4:
+        scratch.layer4_rn = nn.Conv2d(in_shape[3], out_shape4, kernel_size=3, stride=1, padding=1, bias=False, groups=groups)
+
+    return scratch
+
+
+class ResidualConvUnit(nn.Module):
+    """Residual convolution module.
+    """
+
+    def __init__(self, features, activation, bn):
+        """Init.
+
+        Args:
+            features (int): number of features
+        """
+        super().__init__()
+
+        self.bn = bn
+
+        self.groups=1
+
+        self.conv1 = nn.Conv2d(features, features, kernel_size=3, stride=1, padding=1, bias=True, groups=self.groups)
+        
+        self.conv2 = nn.Conv2d(features, features, kernel_size=3, stride=1, padding=1, bias=True, groups=self.groups)
+
+        if self.bn == True:
+            self.bn1 = nn.BatchNorm2d(features)
+            self.bn2 = nn.BatchNorm2d(features)
+
+        self.activation = activation
+
+        self.skip_add = nn.quantized.FloatFunctional()
+
+    def forward(self, x):
+        """Forward pass.
+
+        Args:
+            x (tensor): input
+
+        Returns:
+            tensor: output
+        """
+        
+        out = self.activation(x)
+        out = self.conv1(out)
+        if self.bn == True:
+            out = self.bn1(out)
+       
+        out = self.activation(out)
+        out = self.conv2(out)
+        if self.bn == True:
+            out = self.bn2(out)
+
+        if self.groups > 1:
+            out = self.conv_merge(out)
+
+        return self.skip_add.add(out, x)
+
+
+class FeatureFusionBlock(nn.Module):
+    """Feature fusion block.
+    """
+
+    def __init__(
+        self, 
+        features, 
+        activation, 
+        deconv=False, 
+        bn=False, 
+        expand=False, 
+        align_corners=True,
+        size=None
+    ):
+        """Init.
+        
+        Args:
+            features (int): number of features
+        """
+        super(FeatureFusionBlock, self).__init__()
+
+        self.deconv = deconv
+        self.align_corners = align_corners
+
+        self.groups=1
+
+        self.expand = expand
+        out_features = features
+        if self.expand == True:
+            out_features = features // 2
+        
+        self.out_conv = nn.Conv2d(features, out_features, kernel_size=1, stride=1, padding=0, bias=True, groups=1)
+
+        self.resConfUnit1 = ResidualConvUnit(features, activation, bn)
+        self.resConfUnit2 = ResidualConvUnit(features, activation, bn)
+        
+        self.skip_add = nn.quantized.FloatFunctional()
+
+        self.size=size
+
+    def forward(self, *xs, size=None):
+        """Forward pass.
+
+        Returns:
+            tensor: output
+        """
+        output = xs[0]
+
+        if len(xs) == 2:
+            res = self.resConfUnit1(xs[1])
+            output = self.skip_add.add(output, res)
+
+        output = self.resConfUnit2(output)
+
+        if (size is None) and (self.size is None):
+            modifier = {"scale_factor": 2}
+        elif size is None:
+            modifier = {"size": self.size}
+        else:
+            modifier = {"size": size}
+
+        output = nn.functional.interpolate(output, **modifier, mode="bilinear", align_corners=self.align_corners)
+        
+        output = self.out_conv(output)
+
+        return output

depth_anything/util/dformerv2.py

@@ -0,0 +1,622 @@
+'''
+DFormerv2: Geometry Self-Attention for RGBD Semantic Segmentation
+Code: https://github.com/VCIP-RGBD/DFormer
+
+Author: yinbow
+Email: [email protected]
+
+This source code is licensed under the license found in the
+LICENSE file in the root directory of this source tree.
+'''
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+import torch.utils.checkpoint as checkpoint
+import math
+from timm.models.layers import DropPath, trunc_normal_
+from typing import List
+from mmengine.runner.checkpoint import load_state_dict
+from mmengine.runner.checkpoint import load_checkpoint
+from typing import Tuple
+import sys
+import os
+from collections import OrderedDict
+
+class LayerNorm2d(nn.Module):
+    def __init__(self, dim):
+        super().__init__()
+        self.norm = nn.LayerNorm(dim, eps=1e-6)
+
+    def forward(self, x: torch.Tensor):
+        '''
+        input shape (b c h w)
+        '''
+        x = x.permute(0, 2, 3, 1).contiguous() #(b h w c)
+        x = self.norm(x) #(b h w c)
+        x = x.permute(0, 3, 1, 2).contiguous()
+        return x
+
+class PatchEmbed(nn.Module):
+    """ 
+    Image to Patch Embedding
+    """
+
+    def __init__(self, in_chans=3, embed_dim=96, norm_layer=None):
+        super().__init__()
+        self.in_chans = in_chans
+        self.embed_dim = embed_dim
+
+        self.proj = nn.Sequential(
+            nn.Conv2d(in_chans, embed_dim//2, 3, 2, 1),
+            nn.SyncBatchNorm(embed_dim//2),
+            nn.GELU(),
+            nn.Conv2d(embed_dim//2, embed_dim//2, 3, 1, 1),
+            nn.SyncBatchNorm(embed_dim//2),
+            nn.GELU(),
+            nn.Conv2d(embed_dim//2, embed_dim, 3, 2, 1),
+            nn.SyncBatchNorm(embed_dim),
+            nn.GELU(),
+            nn.Conv2d(embed_dim, embed_dim, 3, 1, 1),
+            nn.SyncBatchNorm(embed_dim)
+        )
+
+    def forward(self, x):
+        B, C, H, W = x.shape
+        x = self.proj(x).permute(0, 2, 3, 1)
+        return x
+
+class DWConv2d(nn.Module):
+
+    def __init__(self, dim, kernel_size, stride, padding):
+        super().__init__()
+        self.dwconv = nn.Conv2d(dim, dim, kernel_size, stride, padding, groups=dim)
+
+    def forward(self, x: torch.Tensor):
+        '''
+        input (b h w c)
+        '''
+        x = x.permute(0, 3, 1, 2) 
+        x = self.dwconv(x)
+        x = x.permute(0, 2, 3, 1)
+        return x
+
+class PatchMerging(nn.Module):
+    """ 
+    Patch Merging Layer.
+    """
+    def __init__(self, dim, out_dim, norm_layer=nn.LayerNorm):
+        super().__init__()
+        self.dim = dim
+        self.reduction = nn.Conv2d(dim, out_dim, 3, 2, 1)
+        self.norm = nn.SyncBatchNorm(out_dim)
+
+    def forward(self, x):
+        '''
+        x: B H W C
+        '''
+        x = x.permute(0, 3, 1, 2).contiguous()  #(b c h w)
+        x = self.reduction(x) #(b oc oh ow)
+        x = self.norm(x)
+        x = x.permute(0, 2, 3, 1) #(b oh ow oc)
+        return x
+
+def angle_transform(x, sin, cos):
+    x1 = x[:, :, :, :, ::2]
+    x2 = x[:, :, :, :, 1::2]
+    return (x * cos) + (torch.stack([-x2, x1], dim=-1).flatten(-2) * sin)
+
+class GeoPriorGen(nn.Module):
+
+    def __init__(self, embed_dim, num_heads, initial_value, heads_range):
+        super().__init__()
+        angle = 1.0 / (10000 ** torch.linspace(0, 1, embed_dim // num_heads // 2))
+        angle = angle.unsqueeze(-1).repeat(1, 2).flatten()
+        self.weight = nn.Parameter(torch.ones(2,1,1,1), requires_grad=True)
+        decay = torch.log(1 - 2 ** (-initial_value - heads_range * torch.arange(num_heads, dtype=torch.float) / num_heads))
+        self.register_buffer('angle', angle)
+        self.register_buffer('decay', decay)
+
+    def generate_depth_decay(self, H: int, W: int, depth_grid):
+        '''
+        generate 2d decay mask, the result is (HW)*(HW)
+        H, W are the numbers of patches at each column and row
+        '''
+        B,_,H,W = depth_grid.shape
+        grid_d = depth_grid.reshape(B, H*W, 1)
+        mask_d = grid_d[:, :, None, :] - grid_d[:, None, :, :] 
+        mask_d = (mask_d.abs()).sum(dim=-1)
+        mask_d = mask_d.unsqueeze(1) * self.decay[None, :, None, None] 
+        return mask_d
+        
+    def generate_pos_decay(self, H: int, W: int):
+        '''
+        generate 2d decay mask, the result is (HW)*(HW)
+        H, W are the numbers of patches at each column and row
+        '''
+        index_h = torch.arange(H).to(self.decay)
+        index_w = torch.arange(W).to(self.decay)
+        grid = torch.meshgrid([index_h, index_w])
+        grid = torch.stack(grid, dim=-1).reshape(H*W, 2) 
+        mask = grid[:, None, :] - grid[None, :, :] 
+        mask = (mask.abs()).sum(dim=-1)
+        mask = mask * self.decay[:, None, None] 
+        return mask
+    
+    def generate_1d_depth_decay(self, H, W, depth_grid):
+        '''
+        generate 1d depth decay mask, the result is l*l
+        '''
+        mask = depth_grid[:, :, :, :, None] - depth_grid[:, :, :, None, :]
+        mask = mask.abs()
+        mask = mask * self.decay[:, None, None, None]
+        assert mask.shape[2:] == (W,H,H)
+        return mask
+    
+    
+    def generate_1d_decay(self, l: int):
+        '''
+        generate 1d decay mask, the result is l*l
+        '''
+        index = torch.arange(l).to(self.decay)
+        mask = index[:, None] - index[None, :] 
+        mask = mask.abs()
+        mask = mask * self.decay[:, None, None] 
+        return mask
+    
+    def forward(self, HW_tuple: Tuple[int], depth_map, split_or_not=False):
+        '''
+        depth_map: depth patches  
+        HW_tuple: (H, W)
+        H * W == l
+        '''
+        depth_map = F.interpolate(depth_map, size=HW_tuple,mode='bilinear',align_corners=False)
+
+        if split_or_not:
+            index = torch.arange(HW_tuple[0]*HW_tuple[1]).to(self.decay)
+            sin = torch.sin(index[:, None] * self.angle[None, :]) 
+            sin = sin.reshape(HW_tuple[0], HW_tuple[1], -1) 
+            cos = torch.cos(index[:, None] * self.angle[None, :]) 
+            cos = cos.reshape(HW_tuple[0], HW_tuple[1], -1) 
+
+            mask_d_h = self.generate_1d_depth_decay(HW_tuple[0], HW_tuple[1], depth_map.transpose(-2,-1))
+            mask_d_w = self.generate_1d_depth_decay(HW_tuple[1], HW_tuple[0], depth_map)
+
+
+            mask_h = self.generate_1d_decay(HW_tuple[0])
+            mask_w = self.generate_1d_decay(HW_tuple[1])
+
+            mask_h = self.weight[0]*mask_h.unsqueeze(0).unsqueeze(2) + self.weight[1]*mask_d_h
+            mask_w = self.weight[0]*mask_w.unsqueeze(0).unsqueeze(2) + self.weight[1]*mask_d_w 
+            
+
+            geo_prior = ((sin, cos), (mask_h, mask_w))
+
+        else:
+            index = torch.arange(HW_tuple[0]*HW_tuple[1]).to(self.decay)
+            sin = torch.sin(index[:, None] * self.angle[None, :]) 
+            sin = sin.reshape(HW_tuple[0], HW_tuple[1], -1) 
+            cos = torch.cos(index[:, None] * self.angle[None, :]) 
+            cos = cos.reshape(HW_tuple[0], HW_tuple[1], -1) 
+            mask = self.generate_pos_decay(HW_tuple[0], HW_tuple[1]) 
+
+            mask_d = self.generate_depth_decay(HW_tuple[0], HW_tuple[1], depth_map)
+            mask = (self.weight[0]*mask+self.weight[1]*mask_d)
+
+            geo_prior = ((sin, cos), mask)
+
+        return geo_prior
+    
+class Decomposed_GSA(nn.Module):
+
+    def __init__(self, embed_dim, num_heads, value_factor=1):
+        super().__init__()
+        self.factor = value_factor
+        self.embed_dim = embed_dim
+        self.num_heads = num_heads
+        self.head_dim = self.embed_dim * self.factor // num_heads
+        self.key_dim = self.embed_dim // num_heads
+        self.scaling = self.key_dim ** -0.5
+        self.q_proj = nn.Linear(embed_dim, embed_dim, bias=True)
+        self.k_proj = nn.Linear(embed_dim, embed_dim, bias=True)
+        self.v_proj = nn.Linear(embed_dim, embed_dim * self.factor, bias=True)
+        self.lepe = DWConv2d(embed_dim, 5, 1, 2)
+
+        self.out_proj = nn.Linear(embed_dim*self.factor, embed_dim, bias=True)
+        self.reset_parameters()
+
+    def forward(self, x: torch.Tensor, rel_pos, split_or_not=False):
+
+        bsz, h, w, _ = x.size()
+
+        (sin, cos), (mask_h, mask_w) = rel_pos
+
+        q = self.q_proj(x)
+        k = self.k_proj(x)
+        v = self.v_proj(x)
+        lepe = self.lepe(v)
+
+        k = k * self.scaling
+        q = q.view(bsz, h, w, self.num_heads, self.key_dim).permute(0, 3, 1, 2, 4) #(b n h w d1)
+        k = k.view(bsz, h, w, self.num_heads, self.key_dim).permute(0, 3, 1, 2, 4) #(b n h w d1)
+        qr = angle_transform(q, sin, cos)
+        kr = angle_transform(k, sin, cos)
+        
+        qr_w = qr.transpose(1, 2)
+        kr_w = kr.transpose(1, 2)
+        v = v.reshape(bsz, h, w, self.num_heads, -1).permute(0, 1, 3, 2, 4)
+
+        qk_mat_w = qr_w @ kr_w.transpose(-1, -2)
+        qk_mat_w = qk_mat_w + mask_w.transpose(1,2)
+        qk_mat_w = torch.softmax(qk_mat_w, -1)
+        v = torch.matmul(qk_mat_w, v)
+
+
+        qr_h = qr.permute(0, 3, 1, 2, 4)
+        kr_h = kr.permute(0, 3, 1, 2, 4)
+        v = v.permute(0, 3, 2, 1, 4)
+
+        qk_mat_h = qr_h @ kr_h.transpose(-1, -2)
+        qk_mat_h = qk_mat_h + mask_h.transpose(1,2)
+        qk_mat_h = torch.softmax(qk_mat_h, -1)
+        output = torch.matmul(qk_mat_h, v)
+        
+        output = output.permute(0, 3, 1, 2, 4).flatten(-2, -1)
+        output = output + lepe
+        output = self.out_proj(output)
+        return output
+    
+    def reset_parameters(self):
+        nn.init.xavier_normal_(self.q_proj.weight, gain=2 ** -2.5)
+        nn.init.xavier_normal_(self.k_proj.weight, gain=2 ** -2.5)
+        nn.init.xavier_normal_(self.v_proj.weight, gain=2 ** -2.5)
+        nn.init.xavier_normal_(self.out_proj.weight)
+        nn.init.constant_(self.out_proj.bias, 0.0)
+    
+class Full_GSA(nn.Module):
+
+    def __init__(self, embed_dim, num_heads, value_factor=1):
+        super().__init__()
+        self.factor = value_factor
+        self.embed_dim = embed_dim
+        self.num_heads = num_heads
+        self.head_dim = self.embed_dim * self.factor // num_heads
+        self.key_dim = self.embed_dim // num_heads
+        self.scaling = self.key_dim ** -0.5
+        self.q_proj = nn.Linear(embed_dim, embed_dim, bias=True)
+        self.k_proj = nn.Linear(embed_dim, embed_dim, bias=True)
+        self.v_proj = nn.Linear(embed_dim, embed_dim * self.factor, bias=True)
+        self.lepe = DWConv2d(embed_dim, 5, 1, 2)
+        self.out_proj = nn.Linear(embed_dim*self.factor, embed_dim, bias=True)
+        self.reset_parameters()
+
+    def forward(self, x: torch.Tensor, rel_pos, split_or_not=False):
+        '''
+        x: (b h w c)
+        rel_pos: mask: (n l l)
+        '''
+        bsz, h, w, _ = x.size()
+        (sin, cos), mask = rel_pos
+        assert h*w == mask.size(3)
+        q = self.q_proj(x)
+        k = self.k_proj(x)
+        v = self.v_proj(x)
+        lepe = self.lepe(v)
+
+        k = k * self.scaling
+        q = q.view(bsz, h, w, self.num_heads, -1).permute(0, 3, 1, 2, 4)
+        k = k.view(bsz, h, w, self.num_heads, -1).permute(0, 3, 1, 2, 4)
+        qr = angle_transform(q, sin, cos)
+        kr = angle_transform(k, sin, cos)
+
+        qr = qr.flatten(2, 3)
+        kr = kr.flatten(2, 3)
+        vr = v.reshape(bsz, h, w, self.num_heads, -1).permute(0, 3, 1, 2, 4)
+        vr = vr.flatten(2, 3)
+        qk_mat = qr @ kr.transpose(-1, -2)
+        qk_mat = qk_mat + mask
+        qk_mat = torch.softmax(qk_mat, -1)
+        output = torch.matmul(qk_mat, vr)
+        output = output.transpose(1, 2).reshape(bsz, h, w, -1)
+        output = output + lepe
+        output = self.out_proj(output)
+        return output
+    
+    def reset_parameters(self):
+        nn.init.xavier_normal_(self.q_proj.weight, gain=2 ** -2.5)
+        nn.init.xavier_normal_(self.k_proj.weight, gain=2 ** -2.5)
+        nn.init.xavier_normal_(self.v_proj.weight, gain=2 ** -2.5)
+        nn.init.xavier_normal_(self.out_proj.weight)
+        nn.init.constant_(self.out_proj.bias, 0.0)
+
+class FeedForwardNetwork(nn.Module):
+    def __init__(
+        self,
+        embed_dim,
+        ffn_dim,
+        activation_fn=F.gelu,
+        dropout=0.0,
+        activation_dropout=0.0,
+        layernorm_eps=1e-6,
+        subln=False,
+        subconv=True
+        ):
+        super().__init__()
+        self.embed_dim = embed_dim
+        self.activation_fn = activation_fn
+        self.activation_dropout_module = torch.nn.Dropout(activation_dropout)
+        self.dropout_module = torch.nn.Dropout(dropout)
+        self.fc1 = nn.Linear(self.embed_dim, ffn_dim)
+        self.fc2 = nn.Linear(ffn_dim, self.embed_dim)
+        self.ffn_layernorm = nn.LayerNorm(ffn_dim, eps=layernorm_eps) if subln else None
+        self.dwconv = DWConv2d(ffn_dim, 3, 1, 1) if subconv else None
+
+    def reset_parameters(self):
+        self.fc1.reset_parameters()
+        self.fc2.reset_parameters()
+        if self.ffn_layernorm is not None:
+            self.ffn_layernorm.reset_parameters()
+
+    def forward(self, x: torch.Tensor):
+        '''
+        input shape: (b h w c)
+        '''
+        x = self.fc1(x)
+        x = self.activation_fn(x)
+        x = self.activation_dropout_module(x)
+        residual = x
+        if self.dwconv is not None:
+            x = self.dwconv(x)
+        if self.ffn_layernorm is not None:
+            x = self.ffn_layernorm(x)
+        x = x + residual
+        x = self.fc2(x)
+        x = self.dropout_module(x)
+        return x
+    
+class RGBD_Block(nn.Module):
+
+    def __init__(self, split_or_not: str, embed_dim: int, num_heads: int, ffn_dim: int, drop_path=0., layerscale=False, layer_init_values=1e-5, init_value=2, heads_range=4):
+        super().__init__()
+        self.layerscale = layerscale
+        self.embed_dim = embed_dim
+        self.layer_norm1 = nn.LayerNorm(self.embed_dim, eps=1e-6)
+        self.layer_norm2 = nn.LayerNorm(self.embed_dim, eps=1e-6)
+        if split_or_not:
+            self.Attention = Decomposed_GSA(embed_dim, num_heads)
+        else:
+            self.Attention = Full_GSA(embed_dim, num_heads)
+        self.drop_path = DropPath(drop_path)
+        # FFN
+        self.ffn = FeedForwardNetwork(embed_dim, ffn_dim)
+        self.cnn_pos_encode = DWConv2d(embed_dim, 3, 1, 1)
+        # the function to generate the geometry prior for the current block
+        self.Geo = GeoPriorGen(embed_dim, num_heads, init_value, heads_range)
+
+        if layerscale:
+            self.gamma_1 = nn.Parameter(layer_init_values * torch.ones(1, 1, 1, embed_dim),requires_grad=True)
+            self.gamma_2 = nn.Parameter(layer_init_values * torch.ones(1, 1, 1, embed_dim),requires_grad=True)
+
+    def forward(
+            self,
+            x: torch.Tensor, 
+            x_e: torch.Tensor,
+            split_or_not=False
+        ):
+        x = x + self.cnn_pos_encode(x)
+        b, h, w, d = x.size()
+
+        geo_prior = self.Geo((h, w), x_e, split_or_not=split_or_not)
+        if self.layerscale:
+            x = x + self.drop_path(self.gamma_1 * self.Attention(self.layer_norm1(x), geo_prior, split_or_not))
+            x = x + self.drop_path(self.gamma_2 * self.ffn(self.layer_norm2(x)))
+        else:
+            x = x + self.drop_path(self.Attention(self.layer_norm1(x), geo_prior, split_or_not))
+            x = x + self.drop_path(self.ffn(self.layer_norm2(x)))
+        return x
+     
+class BasicLayer(nn.Module):
+    """ 
+    A basic RGB-D layer in DFormerv2.
+    """
+
+    def __init__(self, embed_dim, out_dim, depth, num_heads,
+                 init_value: float, heads_range: float,
+                 ffn_dim=96., drop_path=0., norm_layer=nn.LayerNorm, split_or_not=False,
+                 downsample: PatchMerging=None, use_checkpoint=False,
+                 layerscale=False, layer_init_values=1e-5):
+
+        super().__init__()
+        self.embed_dim = embed_dim
+        self.depth = depth
+        self.use_checkpoint = use_checkpoint
+        self.split_or_not = split_or_not
+
+        # build blocks
+        self.blocks = nn.ModuleList([
+            RGBD_Block(split_or_not, embed_dim, num_heads, ffn_dim, 
+                     drop_path[i] if isinstance(drop_path, list) else drop_path, layerscale, layer_init_values, init_value=init_value, heads_range=heads_range)
+            for i in range(depth)])
+
+        # patch merging layer
+        if downsample is not None:
+            self.downsample = downsample(dim=embed_dim, out_dim=out_dim, norm_layer=norm_layer)
+        else:
+            self.downsample = None
+
+    def forward(self, x, x_e):
+        b, h, w, d = x.size()
+        for blk in self.blocks:
+            if self.use_checkpoint:
+                x = checkpoint.checkpoint(blk, x=x, x_e=x_e, split_or_not=self.split_or_not)
+            else:
+                x = blk(x, x_e, split_or_not=self.split_or_not)
+        if self.downsample is not None:
+            x_down = self.downsample(x)
+            return x, x_down
+        else:
+            return x, x
+    
+class dformerv2(nn.Module):
+
+    def __init__(self, out_indices=(0, 1, 2, 3),
+                 embed_dims=[64, 128, 256, 512], depths=[2, 2, 8, 2], num_heads=[4, 4, 8, 16],
+                 init_values=[2, 2, 2, 2], heads_ranges=[4, 4, 6, 6], mlp_ratios=[4, 4, 3, 3], drop_path_rate=0.1, norm_layer=nn.LayerNorm, 
+                 patch_norm=True, use_checkpoint=False, projection=1024, norm_cfg = None,
+                 layerscales=[False, False, False, False], layer_init_values=1e-6, norm_eval=True):
+        super().__init__()
+        self.out_indices = out_indices
+        self.num_layers = len(depths)
+        self.embed_dim = embed_dims[0]
+        self.patch_norm = patch_norm
+        self.num_features = embed_dims[-1]
+        self.mlp_ratios = mlp_ratios
+        self.norm_eval = norm_eval
+
+        # patch embedding
+        self.patch_embed = PatchEmbed(in_chans=3, embed_dim=embed_dims[0],
+            norm_layer=norm_layer if self.patch_norm else None)
+
+
+        # drop path rate
+        dpr = [x.item() for x in torch.linspace(0, drop_path_rate, sum(depths))]  
+
+        # build layers
+        self.layers = nn.ModuleList()
+
+        for i_layer in range(self.num_layers):
+            layer = BasicLayer(
+                embed_dim=embed_dims[i_layer],
+                out_dim=embed_dims[i_layer+1] if (i_layer < self.num_layers - 1) else None,
+                depth=depths[i_layer],
+                num_heads=num_heads[i_layer],
+                init_value=init_values[i_layer],
+                heads_range=heads_ranges[i_layer],
+                ffn_dim=int(mlp_ratios[i_layer]*embed_dims[i_layer]),
+                drop_path=dpr[sum(depths[:i_layer]):sum(depths[:i_layer + 1])],
+                norm_layer=norm_layer,
+                split_or_not=(i_layer!=3),
+                downsample=PatchMerging if (i_layer < self.num_layers - 1) else None,
+                use_checkpoint=use_checkpoint,
+                layerscale=layerscales[i_layer],
+                layer_init_values=layer_init_values
+            )
+            self.layers.append(layer)
+
+        self.extra_norms = nn.ModuleList()
+        for i in range(3):
+            self.extra_norms.append(nn.LayerNorm(embed_dims[i+1]))
+
+        self.apply(self._init_weights)
+
+    def _init_weights(self, m):
+        if isinstance(m, nn.Linear):
+            trunc_normal_(m.weight, std=.02)
+            if isinstance(m, nn.Linear) and m.bias is not None:
+                nn.init.constant_(m.bias, 0)
+        elif isinstance(m, nn.LayerNorm):
+            try:
+                nn.init.constant_(m.bias, 0)
+                nn.init.constant_(m.weight, 1.0)
+            except:
+                pass
+
+    def init_weights(self, pretrained=None):
+        """Initialize the weights in backbone.
+
+        Args:
+            pretrained (str, optional): Path to pre-trained weights.
+                Defaults to None.
+        """
+
+        def _init_weights(m):
+            if isinstance(m, nn.Linear):
+                trunc_normal_(m.weight, std=.02)
+                if isinstance(m, nn.Linear) and m.bias is not None:
+                    nn.init.constant_(m.bias, 0)
+            elif isinstance(m, nn.LayerNorm):
+                nn.init.constant_(m.bias, 0)
+                nn.init.constant_(m.weight, 1.0)
+
+        if isinstance(pretrained, str):
+            self.apply(_init_weights)
+            # logger = get_root_logger()
+            _state_dict = torch.load(pretrained)
+            if 'model' in _state_dict.keys():
+                _state_dict=_state_dict['model']
+            if 'state_dict' in _state_dict.keys():
+                _state_dict=_state_dict['state_dict']
+            state_dict = OrderedDict()
+
+            for k, v in _state_dict.items():
+                if k.startswith('backbone.'):
+                    state_dict[k[9:]] = v
+                else:
+                    state_dict[k] = v
+            print('load '+pretrained)
+            load_state_dict(self, state_dict, strict=False)
+            # load_checkpoint(self, pretrained, strict=False)
+            # load_checkpoint(self, pretrained, strict=False, logger=logger)
+        elif pretrained is None:
+            self.apply(_init_weights)
+        else:
+            raise TypeError('pretrained must be a str or None')
+
+    @torch.jit.ignore
+    def no_weight_decay(self):
+        return {'absolute_pos_embed'}
+
+    @torch.jit.ignore
+    def no_weight_decay_keywords(self):
+        return {'relative_position_bias_table'}
+
+    def forward(self, x, x_e):
+        # rgb input
+        x = self.patch_embed(x)
+        # depth input
+        x_e = x_e[:,0,:,:].unsqueeze(1)
+
+        outs = []
+
+        for i in range(self.num_layers):
+            layer = self.layers[i]
+            x_out, x = layer(x, x_e)
+            if i in self.out_indices:
+                if i != 0:
+                    x_out = self.extra_norms[i-1](x_out)
+                out = x_out.permute(0, 3, 1, 2).contiguous()
+                outs.append(out)
+        
+        return tuple(outs)
+
+    
+    def train(self, mode=True):
+        """Convert the model into training mode while keep normalization layer
+        freezed."""
+        super().train(mode)
+        if mode and self.norm_eval:
+            for m in self.modules():
+                # trick: eval have effect on BatchNorm only
+                if isinstance(m, nn.BatchNorm2d):
+                    m.eval()
+
+def DFormerv2_S(pretrained=False, **kwargs):
+    model = dformerv2(embed_dims=[64, 128, 256, 512], depths=[3, 4, 18, 4], num_heads=[4, 4, 8, 16],
+                heads_ranges=[4, 4, 6, 6], **kwargs)
+    return model
+
+def DFormerv2_B(pretrained=False, **kwargs):
+    model = dformerv2(embed_dims=[80, 160, 320, 512], depths=[4, 8, 25, 8], num_heads=[5, 5, 10, 16],
+                heads_ranges=[5, 5, 6, 6],
+        layerscales=[False, False, True, True],
+        layer_init_values=1e-6,  **kwargs)
+    return model
+
+def DFormerv2_L(pretrained=False, **kwargs):
+    model = dformerv2(embed_dims=[112, 224, 448, 640], depths=[4, 8, 25, 8], num_heads=[7, 7, 14, 20],
+                heads_ranges=[6, 6, 6, 6],
+        layerscales=[False, False, True, True],
+        layer_init_values=1e-6,  **kwargs) 
+    return model
+
+

depth_anything/util/priorgenerate.py

@@ -0,0 +1,103 @@
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from typing import Tuple
+
+
+
+class GeoPriorGen(nn.Module):
+
+    def __init__(self,weight=[0.5,0.5]):
+        super().__init__()
+        self.weight = weight
+        
+
+    def generate_depth_decay(self, H: int, W: int, depth_grid):
+        '''
+        generate 2d decay mask, the result is (HW)*(HW)
+        H, W are the numbers of patches at each column and row
+        '''
+        B,_,H,W = depth_grid.shape
+        grid_d = depth_grid.reshape(B, H*W, 1)
+        mask_d = grid_d[:, :, None, :] - grid_d[:, None, :, :] 
+        mask_d = (mask_d.abs()).sum(dim=-1)
+        mask_d = mask_d.unsqueeze(1) * self.decay[None, :, None, None] 
+        return mask_d
+        
+    def generate_pos_decay(self, H: int, W: int):
+        '''
+        generate 2d decay mask, the result is (HW)*(HW)
+        H, W are the numbers of patches at each column and row
+        '''
+        index_h = torch.arange(H).to(self.decay)
+        index_w = torch.arange(W).to(self.decay)
+        grid = torch.meshgrid([index_h, index_w])
+        grid = torch.stack(grid, dim=-1).reshape(H*W, 2) 
+        mask = grid[:, None, :] - grid[None, :, :] 
+        mask = (mask.abs()).sum(dim=-1)
+        mask = mask * self.decay[:, None, None] 
+        return mask
+    
+    def generate_1d_depth_decay(self, H, W, depth_grid):
+        '''
+        generate 1d depth decay mask, the result is l*l
+        '''
+        mask = depth_grid[:, :, :, :, None] - depth_grid[:, :, :, None, :]
+        mask = mask.abs()
+        mask = mask * self.decay[:, None, None, None]
+        assert mask.shape[2:] == (W,H,H)
+        return mask
+    
+    
+    def generate_1d_decay(self, l: int):
+        '''
+        generate 1d decay mask, the result is l*l
+        '''
+        index = torch.arange(l).to(self.decay)
+        mask = index[:, None] - index[None, :] 
+        mask = mask.abs()
+        mask = mask * self.decay[:, None, None] 
+        return mask
+    
+    def forward(self, HW_tuple: Tuple[int], depth_map, split_or_not=False):
+        '''
+        depth_map: depth patches  
+        HW_tuple: (H, W)
+        H * W == l
+        '''
+        depth_map = F.interpolate(depth_map, size=HW_tuple,mode='bilinear',align_corners=False)
+
+        if split_or_not:
+            index = torch.arange(HW_tuple[0]*HW_tuple[1]).to(self.decay)
+            sin = torch.sin(index[:, None] * self.angle[None, :]) 
+            sin = sin.reshape(HW_tuple[0], HW_tuple[1], -1) 
+            cos = torch.cos(index[:, None] * self.angle[None, :]) 
+            cos = cos.reshape(HW_tuple[0], HW_tuple[1], -1) 
+
+            mask_d_h = self.generate_1d_depth_decay(HW_tuple[0], HW_tuple[1], depth_map.transpose(-2,-1))
+            mask_d_w = self.generate_1d_depth_decay(HW_tuple[1], HW_tuple[0], depth_map)
+
+
+            mask_h = self.generate_1d_decay(HW_tuple[0])
+            mask_w = self.generate_1d_decay(HW_tuple[1])
+
+            mask_h = self.weight[0]*mask_h.unsqueeze(0).unsqueeze(2) + self.weight[1]*mask_d_h
+            mask_w = self.weight[0]*mask_w.unsqueeze(0).unsqueeze(2) + self.weight[1]*mask_d_w 
+            
+
+            geo_prior = ((sin, cos), (mask_h, mask_w))
+
+        else:
+            index = torch.arange(HW_tuple[0]*HW_tuple[1]).to(self.decay)
+            sin = torch.sin(index[:, None] * self.angle[None, :]) 
+            sin = sin.reshape(HW_tuple[0], HW_tuple[1], -1) 
+            cos = torch.cos(index[:, None] * self.angle[None, :]) 
+            cos = cos.reshape(HW_tuple[0], HW_tuple[1], -1) 
+            mask = self.generate_pos_decay(HW_tuple[0], HW_tuple[1]) 
+
+            mask_d = self.generate_depth_decay(HW_tuple[0], HW_tuple[1], depth_map)
+            mask = (self.weight[0]*mask+self.weight[1]*mask_d)
+
+            geo_prior = ((sin, cos), mask)
+
+        return geo_prior

depth_anything/util/transform.py

@@ -0,0 +1,248 @@
+import random
+from PIL import Image, ImageOps, ImageFilter
+import torch
+from torchvision import transforms
+import torch.nn.functional as F
+
+import numpy as np
+import cv2
+import math
+
+
+def apply_min_size(sample, size, image_interpolation_method=cv2.INTER_AREA):
+    """Rezise the sample to ensure the given size. Keeps aspect ratio.
+
+    Args:
+        sample (dict): sample
+        size (tuple): image size
+
+    Returns:
+        tuple: new size
+    """
+    shape = list(sample["disparity"].shape)
+
+    if shape[0] >= size[0] and shape[1] >= size[1]:
+        return sample
+
+    scale = [0, 0]
+    scale[0] = size[0] / shape[0]
+    scale[1] = size[1] / shape[1]
+
+    scale = max(scale)
+
+    shape[0] = math.ceil(scale * shape[0])
+    shape[1] = math.ceil(scale * shape[1])
+
+    # resize
+    sample["image"] = cv2.resize(
+        sample["image"], tuple(shape[::-1]), interpolation=image_interpolation_method
+    )
+
+    sample["disparity"] = cv2.resize(
+        sample["disparity"], tuple(shape[::-1]), interpolation=cv2.INTER_NEAREST
+    )
+    sample["mask"] = cv2.resize(
+        sample["mask"].astype(np.float32),
+        tuple(shape[::-1]),
+        interpolation=cv2.INTER_NEAREST,
+    )
+    sample["mask"] = sample["mask"].astype(bool)
+
+    return tuple(shape)
+
+
+class Resize(object):
+    """Resize sample to given size (width, height).
+    """
+
+    def __init__(
+        self,
+        width,
+        height,
+        resize_target=True,
+        keep_aspect_ratio=False,
+        ensure_multiple_of=1,
+        resize_method="lower_bound",
+        image_interpolation_method=cv2.INTER_AREA,
+    ):
+        """Init.
+
+        Args:
+            width (int): desired output width
+            height (int): desired output height
+            resize_target (bool, optional):
+                True: Resize the full sample (image, mask, target).
+                False: Resize image only.
+                Defaults to True.
+            keep_aspect_ratio (bool, optional):
+                True: Keep the aspect ratio of the input sample.
+                Output sample might not have the given width and height, and
+                resize behaviour depends on the parameter 'resize_method'.
+                Defaults to False.
+            ensure_multiple_of (int, optional):
+                Output width and height is constrained to be multiple of this parameter.
+                Defaults to 1.
+            resize_method (str, optional):
+                "lower_bound": Output will be at least as large as the given size.
+                "upper_bound": Output will be at max as large as the given size. (Output size might be smaller than given size.)
+                "minimal": Scale as least as possible.  (Output size might be smaller than given size.)
+                Defaults to "lower_bound".
+        """
+        self.__width = width
+        self.__height = height
+
+        self.__resize_target = resize_target
+        self.__keep_aspect_ratio = keep_aspect_ratio
+        self.__multiple_of = ensure_multiple_of
+        self.__resize_method = resize_method
+        self.__image_interpolation_method = image_interpolation_method
+
+    def constrain_to_multiple_of(self, x, min_val=0, max_val=None):
+        y = (np.round(x / self.__multiple_of) * self.__multiple_of).astype(int)
+
+        if max_val is not None and y > max_val:
+            y = (np.floor(x / self.__multiple_of) * self.__multiple_of).astype(int)
+
+        if y < min_val:
+            y = (np.ceil(x / self.__multiple_of) * self.__multiple_of).astype(int)
+
+        return y
+
+    def get_size(self, width, height):
+        # determine new height and width
+        scale_height = self.__height / height
+        scale_width = self.__width / width
+
+        if self.__keep_aspect_ratio:
+            if self.__resize_method == "lower_bound":
+                # scale such that output size is lower bound
+                if scale_width > scale_height:
+                    # fit width
+                    scale_height = scale_width
+                else:
+                    # fit height
+                    scale_width = scale_height
+            elif self.__resize_method == "upper_bound":
+                # scale such that output size is upper bound
+                if scale_width < scale_height:
+                    # fit width
+                    scale_height = scale_width
+                else:
+                    # fit height
+                    scale_width = scale_height
+            elif self.__resize_method == "minimal":
+                # scale as least as possbile
+                if abs(1 - scale_width) < abs(1 - scale_height):
+                    # fit width
+                    scale_height = scale_width
+                else:
+                    # fit height
+                    scale_width = scale_height
+            else:
+                raise ValueError(
+                    f"resize_method {self.__resize_method} not implemented"
+                )
+
+        if self.__resize_method == "lower_bound":
+            new_height = self.constrain_to_multiple_of(
+                scale_height * height, min_val=self.__height
+            )
+            new_width = self.constrain_to_multiple_of(
+                scale_width * width, min_val=self.__width
+            )
+        elif self.__resize_method == "upper_bound":
+            new_height = self.constrain_to_multiple_of(
+                scale_height * height, max_val=self.__height
+            )
+            new_width = self.constrain_to_multiple_of(
+                scale_width * width, max_val=self.__width
+            )
+        elif self.__resize_method == "minimal":
+            new_height = self.constrain_to_multiple_of(scale_height * height)
+            new_width = self.constrain_to_multiple_of(scale_width * width)
+        else:
+            raise ValueError(f"resize_method {self.__resize_method} not implemented")
+
+        return (new_width, new_height)
+
+    def __call__(self, sample):
+        width, height = self.get_size(
+            sample["image"].shape[1], sample["image"].shape[0]
+        )
+
+        # resize sample
+        sample["image"] = cv2.resize(
+            sample["image"],
+            (width, height),
+            interpolation=self.__image_interpolation_method,
+        )
+
+        if self.__resize_target:
+            if "disparity" in sample:
+                sample["disparity"] = cv2.resize(
+                    sample["disparity"],
+                    (width, height),
+                    interpolation=cv2.INTER_NEAREST,
+                )
+
+            if "depth" in sample:
+                sample["depth"] = cv2.resize(
+                    sample["depth"], (width, height), interpolation=cv2.INTER_NEAREST
+                )
+
+            if "semseg_mask" in sample:
+                # sample["semseg_mask"] = cv2.resize(
+                #     sample["semseg_mask"], (width, height), interpolation=cv2.INTER_NEAREST
+                # )
+                sample["semseg_mask"] = F.interpolate(torch.from_numpy(sample["semseg_mask"]).float()[None, None, ...], (height, width), mode='nearest').numpy()[0, 0]
+                
+            if "mask" in sample:
+                sample["mask"] = cv2.resize(
+                    sample["mask"].astype(np.float32),
+                    (width, height),
+                    interpolation=cv2.INTER_NEAREST,
+                )
+                # sample["mask"] = sample["mask"].astype(bool)
+
+        # print(sample['image'].shape, sample['depth'].shape)
+        return sample
+
+
+class NormalizeImage(object):
+    """Normlize image by given mean and std.
+    """
+
+    def __init__(self, mean, std):
+        self.__mean = mean
+        self.__std = std
+
+    def __call__(self, sample):
+        sample["image"] = (sample["image"] - self.__mean) / self.__std
+
+        return sample
+
+
+class PrepareForNet(object):
+    """Prepare sample for usage as network input.
+    """
+
+    def __init__(self):
+        pass
+
+    def __call__(self, sample):
+        image = np.transpose(sample["image"], (2, 0, 1))
+        sample["image"] = np.ascontiguousarray(image).astype(np.float32)
+
+        if "mask" in sample:
+            sample["mask"] = sample["mask"].astype(np.float32)
+            sample["mask"] = np.ascontiguousarray(sample["mask"])
+        
+        if "depth" in sample:
+            depth = sample["depth"].astype(np.float32)
+            sample["depth"] = np.ascontiguousarray(depth)
+            
+        if "semseg_mask" in sample:
+            sample["semseg_mask"] = sample["semseg_mask"].astype(np.float32)
+            sample["semseg_mask"] = np.ascontiguousarray(sample["semseg_mask"])
+
+        return sample

requirements.txt

@@ -0,0 +1,16 @@
+gradio==4.44.0
+torch==2.3.0
+torchvision==0.18.0
+torchaudio==2.3.0
+numpy==1.26.4
+opencv-python==4.9.0.80
+Pillow==10.3.0
+matplotlib==3.8.4
+scikit-image==0.24.0
+scikit-learn==1.6.1
+pandas==2.2.3
+scipy==1.13.0
+tqdm==4.66.4
+PyYAML==6.0.1
+tensorboardX==2.6.2.2
+transformers==4.39.0 

util/dist_helper.py

@@ -0,0 +1,24 @@
+import os
+import subprocess
+
+import torch
+import torch.distributed as dist
+
+
+def setup_distributed(backend="nccl", port=None):
+    """AdaHessian Optimizer
+    Lifted from https://github.com/BIGBALLON/distribuuuu/blob/master/distribuuuu/utils.py
+    Originally licensed MIT, Copyright (c) 2020 Wei Li
+    """
+    num_gpus = torch.cuda.device_count()
+    rank = int(os.environ["RANK"])
+    world_size = int(os.environ["WORLD_SIZE"])
+
+    torch.cuda.set_device(rank % num_gpus)
+
+    dist.init_process_group(
+        backend=backend,
+        world_size=world_size,
+        rank=rank,
+    )
+    return rank, world_size

util/utils.py

@@ -0,0 +1,106 @@
+import numpy as np
+import logging
+import os
+
+
+def count_params(model):
+    param_num = sum(p.numel() for p in model.parameters())
+    return param_num / 1e6
+
+
+def color_map(dataset='pascal'):
+    cmap = np.zeros((256, 3), dtype='uint8')
+
+    if dataset == 'pascal' or dataset == 'coco':
+        def bitget(byteval, idx):
+            return (byteval & (1 << idx)) != 0
+
+        for i in range(256):
+            r = g = b = 0
+            c = i
+            for j in range(8):
+                r = r | (bitget(c, 0) << 7-j)
+                g = g | (bitget(c, 1) << 7-j)
+                b = b | (bitget(c, 2) << 7-j)
+                c = c >> 3
+
+            cmap[i] = np.array([r, g, b])
+
+    elif dataset == 'cityscapes':
+        cmap[0] = np.array([128, 64, 128])
+        cmap[1] = np.array([244, 35, 232])
+        cmap[2] = np.array([70, 70, 70])
+        cmap[3] = np.array([102, 102, 156])
+        cmap[4] = np.array([190, 153, 153])
+        cmap[5] = np.array([153, 153, 153])
+        cmap[6] = np.array([250, 170, 30])
+        cmap[7] = np.array([220, 220, 0])
+        cmap[8] = np.array([107, 142, 35])
+        cmap[9] = np.array([152, 251, 152])
+        cmap[10] = np.array([70, 130, 180])
+        cmap[11] = np.array([220, 20, 60])
+        cmap[12] = np.array([255,  0,  0])
+        cmap[13] = np.array([0,  0, 142])
+        cmap[14] = np.array([0,  0, 70])
+        cmap[15] = np.array([0, 60, 100])
+        cmap[16] = np.array([0, 80, 100])
+        cmap[17] = np.array([0,  0, 230])
+        cmap[18] = np.array([119, 11, 32])
+
+    return cmap
+
+
+class AverageMeter(object):
+    """Computes and stores the average and current value"""
+
+    def __init__(self, length=0):
+        self.length = length
+        self.reset()
+
+    def reset(self):
+        if self.length > 0:
+            self.history = []
+        else:
+            self.count = 0
+            self.sum = 0.0
+        self.val = 0.0
+        self.avg = 0.0
+
+    def update(self, val, num=1):
+        if self.length > 0:
+            # currently assert num==1 to avoid bad usage, refine when there are some explict requirements
+            assert num == 1
+            self.history.append(val)
+            if len(self.history) > self.length:
+                del self.history[0]
+
+            self.val = self.history[-1]
+            self.avg = np.mean(self.history)
+        else:
+            self.val = val
+            self.sum += val * num
+            self.count += num
+            self.avg = self.sum / self.count
+
+
+logs = set()
+
+
+def init_log(name, level=logging.INFO):
+    if (name, level) in logs:
+        return
+    logs.add((name, level))
+    logger = logging.getLogger(name)
+    logger.setLevel(level)
+    ch = logging.StreamHandler()
+    ch.setLevel(level)
+    if "SLURM_PROCID" in os.environ:
+        rank = int(os.environ["SLURM_PROCID"])
+        logger.addFilter(lambda record: rank == 0)
+    else:
+        rank = 0
+    format_str = "[%(asctime)s][%(levelname)8s] %(message)s"
+    formatter = logging.Formatter(format_str)
+    ch.setFormatter(formatter)
+    logger.addHandler(ch)
+    return logger

util/visualize_utils.py

@@ -0,0 +1,130 @@
+import cv2
+import numpy as np
+import matplotlib.pyplot as plt
+import torch
+
+def visualize_geo_prior(img, geo_prior, save_path, batch_idx=0, point_coords=None, normalize=True, alpha=0.6):
+    """
+    Visualize geometric prior matrix and overlay the result on the original image
+    Args:
+        img: Original image tensor [B,C,H,W]
+        geo_prior: Geometric prior tensor with shape [B,HW,HW]
+        save_path: Save path
+        batch_idx: Batch index to visualize
+        point_coords: Reference point coordinates in format (h, w). If None, center point will be used
+        normalize: Whether to normalize the display result
+        alpha: Heatmap transparency, 0.0 means completely transparent, 1.0 means completely opaque
+    """
+    B, HW, _ = geo_prior.shape
+    H = int(np.sqrt(HW))
+    W = H  
+    geo_prior_single = geo_prior[batch_idx]  # [HW,HW]
+    
+    if point_coords is None:
+        center_h, center_w = H // 2, W // 2
+        point_idx = center_h * W + center_w
+    else:
+        h, w = point_coords
+        point_idx = h * W + w
+    relation = geo_prior_single[point_idx]  # [HW]
+    relation_map = relation.reshape(H, W)
+    relation_np = relation_map.detach().cpu().numpy()
+    
+    if normalize:
+        relation_np = (relation_np - relation_np.min()) / (relation_np.max() - relation_np.min() + 1e-6)
+    
+    orig_img = img[batch_idx].detach().cpu().numpy()
+    orig_img = np.transpose(orig_img, (1, 2, 0))
+    mean = np.array([0.485, 0.456, 0.406])
+    std = np.array([0.229, 0.224, 0.225])
+    orig_img = std * orig_img + mean
+    orig_img = np.clip(orig_img * 255, 0, 255).astype(np.uint8)
+    orig_img = cv2.cvtColor(orig_img, cv2.COLOR_RGB2BGR)
+    
+    orig_h, orig_w = orig_img.shape[:2]
+    
+    colored_map = cv2.applyColorMap((relation_np * 255).astype(np.uint8), cv2.COLORMAP_RAINBOW)
+    colored_map = cv2.resize(colored_map, (orig_w, orig_h), interpolation=cv2.INTER_LINEAR)
+    
+    overlay = cv2.addWeighted(orig_img, 1-alpha, colored_map, alpha, 0)
+    
+    if point_coords is None:
+        center_w_orig = int(center_w * orig_w / W)
+        center_h_orig = int(center_h * orig_h / H)
+        cv2.drawMarker(overlay, (center_w_orig, center_h_orig), (255, 255, 255), cv2.MARKER_CROSS, 20, 2)
+    else:
+        w_orig = int(w * orig_w / W)
+        h_orig = int(h * orig_h / H)
+        cv2.drawMarker(overlay, (w_orig, h_orig), (255, 255, 255), cv2.MARKER_CROSS, 20, 2)
+    
+    cv2.imwrite(save_path.replace('.png', '_overlay.png'), overlay)
+    
+    colored_map = cv2.applyColorMap((relation_np * 255).astype(np.uint8), cv2.COLORMAP_RAINBOW)
+    cv2.imwrite(save_path.replace('.png', '_heatmap.png'), colored_map)
+    cv2.imwrite(save_path.replace('.png', '_original.png'), orig_img)
+    
+    plt.figure(figsize=(10, 8))
+    plt.imshow(relation_np, cmap='rainbow')
+    plt.colorbar(label='Geometric Prior Strength')
+    
+    if point_coords is None:
+        plt.plot(center_w, center_h, 'w*', markersize=10)
+    else:
+        plt.plot(w, h, 'w*', markersize=10)
+    
+    plt.title(f'Geometric Prior Visualization (Ref Point: {"center" if point_coords is None else f"({point_coords[0]}, {point_coords[1]})"})')
+    plt.savefig(save_path)
+    plt.close()
+    
+    return relation_map
+
+
+def save_feature_visualization(feature_map, save_path):
+    """
+    Visualize feature map by averaging all feature maps into one image and resize to 518*518
+    Args:
+        feature_map: feature map tensor with shape [C,H,W]
+        save_path: save path
+    """
+    
+    if len(feature_map.shape) == 4:
+        feature_map = feature_map.squeeze(0)
+    mean_feature = torch.mean(feature_map, dim=0).detach().cpu().numpy()
+    mean_feature = (mean_feature - mean_feature.min()) / (mean_feature.max() - mean_feature.min() + 1e-6)
+    mean_feature = (mean_feature * 255).astype(np.uint8)
+    mean_feature = cv2.resize(mean_feature, (518, 518), interpolation=cv2.INTER_LINEAR)
+    
+    colored_feature = cv2.applyColorMap(mean_feature, cv2.COLORMAP_VIRIDIS)
+    cv2.imwrite(save_path, colored_feature)
+
+def save_depth_visualization(depth_map, filename):
+    """
+    Save depth map visualization as a colored image.
+    
+    Args:
+        depth_map (torch.Tensor): Depth map tensor with shape [H, W] or [B, H, W]
+        filename (str): Output file path for the visualization
+    """
+    depth_norm = (depth_map - depth_map.min()) / (depth_map.max() - depth_map.min()) * 255.0
+    depth_norm = depth_norm.detach().cpu().numpy().astype(np.uint8)
+    colored_depth = cv2.applyColorMap(depth_norm, cv2.COLORMAP_INFERNO)
+    cv2.imwrite(filename, colored_depth)
+
+def save_image(img_tensor, filename):
+    """
+    Save image tensor as a BGR image file.
+    
+    Args:
+        img_tensor (torch.Tensor): Image tensor with shape [C, H, W] or [B, C, H, W]
+        filename (str): Output file path for the image
+    """
+    img = img_tensor.detach().cpu().numpy()
+
+    if img.shape[0] == 3:  
+        img = np.transpose(img, (1, 2, 0))
+    mean = np.array([0.485, 0.456, 0.406])
+    std = np.array([0.229, 0.224, 0.225])
+    img = std * img + mean
+    img = np.clip(img * 255, 0, 255).astype(np.uint8)
+    img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR)
+    cv2.imwrite(filename, img)