Upload folder using huggingface_hub

.gitattributes

@@ -1,35 +1,54 @@
 *.7z filter=lfs diff=lfs merge=lfs -text
 *.arrow filter=lfs diff=lfs merge=lfs -text
 *.bin filter=lfs diff=lfs merge=lfs -text
+*.bin.* filter=lfs diff=lfs merge=lfs -text
 *.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
 *.ftz filter=lfs diff=lfs merge=lfs -text
 *.gz filter=lfs diff=lfs merge=lfs -text
 *.h5 filter=lfs diff=lfs merge=lfs -text
 *.joblib filter=lfs diff=lfs merge=lfs -text
 *.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
 *.model filter=lfs diff=lfs merge=lfs -text
 *.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
 *.onnx filter=lfs diff=lfs merge=lfs -text
 *.ot filter=lfs diff=lfs merge=lfs -text
 *.parquet filter=lfs diff=lfs merge=lfs -text
 *.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
 *.pt filter=lfs diff=lfs merge=lfs -text
 *.pth filter=lfs diff=lfs merge=lfs -text
 *.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
 saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
 *.tflite filter=lfs diff=lfs merge=lfs -text
 *.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
 *.xz filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
+*.zstandard filter=lfs diff=lfs merge=lfs -text
+*.tfevents* filter=lfs diff=lfs merge=lfs -text
+*.db* filter=lfs diff=lfs merge=lfs -text
+*.ark* filter=lfs diff=lfs merge=lfs -text
+**/*ckpt*data* filter=lfs diff=lfs merge=lfs -text
+**/*ckpt*.meta filter=lfs diff=lfs merge=lfs -text
+**/*ckpt*.index filter=lfs diff=lfs merge=lfs -text
+ 
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.gguf* filter=lfs diff=lfs merge=lfs -text
+*.ggml filter=lfs diff=lfs merge=lfs -text
+*.llamafile* filter=lfs diff=lfs merge=lfs -text
+*.pt2 filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+
+model-00001-of-000001.safetensors filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -textassets/fig1.png filter=lfs diff=lfs merge=lfs -text
+assets/show1.jpg filter=lfs diff=lfs merge=lfs -text
+assets/show2.jpg filter=lfs diff=lfs merge=lfs -text
+assets/show3.jpg filter=lfs diff=lfs merge=lfs -text
+assets/show4.jpg filter=lfs diff=lfs merge=lfs -text

.ipynb_checkpoints/README-checkpoint.md

@@ -0,0 +1,122 @@
+---
+pipeline_tag: image-text-to-text
+language:
+- multilingual
+tags:
+- deepseek
+- vision-language
+- ocr
+- custom_code
+license: mit
+---
+<div align="center">
+  <img src="https://github.com/deepseek-ai/DeepSeek-V2/blob/main/figures/logo.svg?raw=true" width="60%" alt="DeepSeek AI" />
+</div>
+<hr>
+<div align="center">
+  <a href="https://www.deepseek.com/" target="_blank">
+    <img alt="Homepage" src="https://github.com/deepseek-ai/DeepSeek-V2/blob/main/figures/badge.svg?raw=true" />
+  </a>
+  <a href="https://huggingface.co/deepseek-ai/DeepSeek-OCR" target="_blank">
+    <img alt="Hugging Face" src="https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-DeepSeek%20AI-ffc107?color=ffc107&logoColor=white" />
+  </a>
+
+</div>
+
+<div align="center">
+
+  <a href="https://discord.gg/Tc7c45Zzu5" target="_blank">
+    <img alt="Discord" src="https://img.shields.io/badge/Discord-DeepSeek%20AI-7289da?logo=discord&logoColor=white&color=7289da" />
+  </a>
+  <a href="https://twitter.com/deepseek_ai" target="_blank">
+    <img alt="Twitter Follow" src="https://img.shields.io/badge/Twitter-deepseek_ai-white?logo=x&logoColor=white" />
+  </a>
+
+</div>
+
+
+
+<p align="center">
+  <a href="https://github.com/deepseek-ai/DeepSeek-OCR"><b>🌟 Github</b></a> |
+  <a href="https://huggingface.co/deepseek-ai/DeepSeek-OCR"><b>📥 Model Download</b></a> |
+  <a href="https://github.com/deepseek-ai/DeepSeek-OCR/blob/main/DeepSeek_OCR_paper.pdf"><b>📄 Paper Link</b></a> |
+  <a href=""><b>📄 Arxiv Paper Link</b></a> |
+</p>
+<h2>
+<p align="center">
+  <a href="">DeepSeek-OCR: Contexts Optical Compression</a>
+</p>
+</h2>
+<p align="center">
+<img src="assets/fig1.png" style="width: 1000px" align=center>
+</p>
+<p align="center">
+<a href="">Explore the boundaries of visual-text compression.</a>       
+</p>
+
+## Usage
+Inference using Huggingface transformers on NVIDIA GPUs. Requirements tested on python 3.12.9 + CUDA11.8：
+
+```
+torch==2.6.0
+transformers==4.46.3
+tokenizers==0.20.3
+einops
+addict 
+easydict
+pip install flash-attn==2.7.3 --no-build-isolation
+```
+
+```python
+from transformers import AutoModel, AutoTokenizer
+import torch
+import os
+os.environ["CUDA_VISIBLE_DEVICES"] = '0'
+model_name = 'deepseek-ai/DeepSeek-OCR'
+
+tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
+model = AutoModel.from_pretrained(model_name, _attn_implementation='flash_attention_2', trust_remote_code=True, use_safetensors=True)
+model = model.eval().cuda().to(torch.bfloat16)
+
+# prompt = "<image>\nFree OCR. "
+prompt = "<image>\n<|grounding|>Convert the document to markdown. "
+image_file = 'your_image.jpg'
+output_path = 'your/output/dir'
+
+# infer(self, tokenizer, prompt='', image_file='', output_path = ' ', base_size = 1024, image_size = 640, crop_mode = True, test_compress = False, save_results = False):
+
+# Tiny: base_size = 512, image_size = 512, crop_mode = False
+# Small: base_size = 640, image_size = 640, crop_mode = False
+# Base: base_size = 1024, image_size = 1024, crop_mode = False
+# Large: base_size = 1280, image_size = 1280, crop_mode = False
+
+# Gundam: base_size = 1024, image_size = 640, crop_mode = True
+
+res = model.infer(tokenizer, prompt=prompt, image_file=image_file, output_path = output_path, base_size = 1024, image_size = 640, crop_mode=True, save_results = True, test_compress = True)
+```
+
+## vLLM
+Refer to [🌟GitHub](https://github.com/deepseek-ai/DeepSeek-OCR/) for guidance on model inference acceleration and PDF processing, etc.<!--  -->
+
+## Visualizations
+<table>
+<tr>
+<td><img src="assets/show1.jpg" style="width: 500px"></td>
+<td><img src="assets/show2.jpg" style="width: 500px"></td>
+</tr>
+<tr>
+<td><img src="assets/show3.jpg" style="width: 500px"></td>
+<td><img src="assets/show4.jpg" style="width: 500px"></td>
+</tr>
+</table>
+
+
+## Acknowledgement
+
+We would like to thank [Vary](https://github.com/Ucas-HaoranWei/Vary/), [GOT-OCR2.0](https://github.com/Ucas-HaoranWei/GOT-OCR2.0/), [MinerU](https://github.com/opendatalab/MinerU), [PaddleOCR](https://github.com/PaddlePaddle/PaddleOCR), [OneChart](https://github.com/LingyvKong/OneChart), [Slow Perception](https://github.com/Ucas-HaoranWei/Slow-Perception) for their valuable models and ideas.
+
+We also appreciate the benchmarks: [Fox](https://github.com/ucaslcl/Fox), [OminiDocBench](https://github.com/opendatalab/OmniDocBench).
+
+
+## Citation
+Coming soon!

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 DeepSeek
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -1,3 +1,132 @@
----
-license: mit
----
+---
+pipeline_tag: image-text-to-text
+language:
+- multilingual
+tags:
+- mindspore
+- mindnlp
+- deepseek
+- vision-language
+- ocr
+- custom_code
+license: mit
+---
+<div align="center">
+  <img src="https://github.com/deepseek-ai/DeepSeek-V2/blob/main/figures/logo.svg?raw=true" width="60%" alt="DeepSeek AI" />
+</div>
+<hr>
+<div align="center">
+  <a href="https://www.deepseek.com/" target="_blank">
+    <img alt="Homepage" src="https://github.com/deepseek-ai/DeepSeek-V2/blob/main/figures/badge.svg?raw=true" />
+  </a>
+  <a href="https://huggingface.co/lvyufeng/DeepSeek-OCR" target="_blank">
+    <img alt="Hugging Face" src="https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-DeepSeek%20AI-ffc107?color=ffc107&logoColor=white" />
+  </a>
+
+</div>
+
+
+
+
+<p align="center">
+  <a href="https://github.com/mindspore-lab/mindnlp/tree/master/examples/transformers/inference/deepseek-ocr"><b>🌟 Github</b></a> |
+  <a href="https://huggingface.co/lvyufeng/DeepSeek-OCR"><b>📥 Model Download</b></a> |
+  <a href="https://github.com/deepseek-ai/DeepSeek-OCR/blob/main/DeepSeek_OCR_paper.pdf"><b>📄 Paper Link</b></a> |
+  <a href=""><b>📄 Arxiv Paper Link</b></a> |
+</p>
+<h2>
+<p align="center">
+  <a href="">DeepSeek-OCR: Contexts Optical Compression</a>
+</p>
+</h2>
+<p align="center">
+<a href="">Explore the boundaries of visual-text compression.</a>       
+</p>
+
+The official version of DeepSeek-OCR has limited the transformers version to 4.46.3 and has not been adapted to the latest version. Therefore, this community edition has modified the modeling.py module to facilitate user convenience without requiring a transformers downgrade. Additionally, this version has been adapted for MindSpore+MindNLP compatibility, and users are welcome to utilize it on Ascend hardware.
+
+Feel free to opt for various attention implementations such as Flash Attention or SDPA to leverage the latest optimizations in transformers for a performance boost.
+
+## MindSpore Usage
+Inference using Huggingface transformers on NVIDIA GPUs. Requirements tested on python 3.12.9 + CUDA11.8：
+
+```
+mindspore==2.7.0
+mindnlp==0.5.0rc3
+transformers==4.57.1
+tokenizers
+einops
+addict 
+easydict
+```
+
+```python
+import os
+import mindnlp
+import mindspore
+from transformers import AutoModel, AutoTokenizer
+
+model_name = 'lvyufeng/DeepSeek-OCR-Community-Latest'
+
+tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
+model = AutoModel.from_pretrained(model_name, dtype=mindspore.float16, _attn_implementation='sdpa', trust_remote_code=True, use_safetensors=True, device_map='auto')
+model = model.eval()
+
+# prompt = "<image>\nFree OCR. "
+prompt = "<image>\n<|grounding|>Convert the document to markdown. "
+image_file = 'your_image.jpg'
+output_path = 'your/output/dir'
+
+# infer(self, tokenizer, prompt='', image_file='', output_path = ' ', base_size = 1024, image_size = 640, crop_mode = True, test_compress = False, save_results = False):
+
+# Tiny: base_size = 512, image_size = 512, crop_mode = False
+# Small: base_size = 640, image_size = 640, crop_mode = False
+# Base: base_size = 1024, image_size = 1024, crop_mode = False
+# Large: base_size = 1280, image_size = 1280, crop_mode = False
+
+# Gundam: base_size = 1024, image_size = 640, crop_mode = True
+
+res = model.infer(tokenizer, prompt=prompt, image_file=image_file, output_path = output_path, base_size = 1024, image_size = 640, crop_mode=True, save_results = True, test_compress = True)
+```
+
+
+## Pytorch Usage
+Inference using Huggingface transformers on NVIDIA GPUs. Requirements tested on python 3.12.9 + CUDA11.8：
+
+```
+torch
+transformers==4.57.1
+tokenizers
+einops
+addict 
+easydict
+pip install flash-attn
+```
+
+```python
+from transformers import AutoModel, AutoTokenizer
+import torch
+import os
+os.environ["CUDA_VISIBLE_DEVICES"] = '0'
+model_name = 'deepseek-ai/DeepSeek-OCR'
+tokenizer = AutoTokenizer.from_pretrained(model_name, dtype=torch.bfloat16,trust_remote_code=True, device_map='auto')
+model = AutoModel.from_pretrained(model_name, _attn_implementation='flash_attention_2', trust_remote_code=True, use_safetensors=True)
+model = model.eval()
+# prompt = "<image>\nFree OCR. "
+prompt = "<image>\n<|grounding|>Convert the document to markdown. "
+image_file = 'your_image.jpg'
+output_path = 'your/output/dir'
+# infer(self, tokenizer, prompt='', image_file='', output_path = ' ', base_size = 1024, image_size = 640, crop_mode = True, test_compress = False, save_results = False):
+# Tiny: base_size = 512, image_size = 512, crop_mode = False
+# Small: base_size = 640, image_size = 640, crop_mode = False
+# Base: base_size = 1024, image_size = 1024, crop_mode = False
+# Large: base_size = 1280, image_size = 1280, crop_mode = False
+# Gundam: base_size = 1024, image_size = 640, crop_mode = True
+res = model.infer(tokenizer, prompt=prompt, image_file=image_file, output_path = output_path, base_size = 1024, image_size = 640, crop_mode=True, save_results = True, test_compress = True)
+```
+
+## Acknowledgement
+
+We would like to thank [Vary](https://github.com/Ucas-HaoranWei/Vary/), [GOT-OCR2.0](https://github.com/Ucas-HaoranWei/GOT-OCR2.0/), [MinerU](https://github.com/opendatalab/MinerU), [PaddleOCR](https://github.com/PaddlePaddle/PaddleOCR), [OneChart](https://github.com/LingyvKong/OneChart), [Slow Perception](https://github.com/Ucas-HaoranWei/Slow-Perception) for their valuable models and ideas.
+
+We also appreciate the benchmarks: [Fox](https://github.com/ucaslcl/Fox), [OminiDocBench](https://github.com/opendatalab/OmniDocBench).

config.json

@@ -0,0 +1,118 @@
+{
+ "_name_or_path": "deepseek-ai/DeepSeek-OCR",
+  "candidate_resolutions": [
+  [
+   1024,
+   1024
+  ]
+ ],
+ "global_view_pos": "head",
+  "architectures": [
+      "DeepseekOCRForCausalLM"
+    ],
+  "auto_map": {
+    "AutoConfig": "modeling_deepseekocr.DeepseekOCRConfig",
+    "AutoModel": "modeling_deepseekocr.DeepseekOCRForCausalLM"
+  },
+ "language_config": {
+    "architectures": [
+      "DeepseekV2ForCausalLM"
+    ],
+    "auto_map": {
+      "AutoConfig": "configuration_deepseekv2.DeepseekV2Config",
+      "AutoModel": "modeling_deepseek.DeepseekV2Model",
+      "AutoModelForCausalLM": "modeling_deepseek.DeepseekV2ForCausalLM"
+    },
+    "bos_token_id": 0,
+    "eos_token_id": 1,
+    "first_k_dense_replace": 1,
+    "hidden_size": 1280,
+    "intermediate_size": 6848,
+    "kv_lora_rank": null,
+    "lm_head": true,
+    "max_position_embeddings": 8192,
+    "moe_intermediate_size": 896,
+    "n_group": 1,
+    "n_routed_experts": 64,
+    "n_shared_experts": 2,
+    "num_attention_heads": 10,
+    "num_experts_per_tok": 6,
+    "num_hidden_layers": 12,
+    "num_key_value_heads": 10,
+    "q_lora_rank": null,
+    "qk_nope_head_dim": 0,
+    "qk_rope_head_dim": 0,
+    "rm_head": false,
+    "topk_group": 1,
+    "topk_method": "greedy",
+    "torch_dtype": "bfloat16",
+    "use_mla": false,
+    "v_head_dim": 0,
+    "vocab_size": 129280
+  },
+ "model_type": "deepseek_vl_v2",
+ "projector_config": {
+  "input_dim": 2048,
+  "model_type": "mlp_projector",
+  "n_embed": 1280,
+  "projector_type": "linear"
+ },
+ "tile_tag": "2D",
+ "torch_dtype": "bfloat16",
+ "transformers_version": "4.46.3",
+ "vision_config": {
+  "image_size": 1024,
+  "mlp_ratio": 3.7362,
+  "model_name": "deeplip_b_l",
+  "model_type": "vision",
+  "width": {
+   "clip-l-14-224": {
+    "heads": 16,
+    "image_size": 224,
+    "layers": 24,
+    "patch_size": 14,
+    "width": 1024
+   },
+   "sam_vit_b": {
+    "downsample_channels": [
+     512,
+     1024
+    ],
+    "global_attn_indexes": [
+     2,
+     5,
+     8,
+     11
+    ],
+    "heads": 12,
+    "layers": 12,
+    "width": 768
+   }
+  }
+ },
+  "bos_token_id": 0,
+  "eos_token_id": 1,
+  "first_k_dense_replace": 1,
+  "hidden_size": 1280,
+  "intermediate_size": 6848,
+  "kv_lora_rank": null,
+  "lm_head": true,
+  "max_position_embeddings": 8192,
+  "moe_intermediate_size": 896,
+  "n_group": 1,
+  "n_routed_experts": 64,
+  "n_shared_experts": 2,
+  "num_attention_heads": 10,
+  "num_experts_per_tok": 6,
+  "num_hidden_layers": 12,
+  "num_key_value_heads": 10,
+  "q_lora_rank": null,
+  "qk_nope_head_dim": 0,
+  "qk_rope_head_dim": 0,
+  "rm_head": false,
+  "topk_group": 1,
+  "topk_method": "greedy",
+  "use_mla": false,
+  "v_head_dim": 0,
+  "vocab_size": 129280
+}

configuration.json

@@ -0,0 +1 @@
+{"framework": "pytorch", "task": "image-text-to-text", "allow_remote": true}

configuration_deepseek_v2.py

@@ -0,0 +1,210 @@
+from transformers.configuration_utils import PretrainedConfig
+from transformers.utils import logging
+
+logger = logging.get_logger(__name__)
+
+DEEPSEEK_PRETRAINED_CONFIG_ARCHIVE_MAP = {}
+class DeepseekV2Config(PretrainedConfig):
+    r"""
+    This is the configuration class to store the configuration of a [`DeepseekV2Model`]. It is used to instantiate an DeepSeek
+    model according to the specified arguments, defining the model architecture. Instantiating a configuration with the
+    defaults will yield a similar configuration to that of the DeepSeek-V2 with multi-latent attention.
+
+    Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+    documentation from [`PretrainedConfig`] for more information.
+
+
+    Args:
+        vocab_size (`int`, *optional*, defaults to 102400):
+            Vocabulary size of the Deep model. Defines the number of different tokens that can be represented by the
+            `inputs_ids` passed when calling [`DeepseekV2Model`]
+        hidden_size (`int`, *optional*, defaults to 4096):
+            Dimension of the hidden representations.
+        intermediate_size (`int`, *optional*, defaults to 11008):
+            Dimension of the MLP representations.
+        moe_intermediate_size (`int`, *optional*, defaults to 1407):
+            Dimension of the MoE representations.
+        num_hidden_layers (`int`, *optional*, defaults to 32):
+            Number of hidden layers in the Transformer decoder.
+        num_attention_heads (`int`, *optional*, defaults to 32):
+            Number of attention heads for each attention layer in the Transformer decoder.
+        n_shared_experts (`int`, *optional*, defaults to None):
+            Number of shared experts, None means dense model.
+        n_routed_experts (`int`, *optional*, defaults to None):
+            Number of routed experts, None means dense model.
+        routed_scaling_factor (`float`, *optional*, defaults to 1.0):
+            Scaling factor or routed experts.
+        topk_method (`str`, *optional*, defaults to `gready`):
+            Topk method used in routed gate.
+        n_group (`int`, *optional*, defaults to None):
+            Number of groups for routed experts.
+        topk_group (`int`, *optional*, defaults to None):
+            Number of selected groups for each token(for each token, ensuring the selected experts is only within `topk_group` groups).
+        num_experts_per_tok (`int`, *optional*, defaults to None):
+            Number of selected experts, None means dense model.
+        moe_layer_freq (`int`, *optional*, defaults to 1):
+            The frequency of the MoE layer: one expert layer for every `moe_layer_freq - 1` dense layers.
+        first_k_dense_replace (`int`, *optional*, defaults to 0):
+            Number of dense layers in shallow layers(embed->dense->dense->...->dense->moe->moe...->lm_head).
+                                                            \--k dense layers--/
+        norm_topk_prob (`bool`, *optional*, defaults to False):
+            Whether to normalize the weights of the routed experts.
+        scoring_func (`str`, *optional*, defaults to 'softmax'):
+            Method of computing expert weights.
+        aux_loss_alpha (`float`, *optional*, defaults to 0.001):
+            Auxiliary loss weight coefficient.
+        seq_aux = (`bool`, *optional*, defaults to True):
+            Whether to compute the auxiliary loss for each individual sample.
+        num_key_value_heads (`int`, *optional*):
+            This is the number of key_value heads that should be used to implement Grouped Query Attention. If
+            `num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if
+            `num_key_value_heads=1 the model will use Multi Query Attention (MQA) otherwise GQA is used. When
+            converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed
+            by meanpooling all the original heads within that group. For more details checkout [this
+            paper](https://arxiv.org/pdf/2305.13245.pdf). If it is not specified, will default to
+            `num_attention_heads`.
+        hidden_act (`str` or `function`, *optional*, defaults to `"silu"`):
+            The non-linear activation function (function or string) in the decoder.
+        max_position_embeddings (`int`, *optional*, defaults to 2048):
+            The maximum sequence length that this model might ever be used with.
+        initializer_range (`float`, *optional*, defaults to 0.02):
+            The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
+        rms_norm_eps (`float`, *optional*, defaults to 1e-06):
+            The epsilon used by the rms normalization layers.
+        use_cache (`bool`, *optional*, defaults to `True`):
+            Whether or not the model should return the last key/values attentions (not used by all models). Only
+            relevant if `config.is_decoder=True`.
+        pad_token_id (`int`, *optional*):
+            Padding token id.
+        bos_token_id (`int`, *optional*, defaults to 1):
+            Beginning of stream token id.
+        eos_token_id (`int`, *optional*, defaults to 2):
+            End of stream token id.
+        pretraining_tp (`int`, *optional*, defaults to 1):
+            Experimental feature. Tensor parallelism rank used during pretraining. Please refer to [this
+            document](https://huggingface.co/docs/transformers/parallelism) to understand more about it. This value is
+            necessary to ensure exact reproducibility of the pretraining results. Please refer to [this
+            issue](https://github.com/pytorch/pytorch/issues/76232).
+        tie_word_embeddings (`bool`, *optional*, defaults to `False`):
+            Whether to tie weight embeddings
+        rope_theta (`float`, *optional*, defaults to 10000.0):
+            The base period of the RoPE embeddings.
+        rope_scaling (`Dict`, *optional*):
+            Dictionary containing the scaling configuration for the RoPE embeddings. Currently supports two scaling
+            strategies: linear and dynamic. Their scaling factor must be a float greater than 1. The expected format is
+            `{"type": strategy name, "factor": scaling factor}`. When using this flag, don't update
+            `max_position_embeddings` to the expected new maximum.
+        attention_bias (`bool`, defaults to `False`, *optional*, defaults to `False`):
+            Whether to use a bias in the query, key, value and output projection layers during self-attention.
+        attention_dropout (`float`, *optional*, defaults to 0.0):
+            The dropout ratio for the attention probabilities.
+        use_mla (`bool`, *optional*, defaults to `True`): Use multi-latent attention or multi-head attention. If True,
+            the model will use multi-latent attention, otherwise, it will use multi-head attention.
+
+    ```python
+    >>> from transformers import DeepseekV2Model, DeepseekV2Config
+
+    >>> # Initializing a Deepseek-V2 style configuration
+    >>> configuration = DeepseekV2Config()
+
+    >>> # Accessing the model configuration
+    >>> configuration = model.config
+    ```"""
+
+    model_type = "deepseek_v2"
+    keys_to_ignore_at_inference = ["past_key_values"]
+
+    def __init__(
+        self,
+        vocab_size=102400,
+        hidden_size=4096,
+        intermediate_size=11008,
+        moe_intermediate_size = 1407,
+        num_hidden_layers=30,
+        num_attention_heads=32,
+        num_key_value_heads=32,
+        n_shared_experts = None,
+        n_routed_experts = None,
+        ep_size = 1,
+        routed_scaling_factor = 1.0,
+        kv_lora_rank = 512,
+        q_lora_rank = 1536,
+        qk_rope_head_dim = 64,
+        v_head_dim = 128,
+        qk_nope_head_dim = 128,
+        topk_method = 'gready',
+        n_group = None,
+        topk_group = None,
+        num_experts_per_tok = None,
+        moe_layer_freq = 1,
+        first_k_dense_replace = 0,
+        norm_topk_prob = False,
+        scoring_func = 'softmax',
+        aux_loss_alpha = 0.001,
+        seq_aux = True,
+        hidden_act="silu",
+        max_position_embeddings=2048,
+        initializer_range=0.02,
+        rms_norm_eps=1e-6,
+        use_cache=True,
+        pad_token_id=None,
+        bos_token_id=100000,
+        eos_token_id=100001,
+        pretraining_tp=1,
+        tie_word_embeddings=False,
+        rope_theta=10000.0,
+        rope_scaling=None,
+        attention_bias=False,
+        attention_dropout=0.0,
+        use_mla=True,
+        **kwargs,
+    ):
+        self.vocab_size = vocab_size
+        self.max_position_embeddings = max_position_embeddings
+        self.hidden_size = hidden_size
+        self.intermediate_size = intermediate_size
+        self.moe_intermediate_size = moe_intermediate_size
+        self.num_hidden_layers = num_hidden_layers
+        self.num_attention_heads = num_attention_heads
+        self.n_shared_experts = n_shared_experts
+        self.n_routed_experts = n_routed_experts
+        self.ep_size = ep_size
+        self.routed_scaling_factor = routed_scaling_factor
+        self.kv_lora_rank = kv_lora_rank
+        self.q_lora_rank = q_lora_rank
+        self.qk_rope_head_dim = qk_rope_head_dim
+        self.v_head_dim = v_head_dim
+        self.qk_nope_head_dim = qk_nope_head_dim
+        self.topk_method = topk_method
+        self.n_group = n_group
+        self.topk_group = topk_group
+        self.num_experts_per_tok = num_experts_per_tok
+        self.moe_layer_freq = moe_layer_freq
+        self.first_k_dense_replace = first_k_dense_replace
+        self.norm_topk_prob = norm_topk_prob
+        self.scoring_func = scoring_func
+        self.aux_loss_alpha = aux_loss_alpha
+        self.seq_aux = seq_aux
+        # for backward compatibility
+        if num_key_value_heads is None:
+            num_key_value_heads = num_attention_heads
+
+        self.num_key_value_heads = num_key_value_heads
+        self.hidden_act = hidden_act
+        self.initializer_range = initializer_range
+        self.rms_norm_eps = float(rms_norm_eps)
+        self.pretraining_tp = pretraining_tp
+        self.use_cache = use_cache
+        self.rope_theta = rope_theta
+        self.rope_scaling = rope_scaling
+        self.attention_bias = attention_bias
+        self.attention_dropout = attention_dropout
+        self.use_mla = use_mla
+
+        super().__init__(
+            pad_token_id=pad_token_id,
+            bos_token_id=bos_token_id,
+            eos_token_id=eos_token_id,
+            tie_word_embeddings=tie_word_embeddings,
+            **kwargs,
+        )

conversation.py

@@ -0,0 +1,280 @@
+"""
+From https://github.com/lm-sys/FastChat/blob/main/fastchat/conversation.py
+"""
+
+import dataclasses
+from enum import IntEnum, auto
+from typing import Any, Dict, List
+
+
+class SeparatorStyle(IntEnum):
+    """Separator styles."""
+
+    DeepSeek = auto()
+    DeepSeekV2 = auto()
+    PLAIN = auto()
+    ALIGNMENT = auto()
+
+
+@dataclasses.dataclass
+class Conversation:
+    """A class that manages prompt templates and keeps all conversation history."""
+
+    # The name of this template
+    name: str
+    # The template of the system prompt
+    system_template: str = "{system_message}"
+    # The system message
+    system_message: str = ""
+    # The names of two roles
+    roles: List[str] = (("USER", "ASSISTANT"),)
+    # All messages. Each item is (role, message).
+    messages: List[List[str]] = ()
+    # The number of few shot examples
+    offset: int = 0
+    # The separator style and configurations
+    sep_style: SeparatorStyle = SeparatorStyle.DeepSeek
+    sep: str = "\n"
+    sep2: str = None
+    # Stop criteria (the default one is EOS token)
+    stop_str: str = None
+    # Stops generation if meeting any token in this list
+    stop_token_ids: List[int] = None
+
+    def get_prompt(self) -> str:
+        """Get the prompt for generation."""
+        system_prompt = self.system_template.format(system_message=self.system_message)
+        if self.sep_style == SeparatorStyle.DeepSeek:
+            seps = [self.sep, self.sep2]
+            if system_prompt == "" or system_prompt is None:
+                ret = ""
+            else:
+                ret = system_prompt + seps[0]
+            for i, (role, message) in enumerate(self.messages):
+                if message:
+                    ret += role + ": " + message + seps[i % 2]
+                else:
+                    ret += role + ":"
+            return ret
+        elif self.sep_style == SeparatorStyle.DeepSeekV2:
+            seps = [self.sep, self.sep2]
+            if system_prompt == "" or system_prompt is None:
+                ret = ""
+            else:
+                ret = system_prompt + seps[0]
+            for i, (role, message) in enumerate(self.messages):
+                if message:
+                    if role == "User":
+                        ret += "<｜sft▁begin｜>\n" + message + self.sep #<｜sft▁begin｜>User Input<｜sft▁end｜>\nResponse<｜end▁of▁sentence｜>
+                    else:
+                        ret += message + self.sep2
+                else:
+                    ret = ret
+            return ret
+
+        elif self.sep_style == SeparatorStyle.PLAIN:
+            seps = [self.sep, self.sep2]
+            ret = ""
+            for i, (role, message) in enumerate(self.messages):
+                if message:
+                    if type(message) is tuple:
+                        message, _, _ = message
+                    if i % 2 == 0:
+                        ret += message + seps[i % 2]
+                    else:
+                        ret += message + seps[i % 2]
+                else:
+                    ret += ""
+            return ret
+        elif self.sep_style == SeparatorStyle.ALIGNMENT:
+            seps = [self.sep, self.sep2]
+            ret = ""
+            for i, (role, message) in enumerate(self.messages):
+                if message:
+                    if type(message) is tuple:
+                        message, _, _ = message
+                    if i % 2 == 0:
+                        ret += '<image>\n' + seps[i % 2]
+                    else:
+                        ret += message + seps[i % 2]
+                else:
+                    ret += ""
+            return ret
+        else:
+            raise ValueError(f"Invalid style: {self.sep_style}")
+
+    def set_system_message(self, system_message: str):
+        """Set the system message."""
+        self.system_message = system_message
+
+    def append_message(self, role: str, message: str):
+        """Append a new message."""
+        self.messages.append([role, message])
+
+    def update_last_message(self, message: str):
+        """Update the last output.
+
+        The last message is typically set to be None when constructing the prompt,
+        so we need to update it in-place after getting the response from a model.
+        """
+        self.messages[-1][1] = message
+
+    def reset_message(self):
+        """Reset a new message."""
+        self.messages = []
+
+    def to_gradio_chatbot(self):
+        """Convert the conversation to gradio chatbot format."""
+        ret = []
+        for i, (role, msg) in enumerate(self.messages[self.offset :]):
+            if i % 2 == 0:
+                ret.append([msg, None])
+            else:
+                ret[-1][-1] = msg
+        return ret
+
+    def to_openai_api_messages(self):
+        """Convert the conversation to OpenAI chat completion format."""
+        system_prompt = self.system_template.format(system_message=self.system_message)
+        ret = [{"role": "system", "content": system_prompt}]
+
+        for i, (_, msg) in enumerate(self.messages[self.offset :]):
+            if i % 2 == 0:
+                ret.append({"role": "user", "content": msg})
+            else:
+                if msg is not None:
+                    ret.append({"role": "assistant", "content": msg})
+        return ret
+
+    def copy(self):
+        return Conversation(
+            name=self.name,
+            system_template=self.system_template,
+            system_message=self.system_message,
+            roles=self.roles,
+            messages=[[x, y] for x, y in self.messages],
+            offset=self.offset,
+            sep_style=self.sep_style,
+            sep=self.sep,
+            sep2=self.sep2,
+            stop_str=self.stop_str,
+            stop_token_ids=self.stop_token_ids,
+        )
+
+    def dict(self):
+        return {
+            "template_name": self.name,
+            "system_message": self.system_message,
+            "roles": self.roles,
+            "messages": self.messages,
+            "offset": self.offset,
+        }
+
+
+# A global registry for all conversation templates
+conv_templates: Dict[str, Conversation] = {}
+
+
+def register_conv_template(template: Conversation, override: bool = False):
+    """Register a new conversation template."""
+    if not override:
+        assert template.name not in conv_templates, f"{template.name} has been registered."
+
+    conv_templates[template.name] = template
+
+
+def get_conv_template(name: str) -> Conversation:
+    """Get a conversation template."""
+    return conv_templates[name].copy()
+
+
+register_conv_template(
+    Conversation(
+        name="deepseek",
+        system_template="{system_message}",
+        # system_message="You are a helpful assistant. Please answer truthfully and write out your "
+        # "thinking step by step to be sure you get the right answer.",
+        system_message="",
+        roles=("<|User|>", "<|Assistant|>"),
+        messages=(),
+        offset=0,
+        sep_style=SeparatorStyle.DeepSeek,
+        sep="\n\n",
+        sep2="<｜end▁of▁sentence｜>",
+        stop_token_ids=[100001],
+        stop_str=["User:", "<｜end▁of▁sentence｜>"]
+    )
+)
+register_conv_template(
+    Conversation(
+        name="deepseekv2",
+        system_template="{system_message}",
+        # system_message="You are a helpful assistant. Please answer truthfully and write out your "
+        # "thinking step by step to be sure you get the right answer.",
+        system_message="",
+        roles=("<｜User｜>", "<｜Assistant｜>"),
+        messages=(),
+        offset=0,
+        sep_style=SeparatorStyle.DeepSeek,
+        sep="",
+        sep2="<｜end▁of▁sentence｜>",
+        stop_token_ids=[100001],
+        stop_str=["User:", "<｜end▁of▁sentence｜>"]
+    )
+)
+
+
+register_conv_template(
+    Conversation(
+        name="plain",
+        system_template="",
+        system_message="",
+        roles=("", ""),
+        messages=(),
+        offset=0,
+        sep_style=SeparatorStyle.PLAIN,
+        sep="",
+        sep2="",
+        stop_token_ids=[100001],
+        stop_str=['</s>'],
+    )
+)
+
+
+register_conv_template(
+    Conversation(
+        name="alignment",
+        system_template="",
+        system_message="",
+        roles=("", ""),
+        messages=(),
+        offset=0,
+        sep_style=SeparatorStyle.ALIGNMENT,
+        sep="",
+        sep2="",
+        stop_token_ids=[100001],
+        stop_str=['</s>'],
+    )
+)
+
+
+if __name__ == "__main__":
+    print("deepseek template:")
+    conv = get_conv_template("deepseek")
+    conv.append_message(conv.roles[0], "Hello!")
+    conv.append_message(conv.roles[1], "Hi! This is Tony.")
+    conv.append_message(conv.roles[0], "Who are you?")
+    conv.append_message(conv.roles[1], "I am a helpful assistant.")
+    conv.append_message(conv.roles[0], "How are you?")
+    conv.append_message(conv.roles[1], None)
+    print(conv.get_prompt())
+
+    print("deepseekv2 template:")
+    conv = get_conv_template("deepseekv2")
+    conv.append_message(conv.roles[0], "Hello!")
+    conv.append_message(conv.roles[1], "Hi! This is Tony.")
+    conv.append_message(conv.roles[0], "Who are you?")
+    conv.append_message(conv.roles[1], "I am a helpful assistant.")
+    conv.append_message(conv.roles[0], "How are you?")
+    conv.append_message(conv.roles[1], None)
+    print(conv.get_prompt())

deepencoder.py

@@ -0,0 +1,1058 @@
+import torch.nn as nn
+import torch
+import torch.nn.functional as F
+import copy
+
+from contextlib import nullcontext
+import math
+from typing import Optional, Tuple
+# from megatron.model import LayerNorm
+
+from einops import rearrange
+from easydict import EasyDict as adict
+
+
+from typing import Optional, Tuple, Type
+from functools import partial
+
+
+
+class MlpProjector(nn.Module):
+
+    def __init__(self, cfg):
+
+        super().__init__()
+
+        self.cfg = cfg
+
+        if cfg.projector_type == "identity":
+            modules = nn.Identity()
+
+        elif cfg.projector_type == "linear":
+            modules = nn.Linear(cfg.input_dim, cfg.n_embed)
+
+        elif cfg.projector_type == "mlp_gelu":
+            mlp_depth = cfg.get("depth", 1)
+            modules = [nn.Linear(cfg.input_dim, cfg.n_embed)]
+            for _ in range(1, mlp_depth):
+                modules.append(nn.GELU())
+                modules.append(nn.Linear(cfg.n_embed, cfg.n_embed))
+            modules = nn.Sequential(*modules)
+        
+        elif cfg.projector_type == "normlayer_downsample_mlp_gelu":
+            mlp_depth = cfg.get("depth", 1)
+            mlp_ratio = cfg.get("mlp_ratio", 1)
+            modules = [
+                nn.LayerNorm(cfg.input_dim * cfg.downsample_ratio * cfg.downsample_ratio),
+                nn.Linear(cfg.input_dim * cfg.downsample_ratio * cfg.downsample_ratio, cfg.n_embed * mlp_ratio)
+            ]
+            for _ in range(1, mlp_depth - 1):
+                modules.append(nn.GELU())
+                modules.append(nn.Linear(cfg.n_embed * mlp_ratio, cfg.n_embed * mlp_ratio))
+            modules.append(nn.GELU())
+            modules.append(nn.Linear(cfg.n_embed * mlp_ratio, cfg.n_embed))
+            modules = nn.Sequential(*modules)
+        
+        elif cfg.projector_type == "downsample_mlp_gelu":
+            mlp_depth = cfg.get("depth", 1)
+            mlp_ratio = cfg.get("mlp_ratio", 1)
+            modules = [nn.Linear(cfg.input_dim * cfg.downsample_ratio * cfg.downsample_ratio, cfg.n_embed * mlp_ratio)]
+            for _ in range(1, mlp_depth - 1):
+                modules.append(nn.GELU())
+                modules.append(nn.Linear(cfg.n_embed * mlp_ratio, cfg.n_embed * mlp_ratio))
+            modules.append(nn.GELU())
+            modules.append(nn.Linear(cfg.n_embed * mlp_ratio, cfg.n_embed))
+            modules = nn.Sequential(*modules)
+
+        elif cfg.projector_type == "low_high_hybrid_split_mlp_gelu":
+            mlp_depth = cfg.get("depth", 1)
+            self.high_up_proj = nn.Linear(cfg.input_dim, cfg.n_embed // 2)
+            self.low_up_proj = nn.Linear(cfg.input_dim, cfg.n_embed // 2)
+
+            modules = []
+            for _ in range(1, mlp_depth):
+                modules.append(nn.GELU())
+                modules.append(nn.Linear(cfg.n_embed, cfg.n_embed))
+            modules = nn.Sequential(*modules)
+
+        elif cfg.projector_type == "hybrid_split_feature_mlp_gelu":
+            mlp_depth = cfg.get("depth", 1)
+            channel_div = cfg.get("channel_div", 0.5)
+            self.high_up_proj = nn.Linear(cfg.input_dim[0], int(cfg.n_embed * channel_div))
+            self.low_up_proj = nn.Linear(cfg.input_dim[1], cfg.n_embed - int(cfg.n_embed * channel_div))
+
+            modules = []
+            for _ in range(1, mlp_depth):
+                modules.append(nn.GELU())
+                modules.append(nn.Linear(cfg.n_embed, cfg.n_embed))
+            modules = nn.Sequential(*modules)
+
+        elif cfg.projector_type == "low_high_split_mlp_gelu":
+            mlp_depth = cfg.get("depth", 1)
+            modules = []
+            for _ in range(1, mlp_depth):
+                modules.append(nn.GELU())
+                modules.append(nn.Linear(cfg.n_embed // 2, cfg.n_embed // 2))
+            modules = nn.Sequential(*modules)
+            self.high_layers = nn.Sequential(*modules)
+            self.low_layers = copy.deepcopy(modules)
+
+        else:
+            raise ValueError(f"Unknown projector type: {cfg.projector_type}")
+
+        if cfg.get("token_pooling", False):
+            self.token_pooling_layer = nn.Linear(cfg.input_dim * 4, cfg.input_dim)
+
+        if cfg.get("conv_fusion_high_low_features", False):
+            self.fusion_layer = nn.Linear(cfg.input_dim, cfg.input_dim)
+        self.layers = modules
+
+    def forward(self, x):
+        if self.cfg.get("token_pooling", False):
+            batch_size, wxh, channels = x.shape
+            w = h = int(wxh**0.5)
+            x = x.view(batch_size, w, h, channels)
+            x = x.permute(0, 3, 1, 2)
+            # import ipdb; ipdb.set_trace()
+            patches = x.unfold(2, 2, 2).unfold(3, 2, 2)
+            batch_size, channels, h_patches, w_patches, _, _ = patches.size()
+            # 在通道维度上拼接
+            patches = patches.contiguous().view(batch_size, channels, h_patches * w_patches, -1)
+
+            # 通过线性层
+            patches = patches.permute(0, 2, 1, 3).contiguous()
+            patches = patches.view(batch_size, h_patches * w_patches, channels * 4)
+
+            x = self.token_pooling_layer(patches)
+        
+        if self.cfg.get("conv_fusion_high_low_features", False):
+            x = self.fusion_layer(x[:, 0]) + x[:, 1]
+
+        if self.cfg.projector_type == 'low_high_hybrid_split_mlp_gelu':
+            high_x, low_x = x[0], x[1]
+            high_x = self.high_up_proj(high_x)
+            low_x = self.low_up_proj(low_x)
+            x = torch.concat([high_x, low_x], dim=-1)
+        
+        if self.cfg.projector_type == 'hybrid_split_feature_mlp_gelu':
+            high_x = x[...,:self.cfg.input_dim[0]]
+            low_x = x[...,self.cfg.input_dim[0]:]
+            high_x = self.high_up_proj(high_x)
+            low_x = self.low_up_proj(low_x)
+            x = torch.concat([high_x, low_x], dim=-1)
+        
+        if self.cfg.projector_type == 'low_high_split_mlp_gelu':
+            high_x, low_x = x[0], x[1]
+            high_x = self.high_layers(high_x)
+            low_x = self.low_layers(low_x)
+            x = torch.concat([high_x, low_x], dim=-1)
+            return x
+        
+        if self.cfg.projector_type == 'downsample_mlp_gelu' or self.cfg.projector_type == 'normlayer_downsample_mlp_gelu':
+            bs, hw, input_dim = x.shape
+            h = w = int((hw) ** 0.5)
+
+            """compute padding"""
+            if h % self.cfg.downsample_ratio:
+                pad = self.cfg.downsample_ratio - h % self.cfg.downsample_ratio
+            else:
+                pad = 0
+            x = x.reshape(bs, h, w, input_dim)
+            if pad > 0:
+                x = F.pad(x, (0, 0, 0, pad, 0, pad), "constant", 0)
+
+            """4 to 1 concat"""
+            x = x.permute(0, 3, 1, 2)  # B, C, H, W
+            x = F.unfold(x, kernel_size=self.cfg.downsample_ratio, stride=self.cfg.downsample_ratio, padding=0) # B, C*4, HW // 4
+            x = x.permute(0, 2, 1)
+            
+        return self.layers(x)
+
+    @staticmethod
+    def get_flops_per_sample(cfg):
+        if cfg.projector_type == "linear":
+            fwd = 2 * cfg.input_dim * cfg.n_embed
+
+        elif "mlp_gelu" in cfg.projector_type :
+            mlp_depth = cfg.get("depth", 1)
+            downsample_ratio = cfg.get("downsample_ratio", 1)
+            input_dim = sum(cfg.input_dim) if isinstance(cfg.input_dim, list) else cfg.input_dim
+            input_dim = input_dim * downsample_ratio * downsample_ratio
+            fwd = 2 * input_dim * cfg.n_embed + (mlp_depth - 1) * 2 * cfg.n_embed * cfg.n_embed
+        else:
+            fwd = 0
+
+        return fwd * 3
+    
+
+#===================clip============================================================
+
+class LayerNormfp32(torch.nn.LayerNorm):
+    """Subclass torch's LayerNorm to handle fp16."""
+
+    def forward(self, x: torch.Tensor):
+        orig_type = x.dtype
+        ret = super().forward(x.type(torch.float32))
+        return ret.type(orig_type)
+
+
+def get_abs_pos(abs_pos, tgt_size):
+    # abs_pos: L, C
+    # tgt_size: M
+    # return: M, C
+
+    # print(tgt_size)
+    # print(abs_pos.shape)
+    # exit()
+    dim = abs_pos.size(-1)
+    # print(dim)
+    abs_pos_new = abs_pos.squeeze(0)
+    cls_token, old_pos_embed = abs_pos_new[:1], abs_pos_new[1:]
+
+
+
+    src_size = int(math.sqrt(abs_pos_new.shape[0] - 1))
+    tgt_size = int(math.sqrt(tgt_size))
+    dtype = abs_pos.dtype
+
+    if src_size != tgt_size:
+        old_pos_embed = old_pos_embed.view(1, src_size, src_size, dim).permute(0, 3, 1,
+                                                                                    2).contiguous()
+        old_pos_embed = old_pos_embed.to(torch.float32)
+        new_pos_embed = F.interpolate(
+            old_pos_embed,
+            size=(tgt_size, tgt_size),
+            mode='bicubic',
+            antialias=True,
+            align_corners=False,
+        ).to(dtype)
+        new_pos_embed = new_pos_embed.permute(0, 2, 3, 1)
+        new_pos_embed = new_pos_embed.view(tgt_size * tgt_size, dim)
+        vision_pos_embed = torch.cat([cls_token, new_pos_embed], dim=0)
+        vision_pos_embed = vision_pos_embed.view(1, tgt_size * tgt_size + 1, dim)
+        return vision_pos_embed
+    else:
+        return abs_pos
+
+@torch.jit.script
+def quick_gelu(x):
+    return x * torch.sigmoid(1.702 * x)
+
+
+
+class CLIPVisionEmbeddings(nn.Module):
+    def __init__(self, hidden_size=1024, image_size=224, patch_size=14, num_channels=3):
+        super().__init__()
+        self.embed_dim = hidden_size
+        self.image_size = image_size
+        self.patch_size = patch_size
+
+        self.class_embedding = torch.nn.Parameter(torch.randn(self.embed_dim))
+
+        self.patch_embedding = torch.nn.Conv2d(
+            in_channels=num_channels,
+            out_channels=self.embed_dim,
+            kernel_size=self.patch_size,
+            stride=self.patch_size,
+            bias=False,
+        )
+
+        self.num_patches = (self.image_size // self.patch_size) ** 2
+        self.num_positions = self.num_patches + 1
+        self.position_embedding = torch.nn.Embedding(self.num_positions, self.embed_dim)
+        self.register_buffer(
+            "position_ids", torch.arange(self.num_positions).expand((1, -1))
+        )
+
+    def forward(self, pixel_values, patch_embeds):
+        batch_size = pixel_values.shape[0]
+        # patch_embeds = self.patch_embedding(
+        #     pixel_values
+        # )  # shape = [*, width, grid, grid]
+
+
+        if patch_embeds is not None:
+            patch_embeds = patch_embeds
+            # print(patch_embeds.shape)
+        else:
+            patch_embeds = self.patch_embedding(pixel_values)  
+            # print(111111)
+        # shape = [*, width, grid, grid]
+        # patch_embeds = patch_embeds.flatten(2).transpose(1, 2)
+
+        patch_embeds = patch_embeds.flatten(2).transpose(1, 2)
+
+
+        class_embeds = self.class_embedding.expand(batch_size, 1, -1)
+        embeddings = torch.cat([class_embeds, patch_embeds], dim=1)
+
+        # x = torch.cat([cls_token, x], dim=1)
+        embeddings = embeddings + get_abs_pos(self.position_embedding(self.position_ids), embeddings.size(1))
+        # embeddings = embeddings + self.position_embedding(self.position_ids)
+        return embeddings
+
+
+class NoTPFeedForward(nn.Module):
+    def __init__(
+            self,
+            cfg,
+            dim: int,
+            hidden_dim: int,
+    ):
+        super().__init__()
+
+        self.fc1 = torch.nn.Linear(dim, hidden_dim, bias=True)
+        self.fc2 = torch.nn.Linear(hidden_dim, dim, bias=True)
+
+    def forward(self, x):
+        output = self.fc2(quick_gelu(self.fc1(x)))
+        return output
+
+
+
+
+class NoTPAttention(torch.nn.Module):
+    def __init__(self, cfg):
+        super().__init__()
+        self.num_heads = cfg.num_attention_heads
+        self.n_local_heads = cfg.num_attention_heads
+        self.head_dim = cfg.hidden_size // cfg.num_attention_heads
+        self.max_seq_len = cfg.seq_length
+        self.use_flash_attention = cfg.use_flash_attn
+
+        self.qkv_proj = torch.nn.Linear(cfg.hidden_size, cfg.hidden_size * 3, bias=True)
+        self.out_proj = torch.nn.Linear(cfg.hidden_size, cfg.hidden_size, bias=True)
+
+        # self.core_attention = CoreAttention(cfg, AttnType.self_attn)
+
+        self.attn_drop = cfg.attention_dropout
+
+    def forward(
+            self,
+            x: torch.Tensor,
+    ):
+        bsz, seqlen, _ = x.shape
+        xqkv = self.qkv_proj(x)
+        xqkv = xqkv.view(bsz, seqlen, 3, self.num_heads, self.head_dim)
+
+        if self.use_flash_attention:
+
+            xq, xk, xv = torch.split(xqkv, 1, dim=2)
+            xq = xq.squeeze(2)
+            xk = xk.squeeze(2)
+            xv = xv.squeeze(2)
+            # xq, xk, xv = xqkv[:, :, 0, ...], xqkv[:, :, 1, ...], xqkv[:, :, 2, ...]
+
+            # （B, num_head, S, head_size)
+            xq = xq.permute(0, 2, 1, 3)
+            xk = xk.permute(0, 2, 1, 3)
+            xv = xv.permute(0, 2, 1, 3)
+            # with torch.backends.cuda.sdp_kernel(enable_flash=True, enable_math=False, enable_mem_efficient=False):
+            output = torch.nn.functional.scaled_dot_product_attention(xq, xk, xv, attn_mask=None)
+            output = output.permute(0, 2, 1, 3).reshape(bsz, seqlen, -1)
+                # output = output.permute(0, 2, 1, 3).contiguous().view(bsz, seqlen, -1)
+        else:
+            # print(22222)
+            xq, xk, xv = torch.split(xqkv, 1, dim=2)
+            xq = xq.squeeze(2)
+            xk = xk.squeeze(2)
+            xv = xv.squeeze(2)
+            # xq, xk, xv = xqkv[:, :, 0, ...], xqkv[:, :, 1, ...], xqkv[:, :, 2, ...]
+
+            # （B, num_head, S, head_size)
+            xq = xq.permute(0, 2, 1, 3)
+            xk = xk.permute(0, 2, 1, 3)
+            xv = xv.permute(0, 2, 1, 3)
+            # with torch.backends.cuda.sdp_kernel(enable_flash=True, enable_math=False, enable_mem_efficient=False):
+            output = torch.nn.functional.scaled_dot_product_attention(xq, xk, xv, attn_mask=None)
+            output = output.permute(0, 2, 1, 3).reshape(bsz, seqlen, -1)
+            # output = output.permute(0, 2, 1, 3).contiguous().view(bsz, seqlen, -1)
+        output = self.out_proj(output)
+        return output
+
+class NoTPTransformerBlock(nn.Module):
+    def __init__(self, cfg, layer_id: int, multiple_of=256):
+        super().__init__()
+
+        self.n_heads = cfg.num_attention_heads
+        self.dim = cfg.hidden_size
+        self.head_dim = cfg.hidden_size // cfg.num_attention_heads
+        self.self_attn = NoTPAttention(cfg)
+        self.mlp = NoTPFeedForward(
+            cfg, dim=cfg.hidden_size, hidden_dim=cfg.ffn_hidden_size
+        )
+        self.layer_id = layer_id
+        self.layer_norm1 = torch.nn.LayerNorm(
+            cfg.hidden_size, eps=cfg.layernorm_epsilon
+        )
+        self.layer_norm2 = torch.nn.LayerNorm(
+            cfg.hidden_size, eps=cfg.layernorm_epsilon
+        )
+
+    def forward(self, x: torch.Tensor):
+        residual = self.self_attn.forward(self.layer_norm1(x))
+        h = x + residual
+        out = h + self.mlp.forward(self.layer_norm2(h))
+        return out
+
+
+class NoTPTransformer(nn.Module):
+    def __init__(self, cfg):
+        super().__init__()
+
+        self.cfg = cfg
+        # self.recompute_list = self.cfg.get("recompute_list", [])
+        self.num_layers = cfg.num_layers  # _get_num_layers(cfg)
+
+        self.layers = torch.nn.ModuleList()
+        for layer_id in range(self.num_layers):
+            self.layers.append(
+                NoTPTransformerBlock(
+                    cfg,
+                    layer_id + 1,
+                )
+            )
+
+    def forward(
+            self,
+            hidden_states,
+    ):
+
+        for lid, layer in enumerate(self.layers):
+            # if lid in self.recompute_list:
+            #     def custom(layer_id):
+            #         def custom_forward(*args, **kwargs):
+            #             x_ = self.layers[layer_id](*args, **kwargs)
+            #             return x_
+
+            #         return custom_forward
+
+            #     assert hidden_states.requires_grad == True, logger.warning(
+            #         "When using recalculation, the input must have grad fn"
+            #     )
+            #     hidden_states = tensor_parallel.checkpoint(
+            #         custom(lid),
+            #         False,
+            #         hidden_states.contiguous()
+            #     )
+            # else:
+            hidden_states = layer(hidden_states)
+
+        return hidden_states
+
+
+# from megatron.core.tensor_parallel.layers import non_tensor_paralleled, local_dp_reduce, local_dp_scatter
+
+class VitModel(nn.Module):
+    def __init__(
+            self,
+            cfg,
+            freeze_embed=False,
+            freeze_pre_norm=False
+    ) -> None:
+        super().__init__()
+
+        self.embeddings = CLIPVisionEmbeddings(hidden_size=cfg.hidden_size, image_size=cfg.image_size, patch_size=cfg.patch_size)
+
+        if freeze_embed:
+            for name, param in self.embeddings.named_parameters():
+                param.requires_grad = False
+
+        self.transformer = NoTPTransformer(cfg=cfg)
+
+        if cfg.get("fp32norm", False):
+            logger.info("Load fp32 layernorm for ViT.")
+            self.pre_layrnorm = LayerNormfp32(
+                cfg.hidden_size,
+                eps=cfg.get("pre_layernorm_epsilon", 1e-5),
+            )
+        else:
+            self.pre_layrnorm = torch.nn.LayerNorm(
+                cfg.hidden_size,
+                eps=cfg.get("pre_layernorm_epsilon", 1e-5),
+            )
+
+        # self.pre_layrnorm = RMSNorm(
+        #     cfg.hidden_size,
+        #     eps=cfg.get("pre_layernorm_epsilon", 1e-5),
+        #     sequence_parallel=False,
+        #     use_fp32=True,
+        #     use_optimus=True,
+        # )
+
+        if freeze_pre_norm:
+            for name, param in self.pre_layrnorm.named_parameters():
+                param.requires_grad = False
+
+        for p in self.parameters():
+            p.micro_dp = True
+
+    def set_input_tensor(self, input_tensor):
+        if not isinstance(input_tensor, list):
+            input_tensor = [input_tensor]
+        self.transformer.set_input_tensor(input_tensor[0])
+
+    def __str__(self) -> str:
+        return "open_clip"
+
+    def forward(
+            self,
+            x,
+            patch_embeds
+    ):
+        x = self.embeddings(x, patch_embeds)
+        hidden_states = self.pre_layrnorm(x)
+
+        # hidden_states, dis = local_dp_scatter(hidden_states)
+        output = self.transformer(hidden_states)
+
+        # output = local_dp_reduce(output, dis)
+
+        return output
+
+
+vit_model_cfg = adict(
+    num_layers=24,
+    hidden_size=1024,
+    num_heads = 16,
+    num_attention_heads=16,
+    ffn_hidden_size=4096,
+    seq_length=256,
+    max_position_embeddings=256,
+    use_flash_attn=False,
+    understand_projector_stride=2,
+    hidden_dropout = 0.0,
+    attention_dropout = 0.0,
+    no_persist_layer_norm = False,
+    layernorm_epsilon = 1e-5,
+    pre_layernorm_epsilon = 1e-5,
+    image_size = 224,
+    patch_size = 14,
+    recompute_list = []
+)
+
+def build_clip_l():
+    return VitModel(
+        cfg=vit_model_cfg,
+        freeze_embed=False,
+        freeze_pre_norm=False,
+    )
+
+
+
+
+
+#=========================Sam-Vary=================================
+
+
+def get_abs_pos_sam(abs_pos, tgt_size):
+
+    dtype = abs_pos.dtype
+
+    src_size = abs_pos.size(1)
+
+    if src_size != tgt_size:
+        old_pos_embed = abs_pos.permute(0, 3, 1, 2)
+        old_pos_embed = old_pos_embed.to(torch.float32)
+        new_pos_embed = F.interpolate(
+            old_pos_embed,
+            size=(tgt_size, tgt_size),
+            mode='bicubic',
+            antialias=True,
+            align_corners=False,
+        ).to(dtype)
+        new_pos_embed = new_pos_embed.permute(0, 2, 3, 1)
+        return new_pos_embed
+    else:
+        return abs_pos
+
+
+
+
+class MLPBlock(nn.Module):
+    def __init__(
+        self,
+        embedding_dim: int,
+        mlp_dim: int,
+        act: Type[nn.Module] = nn.GELU,
+    ) -> None:
+        super().__init__()
+        self.lin1 = nn.Linear(embedding_dim, mlp_dim)
+        self.lin2 = nn.Linear(mlp_dim, embedding_dim)
+        self.act = act()
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        return self.lin2(self.act(self.lin1(x)))
+
+
+# From https://github.com/facebookresearch/detectron2/blob/main/detectron2/layers/batch_norm.py # noqa
+# Itself from https://github.com/facebookresearch/ConvNeXt/blob/d1fa8f6fef0a165b27399986cc2bdacc92777e40/models/convnext.py#L119  # noqa
+class LayerNorm2d(nn.Module):
+    def __init__(self, num_channels: int, eps: float = 1e-6) -> None:
+        super().__init__()
+        self.weight = nn.Parameter(torch.ones(num_channels))
+        self.bias = nn.Parameter(torch.zeros(num_channels))
+        self.eps = eps
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        u = x.mean(1, keepdim=True)
+        s = (x - u).pow(2).mean(1, keepdim=True)
+        x = (x - u) / torch.sqrt(s + self.eps)
+        x = self.weight[:, None, None] * x + self.bias[:, None, None]
+        return x
+
+
+# This class and its supporting functions below lightly adapted from the ViTDet backbone available at: https://github.com/facebookresearch/detectron2/blob/main/detectron2/modeling/backbone/vit.py # noqa
+class ImageEncoderViT(nn.Module):
+    def __init__(
+        self,
+        img_size: int = 1024,
+        patch_size: int = 16,
+        in_chans: int = 3,
+        embed_dim: int = 768,
+        depth: int = 12,
+        num_heads: int = 12,
+        mlp_ratio: float = 4.0,
+        out_chans: int = 256,
+        qkv_bias: bool = True,
+        norm_layer: Type[nn.Module] = nn.LayerNorm,
+        act_layer: Type[nn.Module] = nn.GELU,
+        use_abs_pos: bool = True,
+        use_rel_pos: bool = False,
+        rel_pos_zero_init: bool = True,
+        window_size: int = 0,
+        global_attn_indexes: Tuple[int, ...] = (),
+    ) -> None:
+        """
+        Args:
+            img_size (int): Input image size.
+            patch_size (int): Patch size.
+            in_chans (int): Number of input image channels.
+            embed_dim (int): Patch embedding dimension.
+            depth (int): Depth of ViT.
+            num_heads (int): Number of attention heads in each ViT block.
+            mlp_ratio (float): Ratio of mlp hidden dim to embedding dim.
+            qkv_bias (bool): If True, add a learnable bias to query, key, value.
+            norm_layer (nn.Module): Normalization layer.
+            act_layer (nn.Module): Activation layer.
+            use_abs_pos (bool): If True, use absolute positional embeddings.
+            use_rel_pos (bool): If True, add relative positional embeddings to the attention map.
+            rel_pos_zero_init (bool): If True, zero initialize relative positional parameters.
+            window_size (int): Window size for window attention blocks.
+            global_attn_indexes (list): Indexes for blocks using global attention.
+        """
+        super().__init__()
+        self.img_size = img_size
+
+        self.patch_embed = PatchEmbed(
+            kernel_size=(patch_size, patch_size),
+            stride=(patch_size, patch_size),
+            in_chans=in_chans,
+            embed_dim=embed_dim,
+        )
+
+        self.pos_embed: Optional[nn.Parameter] = None
+        if use_abs_pos:
+            # Initialize absolute positional embedding with pretrain image size.
+            self.pos_embed = nn.Parameter(
+                torch.zeros(1, img_size // patch_size, img_size // patch_size, embed_dim)
+            )
+
+        self.blocks = nn.ModuleList()
+        for i in range(depth):
+            block = Block(
+                dim=embed_dim,
+                num_heads=num_heads,
+                mlp_ratio=mlp_ratio,
+                qkv_bias=qkv_bias,
+                norm_layer=norm_layer,
+                act_layer=act_layer,
+                use_rel_pos=use_rel_pos,
+                rel_pos_zero_init=rel_pos_zero_init,
+                window_size=window_size if i not in global_attn_indexes else 0,
+                input_size=(img_size // patch_size, img_size // patch_size),
+            )
+            self.blocks.append(block)
+
+        self.neck = nn.Sequential(
+            nn.Conv2d(
+                embed_dim,
+                out_chans,
+                kernel_size=1,
+                bias=False,
+            ),
+            LayerNorm2d(out_chans),
+            nn.Conv2d(
+                out_chans,
+                out_chans,
+                kernel_size=3,
+                padding=1,
+                bias=False,
+            ),
+            LayerNorm2d(out_chans),
+        )
+
+        self.net_2 = nn.Conv2d(256, 512, kernel_size=3, stride=2, padding=1, bias=False)
+        self.net_3 = nn.Conv2d(512, 1024, kernel_size=3, stride=2, padding=1, bias=False)
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        x = self.patch_embed(x)
+        if self.pos_embed is not None:
+            # x = x + self.pos_embed
+            x = x + get_abs_pos_sam(self.pos_embed, x.size(1))
+
+        for blk in self.blocks:
+            x = blk(x)
+
+        x = self.neck(x.permute(0, 3, 1, 2))
+        x2 = self.net_2(x)
+        x3 = self.net_3(x2.clone())
+
+        return x3
+
+
+class Block(nn.Module):
+    """Transformer blocks with support of window attention and residual propagation blocks"""
+
+    def __init__(
+        self,
+        dim: int,
+        num_heads: int,
+        mlp_ratio: float = 4.0,
+        qkv_bias: bool = True,
+        norm_layer: Type[nn.Module] = nn.LayerNorm,
+        act_layer: Type[nn.Module] = nn.GELU,
+        use_rel_pos: bool = False,
+        rel_pos_zero_init: bool = True,
+        window_size: int = 0,
+        input_size: Optional[Tuple[int, int]] = None,
+    ) -> None:
+        """
+        Args:
+            dim (int): Number of input channels.
+            num_heads (int): Number of attention heads in each ViT block.
+            mlp_ratio (float): Ratio of mlp hidden dim to embedding dim.
+            qkv_bias (bool): If True, add a learnable bias to query, key, value.
+            norm_layer (nn.Module): Normalization layer.
+            act_layer (nn.Module): Activation layer.
+            use_rel_pos (bool): If True, add relative positional embeddings to the attention map.
+            rel_pos_zero_init (bool): If True, zero initialize relative positional parameters.
+            window_size (int): Window size for window attention blocks. If it equals 0, then
+                use global attention.
+            input_size (tuple(int, int) or None): Input resolution for calculating the relative
+                positional parameter size.
+        """
+        super().__init__()
+        self.norm1 = norm_layer(dim)
+        self.attn = Attention(
+            dim,
+            num_heads=num_heads,
+            qkv_bias=qkv_bias,
+            use_rel_pos=use_rel_pos,
+            rel_pos_zero_init=rel_pos_zero_init,
+            input_size=input_size if window_size == 0 else (window_size, window_size),
+        )
+
+        self.norm2 = norm_layer(dim)
+        self.mlp = MLPBlock(embedding_dim=dim, mlp_dim=int(dim * mlp_ratio), act=act_layer)
+
+        self.window_size = window_size
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        shortcut = x
+        x = self.norm1(x)
+        # Window partition
+        if self.window_size > 0:
+            H, W = x.shape[1], x.shape[2]
+            x, pad_hw = window_partition(x, self.window_size)
+
+        x = self.attn(x)
+        # Reverse window partition
+        if self.window_size > 0:
+            x = window_unpartition(x, self.window_size, pad_hw, (H, W))
+
+        x = shortcut + x
+        x = x + self.mlp(self.norm2(x))
+
+        return x
+
+
+class Attention(nn.Module):
+    """Multi-head Attention block with relative position embeddings."""
+
+    def __init__(
+        self,
+        dim: int,
+        num_heads: int = 8,
+        qkv_bias: bool = True,
+        use_rel_pos: bool = False,
+        rel_pos_zero_init: bool = True,
+        input_size: Optional[Tuple[int, int]] = None,
+    ) -> None:
+        """
+        Args:
+            dim (int): Number of input channels.
+            num_heads (int): Number of attention heads.
+            qkv_bias (bool):  If True, add a learnable bias to query, key, value.
+            rel_pos (bool): If True, add relative positional embeddings to the attention map.
+            rel_pos_zero_init (bool): If True, zero initialize relative positional parameters.
+            input_size (tuple(int, int) or None): Input resolution for calculating the relative
+                positional parameter size.
+        """
+        super().__init__()
+        self.num_heads = num_heads
+        head_dim = dim // num_heads
+        self.scale = head_dim**-0.5
+
+        self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias)
+        self.proj = nn.Linear(dim, dim)
+
+        self.use_rel_pos = use_rel_pos
+        if self.use_rel_pos:
+            assert (
+                input_size is not None
+            ), "Input size must be provided if using relative positional encoding."
+            # initialize relative positional embeddings
+            self.rel_pos_h = nn.Parameter(torch.zeros(2 * input_size[0] - 1, head_dim))
+            self.rel_pos_w = nn.Parameter(torch.zeros(2 * input_size[1] - 1, head_dim))
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        B, H, W, _ = x.shape
+        # qkv with shape (3, B, nHead, H * W, C)
+        qkv = self.qkv(x).reshape(B, H * W, 3, self.num_heads, -1).permute(2, 0, 3, 1, 4)
+        # q, k, v with shape (B * nHead, H * W, C)
+        q, k, v = qkv.reshape(3, B * self.num_heads, H * W, -1).unbind(0)
+
+        rel_h, rel_w = None, None
+        if self.use_rel_pos:
+            rel_h, rel_w = add_decomposed_rel_pos(q, self.rel_pos_h, self.rel_pos_w, (H, W), (H, W))
+
+        q = q.view(B, self.num_heads, H * W, -1)
+        k = k.view(B, self.num_heads, H * W, -1)
+        v = v.view(B, self.num_heads, H * W, -1)
+
+        if self.use_rel_pos:
+            rel_h = rel_h.view(B, self.num_heads, rel_h.size(1), rel_h.size(2), rel_h.size(3))
+            rel_w = rel_w.view(B, self.num_heads, rel_w.size(1), rel_w.size(2), rel_w.size(3))
+            attn_bias = (rel_h + rel_w).view(B, self.num_heads, rel_h.size(2), rel_h.size(3) * rel_w.size(4))
+            x = torch.nn.functional.scaled_dot_product_attention(q, k, v, attn_mask=attn_bias)
+            # x = _attention_rel_h_rel_w(q, k, v, rel_h, rel_w)
+        else:
+            x = torch.nn.functional.scaled_dot_product_attention(q, k, v)
+
+        x = x.view(B, self.num_heads, H, W, -1).permute(0, 2, 3, 1, 4).reshape(B, H, W, -1)
+
+        x = self.proj(x)
+
+        return x
+
+
+def window_partition(x: torch.Tensor, window_size: int) -> Tuple[torch.Tensor, Tuple[int, int]]:
+    """
+    Partition into non-overlapping windows with padding if needed.
+    Args:
+        x (tensor): input tokens with [B, H, W, C].
+        window_size (int): window size.
+
+    Returns:
+        windows: windows after partition with [B * num_windows, window_size, window_size, C].
+        (Hp, Wp): padded height and width before partition
+    """
+    B, H, W, C = x.shape
+
+    pad_h = (window_size - H % window_size) % window_size
+    pad_w = (window_size - W % window_size) % window_size
+    if pad_h > 0 or pad_w > 0:
+        x = F.pad(x, (0, 0, 0, pad_w, 0, pad_h))
+    Hp, Wp = H + pad_h, W + pad_w
+
+    x = x.view(B, Hp // window_size, window_size, Wp // window_size, window_size, C)
+    windows = x.permute(0, 1, 3, 2, 4, 5).contiguous().view(-1, window_size, window_size, C)
+    return windows, (Hp, Wp)
+
+
+def window_unpartition(
+    windows: torch.Tensor, window_size: int, pad_hw: Tuple[int, int], hw: Tuple[int, int]
+) -> torch.Tensor:
+    """
+    Window unpartition into original sequences and removing padding.
+    Args:
+        windows (tensor): input tokens with [B * num_windows, window_size, window_size, C].
+        window_size (int): window size.
+        pad_hw (Tuple): padded height and width (Hp, Wp).
+        hw (Tuple): original height and width (H, W) before padding.
+
+    Returns:
+        x: unpartitioned sequences with [B, H, W, C].
+    """
+    Hp, Wp = pad_hw
+    H, W = hw
+    B = windows.shape[0] // (Hp * Wp // window_size // window_size)
+    x = windows.view(B, Hp // window_size, Wp // window_size, window_size, window_size, -1)
+    x = x.permute(0, 1, 3, 2, 4, 5).contiguous().view(B, Hp, Wp, -1)
+
+    if Hp > H or Wp > W:
+        x = x[:, :H, :W, :].contiguous()
+    return x
+
+
+def get_rel_pos(q_size: int, k_size: int, rel_pos: torch.Tensor) -> torch.Tensor:
+    """
+    Get relative positional embeddings according to the relative positions of
+        query and key sizes.
+    Args:
+        q_size (int): size of query q.
+        k_size (int): size of key k.
+        rel_pos (Tensor): relative position embeddings (L, C).
+
+    Returns:
+        Extracted positional embeddings according to relative positions.
+    """
+    max_rel_dist = int(2 * max(q_size, k_size) - 1)
+    # Interpolate rel pos if needed.
+    if rel_pos.shape[0] != max_rel_dist:
+        # Interpolate rel pos.
+        dtype = rel_pos.dtype
+        rel_pos = rel_pos.to(torch.float32)
+        rel_pos_resized = F.interpolate(
+            rel_pos.reshape(1, rel_pos.shape[0], -1).permute(0, 2, 1),
+            size=max_rel_dist,
+            mode="linear",
+        ).to(dtype)
+        rel_pos_resized = rel_pos_resized.reshape(-1, max_rel_dist).permute(1, 0)
+    else:
+        rel_pos_resized = rel_pos
+
+    # Scale the coords with short length if shapes for q and k are different.
+    q_coords = torch.arange(q_size, device=rel_pos.device)[:, None] * max(k_size / q_size, 1.0)
+    k_coords = torch.arange(k_size, device=rel_pos.device)[None, :] * max(q_size / k_size, 1.0)
+    relative_coords = (q_coords - k_coords) + (k_size - 1) * max(q_size / k_size, 1.0)
+
+    return rel_pos_resized[relative_coords.long()]
+
+
+def add_decomposed_rel_pos(
+    q: torch.Tensor,
+    rel_pos_h: torch.Tensor,
+    rel_pos_w: torch.Tensor,
+    q_size: Tuple[int, int],
+    k_size: Tuple[int, int],
+) -> torch.Tensor:
+    """
+    Calculate decomposed Relative Positional Embeddings from :paper:`mvitv2`.
+    https://github.com/facebookresearch/mvit/blob/19786631e330df9f3622e5402b4a419a263a2c80/mvit/models/attention.py   # noqa B950
+    Args:
+        q (Tensor): query q in the attention layer with shape (B, q_h * q_w, C).
+        rel_pos_h (Tensor): relative position embeddings (Lh, C) for height axis.
+        rel_pos_w (Tensor): relative position embeddings (Lw, C) for width axis.
+        q_size (Tuple): spatial sequence size of query q with (q_h, q_w).
+        k_size (Tuple): spatial sequence size of key k with (k_h, k_w).
+
+    Returns:
+        attn (Tensor): attention map with added relative positional embeddings.
+    """
+    q_h, q_w = q_size
+    k_h, k_w = k_size
+    Rh = get_rel_pos(q_h, k_h, rel_pos_h)
+    Rw = get_rel_pos(q_w, k_w, rel_pos_w)
+
+    B, _, dim = q.shape
+    r_q = q.reshape(B, q_h, q_w, dim)
+    rel_h = torch.einsum("bhwc,hkc->bhwk", r_q, Rh)
+    rel_w = torch.einsum("bhwc,wkc->bhwk", r_q, Rw)
+    rel_h = rel_h.unsqueeze(-1)
+    rel_w = rel_w.unsqueeze(-2)
+    rel_h = rel_h.reshape(B, q_h * q_w, k_h, 1)
+    rel_w = rel_w.reshape(B, q_h * q_w, 1, k_w)
+
+    return rel_h, rel_w
+
+
+class PatchEmbed(nn.Module):
+    """
+    Image to Patch Embedding.
+    """
+
+    def __init__(
+        self,
+        kernel_size: Tuple[int, int] = (16, 16),
+        stride: Tuple[int, int] = (16, 16),
+        padding: Tuple[int, int] = (0, 0),
+        in_chans: int = 3,
+        embed_dim: int = 768,
+    ) -> None:
+        """
+        Args:
+            kernel_size (Tuple): kernel size of the projection layer.
+            stride (Tuple): stride of the projection layer.
+            padding (Tuple): padding size of the projection layer.
+            in_chans (int): Number of input image channels.
+            embed_dim (int): Patch embedding dimension.
+        """
+        super().__init__()
+
+        self.proj = nn.Conv2d(
+            in_chans, embed_dim, kernel_size=kernel_size, stride=stride, padding=padding
+        )
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        x = self.proj(x)
+        # B C H W -> B H W C
+        x = x.permute(0, 2, 3, 1)
+        return x
+
+
+def build_sam_vit_b(checkpoint=None):
+    return _build_sam(
+        encoder_embed_dim=768,
+        encoder_depth=12,
+        encoder_num_heads=12,
+        encoder_global_attn_indexes=[2, 5, 8, 11],
+        checkpoint=checkpoint,
+    )
+
+def build_sam_fast_vit_b(checkpoint=None, compile_mode='max-autotune', dtype=torch.bfloat16):
+    image_encoder = build_sam_vit_b(checkpoint).eval().to(dtype)
+    # sam = _apply_eval_dtype_sam(sam, dtype)
+    image_encoder = torch.compile(image_encoder, mode=compile_mode)
+    return image_encoder
+
+
+def _build_sam(
+    encoder_embed_dim,
+    encoder_depth,
+    encoder_num_heads,
+    encoder_global_attn_indexes,
+    checkpoint=None,
+):
+    prompt_embed_dim = 256
+    image_size = 1024
+    vit_patch_size = 16
+    image_embedding_size = image_size // vit_patch_size
+    image_encoder=ImageEncoderViT(
+            depth=encoder_depth,
+            embed_dim=encoder_embed_dim,
+            img_size=image_size,
+            mlp_ratio=4,
+            norm_layer=partial(torch.nn.LayerNorm, eps=1e-6),
+            num_heads=encoder_num_heads,
+            patch_size=vit_patch_size,
+            qkv_bias=True,
+            use_rel_pos=True,
+            global_attn_indexes=encoder_global_attn_indexes,
+            window_size=14,
+            out_chans=prompt_embed_dim,
+        )
+    image_encoder.eval()
+    if checkpoint is not None:
+        # with open(checkpoint, "rb") as f:
+        state_dict = torch.load(checkpoint)
+        # print(state_dict.keys())
+        # for key in state_dict:
+        # image_encoder.load_state_dict({k[14:]: v for k, v in state_dict.items() if 'image_encoder' in k}, strict=False)
+        # ocr-anyting
+        # image_encoder.load_state_dict(state_dict, strict=True)
+        # tob
+        image_encoder.load_state_dict({k[30:]: v for k, v in state_dict.items() if 'vision_tower_high' in k}, strict=True)
+        print(checkpoint)
+    return image_encoder

model-00001-of-000001.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1169e7cdc28ff2fb6186556acb2175db148ad26a62097df4c45a17e523180d3f
+size 6672547120

modeling_deepseekocr.py

@@ -0,0 +1,1044 @@
+import os
+import math
+import re
+from tqdm import tqdm
+from abc import ABC
+from typing import List, Optional, Tuple, Union
+
+from addict import Dict
+from PIL import Image, ImageOps, ImageDraw, ImageFont
+import numpy as np
+
+import torch
+import torch.nn as nn
+from torch.nn import CrossEntropyLoss
+from torchvision import transforms
+
+from transformers.cache_utils import Cache
+from transformers.modeling_outputs import BaseModelOutputWithPast, CausalLMOutputWithPast
+from transformers import DeepseekV2Model, DeepseekV2ForCausalLM
+from transformers import DeepseekV2Config
+from transformers.models.deepseek_v2.modeling_deepseek_v2 import (
+    DeepseekV2Attention, DeepseekV2MLP, DeepseekV2MoE, DeepseekV2RMSNorm, DeepseekV2DecoderLayer)
+from transformers.models.llama.modeling_llama import LlamaAttention, LlamaRotaryEmbedding
+from transformers import TextStreamer
+from .deepencoder import build_sam_vit_b, build_clip_l, MlpProjector
+from .conversation import get_conv_template
+
+
+def load_image(image_path):
+
+    try:
+        image = Image.open(image_path)
+        
+        corrected_image = ImageOps.exif_transpose(image)
+        
+        return corrected_image
+        
+    except Exception as e:
+        print(f"error: {e}")
+        try:
+            return Image.open(image_path)
+        except:
+            return None
+
+
+def re_match(text):
+    pattern = r'(<\|ref\|>(.*?)<\|/ref\|><\|det\|>(.*?)<\|/det\|>)'
+    matches = re.findall(pattern, text, re.DOTALL)
+
+    # pattern1 = r'<\|ref\|>.*?<\|/ref\|>\n'
+    # new_text1 = re.sub(pattern1, '', text, flags=re.DOTALL)
+
+    mathes_image = []
+    mathes_other = []
+    for a_match in matches:
+        if '<|ref|>image<|/ref|>' in a_match[0]:
+            mathes_image.append(a_match[0])
+        else:
+            mathes_other.append(a_match[0])
+    return matches, mathes_image, mathes_other
+
+
+def extract_coordinates_and_label(ref_text, image_width, image_height):
+
+    try:
+        label_type = ref_text[1]
+        cor_list = eval(ref_text[2])
+    except Exception as e:
+        print(e)
+        return None
+
+    return (label_type, cor_list)
+
+
+def draw_bounding_boxes(image, refs, ouput_path):
+
+    image_width, image_height = image.size
+    
+    img_draw = image.copy()
+    draw = ImageDraw.Draw(img_draw)
+
+    overlay = Image.new('RGBA', img_draw.size, (0, 0, 0, 0))
+    draw2 = ImageDraw.Draw(overlay)
+    
+    # try:
+    # except IOError:
+    #     try:
+    #         font = ImageFont.truetype("DejaVuSans.ttf", 20) 
+    #     except IOError:
+    font = ImageFont.load_default()
+
+    img_idx = 0
+    
+    for i, ref in enumerate(refs):
+        try:
+            result = extract_coordinates_and_label(ref, image_width, image_height)
+            if result:
+                label_type, points_list = result
+                
+                color = (np.random.randint(0, 200), np.random.randint(0, 200), np.random.randint(0, 255))
+
+                color_a = color + (20, )
+                for points in points_list:
+                    x1, y1, x2, y2 = points
+
+                    x1 = int(x1 / 999 * image_width)
+                    y1 = int(y1 / 999 * image_height)
+
+                    x2 = int(x2 / 999 * image_width)
+                    y2 = int(y2 / 999 * image_height)
+
+                    if label_type == 'image':
+                        try:
+                            cropped = image.crop((x1, y1, x2, y2))
+                            cropped.save(f"{ouput_path}/images/{img_idx}.jpg")
+                        except Exception as e:
+                            print(e)
+                            pass
+                        img_idx += 1
+                        
+                    try:
+                        if label_type == 'title':
+                            draw.rectangle([x1, y1, x2, y2], outline=color, width=4)
+                            draw2.rectangle([x1, y1, x2, y2], fill=color_a, outline=(0, 0, 0, 0), width=1)
+                        else:
+                            draw.rectangle([x1, y1, x2, y2], outline=color, width=2)
+                            draw2.rectangle([x1, y1, x2, y2], fill=color_a, outline=(0, 0, 0, 0), width=1)
+                        text_x = x1
+                        text_y = max(0, y1 - 15)
+                            
+                        
+                        text_bbox = draw.textbbox((0, 0), label_type, font=font)
+                        text_width = text_bbox[2] - text_bbox[0]
+                        text_height = text_bbox[3] - text_bbox[1]
+                        draw.rectangle([text_x, text_y, text_x + text_width, text_y + text_height], 
+                                    fill=(255, 255, 255, 30))
+                        
+                        draw.text((text_x, text_y), label_type, font=font, fill=color)
+                    except:
+                        pass
+        except:
+            continue
+    img_draw.paste(overlay, (0, 0), overlay)
+    return img_draw
+
+
+def process_image_with_refs(image, ref_texts, output_path):
+
+    result_image = draw_bounding_boxes(image, ref_texts, output_path)
+    
+    return result_image
+
+
+
+
+
+def find_closest_aspect_ratio(aspect_ratio, target_ratios, width, height, image_size):
+    best_ratio_diff = float('inf')
+    best_ratio = (1, 1)
+    area = width * height
+    for ratio in target_ratios:
+        target_aspect_ratio = ratio[0] / ratio[1]
+        ratio_diff = abs(aspect_ratio - target_aspect_ratio)
+        if ratio_diff < best_ratio_diff:
+            best_ratio_diff = ratio_diff
+            best_ratio = ratio
+        elif ratio_diff == best_ratio_diff:
+            if area > 0.5 * image_size * image_size * ratio[0] * ratio[1]:
+                best_ratio = ratio
+    # print(f'width: {width}, height: {height}, best_ratio: {best_ratio}')
+    return best_ratio
+
+
+def dynamic_preprocess(image, min_num=2, max_num=9, image_size=640, use_thumbnail=False):
+    orig_width, orig_height = image.size
+    aspect_ratio = orig_width / orig_height
+
+    # calculate the existing image aspect ratio
+    target_ratios = set(
+        (i, j) for n in range(min_num, max_num + 1) for i in range(1, n + 1) for j in range(1, n + 1) if
+        i * j <= max_num and i * j >= min_num)
+    # print(target_ratios)
+    target_ratios = sorted(target_ratios, key=lambda x: x[0] * x[1])
+
+    # find the closest aspect ratio to the target
+    target_aspect_ratio = find_closest_aspect_ratio(
+        aspect_ratio, target_ratios, orig_width, orig_height, image_size)
+
+    # print(target_aspect_ratio)
+    # calculate the target width and height
+    target_width = image_size * target_aspect_ratio[0]
+    target_height = image_size * target_aspect_ratio[1]
+    blocks = target_aspect_ratio[0] * target_aspect_ratio[1]
+
+    # resize the image
+    resized_img = image.resize((target_width, target_height))
+    processed_images = []
+    for i in range(blocks):
+        box = (
+            (i % (target_width // image_size)) * image_size,
+            (i // (target_width // image_size)) * image_size,
+            ((i % (target_width // image_size)) + 1) * image_size,
+            ((i // (target_width // image_size)) + 1) * image_size
+        )
+        # split the image
+        split_img = resized_img.crop(box)
+        processed_images.append(split_img)
+    assert len(processed_images) == blocks
+    if use_thumbnail and len(processed_images) != 1:
+        thumbnail_img = image.resize((image_size, image_size))
+        processed_images.append(thumbnail_img)
+    return processed_images, target_aspect_ratio
+
+
+
+def normalize_transform(mean, std):
+    if mean is None and std is None:
+        transform = None
+    elif mean is None and std is not None:
+        mean = [0.] * len(std)
+        transform = transforms.Normalize(mean=mean, std=std)
+    elif mean is not None and std is None:
+        std = [1.] * len(mean)
+        transform = transforms.Normalize(mean=mean, std=std)
+    else:
+        transform = transforms.Normalize(mean=mean, std=std)
+
+    return transform
+
+
+
+def format_messages(
+        conversations: List[Dict[str, str]],
+        sft_format: str = "deepseek",
+        system_prompt: str = "",
+):
+    """
+    Applies the SFT template to conversation.
+
+    Args:
+        conversations (List[Dict]): A List of messages.
+        sft_format (str, optional): The format of the SFT template to use. Defaults to "deepseek".
+        system_prompt (str, optional): The system prompt to use in the SFT template. Defaults to "".
+
+    Returns:
+        sft_prompt (str): The formatted text.
+    """
+
+    conv = get_conv_template(sft_format)
+    conv.set_system_message(system_prompt)
+    for message in conversations:
+        conv.append_message(message["role"], message["content"].strip())
+    sft_prompt = conv.get_prompt().strip()
+
+    return sft_prompt
+
+
+def text_encode(tokenizer, text: str, bos: bool = True, eos: bool = False):
+    t = tokenizer.encode(text, add_special_tokens=False)
+    bos_id = 0
+    eos_id = 1
+    if bos:
+        t = [bos_id] + t
+    if eos:
+        t = t + [eos_id]
+
+    return t
+
+def load_pil_images(conversations: List[Dict[str, str]]) -> List[Image.Image]:
+    """
+
+    Args:
+        conversations (List[Dict[str, str]]): the conversations with a list of messages. An example is :
+            [
+                {
+                    "role": "User",
+                    "content": "<image_placeholder>\nExtract all information from this image and convert them into markdown format.",
+                    "images": ["./examples/table_datasets.png"]
+                },
+                {"role": "Assistant", "content": ""},
+            ]
+
+    Returns:
+        pil_images (List[PIL.Image.Image]): the list of PIL images.
+
+    """
+
+    pil_images = []
+
+    for message in conversations:
+        if "images" not in message:
+            continue
+
+        for image_path in message["images"]:
+            # print('----------------')
+            # print(image_path)
+            # print('----------------')
+            # exit()
+            
+            # pil_img = Image.open(image_path)
+            pil_img = load_image(image_path)
+            pil_img = pil_img.convert("RGB")
+            pil_images.append(pil_img)
+
+    return pil_images
+
+
+class BaseTransform(ABC):
+
+    def set_rng(self, *args, **kwargs):
+        pass
+
+    def __call__(self, *args, **kwargs) -> torch.Tensor:
+        pass
+
+    @property
+    def default_shape(self):
+        raise NotImplementedError
+
+
+class BasicImageTransform(BaseTransform):
+    def __init__(
+        self, 
+        mean: Optional[Tuple[float, float, float]] = (0.5, 0.5, 0.5),
+        std: Optional[Tuple[float, float, float]] = (0.5, 0.5, 0.5),
+        normalize: bool = True
+    ):
+        self.mean = mean
+        self.std = std
+    
+        transform_pipelines = [
+            transforms.ToTensor()
+        ]
+
+        normalize = normalize_transform(mean, std) if normalize else nn.Identity()
+        if normalize is not None:
+            transform_pipelines.append(normalize)
+
+        self.transform = transforms.Compose(transform_pipelines)
+    
+    def __call__(self, x):
+        x = self.transform(x)
+        return x
+
+class NoEOSTextStreamer(TextStreamer):
+    def on_finalized_text(self, text: str, stream_end: bool = False):
+
+        eos_text = self.tokenizer.decode([self.tokenizer.eos_token_id], skip_special_tokens=False)
+        text = text.replace(eos_text, "\n")
+        print(text, flush=True, end="")
+
+
+def decoder_layer_init(self, config: DeepseekV2Config, layer_idx: int):
+    nn.Module.__init__(self)
+    self.hidden_size = config.hidden_size
+
+    if config.use_mla:
+        self.self_attn = DeepseekV2Attention(config=config, layer_idx=layer_idx)
+    else:
+        config.head_dim = config.hidden_size // config.num_attention_heads
+        self.self_attn = LlamaAttention(config, layer_idx)
+    self.mlp = DeepseekV2MoE(config) if layer_idx >= config.first_k_dense_replace else DeepseekV2MLP(config)
+
+    self.input_layernorm = DeepseekV2RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+    self.post_attention_layernorm = DeepseekV2RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+
+
+DeepseekV2DecoderLayer.__init__ = decoder_layer_init
+
+class DeepseekOCRConfig(DeepseekV2Config):
+    model_type = "DeepseekOCR"
+
+class DeepseekOCRModel(DeepseekV2Model):
+    config_class = DeepseekOCRConfig
+
+    def __init__(self, config: DeepseekV2Config):
+        super(DeepseekOCRModel, self).__init__(config)
+
+        self.sam_model = build_sam_vit_b()
+        self.vision_model = build_clip_l()
+        # self.conv_2 = nn.Conv2d(in_channels=1024, out_channels=2048, kernel_size=2, stride=2)
+        n_embed = 1280
+        self.projector =  MlpProjector(Dict(projector_type="linear", input_dim=2048, n_embed=n_embed))
+        embed_std = 1 / torch.sqrt(torch.tensor(n_embed, dtype=torch.float32))
+        self.image_newline = nn.Parameter(torch.randn(n_embed) * embed_std)
+        self.view_seperator = nn.Parameter(torch.randn(n_embed) * embed_std)
+
+        self.rotary_emb = LlamaRotaryEmbedding(config=config)
+    
+    def forward(
+        self,
+        input_ids: torch.LongTensor = None,
+        attention_mask: Optional[torch.Tensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        past_key_values: Optional[List[torch.FloatTensor]] = None,
+        inputs_embeds: Optional[torch.FloatTensor] = None,
+        use_cache: Optional[bool] = None,
+        output_attentions: Optional[bool] = None,
+        output_hidden_states: Optional[bool] = None,
+        images: Optional[torch.FloatTensor] = None,
+        images_seq_mask: Optional[torch.FloatTensor] = None,
+        images_spatial_crop: Optional[torch.FloatTensor] = None,
+        return_dict: Optional[bool] = None,
+    ) -> Union[Tuple, BaseModelOutputWithPast]:
+
+
+
+        if inputs_embeds is None:
+            # inputs_embeds = self.embed_tokens(input_ids)
+            inputs_embeds = self.get_input_embeddings()(input_ids)
+
+
+
+        sam_model = getattr(self, 'sam_model', None)
+        # sam_model = self.sam_model
+        vision_model = getattr(self, 'vision_model', None)
+
+
+
+        if sam_model is not None and (input_ids.shape[1] != 1 or self.training) and torch.sum(images[0][1]).item() != 0:
+
+            idx = 0
+            
+            # sam_model = torch.jit.script(sam_model)
+            
+            # start_time = time.time()
+            for image, crop_shape in zip(images, images_spatial_crop):
+                images_in_this_batch = []
+
+                patches = image[0]
+                image_ori = image[1]
+
+                with torch.no_grad():
+                # with torch.inference_mode(): 
+                    
+                    if torch.sum(patches).item() != 0:
+                        # P, C, H, W = patches.shape
+                        crop_flag = 1
+                        local_features_1 = sam_model(patches)
+
+                        local_features_2 = vision_model(patches, local_features_1)  
+                        # vit_time = time.time()
+                        local_features = torch.cat((local_features_2[:, 1:], local_features_1.flatten(2).permute(0, 2, 1)), dim=-1) 
+                        local_features = self.projector(local_features)
+
+
+                        global_features_1 = sam_model(image_ori)
+                        global_features_2 = vision_model(image_ori, global_features_1) 
+                        global_features = torch.cat((global_features_2[:, 1:], global_features_1.flatten(2).permute(0, 2, 1)), dim=-1) 
+                        global_features = self.projector(global_features)
+
+                        print('=====================')
+                        print('BASE: ', global_features.shape)
+                        print('PATCHES: ', local_features.shape)
+                        print('=====================')
+
+                        _, hw, n_dim = global_features.shape
+                        h = w = int(hw ** 0.5)
+
+                        _2, hw2, n_dim2 = local_features.shape
+                        h2 = w2 = int(hw2 ** 0.5)
+
+                        width_crop_num, height_crop_num = crop_shape[0], crop_shape[1]
+
+                        global_features = global_features.view(h, w, n_dim)
+
+                        global_features = torch.cat(
+                            [global_features, self.image_newline[None, None, :].expand(h, 1, n_dim)], dim=1
+                        )
+
+                        global_features = global_features.view(-1, n_dim)
+
+
+                        local_features = local_features.view(height_crop_num, width_crop_num, h2, w2, n_dim2).permute(0, 2, 1, 3, 4).reshape(height_crop_num*h2, width_crop_num*w2, n_dim2)
+                        local_features = torch.cat(
+                            [local_features, self.image_newline[None, None, :].expand(height_crop_num * h2, 1, n_dim2)], dim=1
+                        )
+                        local_features = local_features.view(-1, n_dim2)
+
+                        global_local_features = torch.cat([local_features, global_features, self.view_seperator[None, :]], dim=0)
+
+                        # end_time = time.time()
+
+                        # print('sam: ', sam_time - start_time)
+                        # print('vit: ', vit_time - sam_time)
+                        # print('all: ', end_time - start_time)
+
+                        # exit()
+                   
+                    else:
+                        global_features_1 = sam_model(image_ori)
+                        global_features_2 = vision_model(image_ori, global_features_1) 
+                        global_features = torch.cat((global_features_2[:, 1:], global_features_1.flatten(2).permute(0, 2, 1)), dim=-1) 
+                        global_features = self.projector(global_features)
+                        print('=====================')
+                        print('BASE: ', global_features.shape)
+                        print('NO PATCHES')
+                        print('=====================')
+                        _, hw, n_dim = global_features.shape
+                        h = w = int(hw ** 0.5)
+
+
+                        global_features = global_features.view(h, w, n_dim)
+
+                        global_features = torch.cat(
+                            [global_features, self.image_newline[None, None, :].expand(h, 1, n_dim)], dim=1
+                        )
+
+                        global_features = global_features.view(-1, n_dim)
+
+                        global_local_features = torch.cat([global_features, self.view_seperator[None, :]], dim=0)
+
+                    images_in_this_batch.append(global_local_features)
+                
+
+                if images_in_this_batch:
+                    images_in_this_batch = torch.cat(images_in_this_batch, dim=0)
+                    # exit()
+
+                    inputs_embeds[idx].masked_scatter_(images_seq_mask[idx].unsqueeze(-1).cuda(), images_in_this_batch)
+
+                idx += 1
+            
+        return super(DeepseekOCRModel, self).forward(
+            input_ids=None, attention_mask=attention_mask, past_key_values=past_key_values,
+            inputs_embeds=inputs_embeds, use_cache=use_cache, position_ids = position_ids,
+            output_attentions=output_attentions, output_hidden_states=output_hidden_states,
+            return_dict=return_dict
+        )
+    
+
+class DeepseekOCRForCausalLM(DeepseekV2ForCausalLM):
+
+    config_class = DeepseekOCRConfig
+    # supports_gradient_checkpointing = True
+
+    def __init__(self, config):
+        super(DeepseekV2ForCausalLM, self).__init__(config)
+        self.model = DeepseekOCRModel(config)
+
+        self.vocab_size = config.vocab_size
+
+        self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)
+
+        # self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)
+
+        # Initialize weights and apply final processing
+        self.post_init()
+
+    def get_model(self):
+        return self.model
+
+
+    def forward(
+        self,
+        input_ids: torch.LongTensor = None,
+        attention_mask: Optional[torch.Tensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        past_key_values: Optional[List[torch.FloatTensor]] = None,
+        inputs_embeds: Optional[torch.FloatTensor] = None,
+        labels: Optional[torch.LongTensor] = None,
+        use_cache: Optional[bool] = None,
+        output_attentions: Optional[bool] = None,
+        output_hidden_states: Optional[bool] = None,
+        images: Optional[torch.FloatTensor] = None,
+        images_seq_mask: Optional[torch.FloatTensor] = None,
+        images_spatial_crop: Optional[torch.FloatTensor] = None,
+        return_dict: Optional[bool] = None,
+        
+    ) -> Union[Tuple, CausalLMOutputWithPast]:
+        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
+        output_hidden_states = (
+            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+        )
+        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+
+
+        outputs  = self.model(
+            input_ids=input_ids,
+            past_key_values=past_key_values,
+            attention_mask=attention_mask,
+            position_ids=position_ids,
+            inputs_embeds=inputs_embeds,
+            use_cache=use_cache,
+            output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
+            images=images,
+            images_seq_mask = images_seq_mask,
+            images_spatial_crop = images_spatial_crop,
+            return_dict=return_dict
+            
+        )
+
+        hidden_states = outputs[0]
+        logits = self.lm_head(hidden_states)
+        logits = logits.float()
+
+        # logits
+
+        loss = None
+        if labels is not None:
+            # Shift so that tokens < n predict n
+            shift_logits = logits[..., :-1, :].contiguous()
+            shift_labels = labels[..., 1:].contiguous()
+            # Flatten the tokens
+            loss_fct = CrossEntropyLoss()
+            shift_logits = shift_logits.view(-1, self.config.vocab_size)
+            shift_labels = shift_labels.view(-1)
+            # Enable model parallelism
+            shift_labels = shift_labels.to(shift_logits.device)
+            loss = loss_fct(shift_logits, shift_labels)
+
+        if not return_dict:
+            output = (logits,) + outputs[1:]
+            return (loss,) + output if loss is not None else output
+
+        return CausalLMOutputWithPast(
+            loss=loss,
+            logits=logits,
+            past_key_values=outputs.past_key_values,
+            hidden_states=outputs.hidden_states,
+            attentions=outputs.attentions,
+        )
+
+
+    def prepare_inputs_for_generation(
+        self, input_ids, past_key_values=None, attention_mask=None, inputs_embeds=None, **kwargs
+    ):
+        # Omit tokens covered by past_key_values
+        past_length = 0
+        if past_key_values is not None:
+            if isinstance(past_key_values, Cache):
+                cache_length = past_key_values.get_seq_length()
+                past_length = past_key_values.get_seq_length()
+                max_cache_length = None
+            else:
+                cache_length = past_length = past_key_values[0][0].shape[2]
+                max_cache_length = None
+
+            # Keep only the unprocessed tokens:
+            # 1 - If the length of the attention_mask exceeds the length of input_ids, then we are in a setting where
+            # some of the inputs are exclusively passed as part of the cache (e.g. when passing input_embeds as
+            # input)
+            if attention_mask is not None and attention_mask.shape[1] > input_ids.shape[1]:
+                input_ids = input_ids[:, -(attention_mask.shape[1] - past_length) :]
+            # 2 - If the past_length is smaller than input_ids', then input_ids holds all input tokens. We can discard
+            # input_ids based on the past_length.
+            elif past_length < input_ids.shape[1]:
+                input_ids = input_ids[:, past_length:]
+            # 3 - Otherwise (past_length >= input_ids.shape[1]), let's assume input_ids only has unprocessed tokens.
+
+            # If we are about to go beyond the maximum cache length, we need to crop the input attention mask.
+            if (
+                max_cache_length is not None
+                and attention_mask is not None
+                and cache_length + input_ids.shape[1] > max_cache_length
+            ):
+                attention_mask = attention_mask[:, -max_cache_length:]
+
+        position_ids = kwargs.get("position_ids", None)
+        if attention_mask is not None and position_ids is None:
+            # create position_ids on the fly for batch generation
+            position_ids = attention_mask.long().cumsum(-1) - 1
+            position_ids.masked_fill_(attention_mask == 0, 1)
+            if past_key_values:
+                position_ids = position_ids[:, -input_ids.shape[1] :]
+
+        # if self.generation_config.cache_implementation == "static":
+        #     # generation with static cache
+        #     cache_position = kwargs.get("cache_position", None)
+        #     if cache_position is None:
+        #         past_length = 0
+        #     else:
+        #         past_length = cache_position[-1] + 1
+        #     input_ids = input_ids[:, past_length:]
+        #     position_ids = position_ids[:, past_length:]
+
+        # TODO @gante we should only keep a `cache_position` in generate, and do +=1.
+        # same goes for position ids. Could also help with continued generation.
+        cache_position = torch.arange(past_length, past_length + position_ids.shape[-1], device=position_ids.device)
+
+        # if `inputs_embeds` are passed, we only want to use them in the 1st generation step
+        if inputs_embeds is not None and past_key_values is None:
+            model_inputs = {"inputs_embeds": inputs_embeds}
+        else:
+            model_inputs = {"input_ids": input_ids}
+
+        model_inputs.update(
+            {
+                "position_ids": position_ids,
+                "past_key_values": past_key_values,
+                "use_cache": kwargs.get("use_cache"),
+                "attention_mask": attention_mask,
+                "images": kwargs.get("images", None),
+                "images_seq_mask": kwargs.get("images_seq_mask", None),
+                "images_spatial_crop": kwargs.get("images_spatial_crop", None),
+            }
+        )
+        return model_inputs
+    
+
+    def disable_torch_init(self):
+        """
+        Disable the redundant torch default initialization to accelerate model creation.
+        """
+        import torch
+        setattr(torch.nn.Linear, "reset_parameters", lambda self: None)
+        setattr(torch.nn.LayerNorm, "reset_parameters", lambda self: None)
+
+
+
+    def infer(self, tokenizer, prompt='', image_file='', output_path = '', base_size=1024, image_size=640, crop_mode=True, test_compress=False, save_results=False, eval_mode=False):
+        self.disable_torch_init()
+
+        os.makedirs(output_path, exist_ok=True)
+        os.makedirs(f'{output_path}/images', exist_ok=True)
+
+        if prompt and image_file:
+            conversation = [
+                {
+                    "role": "<|User|>",
+                    # "content": "<image>\n<|grounding|>Given the layout of the image. ",
+                    "content": f'{prompt}',
+                    # "content": "君不见黄河之水天上来的下一句是什么？",
+                    # "content": "<image>\nFree OCR. ",
+                    # "content": "<image>\nParse the figure. ",
+                    # "content": "<image>\nExtract the text in the image. ",
+                    "images": [f'{image_file}'],
+                },
+                {"role": "<|Assistant|>", "content": ""},
+            ]
+        
+        elif prompt:
+            conversation = [
+                {
+                    "role": "<|User|>",
+                    # "content": "<image>\n<|grounding|>Given the layout of the image. ",
+                    "content": f'{prompt}',
+                    # "content": "君不见黄河之水天上来的下一句是什么？",
+                    # "content": "<image>\nFree OCR. ",
+                    # "content": "<image>\nParse the figure. ",
+                    # "content": "<image>\nExtract the text in the image. ",
+                    # "images": [f'{image_file}'],
+                },
+                {"role": "<|Assistant|>", "content": ""},
+            ]
+        else:
+            assert False, f'prompt is none!'
+        
+        prompt = format_messages(conversations=conversation, sft_format='plain', system_prompt='')
+
+        patch_size = 16
+        downsample_ratio = 4
+        images = load_pil_images(conversation)
+
+        valid_img_tokens = 0
+        ratio = 1
+
+        image_draw = images[0].copy()
+
+        w,h = image_draw.size
+        # print(w, h)
+        ratio = 1 - ((max(w, h) - min(w, h)) / (max(w, h)))
+    
+
+        image_transform=BasicImageTransform(mean=(0.5, 0.5, 0.5), std=(0.5, 0.5, 0.5), normalize=True)
+        images_seq_mask = []
+
+        image_token = '<image>'
+        image_token_id = 128815
+        text_splits = prompt.split(image_token)
+
+        images_list, images_crop_list, images_seq_mask = [], [], []
+        tokenized_str = []
+        images_spatial_crop = []
+        for text_sep, image in zip(text_splits, images):
+
+            tokenized_sep = text_encode(tokenizer, text_sep, bos=False, eos=False)
+            tokenized_str += tokenized_sep
+            images_seq_mask += [False] * len(tokenized_sep)
+
+            if crop_mode:
+
+                if image.size[0] <= 640 and image.size[1] <= 640:
+                    crop_ratio = [1, 1]
+
+                else:
+                    if crop_mode:
+                        # best_width, best_height = select_best_resolution(image.size, self.candidate_resolutions)
+                        images_crop_raw, crop_ratio = dynamic_preprocess(image)
+                    else:
+                        # best_width, best_height = self.image_size, self.image_size
+                        crop_ratio = [1, 1]
+                
+                """process the global view"""
+                # image = image.resize((base_size, base_size))
+                global_view = ImageOps.pad(image, (base_size, base_size),
+                                        color=tuple(int(x * 255) for x in image_transform.mean))
+                
+                if base_size == 1024:
+                    valid_img_tokens += int(256 * ratio)
+                elif base_size == 1280:
+                    valid_img_tokens += int(400 * ratio)
+                # elif base_size == 640:
+                #     valid_img_tokens += int(100 * ratio)
+                
+
+
+
+                
+                images_list.append(image_transform(global_view).to(torch.bfloat16))
+
+                # global_view_tensor = image_transform(global_view).to(torch.bfloat16)
+
+                width_crop_num, height_crop_num = crop_ratio
+
+                images_spatial_crop.append([width_crop_num, height_crop_num])
+                
+                
+                if width_crop_num > 1 or height_crop_num > 1:
+                    """process the local views"""
+                    
+                    for i in range(len(images_crop_raw)):
+                        images_crop_list.append(image_transform(images_crop_raw[i]).to(torch.bfloat16))
+                
+                if image_size == 640:
+                    valid_img_tokens += len(images_crop_list) * 100
+
+                num_queries = math.ceil((image_size // patch_size) / downsample_ratio)
+                num_queries_base = math.ceil((base_size // patch_size) / downsample_ratio)
+
+
+
+                """add image tokens"""
+
+                
+
+                tokenized_image = ([image_token_id] * num_queries_base + [image_token_id]) * num_queries_base
+                tokenized_image += [image_token_id]
+                if width_crop_num > 1 or height_crop_num > 1:
+                    tokenized_image += ([image_token_id] * (num_queries * width_crop_num) + [image_token_id]) * (
+                                num_queries * height_crop_num)
+                tokenized_str += tokenized_image
+                images_seq_mask += [True] * len(tokenized_image)
+                # num_image_tokens.append(len(tokenized_image))
+
+            else:
+                # best_width, best_height = self.image_size, self.image_size
+                # print(image.size, (best_width, best_height)) # check the select_best_resolutions func
+
+                """process the global view"""
+                if image_size <= 640:
+                    print('directly resize')
+                    image = image.resize((image_size, image_size))
+                # else:
+                global_view = ImageOps.pad(image, (image_size, image_size),
+                                        color=tuple(int(x * 255) for x in image_transform.mean))
+                images_list.append(image_transform(global_view).to(torch.bfloat16))
+
+                if base_size == 1024:
+                    valid_img_tokens += int(256 * ratio)
+                elif base_size == 1280:
+                    valid_img_tokens += int(400 * ratio)
+                elif base_size == 640:
+                    valid_img_tokens += int(100 * 1)
+                elif base_size == 512:
+                    valid_img_tokens += int(64 * 1)
+
+                width_crop_num, height_crop_num = 1, 1
+
+                images_spatial_crop.append([width_crop_num, height_crop_num])
+
+
+                """add image tokens"""
+                num_queries = math.ceil((image_size // patch_size) / downsample_ratio)
+
+                tokenized_image = ([image_token_id] * num_queries + [image_token_id]) * num_queries
+                tokenized_image += [image_token_id]
+                # tokenized_image += ([self.image_token_id] * (num_queries * width_crop_num) + [self.image_token_id]) * (
+                #             num_queries * height_crop_num)
+                tokenized_str += tokenized_image
+                images_seq_mask += [True] * len(tokenized_image)
+                # num_image_tokens.append(len(tokenized_image))
+        
+
+        """process the last text split"""
+        tokenized_sep = text_encode(tokenizer, text_splits[-1], bos=False, eos=False)
+        tokenized_str += tokenized_sep
+        images_seq_mask += [False] * len(tokenized_sep)
+
+        """add the bos tokens"""
+        bos_id = 0
+        tokenized_str = [bos_id] + tokenized_str 
+        images_seq_mask = [False] + images_seq_mask
+
+
+
+        input_ids = torch.LongTensor(tokenized_str)
+
+        images_seq_mask = torch.tensor(images_seq_mask, dtype=torch.bool)
+
+
+        if len(images_list) == 0:
+            images_ori = torch.zeros((1, 3, image_size, image_size))
+            images_spatial_crop = torch.zeros((1, 2), dtype=torch.long)
+            images_crop = torch.zeros((1, 3, base_size, base_size))
+
+        else:
+            images_ori = torch.stack(images_list, dim=0)
+            images_spatial_crop = torch.tensor(images_spatial_crop, dtype=torch.long)
+            if images_crop_list:
+                images_crop = torch.stack(images_crop_list, dim=0)
+            else:
+                images_crop = torch.zeros((1, 3, base_size, base_size))
+
+
+
+        if not eval_mode:
+            streamer = NoEOSTextStreamer(tokenizer, skip_prompt=True, skip_special_tokens=False)
+            with torch.autocast("cuda", dtype=torch.bfloat16):
+                with torch.no_grad():
+                    output_ids = self.generate(
+                        input_ids.unsqueeze(0).cuda(),
+                        images=[(images_crop.cuda(), images_ori.cuda())],
+                        images_seq_mask = images_seq_mask.unsqueeze(0).cuda(),
+                        images_spatial_crop = images_spatial_crop,
+                        # do_sample=False,
+                        # num_beams = 1,
+                        temperature=0.0,
+                        eos_token_id=tokenizer.eos_token_id,
+                        streamer=streamer,
+                        max_new_tokens=8192,
+                        no_repeat_ngram_size = 20,
+                        use_cache = True
+                        )
+
+        else:
+            with torch.autocast("cuda", dtype=torch.bfloat16):
+                with torch.no_grad():
+                    output_ids = self.generate(
+                        input_ids.unsqueeze(0).cuda(),
+                        images=[(images_crop.cuda(), images_ori.cuda())],
+                        images_seq_mask = images_seq_mask.unsqueeze(0).cuda(),
+                        images_spatial_crop = images_spatial_crop,
+                        # do_sample=False,
+                        # num_beams = 1,
+                        temperature=0.0,
+                        eos_token_id=tokenizer.eos_token_id,
+                        max_new_tokens=8192,
+                        no_repeat_ngram_size = 35,
+                        use_cache = True
+                        )
+                
+
+        if '<image>' in conversation[0]['content'] and eval_mode:
+                outputs = tokenizer.decode(output_ids[0, input_ids.unsqueeze(0).cuda().shape[1]:])
+                stop_str = '<｜end▁of▁sentence｜>'
+                if outputs.endswith(stop_str):
+                    outputs = outputs[:-len(stop_str)]
+                # re_match
+                outputs = outputs.strip()
+
+                return outputs
+        
+        if '<image>' in conversation[0]['content'] and test_compress:
+            outputs = tokenizer.decode(output_ids[0, input_ids.unsqueeze(0).cuda().shape[1]:])
+            pure_texts_outputs_token_length = len(text_encode(tokenizer, outputs, bos=False, eos=False))
+            print('='*50)
+            print('image size: ', (w, h))
+            print('valid image tokens: ', int(valid_img_tokens))
+            print('output texts tokens (valid): ', pure_texts_outputs_token_length)
+            print('compression ratio: ', round(pure_texts_outputs_token_length/valid_img_tokens, 2))
+            print('='*50)
+
+
+        if '<image>' in conversation[0]['content'] and save_results:
+            outputs = tokenizer.decode(output_ids[0, input_ids.unsqueeze(0).cuda().shape[1]:])
+            stop_str = '<｜end▁of▁sentence｜>'
+
+            print('='*15 + 'save results:' + '='*15)
+            
+            # # # # conv.messages[-1][-1] = outputs
+            if outputs.endswith(stop_str):
+                outputs = outputs[:-len(stop_str)]
+            outputs = outputs.strip()
+
+            matches_ref, matches_images, mathes_other = re_match(outputs)
+            # print(matches_ref)
+            result = process_image_with_refs(image_draw, matches_ref, output_path)
+
+
+            for idx, a_match_image in enumerate(tqdm(matches_images, desc="image")):
+                outputs = outputs.replace(a_match_image, '![](images/' + str(idx) + '.jpg)\n')
+            
+            for idx, a_match_other in enumerate(tqdm(mathes_other, desc="other")):
+                outputs = outputs.replace(a_match_other, '').replace('\\coloneqq', ':=').replace('\\eqqcolon', '=:')
+
+
+            # if 'structural formula' in conversation[0]['content']:
+            #     outputs = '<smiles>' + outputs + '</smiles>'
+            with open(f'{output_path}/result.mmd', 'w', encoding = 'utf-8') as afile:
+                afile.write(outputs)
+
+            if 'line_type' in outputs:
+                import matplotlib.pyplot as plt
+                lines = eval(outputs)['Line']['line']
+
+                line_type = eval(outputs)['Line']['line_type']
+                # print(lines)
+
+                endpoints = eval(outputs)['Line']['line_endpoint']
+
+                fig, ax = plt.subplots(figsize=(3,3), dpi=200)
+                ax.set_xlim(-15, 15)
+                ax.set_ylim(-15, 15)
+
+                for idx, line in enumerate(lines):
+                    try:
+                        p0 = eval(line.split(' -- ')[0])
+                        p1 = eval(line.split(' -- ')[-1])
+
+                        if line_type[idx] == '--':
+                            ax.plot([p0[0], p1[0]], [p0[1], p1[1]], linewidth=0.8, color='k')
+                        else:
+                            ax.plot([p0[0], p1[0]], [p0[1], p1[1]], linewidth = 0.8, color = 'k')
+
+                        ax.scatter(p0[0], p0[1], s=5, color = 'k')
+                        ax.scatter(p1[0], p1[1], s=5, color = 'k')
+                    except:
+                        pass
+
+                for endpoint in endpoints:
+
+                    label = endpoint.split(': ')[0]
+                    (x, y) = eval(endpoint.split(': ')[1])
+                    ax.annotate(label, (x, y), xytext=(1, 1), textcoords='offset points', 
+                                fontsize=5, fontweight='light')
+                
+
+                plt.savefig(f'{output_path}/geo.jpg')
+                plt.close()
+
+            result.save(f"{output_path}/result_with_boxes.jpg")

processor_config.json

@@ -0,0 +1,28 @@
+{
+  "add_special_token": false,
+  "candidate_resolutions": [
+    [
+      1024,
+      1024
+    ]
+  ],
+  "downsample_ratio": 4,
+  "ignore_id": -100,
+  "image_mean": [
+    0.5,
+    0.5,
+    0.5
+  ],
+  "image_std": [
+    0.5,
+    0.5,
+    0.5
+  ],
+  "image_token": "<image>",
+  "mask_prompt": false,
+  "normalize": true,
+  "pad_token": "<\uff5c\u2581pad\u2581\uff5c>",
+  "patch_size": 16,
+  "processor_class": "DeepseekVLV2Processor",
+  "sft_format": "deepseek"
+}

special_tokens_map.json

@@ -0,0 +1,39 @@
+{
+  "additional_special_tokens": [
+    {
+      "content": "<|User|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|Assistant|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    }
+  ],
+  "bos_token": {
+    "content": "<｜begin▁of▁sentence｜>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<｜end▁of▁sentence｜>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<｜▁pad▁｜>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}