Upload files with `vila-upload`.

Upload media_encoder.py
Upload media.py
Upload modeling_vila.py
Upload configuration_vila.py
Upload builder.py
Upload mm_utils.py
Upload tokenizer_utils.py
Upload siglip_encoder.py

builder.py

@@ -22,9 +22,9 @@ from dataclasses import asdict
 from typing import Any, Dict, List, Optional, Sequence, Tuple
 
 import torch
+import transformers
 from huggingface_hub import file_exists, repo_exists
 from huggingface_hub.utils import HFValidationError
-import transformers
 from transformers import (
     AutoConfig,
     AutoModelForCausalLM,
@@ -33,8 +33,9 @@ from transformers import (
     PreTrainedModel,
     PreTrainedTokenizer,
 )
+
 # from .conversation import *
-from .conversation import default_conversation, SeparatorStyle
+from .conversation import SeparatorStyle, default_conversation
 
 SENTINEL_TOKEN = "<vila/sentinel>"
 MEDIA_TOKENS = {
@@ -51,9 +52,11 @@ DUMMY_CONVERSATION = [
     {"from": "gpt", "value": "answer"},
 ] * 10
 
+
 def tokenizer_image_token(prompt, tokenizer, return_tensors=None):
     return tokenizer(prompt, return_tensors=return_tensors).input_ids[0]
-    
+
+
 def has_tokenizer(repo_id_or_path: str) -> bool:
     # Check if the tokenizer is in a local directory
     if osp.exists(osp.join(repo_id_or_path, "tokenizer_config.json")):
@@ -65,12 +68,14 @@ def has_tokenizer(repo_id_or_path: str) -> bool:
     except HFValidationError:
         return False
 
+
 def _maybe_add_sentinel_token(tokenizer: transformers.PreTrainedTokenizer) -> None:
     if not hasattr(tokenizer, "sentinel_token"):
         tokenizer.add_tokens([SENTINEL_TOKEN], special_tokens=True)
         tokenizer.sentinel_token = SENTINEL_TOKEN
         tokenizer.sentinel_token_id = tokenizer.convert_tokens_to_ids(SENTINEL_TOKEN)
 
+
 def tokenize_conversation_legacy(
     messages: Sequence[Dict[str, str]],
     tokenizer: transformers.PreTrainedTokenizer,
@@ -103,6 +108,7 @@ def tokenize_conversation_legacy(
 
     return tokenizer_image_token(conv.get_prompt(), tokenizer, return_tensors="pt")
 
+
 def tokenize_conversation(
     messages: Sequence[Dict[str, str]],
     tokenizer: transformers.PreTrainedTokenizer,
@@ -148,6 +154,7 @@ def tokenize_conversation(
     )
     return tokenizer_image_token(text, tokenizer, return_tensors="pt")
 
+
 def infer_stop_tokens(tokenizer: transformers.PreTrainedTokenizer) -> List[str]:
     _maybe_add_sentinel_token(tokenizer)
     template = tokenize_conversation(DUMMY_CONVERSATION, tokenizer, overrides={"gpt": SENTINEL_TOKEN})
@@ -159,6 +166,7 @@ def infer_stop_tokens(tokenizer: transformers.PreTrainedTokenizer) -> List[str]:
             stop_tokens.add(stop_token)
     return list(stop_tokens)
 
+
 def context_length_extension(config):
     orig_ctx_len = getattr(config, "max_position_embeddings", None)
     model_max_length = getattr(config, "model_max_length", None)
@@ -186,7 +194,7 @@ def build_llm_and_tokenizer(
 
     # Quantization related
     quantization_restore_from_checkpoint = False
-  
+
     if quantization_restore_from_checkpoint:
         fp8_model_name_or_path = kwargs.pop("fp8_llm_cfg", None)
 
@@ -215,6 +223,8 @@ def build_llm_and_tokenizer(
     if getattr(config, "chat_template", None) is not None:
         print(f"Using chat template: {config.chat_template}")
         fpath = os.path.join(os.path.dirname(__file__), "chat_templates", f"{config.chat_template}.jinja")
+        if not os.path.exists(fpath):
+            fpath = os.path.join(os.path.dirname(model_name_or_path), f"{config.chat_template}.jinja")
         with open(fpath) as fd:
             chat_template = fd.read()
         tokenizer.chat_template = chat_template.replace("    ", "").replace("\n", "")

configuration_vila.py

@@ -1,15 +1,24 @@
+import json
 import math
+import os
+import os.path as osp
+from copy import deepcopy
+from threading import Thread
 from typing import List, Optional
-import json
+
 import torch
 import torchvision
-import os, os.path as osp
-
-from threading import Thread
-from copy import deepcopy
 from PIL import Image
-from transformers import Qwen2Config, PretrainedConfig, PreTrainedModel
-from transformers import AutoProcessor, Qwen2PreTrainedModel, Qwen2ForCausalLM, TextIteratorStreamer
+from transformers import (
+    AutoProcessor,
+    PretrainedConfig,
+    PreTrainedModel,
+    Qwen2Config,
+    Qwen2ForCausalLM,
+    Qwen2PreTrainedModel,
+    TextIteratorStreamer,
+)
+
 
 class VILAConfig(PretrainedConfig):
     model_type = "vila"
@@ -82,4 +91,3 @@ class VILAConfig(PretrainedConfig):
         self.video_encoder = video_encoder
 
         super().__init__(**kwargs)
-

media.py

@@ -20,13 +20,16 @@ MEDIA_TOKENS = {
     "video": "<vila/video>",
 }
 
+
 class Media:
     pass
 
+
 class File(Media):
     def __init__(self, path: str) -> None:
         self.path = path
 
+
 class Image(File):
     pass
 
@@ -34,6 +37,7 @@ class Image(File):
 class Video(File):
     pass
 
+
 def make_list(obj: Any) -> List:
     return obj if isinstance(obj, list) else [obj]
 

media_encoder.py

@@ -1,8 +1,9 @@
-import torch
-from torch import nn
 from functools import partial
 from typing import Any, Dict, List, Optional
 
+import torch
+from torch import nn
+
 
 class BaseEncoder(nn.Module):
     def __init__(self, parent: nn.Module) -> None:

mm_utils.py

@@ -26,7 +26,7 @@ import torch
 from PIL import Image
 from transformers import StoppingCriteria
 
-from llava.constants import DEFAULT_IMAGE_TOKEN
+from .constants import DEFAULT_IMAGE_TOKEN
 
 
 def get_frame_from_vcap(vidcap, num_frames=10, max_fps=0.0, fps=None, frame_count=None, video_file_name=None):

modeling_vila.py

@@ -1,4 +1,3 @@
-import shutil
 import copy
 import json
 import logging
@@ -6,6 +5,7 @@ import math
 import os
 import os.path
 import os.path as osp
+import shutil
 import warnings
 from abc import ABC
 from collections import OrderedDict, defaultdict, deque
@@ -15,13 +15,12 @@ from threading import Thread
 from typing import Any, Dict, List, Optional, Tuple, Union
 
 import torch
-import torch.nn as nn
 import torch.distributed as dist
+import torch.nn as nn
 import torch.nn.functional as F
 import torchvision
 from einops import rearrange
 from PIL import Image
-
 from transformers import (
     AutoConfig,
     AutoModel,
@@ -34,28 +33,30 @@ from transformers import (
     Qwen2Config,
     Qwen2ForCausalLM,
     Qwen2PreTrainedModel,
-    TextIteratorStreamer
+    TextIteratorStreamer,
 )
-from transformers.modeling_utils import ContextManagers, no_init_weights
 from transformers.modeling_outputs import CausalLMOutputWithPast
+from transformers.modeling_utils import ContextManagers, no_init_weights
 
 from .base_projector import MultimodalProjector, MultimodalProjectorConfig
 from .builder import build_llm_and_tokenizer
 from .configuration_vila import VILAConfig
-from .media_encoder import BasicImageEncoder, BasicVideoEncoder
-from .siglip_encoder import SiglipVisionTower, SiglipVisionTowerDynamicS2, SiglipVisionTowerS2
-from .utils import get_model_config
+from .constants import *
+from .conversation import SeparatorStyle, default_conversation
 from .media import extract_media
+from .media_encoder import BasicImageEncoder, BasicVideoEncoder
 from .mm_utils import process_image, process_images
+from .siglip_encoder import SiglipVisionTower, SiglipVisionTowerDynamicS2, SiglipVisionTowerS2
 from .tokenizer_utils import tokenize_conversation
-from .constants import *
-from .conversation import default_conversation, SeparatorStyle
+from .utils import get_model_config
+
 
 # from llava.constants import DEFAULT_IMAGE_TOKEN, IGNORE_INDEX, NUM_EXTRA_TOKENS
 # quick hack for remote code
 def get_pg_manager():
     return None
 
+
 def get_model_weights_dtype(model: nn.Module):
     pass
 
@@ -72,7 +73,77 @@ def build_mm_projector(model_type_or_path: str, config: PretrainedConfig) -> Pre
         mm_projector_cfg = MultimodalProjectorConfig(model_type_or_path)
         mm_projector = MultimodalProjector(mm_projector_cfg, config)
         return mm_projector
-    
+
+
+def check_dot_in_model_path(model_path: str):
+    """Check if the model path contains dot, which will affect the remote code loading."""
+    if osp.isdir(model_path):  # local model
+        if "." in osp.abspath(model_path):
+            return True
+    else:  # remote model
+        if "." in model_path:
+            return True
+    return False
+
+
+def get_vila_version(model_path: str) -> str:
+    VERSIONS = ["vila1.5", "vila-u", "longvila", "nvila", "vila-m3"]
+    for version in VERSIONS:
+        if version in model_path.lower():
+            return version
+    return None
+
+
+def generate_jinja_template(conv_mode: str) -> str:
+    if conv_mode == "vicuna_v1":
+        return """{% set system_prompt = "A chat between a curious user and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the user's questions." %}
+{% set roles = ["USER", "ASSISTANT"] %}
+{% set sep = " " %}
+{% set sep2 = "</s>" %}
+
+{{ system_prompt }}
+
+{% for message in messages %}
+    {% if message['role'] == roles[0] %}
+        {{ roles[0] }}{{ sep }}{{ message['content'] }}{{ sep2 }}
+    {% else %}
+        {{ roles[1] }}{{ sep }}{{ message['content'] }}{{ sep2 }}
+    {% endif %}
+{% endfor %}"""
+    elif conv_mode == "llama_3":
+        return """{% set system_prompt = "<|begin_of_text|><|start_header_id|>system<|end_header_id|>\n\nYou are a helpful language and vision assistant. You are able to understand the visual content that the user provides, and assist the user with a variety of tasks using natural language." %}
+{% set roles = ["<|start_header_id|>user<|end_header_id|>\n\n", "<|start_header_id|>assistant<|end_header_id|>\n\n"] %}
+{% set sep = "<|eot_id|>" %}
+{% set sep2 = "<|end_of_text|>" %}
+
+{{ system_prompt }}
+
+{% for message in messages %}
+    {% if message['role'] == 'user' %}
+        {{ roles[0] }}{{ message['content'] }}{{ sep }}
+    {% else %}
+        {{ roles[1] }}{{ message['content'] }}{{ sep }}
+    {% endif %}
+{% endfor %}
+
+{{ sep2 }}"""
+    elif conv_mode == "hermes_2":
+        return """{% set system_prompt = "<|im_start|>system\nAnswer the questions." %}
+{% set roles = ["<|im_start|>user\n", "<|im_start|>assistant\n"] %}
+{% set sep = "<|im_end|>" %}
+
+{{ system_prompt }}{{ sep }}
+
+{% for message in messages %}
+    {% if message['role'] == 'user' %}
+        {{ roles[0] }}{{ message['content'] }}{{ sep }}
+    {% else %}
+        {{ roles[1] }}{{ message['content'] }}{{ sep }}
+    {% endif %}
+{% endfor %}"""
+    else:
+        raise NotImplementedError(f"Jinja template generation is not implemented for {conv_mode}.")
+
 
 def build_vision_tower(model_name_or_path: str, config: PretrainedConfig) -> PreTrainedModel:
     ## skip vision tower instantiation
@@ -110,7 +181,7 @@ class VILAPretrainedModel(PreTrainedModel):
     main_input_name = "input_embeds"
     supports_gradient_checkpointing = True
     _supports_flash_attn_2 = True
-    
+
     def __init__(self, config: VILAConfig, *args, **kwargs):
         super().__init__(config)
         self.config = config
@@ -119,22 +190,19 @@ class VILAPretrainedModel(PreTrainedModel):
             llm_cfg, vision_tower_cfg, mm_projector_cfg = cfgs
         else:
             raise ValueError("`llm_cfg` `mm_projector_cfg` `vision_tower_cfg` not found in the config.")
-        
+
         # loading on cpu by default
         device_map = kwargs.get("device_map", "cpu")
         self.mm_projector = build_mm_projector(mm_projector_cfg, config)
         self.vision_tower = build_vision_tower(vision_tower_cfg, config)
         if "auto" in device_map or "cuda" in device_map:
             self.mm_projector = self.mm_projector.cuda()
-            self.vision_tower = self.vision_tower.cuda() 
+            self.vision_tower = self.vision_tower.cuda()
         # set device_map auto can autoamtically shard llm to different devices
         self.llm, self.tokenizer = self.init_llm(llm_cfg, config, device_map=device_map)
-        
-        self.encoders = {
-            "image": BasicImageEncoder(self),
-            "video": BasicVideoEncoder(self)
-        }
-        
+
+        self.encoders = {"image": BasicImageEncoder(self), "video": BasicVideoEncoder(self)}
+
         self.post_config()
         self.is_loaded = True
 
@@ -143,37 +211,65 @@ class VILAPretrainedModel(PreTrainedModel):
         ), "At least one of the components must be instantiated."
 
     @classmethod
-    def convert_vila_dev_ckpt_to_remote(self, model_path: str, output_dir:str = None, *model_args, **kwargs):
+    def convert_vila_dev_ckpt_to_remote(
+        self,
+        model_path: str,
+        output_dir: str = None,
+        vila_version: str | None = None,
+        conv_mode: str | None = None,
+        *model_args,
+        **kwargs,
+    ):
         # assert type(self) == VILAForCasualLM, "This method is only available for VILAForCasualLM."
         from huggingface_hub import HfApi, snapshot_download
 
         if os.path.isdir(model_path):
             model_path = model_path
         api = HfApi()
+
+        if check_dot_in_model_path(model_path) and output_dir is None:
+            raise ValueError(
+                f"Model path {model_path} contains a dot, which will affect the remote code loading. Please specify the output directory without dot in the path to fix this issue."
+            )
+        if output_dir is not None and "." in output_dir:
+            raise ValueError(
+                f"Output directory {output_dir} contains a dot, which will affect the remote code loading. Please specify a valid output directory without dots."
+            )
+        if vila_version is None:
+            vila_version = get_vila_version(model_path)
+
         if api.repo_exists(model_path):
             model_path = snapshot_download(model_path, local_dir=output_dir)
             print("downloading HF model to", model_path)
-        
+
         cfg_path = os.path.join(model_path, "config.json")
         config = json.load(open(cfg_path))
-        config["version"] = "2.0" # nvila tag
+        config["version"] = "2.0"  # nvila tag
         config["architectures"] = ["VILAForCasualLM"]
         config["auto_map"] = {
             "AutoConfig": "modeling_vila.VILAConfig",
             "AutoModel": "modeling_vila.VILAForCasualLM",
-            "AutoModelForCausalLM": "modeling_vila.VILAForCasualLM"
+            "AutoModelForCausalLM": "modeling_vila.VILAForCasualLM",
         }
         config["model_type"] = "vila"
+        if vila_version in ["vila1.5", "vila-m3"]:
+            if conv_mode is None:
+                raise ValueError(f"Please specify the conversation mode for {model_path}.")
+            config["chat_template"] = conv_mode
+            jinja_template = generate_jinja_template(conv_mode)
+            jinja_path = os.path.join(model_path, f"{conv_mode}.jinja")
+            with open(jinja_path, "w") as f:
+                f.write(jinja_template)
         json.dump(config, open(cfg_path, "w"), indent=2)
         self.copy_remote_py_files(model_path)
-        
+
     @classmethod
     def copy_remote_py_files(cls, output_dir):
         ## copy .py and REAMDE for next loading remote code
         current_file_path = os.path.abspath(__file__)
         current_folder = os.path.dirname(current_file_path)
         for file_name in os.listdir(current_folder):
-            if file_name.endswith(".py"):
+            if file_name.endswith(".py") or file_name.endswith(".jinja"):
                 full_file_name = os.path.join(current_folder, file_name)
                 if os.path.isfile(full_file_name):
                     shutil.copy(full_file_name, output_dir)
@@ -222,17 +318,15 @@ class VILAPretrainedModel(PreTrainedModel):
                 state_dict=mm_projector_state_dict,
             )
             self.config.mm_projector_cfg = self.mm_projector.config
-        
+
         ## update and save top-level config
         self.config._name_or_path = output_dir
         self.config.architectures = [self.__class__.__name__]
         self.config.save_pretrained(output_dir)
-        
+
         ## copy .py and REAMDE for next loading remote code
         self.copy_remote_py_files(output_dir)
 
-        
-
     @classmethod
     def from_pretrained(
         cls,
@@ -258,7 +352,7 @@ class VILAPretrainedModel(PreTrainedModel):
         # variables for XGrammar
         # print("DEBUG", len(self.tokenizer.added_tokens_encoder.keys()), self.tokenizer.added_tokens_encoder.keys())
         NUM_EXTRA_TOKENS = len(self.tokenizer.added_tokens_encoder.keys())
-        
+
         # TODO: SENTINEL_TOKEN is not added, need to check with Zhijian
         self.vocab_size = self.tokenizer.vocab_size + NUM_EXTRA_TOKENS
         # XGrammar tokenizer and grammar compiler
@@ -318,11 +412,12 @@ class VILAPretrainedModel(PreTrainedModel):
                 self.get_vision_tower().eval()
             if self.get_mm_projector() and not getattr(self.config, "tune_mm_projector", False):
                 self.get_mm_projector().eval()
-    
+
+
 class VILAForCasualLM(VILAPretrainedModel):
     def __init__(self, config: VILAConfig, *args, **kwargs):
         super().__init__(config, *args, **kwargs)
-    
+
     def merge_features_for_dynamic_s2(self, image_features, block_sizes):
         scales = self.get_vision_tower().scales
         resize_output_to_scale_idx = self.get_vision_tower().resize_output_to_scale_idx
@@ -395,7 +490,7 @@ class VILAForCasualLM(VILAPretrainedModel):
         if getattr(self.config, "dynamic_s2", False):
             image_features = self.get_vision_tower()(images)
             image_features, new_block_sizes = self.merge_features_for_dynamic_s2(image_features, block_sizes)
-            
+
             image_features = [
                 self.split_chessboard(x, block_size[0], block_size[1])
                 for x, block_size in zip(image_features, new_block_sizes)
@@ -881,6 +976,7 @@ class VILAForCasualLM(VILAPretrainedModel):
             return outputs.logits, labels
 
         return outputs
+
     @torch.inference_mode()
     def generate(
         self,
@@ -898,7 +994,7 @@ class VILAForCasualLM(VILAPretrainedModel):
         self,
         prompt: Union[str, List],
         generation_config: Optional[GenerationConfig] = None,
-        response_format = None,
+        response_format=None,
     ) -> str:
         # TODO(zhijianl): Support directly taking conversation as input
         conversation = [{"from": "human", "value": prompt}]

siglip_encoder.py

@@ -20,11 +20,11 @@ import torch.nn.functional as F
 from accelerate.hooks import add_hook_to_module
 from einops import rearrange
 from s2wrapper import forward as multiscale_forward
-from transformers import AutoConfig, PreTrainedModel
+from transformers import AutoConfig, PretrainedConfig, PreTrainedModel, SiglipImageProcessor
 from transformers.image_processing_utils import BaseImageProcessor
 from transformers.integrations.deepspeed import is_deepspeed_zero3_enabled
 from transformers.models.siglip import SiglipVisionModel
-from transformers import PretrainedConfig, SiglipImageProcessor
+
 
 class VisionTower(nn.Module):
     def __init__(self, vision_tower, args, delay_load=False):
@@ -146,7 +146,6 @@ class VisionTower(nn.Module):
 
         return image_features
 
-
     @property
     def dummy_feature(self):
         return torch.zeros(1, self.hidden_size, device=self.device, dtype=self.dtype)

tokenizer_utils.py

@@ -19,9 +19,9 @@ from typing import Any, Dict, List, Optional, Sequence
 import torch
 import transformers
 
-from .conversation import default_conversation, SeparatorStyle
-from .mm_utils import tokenizer_image_token
 from .constants import IGNORE_INDEX, SENTINEL_TOKEN
+from .conversation import SeparatorStyle, default_conversation
+from .mm_utils import tokenizer_image_token
 
 # __all__ = [
 #     "tokenize_conversation",