Update model for latest `transformers` changes (#14)

- update the files (92cb457daed1ceefd6fd7f53294ee42085575cdc)
- add video token index also (4fd0ddbb2172352d120b7ff982274f32f6fe278f)
- add pipeline (190f6f4c0fad60d9d597a011c67cd9c213cd60af)


Co-authored-by: Raushan Turganbay <[email protected]>

README.md

@@ -4,7 +4,7 @@ license: other
 tags:
 - vision
 - image-captioning
-pipeline_tag: image-to-text
+pipeline_tag: image-text-to-text
 ---
 
 # InstructBLIP model

added_tokens.json

@@ -1,3 +1,5 @@
 {
+  "<image>": 32001,
+  "<video>": 32002,
   "[PAD]": 32000
 }

config.json

@@ -1,244 +1,49 @@
 {
-  "_commit_hash": null,
   "architectures": [
     "InstructBlipForConditionalGeneration"
   ],
+  "image_token_index": 32001,
   "initializer_factor": 1.0,
   "initializer_range": 0.02,
   "model_type": "instructblip",
   "num_query_tokens": 32,
   "qformer_config": {
-    "_name_or_path": "",
-    "add_cross_attention": false,
-    "architectures": null,
-    "attention_probs_dropout_prob": 0.1,
-    "bad_words_ids": null,
-    "begin_suppress_tokens": null,
-    "bos_token_id": null,
-    "chunk_size_feed_forward": 0,
-    "cross_attention_frequency": 2,
-    "cross_attention_hidden_size": null,
-    "decoder_start_token_id": null,
-    "diversity_penalty": 0.0,
-    "do_sample": false,
-    "early_stopping": false,
-    "encoder_hidden_size": 1408,
-    "encoder_no_repeat_ngram_size": 0,
-    "eos_token_id": null,
-    "exponential_decay_length_penalty": null,
-    "finetuning_task": null,
-    "forced_bos_token_id": null,
-    "forced_eos_token_id": null,
-    "hidden_act": "gelu",
-    "hidden_dropout_prob": 0.1,
-    "hidden_size": 768,
-    "id2label": {
-      "0": "LABEL_0",
-      "1": "LABEL_1"
-    },
-    "initializer_range": 0.02,
-    "intermediate_size": 3072,
-    "is_decoder": false,
-    "is_encoder_decoder": false,
-    "label2id": {
-      "LABEL_0": 0,
-      "LABEL_1": 1
-    },
-    "layer_norm_eps": 1e-12,
-    "length_penalty": 1.0,
-    "max_length": 20,
-    "max_position_embeddings": 512,
-    "min_length": 0,
     "model_type": "instructblip_qformer",
-    "no_repeat_ngram_size": 0,
-    "num_attention_heads": 12,
-    "num_beam_groups": 1,
-    "num_beams": 1,
-    "num_hidden_layers": 12,
-    "num_return_sequences": 1,
-    "output_attentions": false,
-    "output_hidden_states": false,
-    "output_scores": false,
-    "pad_token_id": 0,
-    "position_embedding_type": "absolute",
-    "prefix": null,
-    "problem_type": null,
-    "pruned_heads": {},
-    "remove_invalid_values": false,
-    "repetition_penalty": 1.0,
-    "return_dict": true,
-    "return_dict_in_generate": false,
-    "sep_token_id": null,
-    "suppress_tokens": null,
-    "task_specific_params": null,
-    "temperature": 1.0,
-    "tf_legacy_loss": false,
-    "tie_encoder_decoder": false,
-    "tie_word_embeddings": true,
-    "tokenizer_class": null,
-    "top_k": 50,
-    "top_p": 1.0,
-    "torch_dtype": null,
-    "torchscript": false,
-    "transformers_version": "4.31.0.dev0",
-    "typical_p": 1.0,
-    "use_bfloat16": false,
     "vocab_size": 30523
   },
   "text_config": {
-    "_name_or_path": "",
-    "add_cross_attention": false,
     "architectures": [
       "LLaMAForCausalLM"
     ],
-    "bad_words_ids": null,
-    "begin_suppress_tokens": null,
+    "attention_bias": false,
     "bos_token_id": 0,
-    "chunk_size_feed_forward": 0,
-    "cross_attention_hidden_size": null,
-    "decoder_start_token_id": null,
-    "diversity_penalty": 0.0,
-    "do_sample": false,
-    "early_stopping": false,
-    "encoder_no_repeat_ngram_size": 0,
     "eos_token_id": 1,
-    "exponential_decay_length_penalty": null,
-    "finetuning_task": null,
-    "forced_bos_token_id": null,
-    "forced_eos_token_id": null,
+    "head_dim": 128,
     "hidden_act": "silu",
     "hidden_size": 4096,
-    "id2label": {
-      "0": "LABEL_0",
-      "1": "LABEL_1"
-    },
     "initializer_range": 0.02,
     "intermediate_size": 11008,
-    "is_decoder": false,
-    "is_encoder_decoder": false,
-    "label2id": {
-      "LABEL_0": 0,
-      "LABEL_1": 1
-    },
-    "length_penalty": 1.0,
-    "max_length": 20,
-    "max_position_embeddings": 2048,
     "max_sequence_length": 2048,
-    "min_length": 0,
+    "mlp_bias": false,
     "model_type": "llama",
-    "no_repeat_ngram_size": 0,
     "num_attention_heads": 32,
-    "num_beam_groups": 1,
-    "num_beams": 1,
     "num_hidden_layers": 32,
-    "num_return_sequences": 1,
-    "output_attentions": false,
-    "output_hidden_states": false,
-    "output_scores": false,
+    "num_key_value_heads": 32,
     "pad_token_id": 32000,
-    "prefix": null,
-    "problem_type": null,
-    "pruned_heads": {},
-    "remove_invalid_values": false,
-    "repetition_penalty": 1.0,
-    "return_dict": true,
-    "return_dict_in_generate": false,
+    "pretraining_tp": 1,
     "rms_norm_eps": 1e-06,
-    "sep_token_id": null,
-    "suppress_tokens": null,
-    "task_specific_params": null,
-    "temperature": 1.0,
-    "tf_legacy_loss": false,
-    "tie_encoder_decoder": false,
+    "rope_scaling": null,
+    "rope_theta": 10000.0,
     "tie_word_embeddings": false,
-    "tokenizer_class": null,
-    "top_k": 50,
-    "top_p": 1.0,
     "torch_dtype": "float16",
-    "torchscript": false,
-    "transformers_version": "4.31.0.dev0",
-    "typical_p": 1.0,
-    "use_bfloat16": false,
-    "use_cache": true,
-    "vocab_size": 32001
+    "vocab_size": 32064
   },
   "tie_word_embeddings": false,
   "torch_dtype": "float32",
-  "transformers_version": null,
+  "transformers_version": "4.47.0.dev0",
   "use_decoder_only_language_model": true,
+  "video_token_index": 32002,
   "vision_config": {
-    "_name_or_path": "",
-    "add_cross_attention": false,
-    "architectures": null,
-    "attention_dropout": 0.0,
-    "bad_words_ids": null,
-    "begin_suppress_tokens": null,
-    "bos_token_id": null,
-    "chunk_size_feed_forward": 0,
-    "cross_attention_hidden_size": null,
-    "decoder_start_token_id": null,
-    "diversity_penalty": 0.0,
-    "do_sample": false,
-    "early_stopping": false,
-    "encoder_no_repeat_ngram_size": 0,
-    "eos_token_id": null,
-    "exponential_decay_length_penalty": null,
-    "finetuning_task": null,
-    "forced_bos_token_id": null,
-    "forced_eos_token_id": null,
-    "hidden_act": "gelu",
-    "hidden_size": 1408,
-    "id2label": {
-      "0": "LABEL_0",
-      "1": "LABEL_1"
-    },
-    "image_size": 224,
-    "initializer_range": 1e-10,
-    "intermediate_size": 6144,
-    "is_decoder": false,
-    "is_encoder_decoder": false,
-    "label2id": {
-      "LABEL_0": 0,
-      "LABEL_1": 1
-    },
-    "layer_norm_eps": 1e-06,
-    "length_penalty": 1.0,
-    "max_length": 20,
-    "min_length": 0,
-    "model_type": "instructblip_vision_model",
-    "no_repeat_ngram_size": 0,
-    "num_attention_heads": 16,
-    "num_beam_groups": 1,
-    "num_beams": 1,
-    "num_hidden_layers": 39,
-    "num_return_sequences": 1,
-    "output_attentions": false,
-    "output_hidden_states": false,
-    "output_scores": false,
-    "pad_token_id": null,
-    "patch_size": 14,
-    "prefix": null,
-    "problem_type": null,
-    "pruned_heads": {},
-    "qkv_bias": true,
-    "remove_invalid_values": false,
-    "repetition_penalty": 1.0,
-    "return_dict": true,
-    "return_dict_in_generate": false,
-    "sep_token_id": null,
-    "suppress_tokens": null,
-    "task_specific_params": null,
-    "temperature": 1.0,
-    "tf_legacy_loss": false,
-    "tie_encoder_decoder": false,
-    "tie_word_embeddings": true,
-    "tokenizer_class": null,
-    "top_k": 50,
-    "top_p": 1.0,
-    "torch_dtype": null,
-    "torchscript": false,
-    "transformers_version": "4.31.0.dev0",
-    "typical_p": 1.0,
-    "use_bfloat16": false
+    "model_type": "instructblip_vision_model"
   }
 }

generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 0,
+  "eos_token_id": 1,
+  "pad_token_id": 32000,
+  "transformers_version": "4.47.0.dev0"
+}

model-00001-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a1773d66d8e56dfd4cf5fa4ccfe72be2fe24a5798cdaf88be26bfba9c0942f61
-size 9900377640
+oid sha256:da825eee67c127052d38bc429e077571dfba65860dabbbd84d9fdaf5cd496666
+size 9901403320

model-00002-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a0fbf9da35fb9728e5863ec12879efbfb5a9595b5097f48aaa665cfdb846ca24
-size 9961917232
+oid sha256:9e7afd5d1bc89391d3a6735ca68e4539a7a65d6d1ae0f5151b883c57b0ee649c
+size 9961912600

model-00003-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0d53b9fcde48bc699749d187827ee4c3d9a8e690371384c8b4837f46d9e38f06
-size 9915747424
+oid sha256:68637c404c6e8953ab85247ee311dfefe102830c0a289f13447fa8a1ccabb1bb
+size 9915742400

model-00004-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e9cf86c9c7f7318b6604598f20eed517074b60de47bfd14817a4cb35550b2c11
-size 1874954648
+oid sha256:46d674bda7114f639c411b6341716c03e0f821407b1c4cd58f28aa1c64b34481
+size 1875986448

processor_config.json

@@ -0,0 +1,4 @@
+{
+  "num_query_tokens": 32,
+  "processor_class": "InstructBlipVideoProcessor"
+}

qformer_tokenizer/special_tokens_map.json

@@ -1,8 +1,44 @@
 {
-  "bos_token": "[DEC]",
-  "cls_token": "[CLS]",
-  "mask_token": "[MASK]",
-  "pad_token": "[PAD]",
-  "sep_token": "[SEP]",
-  "unk_token": "[UNK]"
+  "bos_token": {
+    "content": "[DEC]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "cls_token": {
+    "content": "[CLS]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "mask_token": {
+    "content": "[MASK]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "[PAD]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "sep_token": {
+    "content": "[SEP]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "[UNK]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
 }

qformer_tokenizer/tokenizer_config.json

@@ -1,7 +1,59 @@
 {
+  "added_tokens_decoder": {
+    "0": {
+      "content": "[PAD]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100": {
+      "content": "[UNK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "101": {
+      "content": "[CLS]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "102": {
+      "content": "[SEP]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "103": {
+      "content": "[MASK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "30522": {
+      "content": "[DEC]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "[DEC]",
   "clean_up_tokenization_spaces": true,
   "cls_token": "[CLS]",
   "do_lower_case": true,
+  "extra_special_tokens": {},
   "mask_token": "[MASK]",
   "model_max_length": 512,
   "pad_token": "[PAD]",

special_tokens_map.json

@@ -1,5 +1,11 @@
 {
-  "bos_token": "</s>",
+  "bos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
   "eos_token": {
     "content": "</s>",
     "lstrip": false,
@@ -7,6 +13,18 @@
     "rstrip": false,
     "single_word": false
   },
-  "pad_token": "[PAD]",
-  "unk_token": "</s>"
+  "pad_token": {
+    "content": "[PAD]",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  }
 }

tokenizer_config.json

@@ -1,19 +1,68 @@
 {
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "add_prefix_space": null,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32000": {
+      "content": "[PAD]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32001": {
+      "content": "<image>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32002": {
+      "content": "<video>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
   "bos_token": "</s>",
   "clean_up_tokenization_spaces": false,
-  "eos_token": {
-    "__type": "AddedToken",
-    "content": "</s>",
-    "lstrip": false,
-    "normalized": true,
-    "rstrip": false,
-    "single_word": false
-  },
+  "eos_token": "</s>",
+  "extra_special_tokens": {},
+  "legacy": true,
   "model_max_length": 2048,
-  "pad_token": null,
+  "pad_token": "[PAD]",
   "processor_class": "InstructBlipProcessor",
   "sp_model_kwargs": {},
   "tokenizer_class": "LlamaTokenizer",
   "truncation_side": "left",
-  "unk_token": "</s>"
+  "unk_token": "</s>",
+  "use_default_system_prompt": false
 }