Upload Gemma3ForConditionalGeneration

config.json

@@ -3,6 +3,7 @@
     "Gemma3ForConditionalGeneration"
   ],
   "boi_token_index": 255999,
+  "dtype": "bfloat16",
   "eoi_token_index": 256000,
   "eos_token_id": [
     1,
@@ -13,12 +14,15 @@
   "mm_tokens_per_image": 256,
   "model_type": "gemma3",
   "quantization_config": {
+    "include_input_output_embeddings": false,
     "modules_to_not_convert": null,
     "quant_method": "torchao",
     "quant_type": {
       "default": {
         "_data": {
           "group_size": 128,
+          "int4_choose_qparams_algorithm": "hqq",
+          "int4_packing_format": "tile_packed_to_4d",
           "layout": {
             "_data": {
               "inner_k_tiles": 8
@@ -26,10 +30,6 @@
             "_type": "TensorCoreTiledLayout",
             "_version": 1
           },
-          "packing_format": {
-            "_data": "PLAIN",
-            "_type": "PackingFormat"
-          },
           "preserve_zero": null,
           "set_inductor_config": true,
           "use_hqq": false,
@@ -42,19 +42,71 @@
         "_version": 2
       }
     },
-    "quant_type_kwargs": {}
+    "quant_type_kwargs": {},
+    "untie_embedding_weights": false
   },
   "text_config": {
+    "_sliding_window_pattern": 6,
     "attention_bias": false,
     "attention_dropout": 0.0,
     "attn_logit_softcapping": null,
-    "cache_implementation": "hybrid",
+    "dtype": "bfloat16",
     "final_logit_softcapping": null,
     "head_dim": 256,
     "hidden_activation": "gelu_pytorch_tanh",
     "hidden_size": 3840,
     "initializer_range": 0.02,
     "intermediate_size": 15360,
+    "layer_types": [
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "full_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "full_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "full_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "full_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "full_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "full_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "full_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "full_attention"
+    ],
     "max_position_embeddings": 131072,
     "model_type": "gemma3_text",
     "num_attention_heads": 16,
@@ -69,15 +121,13 @@
     },
     "rope_theta": 1000000.0,
     "sliding_window": 1024,
-    "sliding_window_pattern": 6,
-    "torch_dtype": "bfloat16",
     "use_cache": true,
     "vocab_size": 262208
   },
-  "torch_dtype": "bfloat16",
-  "transformers_version": "4.52.0.dev0",
+  "transformers_version": "4.56.1",
   "vision_config": {
     "attention_dropout": 0.0,
+    "dtype": "bfloat16",
     "hidden_act": "gelu_pytorch_tanh",
     "hidden_size": 1152,
     "image_size": 896,
@@ -88,7 +138,6 @@
     "num_channels": 3,
     "num_hidden_layers": 27,
     "patch_size": 14,
-    "torch_dtype": "bfloat16",
     "vision_use_head": false
   }
 }

generation_config.json

@@ -9,5 +9,5 @@
   "pad_token_id": 0,
   "top_k": 64,
   "top_p": 0.95,
-  "transformers_version": "4.52.0.dev0"
+  "transformers_version": "4.56.1"
 }

pytorch_model-00001-of-00002.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:538b990ced48d6dca85707e82960f5d82522e8fa1dee7ce94c6bf6d36f8026c4
-size 4975899385
+oid sha256:44f3a62d7487c8181bb87268e2bfd6ca40e0f9a7973b6f7df65d946ec0e39249
+size 4987219422

pytorch_model-00002-of-00002.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:baa4d2f81b76200df8e943c0e4eab95336b5782b36c09e8a731e54191ff2a0dd
-size 3190968481
+oid sha256:62ac4c77cfe8753a7375fc0fd69deea6326dcab87451b42455663741f32a381a
+size 3548065336

pytorch_model.bin.index.json

@@ -1,6 +1,7 @@
 {
   "metadata": {
-    "total_size": 8166129312
+    "total_parameters": 12187325040,
+    "total_size": 8534683872
   },
   "weight_map": {
     "language_model.lm_head.weight": "pytorch_model-00001-of-00002.bin",
@@ -148,13 +149,13 @@
     "language_model.model.layers.18.self_attn.q_norm.weight": "pytorch_model-00001-of-00002.bin",
     "language_model.model.layers.18.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
     "language_model.model.layers.18.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
-    "language_model.model.layers.19.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
-    "language_model.model.layers.19.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "language_model.model.layers.19.input_layernorm.weight": "pytorch_model-00002-of-00002.bin",
+    "language_model.model.layers.19.mlp.down_proj.weight": "pytorch_model-00002-of-00002.bin",
     "language_model.model.layers.19.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
-    "language_model.model.layers.19.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
-    "language_model.model.layers.19.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
-    "language_model.model.layers.19.post_feedforward_layernorm.weight": "pytorch_model-00001-of-00002.bin",
-    "language_model.model.layers.19.pre_feedforward_layernorm.weight": "pytorch_model-00001-of-00002.bin",
+    "language_model.model.layers.19.mlp.up_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "language_model.model.layers.19.post_attention_layernorm.weight": "pytorch_model-00002-of-00002.bin",
+    "language_model.model.layers.19.post_feedforward_layernorm.weight": "pytorch_model-00002-of-00002.bin",
+    "language_model.model.layers.19.pre_feedforward_layernorm.weight": "pytorch_model-00002-of-00002.bin",
     "language_model.model.layers.19.self_attn.k_norm.weight": "pytorch_model-00001-of-00002.bin",
     "language_model.model.layers.19.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
     "language_model.model.layers.19.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
@@ -174,19 +175,19 @@
     "language_model.model.layers.2.self_attn.q_norm.weight": "pytorch_model-00001-of-00002.bin",
     "language_model.model.layers.2.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
     "language_model.model.layers.2.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
-    "language_model.model.layers.20.input_layernorm.weight": "pytorch_model-00001-of-00002.bin",
-    "language_model.model.layers.20.mlp.down_proj.weight": "pytorch_model-00001-of-00002.bin",
-    "language_model.model.layers.20.mlp.gate_proj.weight": "pytorch_model-00001-of-00002.bin",
-    "language_model.model.layers.20.mlp.up_proj.weight": "pytorch_model-00001-of-00002.bin",
-    "language_model.model.layers.20.post_attention_layernorm.weight": "pytorch_model-00001-of-00002.bin",
-    "language_model.model.layers.20.post_feedforward_layernorm.weight": "pytorch_model-00001-of-00002.bin",
-    "language_model.model.layers.20.pre_feedforward_layernorm.weight": "pytorch_model-00001-of-00002.bin",
-    "language_model.model.layers.20.self_attn.k_norm.weight": "pytorch_model-00001-of-00002.bin",
-    "language_model.model.layers.20.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
-    "language_model.model.layers.20.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
-    "language_model.model.layers.20.self_attn.q_norm.weight": "pytorch_model-00001-of-00002.bin",
-    "language_model.model.layers.20.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
-    "language_model.model.layers.20.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "language_model.model.layers.20.input_layernorm.weight": "pytorch_model-00002-of-00002.bin",
+    "language_model.model.layers.20.mlp.down_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "language_model.model.layers.20.mlp.gate_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "language_model.model.layers.20.mlp.up_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "language_model.model.layers.20.post_attention_layernorm.weight": "pytorch_model-00002-of-00002.bin",
+    "language_model.model.layers.20.post_feedforward_layernorm.weight": "pytorch_model-00002-of-00002.bin",
+    "language_model.model.layers.20.pre_feedforward_layernorm.weight": "pytorch_model-00002-of-00002.bin",
+    "language_model.model.layers.20.self_attn.k_norm.weight": "pytorch_model-00002-of-00002.bin",
+    "language_model.model.layers.20.self_attn.k_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "language_model.model.layers.20.self_attn.o_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "language_model.model.layers.20.self_attn.q_norm.weight": "pytorch_model-00002-of-00002.bin",
+    "language_model.model.layers.20.self_attn.q_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "language_model.model.layers.20.self_attn.v_proj.weight": "pytorch_model-00002-of-00002.bin",
     "language_model.model.layers.21.input_layernorm.weight": "pytorch_model-00002-of-00002.bin",
     "language_model.model.layers.21.mlp.down_proj.weight": "pytorch_model-00002-of-00002.bin",
     "language_model.model.layers.21.mlp.gate_proj.weight": "pytorch_model-00002-of-00002.bin",
@@ -194,12 +195,12 @@
     "language_model.model.layers.21.post_attention_layernorm.weight": "pytorch_model-00002-of-00002.bin",
     "language_model.model.layers.21.post_feedforward_layernorm.weight": "pytorch_model-00002-of-00002.bin",
     "language_model.model.layers.21.pre_feedforward_layernorm.weight": "pytorch_model-00002-of-00002.bin",
-    "language_model.model.layers.21.self_attn.k_norm.weight": "pytorch_model-00001-of-00002.bin",
-    "language_model.model.layers.21.self_attn.k_proj.weight": "pytorch_model-00001-of-00002.bin",
-    "language_model.model.layers.21.self_attn.o_proj.weight": "pytorch_model-00001-of-00002.bin",
-    "language_model.model.layers.21.self_attn.q_norm.weight": "pytorch_model-00001-of-00002.bin",
-    "language_model.model.layers.21.self_attn.q_proj.weight": "pytorch_model-00001-of-00002.bin",
-    "language_model.model.layers.21.self_attn.v_proj.weight": "pytorch_model-00001-of-00002.bin",
+    "language_model.model.layers.21.self_attn.k_norm.weight": "pytorch_model-00002-of-00002.bin",
+    "language_model.model.layers.21.self_attn.k_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "language_model.model.layers.21.self_attn.o_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "language_model.model.layers.21.self_attn.q_norm.weight": "pytorch_model-00002-of-00002.bin",
+    "language_model.model.layers.21.self_attn.q_proj.weight": "pytorch_model-00002-of-00002.bin",
+    "language_model.model.layers.21.self_attn.v_proj.weight": "pytorch_model-00002-of-00002.bin",
     "language_model.model.layers.22.input_layernorm.weight": "pytorch_model-00002-of-00002.bin",
     "language_model.model.layers.22.mlp.down_proj.weight": "pytorch_model-00002-of-00002.bin",
     "language_model.model.layers.22.mlp.gate_proj.weight": "pytorch_model-00002-of-00002.bin",