Upload tokenizer

added_tokens.json

@@ -1,3 +1,5 @@
 {
-  "<image_soft_token>": 262144
+  "<image_soft_token>": 262144,
+  "<|eom_id|>": 262146,
+  "<|eot_id|>": 262145
 }

special_tokens_map.json

@@ -1,6 +1,7 @@
 {
   "additional_special_tokens": [
-    "<end_of_turn>"
+    "<end_of_turn>",
+    "<|eom_id|>"
   ],
   "boi_token": "<start_of_image>",
   "bos_token": {
@@ -12,7 +13,7 @@
   },
   "eoi_token": "<end_of_image>",
   "eos_token": {
-    "content": "<end_of_turn>",
+    "content": "<|eot_id|>",
     "lstrip": false,
     "normalized": false,
     "rstrip": false,

tokenizer.json

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4667f2089529e8e7657cfb6d1c19910ae71ff5f28aa7ab2ff2763330affad795
-size 33384568
+oid sha256:c90ceda7b4cf186df162bb142751be5a9a1d5a7e0bf8c8afe20c050f67aa1e8e
+size 33384942

tokenizer_config.json

@@ -51321,16 +51321,34 @@
       "rstrip": false,
       "single_word": false,
       "special": true
+    },
+    "262145": {
+      "content": "<|eot_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "262146": {
+      "content": "<|eom_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
     }
   },
   "additional_special_tokens": [
-    "<end_of_turn>"
+    "<end_of_turn>",
+    "<|eom_id|>"
   ],
   "boi_token": "<start_of_image>",
   "bos_token": "<bos>",
+  "chat_template": "{{ bos_token }}\n{%- if messages[0]['role'] == 'system' -%}\n    {%- if messages[0]['content'] is string -%}\n        {%- set first_user_prefix = messages[0]['content'] + '\n\n' -%}\n    {%- else -%}\n        {%- set first_user_prefix = messages[0]['content'][0]['text'] + '\n\n' -%}\n    {%- endif -%}\n    {%- set loop_messages = messages[1:] -%}\n{%- else -%}\n    {%- set first_user_prefix = \"\" -%}\n    {%- set loop_messages = messages -%}\n{%- endif -%}\n{%- for message in loop_messages -%}\n    {%- if (message['role'] == 'user') != (loop.index0 % 2 == 0) -%}\n        {{ raise_exception(\"Conversation roles must alternate user/assistant/user/assistant/...\") }}\n    {%- endif -%}\n    {%- if (message['role'] == 'assistant') -%}\n        {%- set role = \"model\" -%}\n    {%- else -%}\n        {%- set role = message['role'] -%}\n    {%- endif -%}\n    {{ '<start_of_turn>' + role + '\n' + (first_user_prefix if loop.first else \"\") }}\n    {%- if message['content'] is string -%}\n        {{ message['content'] | trim }}\n    {%- elif message['content'] is iterable -%}\n        {%- for item in message['content'] -%}\n            {%- if item['type'] == 'image' -%}\n                {{ '<start_of_image>' }}\n            {%- elif item['type'] == 'text' -%}\n                {{ item['text'] | trim }}\n            {%- endif -%}\n        {%- endfor -%}\n    {%- else -%}\n        {{ raise_exception(\"Invalid content type\") }}\n    {%- endif -%}\n    {{ '<end_of_turn>\n' }}\n{%- endfor -%}\n{%- if add_generation_prompt -%}\n    {{'<start_of_turn>model\n'}}\n{%- endif -%}\n",
   "clean_up_tokenization_spaces": false,
   "eoi_token": "<end_of_image>",
-  "eos_token": "<end_of_turn>",
+  "eos_token": "<|eot_id|>",
   "extra_special_tokens": {
     "boi_token": "<start_of_image>",
     "eoi_token": "<end_of_image>",
@@ -51339,7 +51357,7 @@
   "image_token": "<image_soft_token>",
   "model_max_length": 1000000000000000019884624838656,
   "pad_token": "<pad>",
-  "padding_side": "right",
+  "padding_side": "left",
   "processor_class": "Gemma3Processor",
   "sp_model_kwargs": null,
   "spaces_between_special_tokens": false,