Add test tokenization script

test_tokenization.py

@@ -0,0 +1,26 @@
+# /// script
+# requires-python = ">=3.12"
+# dependencies = ["transformers", "jinja2"]
+# ///
+
+
+from transformers import AutoTokenizer
+
+# Initialize tokenizer
+local_tokenizer = AutoTokenizer.from_pretrained(".")
+qwen3_tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen3-Coder-30B-A3B-Instruct")
+
+
+# User message with custom system message
+messages = [
+    {"role": "system", "content": "You are a helpful assistant."},
+    {"role": "user", "content": "What is the capital of France?"},
+]
+
+print("Local")
+print(local_tokenizer.apply_chat_template(messages, tokenize=False))
+print(local_tokenizer.apply_chat_template(messages, tokenize=True))
+
+print("\n\nQwen3-Coder")
+print(qwen3_tokenizer.apply_chat_template(messages, tokenize=False))
+print(qwen3_tokenizer.apply_chat_template(messages, tokenize=True))