tokenizer

.gitattributes

@@ -14,3 +14,4 @@
 *.pb filter=lfs diff=lfs merge=lfs -text
 *.pt filter=lfs diff=lfs merge=lfs -text
 *.pth filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

train_tokenizer.py

@@ -0,0 +1,26 @@
+from datasets import load_dataset
+from tokenizers import ByteLevelBPETokenizer
+
+# load dataset
+dataset = load_dataset("mc4", "id", split="train")
+
+# Instantiate tokenizer
+tokenizer = ByteLevelBPETokenizer()
+
+
+def batch_iterator(batch_size=1000):
+    for i in range(0, len(dataset), batch_size):
+        yield dataset[i : i + batch_size]["text"]
+
+
+# Customized training
+tokenizer.train_from_iterator(
+    batch_iterator(),
+    vocab_size=50265,
+    min_frequency=2,
+    special_tokens=["<s>", "<pad>", "</s>", "<unk>", "<mask>",],
+)
+
+# Save files to disk
+tokenizer.save(f"./tokenizer.json")
+