update tokenizer

README.md

@@ -1,9 +1,9 @@
 ---
+datasets:
+- BEE-spoke-data/bees-internal
 language:
 - en
 license: apache-2.0
-datasets:
-- BEE-spoke-data/bees-internal
 ---
 
 # BeeTokenizer

special_tokens_map.json

@@ -1,4 +1,11 @@
 {
+  "bos_token": {
+    "content": "<bos>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
   "cls_token": {
     "content": "<cls>",
     "lstrip": false,

tokenizer_config.json

@@ -1,7 +1,7 @@
 {
   "added_tokens_decoder": {
     "0": {
-      "content": "<s>",
+      "content": "<bos>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -17,14 +17,6 @@
       "special": true
     },
     "2": {
-      "content": "</s>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "3": {
       "content": "<unk>",
       "lstrip": false,
       "normalized": false,
@@ -32,7 +24,7 @@
       "single_word": false,
       "special": true
     },
-    "4": {
+    "3": {
       "content": "<cls>",
       "lstrip": false,
       "normalized": false,
@@ -40,7 +32,7 @@
       "single_word": false,
       "special": true
     },
-    "5": {
+    "4": {
       "content": "<sep>",
       "lstrip": false,
       "normalized": false,
@@ -48,7 +40,7 @@
       "single_word": false,
       "special": true
     },
-    "6": {
+    "5": {
       "content": "<mask>",
       "lstrip": false,
       "normalized": false,
@@ -56,7 +48,7 @@
       "single_word": false,
       "special": true
     },
-    "7": {
+    "6": {
       "content": "<|endoftext|>",
       "lstrip": false,
       "normalized": false,
@@ -64,7 +56,7 @@
       "single_word": false,
       "special": true
     },
-    "32072": {
+    "31972": {
       "content": "  ",
       "lstrip": false,
       "normalized": true,
@@ -72,7 +64,7 @@
       "single_word": false,
       "special": false
     },
-    "32073": {
+    "31973": {
       "content": "   ",
       "lstrip": false,
       "normalized": true,
@@ -80,7 +72,7 @@
       "single_word": false,
       "special": false
     },
-    "32074": {
+    "31974": {
       "content": "    ",
       "lstrip": false,
       "normalized": true,
@@ -88,7 +80,7 @@
       "single_word": false,
       "special": false
     },
-    "32075": {
+    "31975": {
       "content": "     ",
       "lstrip": false,
       "normalized": true,
@@ -96,7 +88,7 @@
       "single_word": false,
       "special": false
     },
-    "32076": {
+    "31976": {
       "content": "      ",
       "lstrip": false,
       "normalized": true,
@@ -104,7 +96,7 @@
       "single_word": false,
       "special": false
     },
-    "32077": {
+    "31977": {
       "content": "       ",
       "lstrip": false,
       "normalized": true,
@@ -112,7 +104,7 @@
       "single_word": false,
       "special": false
     },
-    "32078": {
+    "31978": {
       "content": "        ",
       "lstrip": false,
       "normalized": true,
@@ -120,7 +112,7 @@
       "single_word": false,
       "special": false
     },
-    "32079": {
+    "31979": {
       "content": "         ",
       "lstrip": false,
       "normalized": true,
@@ -128,7 +120,7 @@
       "single_word": false,
       "special": false
     },
-    "32080": {
+    "31980": {
       "content": "          ",
       "lstrip": false,
       "normalized": true,
@@ -136,7 +128,7 @@
       "single_word": false,
       "special": false
     },
-    "32081": {
+    "31981": {
       "content": "           ",
       "lstrip": false,
       "normalized": true,
@@ -144,7 +136,7 @@
       "single_word": false,
       "special": false
     },
-    "32082": {
+    "31982": {
       "content": "            ",
       "lstrip": false,
       "normalized": true,
@@ -152,7 +144,7 @@
       "single_word": false,
       "special": false
     },
-    "32083": {
+    "31983": {
       "content": "             ",
       "lstrip": false,
       "normalized": true,
@@ -160,7 +152,7 @@
       "single_word": false,
       "special": false
     },
-    "32084": {
+    "31984": {
       "content": "              ",
       "lstrip": false,
       "normalized": true,
@@ -168,7 +160,7 @@
       "single_word": false,
       "special": false
     },
-    "32085": {
+    "31985": {
       "content": "               ",
       "lstrip": false,
       "normalized": true,
@@ -176,7 +168,7 @@
       "single_word": false,
       "special": false
     },
-    "32086": {
+    "31986": {
       "content": "                ",
       "lstrip": false,
       "normalized": true,
@@ -184,7 +176,7 @@
       "single_word": false,
       "special": false
     },
-    "32087": {
+    "31987": {
       "content": "                 ",
       "lstrip": false,
       "normalized": true,
@@ -192,7 +184,7 @@
       "single_word": false,
       "special": false
     },
-    "32088": {
+    "31988": {
       "content": "                  ",
       "lstrip": false,
       "normalized": true,
@@ -200,7 +192,7 @@
       "single_word": false,
       "special": false
     },
-    "32089": {
+    "31989": {
       "content": "                   ",
       "lstrip": false,
       "normalized": true,
@@ -208,7 +200,7 @@
       "single_word": false,
       "special": false
     },
-    "32090": {
+    "31990": {
       "content": "                    ",
       "lstrip": false,
       "normalized": true,
@@ -216,7 +208,7 @@
       "single_word": false,
       "special": false
     },
-    "32091": {
+    "31991": {
       "content": "                     ",
       "lstrip": false,
       "normalized": true,
@@ -224,7 +216,7 @@
       "single_word": false,
       "special": false
     },
-    "32092": {
+    "31992": {
       "content": "                      ",
       "lstrip": false,
       "normalized": true,
@@ -232,7 +224,7 @@
       "single_word": false,
       "special": false
     },
-    "32093": {
+    "31993": {
       "content": "                       ",
       "lstrip": false,
       "normalized": true,
@@ -240,7 +232,7 @@
       "single_word": false,
       "special": false
     },
-    "32094": {
+    "31994": {
       "content": "                        ",
       "lstrip": false,
       "normalized": true,
@@ -248,7 +240,7 @@
       "single_word": false,
       "special": false
     },
-    "32095": {
+    "31995": {
       "content": "                         ",
       "lstrip": false,
       "normalized": true,
@@ -256,7 +248,7 @@
       "single_word": false,
       "special": false
     },
-    "32096": {
+    "31996": {
       "content": "                          ",
       "lstrip": false,
       "normalized": true,
@@ -264,7 +256,7 @@
       "single_word": false,
       "special": false
     },
-    "32097": {
+    "31997": {
       "content": "                           ",
       "lstrip": false,
       "normalized": true,
@@ -272,7 +264,7 @@
       "single_word": false,
       "special": false
     },
-    "32098": {
+    "31998": {
       "content": "                            ",
       "lstrip": false,
       "normalized": true,
@@ -280,7 +272,7 @@
       "single_word": false,
       "special": false
     },
-    "32099": {
+    "31999": {
       "content": "                             ",
       "lstrip": false,
       "normalized": true,
@@ -289,6 +281,7 @@
       "special": false
     }
   },
+  "bos_token": "<bos>",
   "clean_up_tokenization_spaces": true,
   "cls_token": "<cls>",
   "eos_token": "<|endoftext|>",