Update core/loader.py

core/loader.py

@@ -76,6 +76,13 @@ class MVI_AI:
             vocab_path=os.path.join(ARTIFACTS_DIR, "vocab.json")
         )
 
+        # Ensure special tokens exist in fallback vocab
+        for tok in [HybridTokenizer.PAD_TOKEN, HybridTokenizer.UNK_TOKEN,
+                    HybridTokenizer.BOS_TOKEN, HybridTokenizer.EOS_TOKEN]:
+            if tok not in self.tokenizer.vocab:
+                self.tokenizer.vocab[tok] = len(self.tokenizer.vocab)
+        self.tokenizer.inv_vocab = {idx: token for token, idx in self.tokenizer.vocab.items()}
+
         # ===== LANGUAGE CORE =====
         self.embedder = EmbeddingLayer(
             self.tokenizer.vocab_size,