Update utils.py

utils.py

@@ -78,6 +78,17 @@ def load_tokenizer_and_model(base_model, load_8bit=False):
     model.eval()
     return tokenizer,model, device
 
+
+def load_tokenizer(base_model):
+    if torch.cuda.is_available():
+        device = "cuda"
+    else:
+        device = "cpu"
+
+    tokenizer = AutoTokenizer.from_pretrained(base_model, use_fast = True)
+    return tokenizer
+
+
 # Greedy Search
 def greedy_search(input_ids: torch.Tensor,
                   model: torch.nn.Module,