Update utils.py

utils.py

@@ -63,7 +63,8 @@ def load_tokenizer_and_model_setFit(base_model, load_8bit=False):
     
     #tokenizer = AutoTokenizer.from_pretrained(base_model, use_fast = True, use_auth_token=True, bos_token='<|startoftext|>', eos_token='<|endoftext|>', pad_token='<|pad|>')
     tokenizer = AutoTokenizer.from_pretrained(base_model, use_fast = True, use_auth_token=True)
-    tokenizer.add_special_tokens({'pad_token': '[PAD]'})
+    tokenizer.pad_token = tokenizer.eos_token
+    tokenizer.pad_token_id = tokenizer.eos_token_id
     if device == "cuda":
         model = SetFitModel.from_pretrained(
             base_model,
@@ -93,7 +94,8 @@ def load_tokenizer_and_model(base_model, load_8bit=False):
     
     #tokenizer = AutoTokenizer.from_pretrained(base_model, use_fast = True, use_auth_token=True, bos_token='<|startoftext|>', eos_token='<|endoftext|>', pad_token='<|pad|>')
     tokenizer = AutoTokenizer.from_pretrained(base_model, use_fast = True, use_auth_token=True)
-    tokenizer.add_special_tokens({'pad_token': '[PAD]'})
+    tokenizer.pad_token = tokenizer.eos_token
+    tokenizer.pad_token_id = tokenizer.eos_token_id
     if device == "cuda":
         model = AutoModelForCausalLM.from_pretrained( 
             base_model,