Hugging Face's logo

Spaces:
alexkueck
/
LIFineTuned
Paused

App Files Community
alexkueck commited on
Commit
6a0b199
·
1 Parent(s): 90366fa

Update utils.py

Browse files
Files changed (1) hide show
  1. utils.py +4 -2
utils.py CHANGED
@@ -61,7 +61,8 @@ def load_tokenizer_and_model_setFit(base_model, load_8bit=False):
61
  device = "cpu"
62
 
63
 
64
- tokenizer = AutoTokenizer.from_pretrained(base_model, use_fast = True, use_auth_token=True, bos_token='<|startoftext|>', eos_token='<|endoftext|>', pad_token='<|pad|>')
 
65
  if device == "cuda":
66
  model = SetFitModel.from_pretrained(
67
  base_model,
@@ -89,7 +90,8 @@ def load_tokenizer_and_model(base_model, load_8bit=False):
89
  device = "cpu"
90
 
91
 
92
- tokenizer = AutoTokenizer.from_pretrained(base_model, use_fast = True, use_auth_token=True, bos_token='<|startoftext|>', eos_token='<|endoftext|>', pad_token='<|pad|>')
 
93
  if device == "cuda":
94
  model = AutoModelForCausalLM.from_pretrained(
95
  base_model,
 
61
  device = "cpu"
62
 
63
 
64
+ #tokenizer = AutoTokenizer.from_pretrained(base_model, use_fast = True, use_auth_token=True, bos_token='<|startoftext|>', eos_token='<|endoftext|>', pad_token='<|pad|>')
65
+ tokenizer = AutoTokenizer.from_pretrained(base_model, use_fast = True, use_auth_token=True)
66
  if device == "cuda":
67
  model = SetFitModel.from_pretrained(
68
  base_model,
 
90
  device = "cpu"
91
 
92
 
93
+ #tokenizer = AutoTokenizer.from_pretrained(base_model, use_fast = True, use_auth_token=True, bos_token='<|startoftext|>', eos_token='<|endoftext|>', pad_token='<|pad|>')
94
+ tokenizer = AutoTokenizer.from_pretrained(base_model, use_fast = True, use_auth_token=True)
95
  if device == "cuda":
96
  model = AutoModelForCausalLM.from_pretrained(
97
  base_model,