Spaces:
Paused
Paused
Update utils.py
Browse files
utils.py
CHANGED
|
@@ -24,6 +24,7 @@ from datasets import load_dataset
|
|
| 24 |
import evaluate
|
| 25 |
|
| 26 |
|
|
|
|
| 27 |
def is_stop_word_or_prefix(s: str, stop_words: list) -> bool:
|
| 28 |
for stop_word in stop_words:
|
| 29 |
if s.endswith(stop_word):
|
|
@@ -59,6 +60,7 @@ def load_tokenizer_and_model(base_model, load_8bit=False):
|
|
| 59 |
else:
|
| 60 |
device = "cpu"
|
| 61 |
|
|
|
|
| 62 |
tokenizer = AutoTokenizer.from_pretrained(base_model, use_fast = True, use_auth_token=True)
|
| 63 |
if device == "cuda":
|
| 64 |
model = AutoModelForCausalLM.from_pretrained(
|
|
|
|
| 24 |
import evaluate
|
| 25 |
|
| 26 |
|
| 27 |
+
|
| 28 |
def is_stop_word_or_prefix(s: str, stop_words: list) -> bool:
|
| 29 |
for stop_word in stop_words:
|
| 30 |
if s.endswith(stop_word):
|
|
|
|
| 60 |
else:
|
| 61 |
device = "cpu"
|
| 62 |
|
| 63 |
+
|
| 64 |
tokenizer = AutoTokenizer.from_pretrained(base_model, use_fast = True, use_auth_token=True)
|
| 65 |
if device == "cuda":
|
| 66 |
model = AutoModelForCausalLM.from_pretrained(
|