Spaces:
Paused
Paused
Update app.py
Browse files
app.py
CHANGED
|
@@ -89,11 +89,15 @@ base_model = "project-baize/baize-v2-7b" #load_8bit = True (in load_tokenizer_a
|
|
| 89 |
|
| 90 |
|
| 91 |
# Load model directly
|
| 92 |
-
|
| 93 |
#Tokenizer und Model laden
|
| 94 |
tokenizer,model,device = load_tokenizer_and_model(base_model, True)
|
| 95 |
-
tokenizer.add_special_tokens({'pad_token': '[PAD]'}) #not necessary with fast Toekenizers like GPT2
|
|
|
|
|
|
|
|
|
|
| 96 |
|
|
|
|
| 97 |
#Datensets für Finetuning laden
|
| 98 |
dataset_neu = daten_laden("alexkueck/tis")
|
| 99 |
#dataset_neu = daten_laden("EleutherAI/pile")
|
|
|
|
| 89 |
|
| 90 |
|
| 91 |
# Load model directly
|
| 92 |
+
#####################################################
|
| 93 |
#Tokenizer und Model laden
|
| 94 |
tokenizer,model,device = load_tokenizer_and_model(base_model, True)
|
| 95 |
+
#tokenizer.add_special_tokens({'pad_token': '[PAD]'}) #not necessary with fast Toekenizers like GPT2
|
| 96 |
+
#für Blaize....
|
| 97 |
+
tokenizer,model,device = load_tokenizer_and_model_Blaize(base_model, True)
|
| 98 |
+
tokenizer.pad_token_id = 0
|
| 99 |
|
| 100 |
+
####################################################
|
| 101 |
#Datensets für Finetuning laden
|
| 102 |
dataset_neu = daten_laden("alexkueck/tis")
|
| 103 |
#dataset_neu = daten_laden("EleutherAI/pile")
|