Update app.py

app.py

@@ -8,7 +8,7 @@ import gradio as gr
 import torch
 from utils import *
 from presets import *
-from transformers import Trainer, TrainingArguments, DataCollatorWithPadding
+from transformers import Trainer, TrainingArguments, DataCollatorForLanguageModeling
 import numpy as np
 import evaluate
 import pandas as pd
@@ -197,7 +197,8 @@ lm_datasets = tokenized_datasets.map(
 )
 
 # Batches von Daten zusammenfassen
-data_collator = DataCollatorWithPadding(tokenizer=tokenizer)
+tokenizer.pad_token = tokenizer.eos_token
+data_collator = DataCollatorForLanguageModeling(tokenizer, mlm=False)
 
 
 print ("###############lm datasets####################")