Update utils.py

utils.py

@@ -22,22 +22,6 @@ from transformers import AutoTokenizer, AutoModelForCausalLM
 import datasets
 from datasets import load_dataset
 
-def reset_state():
-    return [], [], "Reset Done"
-
-def reset_textbox():
-    return gr.update(value=""),""
-
-def cancel_outputing():
-    return "Stop Done"
-
-def transfer_input(inputs):
-    textbox = reset_textbox()
-    return (
-        inputs,
-        gr.update(value=""),
-        gr.Button.update(visible=True),
-    )
 
 def is_stop_word_or_prefix(s: str, stop_words: list) -> bool:
     for stop_word in stop_words:
@@ -206,7 +190,34 @@ def predict(text,
         yield a,b,"Generate: Success"
     except:
         pass
-                   
+
+
+#Funktion, die den gegebenen Text aus dem Datenset gruppiert
+def group_texts(examples):
+    # Concatenate all texts.
+    concatenated_examples = {k: sum(examples[k], []) for k in examples.keys()}
+    total_length = len(concatenated_examples[list(examples.keys())[0]])
+    # We drop the small remainder, we could add padding if the model supported it instead of this drop, you can
+    # customize this part to your needs.
+    total_length = (total_length // block_size) * block_size
+    # Split by chunks of max_len.
+    result = {
+        k: [t[i : i + block_size] for i in range(0, total_length, block_size)]
+        for k, t in concatenated_examples.items()
+    }
+    result["labels"] = result["input_ids"].copy()
+    return result
+
+#Funktion, die der trainer braucht, um das Training zu evaluieren - mit einer Metrik
+def compute_metrics(eval_pred):
+    #Metrik berechnen, um das training messen zu können - wird es besser???
+    metric = evaluate.load("accuracy")   #3 Arten von gegebener Metrik: f1  oder roc_auc  oder accuracy
+    logits, labels = eval_pred
+    predictions = np.argmax(logits, axis=-1)
+    #Call compute on metric to calculate the accuracy of your predictions. 
+    #Before passing your predictions to compute, you need to convert the predictions to logits (remember all Transformers models return logits):
+    return metric.compute(predictions=predictions, references=labels)
+
                 
 def convert_to_markdown(text):
     text = text.replace("$","$")