Fix train script for NPSC

run_speech_recognition_ctc.py

@@ -391,6 +391,23 @@ def main():
     # Set seed before initializing model.
     set_seed(training_args.seed)
 
+    # Pre-processing dataset
+    def preprocess_dataset(entry):
+        return (
+            "<INAUDIBLE>" not in entry["text"]
+            and entry["sentence_language_code"].lower() == "nb-no"
+        )
+    
+    def map_dataset(entry):
+        return {"text": (entry["text"]
+            .lower()
+            .replace("<ee>", "eee")
+            .replace("<mm>", "mmm")
+            .replace("<qq>", "qqq")
+            .replace("ó", "o")
+            .replace("é", "e")
+        )}
+
     # 1. First, let's load the dataset
     raw_datasets = DatasetDict()
 
@@ -401,6 +418,8 @@ def main():
             split=data_args.train_split_name,
             use_auth_token=data_args.use_auth_token,
         )
+        raw_datasets["train"] = raw_datasets["train"].filter(preprocess_dataset)
+        raw_datasets["train"] = raw_datasets["train"].map(map_dataset)
 
         if data_args.audio_column_name not in raw_datasets["train"].column_names:
             raise ValueError(
@@ -426,6 +445,8 @@ def main():
             split=data_args.eval_split_name,
             use_auth_token=data_args.use_auth_token,
         )
+        raw_datasets["eval"] = raw_datasets["eval"].filter(preprocess_dataset)
+        raw_datasets["eval"] = raw_datasets["eval"].map(map_dataset)
 
         if data_args.max_eval_samples is not None:
             raw_datasets["eval"] = raw_datasets["eval"].select(range(data_args.max_eval_samples))