Update README.md

README.md

@@ -132,22 +132,30 @@ test_dataset = test_dataset.map(speech_file_to_array_fn)
 # Preprocessing the datasets.
 # We need to read the audio files as arrays
 def evaluate(batch):
-	inputs = processor(batch["speech"], sampling_rate=16_000, return_tensors="pt", padding=True)
+\tinputs = processor(batch["speech"], sampling_rate=16_000, return_tensors="pt", padding=True)
 
-	with torch.no_grad():
-		logits = model(inputs.input_values.to(DEVICE), attention_mask=inputs.attention_mask.to(DEVICE)).logits
+\twith torch.no_grad():
+\t\tlogits = model(inputs.input_values.to(DEVICE), attention_mask=inputs.attention_mask.to(DEVICE)).logits
 
-	pred_ids = torch.argmax(logits, dim=-1)
-	batch["pred_strings"] = processor.batch_decode(pred_ids)
-	return batch
+\tpred_ids = torch.argmax(logits, dim=-1)
+\tbatch["pred_strings"] = processor.batch_decode(pred_ids)
+\treturn batch
 
 result = test_dataset.map(evaluate, batched=True, batch_size=8)
 
-print("WER: {:2f}".format(100 * wer.compute(predictions=result["pred_strings"], references=result["sentence"], chunk_size=1000)))
-print("CER: {:2f}".format(100 * cer.compute(predictions=result["pred_strings"], references=result["sentence"], chunk_size=1000)))
+predictions = [x.upper() for x in result["pred_strings"]]
+references = [x.upper() for x in result["sentence"]]
+
+print(f"WER: {wer.compute(predictions=predictions, references=references, chunk_size=1000) * 100}")
+print(f"CER: {cer.compute(predictions=predictions, references=references, chunk_size=1000) * 100}")
 ```
 
 **Test Result**:
 
-- WER: 93.35%
-- CER: 29.24%
+My model may report better scores than others because of some specificity of my evaluation script, so I ran the same evaluation script on other models (on 2021-04-22) to make a fairer comparison.
+
+| Model | WER | CER |
+| ------------- | ------------- | ------------- |
+| vumichien/wav2vec2-large-xlsr-japanese | 1108.86% | **25.92%** |
+| jonatasgrosman/wav2vec2-large-xlsr-53-japanese | **93.35%** | 29.24% |
+| qqhann/w2v_hf_jsut_xlsr53 | 1012.18% | 69.46% |