add eval script

eval.py

@@ -2,6 +2,7 @@
 from datasets import load_dataset, load_metric, Audio
 from transformers import AutoModelForCTC, AutoProcessor, Wav2Vec2Processor
 import torch
+import re
 
 lang = "sv-SE"
 model_id = "./xls-r-300m-sv"
@@ -11,12 +12,13 @@ device = "cuda" if torch.cuda.is_available() else "cpu"
 dataset = load_dataset("mozilla-foundation/common_voice_7_0", lang, split="test", use_auth_token=True)
 wer = load_metric("wer")
 
-dataset = dataset.select(range(100))
-
 dataset = dataset.cast_column("audio", Audio(sampling_rate=16_000))
 
 model = AutoModelForCTC.from_pretrained(model_id).to(device)
-processor = Wav2Vec2Processor.from_pretrained(model_id)
+processor = AutoProcessor.from_pretrained(model_id)
+
+
+chars_to_ignore_regex = '[,?.!\-\;\:\"“%‘”�—’…–]'  # change to the ignored characters of your fine-tuned model
 
 
 def map_to_pred(batch):
@@ -25,15 +27,19 @@ def map_to_pred(batch):
     with torch.no_grad():
         logits = model(input_values.to(device)).logits
 
-    predicted_ids = torch.argmax(logits, dim=-1)
-    transcription = processor.batch_decode(predicted_ids)[0]
+    if processor.__class__.__name__ == "Wav2Vec2Processor":
+        predicted_ids = torch.argmax(logits, dim=-1)
+        transcription = processor.batch_decode(predicted_ids)[0]
+    else:
+        transcription = processor.batch_decode(logits.cpu().numpy()).text[0]
+
     batch["transcription"] = transcription
+    batch["text"] = re.sub(chars_to_ignore_regex, "", batch["sentence"].lower())
     return batch
 
 
 result = dataset.map(map_to_pred, remove_columns=["audio"])
 
-import ipdb; ipdb.set_trace()
-wer_result = wer.compute(references=result["sentence"], predictions=result["transcription"])
+wer_result = wer.compute(references=result["text"], predictions=result["transcription"])
 
 print("WER", wer_result)

preprocessor_config.json

@@ -6,5 +6,6 @@
   "padding_side": "right",
   "padding_value": 0,
   "return_attention_mask": true,
-  "sampling_rate": 16000
+  "sampling_rate": 16000,
+	"processor_class": "Wav2Vec2ProcessorWithLM"
 }