Updating the example code

README.md

@@ -137,57 +137,56 @@ The fine-tuning process was perform during November (2022) in the servers of the
 import torch
 from transformers import Wav2Vec2Processor
 from transformers import Wav2Vec2ForCTC
+
 #Load the processor and model.
 MODEL_NAME="carlosdanielhernandezmena/wav2vec2-large-xlsr-53-spanish-ep5-944h"
 processor = Wav2Vec2Processor.from_pretrained(MODEL_NAME)
 model = Wav2Vec2ForCTC.from_pretrained(MODEL_NAME)
+
 #Load the dataset
 from datasets import load_dataset, load_metric, Audio
 ds=load_dataset("ciempiess/ciempiess_test", split="test")
-#Normalize the transcriptions
-import re
-chars_to_ignore_regex = '[\\,\\?\\.\\!\\\;\\:\\"\\“\\%\\‘\\”\\�\\)\\(\\*)]'
-def remove_special_characters(batch):
-	batch["normalized_text"] = re.sub(chars_to_ignore_regex, '', batch["normalized_text"]).lower()
-	return batch
-ds = ds.map(remove_special_characters)
+
 #Downsample to 16kHz
 ds = ds.cast_column("audio", Audio(sampling_rate=16_000))
+
 #Process the dataset
 def prepare_dataset(batch):
-	audio = batch["audio"]
-	#Batched output is "un-batched" to ensure mapping is correct
-	batch["input_values"] = processor(audio["array"], sampling_rate=audio["sampling_rate"]).input_values[0]
-	with processor.as_target_processor():
-		batch["labels"] = processor(batch["normalized_text"]).input_ids
-	return batch
+    audio = batch["audio"]
+    #Batched output is "un-batched" to ensure mapping is correct
+    batch["input_values"] = processor(audio["array"], sampling_rate=audio["sampling_rate"]).input_values[0]
+    with processor.as_target_processor():
+        batch["labels"] = processor(batch["normalized_text"]).input_ids
+    return batch
 ds = ds.map(prepare_dataset, remove_columns=ds.column_names,num_proc=1)
+
 #Define the evaluation metric
 import numpy as np
 wer_metric = load_metric("wer")
 def compute_metrics(pred):
-	pred_logits = pred.predictions
-	pred_ids = np.argmax(pred_logits, axis=-1)
-	pred.label_ids[pred.label_ids == -100] = processor.tokenizer.pad_token_id
-	pred_str = processor.batch_decode(pred_ids)
-	#We do not want to group tokens when computing the metrics
-	label_str = processor.batch_decode(pred.label_ids, group_tokens=False)
-	wer = wer_metric.compute(predictions=pred_str, references=label_str)
-	return {"wer": wer}
+    pred_logits = pred.predictions
+    pred_ids = np.argmax(pred_logits, axis=-1)
+    pred.label_ids[pred.label_ids == -100] = processor.tokenizer.pad_token_id
+    pred_str = processor.batch_decode(pred_ids)
+    #We do not want to group tokens when computing the metrics
+    label_str = processor.batch_decode(pred.label_ids, group_tokens=False)
+    wer = wer_metric.compute(predictions=pred_str, references=label_str)
+    return {"wer": wer}
+
 #Do the evaluation (with batch_size=1)
 model = model.to(torch.device("cuda"))
 def map_to_result(batch):
-	with torch.no_grad():
-		input_values = torch.tensor(batch["input_values"], device="cuda").unsqueeze(0)
-		logits = model(input_values).logits
-	pred_ids = torch.argmax(logits, dim=-1)
-	batch["pred_str"] = processor.batch_decode(pred_ids)[0]
-	batch["normalized_text"] = processor.decode(batch["labels"], group_tokens=False)
-	return batch
+    with torch.no_grad():
+        input_values = torch.tensor(batch["input_values"], device="cuda").unsqueeze(0)
+        logits = model(input_values).logits
+    pred_ids = torch.argmax(logits, dim=-1)
+    batch["pred_str"] = processor.batch_decode(pred_ids)[0]
+    batch["sentence"] = processor.decode(batch["labels"], group_tokens=False)
+    return batch
 results = ds.map(map_to_result,remove_columns=ds.column_names)
-#Compute the overall WER now.
-print("Test WER: {:.3f}".format(wer_metric.compute(predictions=results["pred_str"], references=results["normalized_text"])))
 
+#Compute the overall WER now.
+print("Test WER: {:.3f}".format(wer_metric.compute(predictions=results["pred_str"], references=results["sentence"])))
 ```
 **Test Result**: 0.112
 # BibTeX entry and citation info