Update README.md

README.md

@@ -66,20 +66,22 @@ with open('./FonAudio/pyFongbe-master/data/test.csv', newline='',encoding='UTF-8
       
       
 #Get valid indices
-random.seed(42) #this seed was used specifically to compare with Okwugbe model
+random.seed(42) #this seed was used specifically to compare
+                # with Okwugbe model (https://arxiv.org/abs/2103.07762)
 
 
-v = 1500 #200 samples for valid. Change as you want
+v = 1500 
 test_list = [i for i in range(len(t_data))]
 valid_indices = random.choices(test_list, k=v)
 
 test_data = [t_data[i] for i in range(len(t_data)) if i not in valid_indices] 
 valid_data = [t_data[i] for i in range(len(t_data)) if i in valid_indices]
 
-#Length of validation_dataset -> 1107
-#Length of test_dataset -> 1061
+#Final length of validation_dataset -> 1107
+#Final length of test_dataset -> 1061
 
-#Please note, the final validation size is is smaller than the expected (1500) because we used random.choices which could contain duplicates.
+#Please note, the final validation size is is smaller than the
+#expected (1500) because we used random.choices which could contain duplicates.
 
 #Create JSON files 
 def create_json_file(d):
@@ -135,7 +137,7 @@ for root, dirs, files in os.walk(test_path):
   test_dataset= load_dataset("json", data_files=[os.path.join(root,i) for i in files],split="train")
 
 #Remove unnecessary chars
-chars_to_ignore_regex = '[\\\\,\\\\?\\\\.\\\\!\\\\-\\\\;\\\\:\\\\"\\\\“\\\\%\\\\‘\\\\”\\\\�]'
+chars_to_ignore_regex = '[\\\\\\\\,\\\\\\\\?\\\\\\\\.\\\\\\\\!\\\\\\\\-\\\\\\\\;\\\\\\\\:\\\\\\\\"\\\\\\\\“\\\\\\\\%\\\\\\\\‘\\\\\\\\”\\\\\\\\�]'
 def remove_special_characters(batch):
     batch["sentence"] = re.sub(chars_to_ignore_regex, '', batch["sentence"]).lower() + " "
     return batch
@@ -151,15 +153,15 @@ model = Wav2Vec2ForCTC.from_pretrained("chrisjay/wav2vec2-large-xlsr-53-fon")
 # Preprocessing the datasets.
 # We need to read the audio files as arrays
 def speech_file_to_array_fn(batch):
-\\tspeech_array, sampling_rate = torchaudio.load(batch["path"])
-\\tbatch["speech"]=speech_array.squeeze().numpy()
-\\treturn batch
+\\\\tspeech_array, sampling_rate = torchaudio.load(batch["path"])
+\\\\tbatch["speech"]=speech_array.squeeze().numpy()
+\\\\treturn batch
 
 test_dataset = test_dataset.map(speech_file_to_array_fn)
 inputs = processor(test_dataset["speech"][:2], sampling_rate=16_000, return_tensors="pt", padding=True)
 
 with torch.no_grad():
-\\tlogits = model(inputs.input_values, attention_mask=inputs.attention_mask).logits
+\\\\tlogits = model(inputs.input_values, attention_mask=inputs.attention_mask).logits
 
 predicted_ids = torch.argmax(logits, dim=-1)
 
@@ -182,7 +184,7 @@ import re
 for root, dirs, files in os.walk(test_path):
   test_dataset = load_dataset("json", data_files=[os.path.join(root,i) for i in files],split="train")
 
-chars_to_ignore_regex = '[\\\\,\\\\?\\\\.\\\\!\\\\-\\\\;\\\\:\\\\"\\\\“\\\\%\\\\‘\\\\”\\\\�]'
+chars_to_ignore_regex = '[\\\\\\\\,\\\\\\\\?\\\\\\\\.\\\\\\\\!\\\\\\\\-\\\\\\\\;\\\\\\\\:\\\\\\\\"\\\\\\\\“\\\\\\\\%\\\\\\\\‘\\\\\\\\”\\\\\\\\�]'
 def remove_special_characters(batch):
     batch["sentence"] = re.sub(chars_to_ignore_regex, '', batch["sentence"]).lower() + " "
     return batch