File size: 2,008 Bytes
e4389e7 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 |
import os
import soundfile as sf
import csv
from datasets import load_dataset
# Load the Italian subset of the Multilingual LibriSpeech dataset
dataset = load_dataset("facebook/multilingual_librispeech", "italian")
# Define the output directory
output_dir = "multilingual_librispeech_italian"
os.makedirs(output_dir, exist_ok=True)
def save_split(split_name, dry_run=False):
split = dataset[split_name]
split_dir = os.path.join(output_dir, split_name)
os.makedirs(split_dir, exist_ok=True)
wavs_dir = os.path.join(split_dir, "wavs")
os.makedirs(wavs_dir, exist_ok=True)
COLUMNS_TO_KEEP = ["transcript", "audio", "sampling_rate"]
all_columns = split.column_names
if dry_run:
print(split)
columns_to_remove = set(all_columns) - set(COLUMNS_TO_KEEP)
split = split.remove_columns(columns_to_remove)
print(split[0])
return
columns_to_remove = set(all_columns) - set(COLUMNS_TO_KEEP)
split = split.remove_columns(columns_to_remove)
metadata_path = os.path.join(split_dir, "metadata.csv")
with open(metadata_path, mode='w', newline='', encoding='utf-8') as file:
writer = csv.writer(file, delimiter='|')
for i, example in enumerate(split):
# Extract audio data and sampling rate
audio = example["audio"]
audio_array = audio["array"]
sampling_rate = audio["sampling_rate"]
# Define file paths
audio_path = os.path.join(wavs_dir, f"{i}.wav")
# Save audio file in WAV format
sf.write(audio_path, audio_array, sampling_rate)
# Save transcription
# transcription_path = os.path.join(split_dir, f"{i}.txt")
# with open(transcription_path, "w", encoding="utf-8") as f:
# f.write(example["transcript"])
# Save metadata
writer.writerow([audio_path, example["transcript"]])
# save_split("1_hours", dry_run=True)
save_split("9_hours")
|