Spaces:
Sleeping
Sleeping
| # !/usr/bin/env python3 | |
| # -*- coding: utf-8 -*- | |
| # | |
| # Copyright 2016-2099 Ailemon.net | |
| # | |
| # This file is part of ASRT Speech Recognition Tool. | |
| # | |
| # ASRT is free software: you can redistribute it and/or modify | |
| # it under the terms of the GNU General Public License as published by | |
| # the Free Software Foundation, either version 3 of the License, or | |
| # (at your option) any later version. | |
| # ASRT is distributed in the hope that it will be useful, | |
| # but WITHOUT ANY WARRANTY; without even the implied warranty of | |
| # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
| # GNU General Public License for more details. | |
| # | |
| # You should have received a copy of the GNU General Public License | |
| # along with ASRT. If not, see <https://www.gnu.org/licenses/>. | |
| # ============================================================================ | |
| """ | |
| @author: nl8590687 | |
| 加载语音数据集用的数据加载器的定义 | |
| """ | |
| import os | |
| import random | |
| import numpy as np | |
| from utils.config import load_config_file, DEFAULT_CONFIG_FILENAME, load_pinyin_dict | |
| from utils.ops import read_wav_data | |
| class DataLoader: | |
| """ | |
| 数据加载器 | |
| 参数：\\ | |
| config: 配置信息字典 | |
| dataset_type: 要加载的数据集类型，包含('train', 'dev', 'test')三种 | |
| """ | |
| def __init__(self, dataset_type:str): | |
| self.dataset_type = dataset_type | |
| self.data_list = list() | |
| self.wav_dict = dict() | |
| self.label_dict = dict() | |
| self.pinyin_list = list() | |
| self.pinyin_dict = dict() | |
| self.hanzi_lisst = list() | |
| self.hanzi_dict = dict() | |
| self._load_data() | |
| def _load_data(self): | |
| config = load_config_file(DEFAULT_CONFIG_FILENAME) | |
| self.pinyin_list, self.pinyin_dict = load_pinyin_dict(config['dict_filename']) | |
| for index in range(len(config['dataset'][self.dataset_type])): | |
| filename_datalist = config['dataset'][self.dataset_type][index]['data_list'] | |
| filename_datapath = config['dataset'][self.dataset_type][index]['data_path'] | |
| with open(filename_datalist, 'r', encoding='utf-8') as file_pointer: | |
| lines = file_pointer.read().split('\n') | |
| for line in lines: | |
| if len(line) == 0: | |
| continue | |
| tokens = line.split(' ') | |
| self.data_list.append(tokens[0]) | |
| self.wav_dict[tokens[0]] = os.path.join(filename_datapath, tokens[1]) | |
| filename_labellist = config['dataset'][self.dataset_type][index]['label_list'] | |
| with open(filename_labellist, 'r', encoding='utf-8') as file_pointer: | |
| lines = file_pointer.read().split('\n') | |
| for line in lines: | |
| if len(line) == 0: | |
| continue | |
| tokens = line.split(' ') | |
| self.label_dict[tokens[0]] = tokens[1:] | |
| def get_data_count(self) -> int: | |
| """ | |
| 获取数据集总数量 | |
| """ | |
| return len(self.data_list) | |
| def get_data(self, index: int) -> tuple: | |
| """ | |
| 按下标获取一条数据 | |
| """ | |
| mark = self.data_list[index] | |
| wav_signal, sample_rate, _, _ = read_wav_data(self.wav_dict[mark]) | |
| labels = list() | |
| for item in self.label_dict[mark]: | |
| if len(item) == 0: | |
| continue | |
| labels.append(self.pinyin_dict[item]) | |
| data_label = np.array(labels) | |
| return wav_signal, sample_rate, data_label | |
| def shuffle(self) -> None: | |
| """ | |
| 随机打乱数据 | |
| """ | |
| random.shuffle(self.data_list) | |