# Copyright (c) Alibaba, Inc. and its affiliates. """Some implementations are adapted from https://github.com/yuyq96/D-TDNN""" import io from typing import Union import librosa as sf import numpy as np import torch import torch.nn.functional as F import torchaudio.compliance.kaldi as Kaldi from torch import nn from funasr_detach.utils.modelscope_file import File def check_audio_list(audio: list): audio_dur = 0 for i in range(len(audio)): seg = audio[i] assert seg[1] >= seg[0], "modelscope error: Wrong time stamps." assert isinstance(seg[2], np.ndarray), "modelscope error: Wrong data type." assert ( int(seg[1] * 16000) - int(seg[0] * 16000) == seg[2].shape[0] ), "modelscope error: audio data in list is inconsistent with time length." if i > 0: assert seg[0] >= audio[i - 1][1], "modelscope error: Wrong time stamps." audio_dur += seg[1] - seg[0] return audio_dur # assert audio_dur > 5, 'modelscope error: The effective audio duration is too short.' def sv_preprocess(inputs: Union[np.ndarray, list]): output = [] for i in range(len(inputs)): if isinstance(inputs[i], str): file_bytes = File.read(inputs[i]) data, fs = sf.load(io.BytesIO(file_bytes), dtype="float32") if len(data.shape) == 2: data = data[:, 0] data = torch.from_numpy(data).unsqueeze(0) data = data.squeeze(0) elif isinstance(inputs[i], np.ndarray): assert ( len(inputs[i].shape) == 1 ), "modelscope error: Input array should be [N, T]" data = inputs[i] if data.dtype in ["int16", "int32", "int64"]: data = (data / (1 << 15)).astype("float32") else: data = data.astype("float32") data = torch.from_numpy(data) else: raise ValueError( "modelscope error: The input type is restricted to audio address and nump array." ) output.append(data) return output def sv_chunk(vad_segments: list, fs=16000) -> list: config = { "seg_dur": 1.5, "seg_shift": 0.75, } def seg_chunk(seg_data): seg_st = seg_data[0] data = seg_data[2] chunk_len = int(config["seg_dur"] * fs) chunk_shift = int(config["seg_shift"] * fs) last_chunk_ed = 0 seg_res = [] for chunk_st in range(0, data.shape[0], chunk_shift): chunk_ed = min(chunk_st + chunk_len, data.shape[0]) if chunk_ed <= last_chunk_ed: break last_chunk_ed = chunk_ed chunk_st = max(0, chunk_ed - chunk_len) chunk_data = data[chunk_st:chunk_ed] if chunk_data.shape[0] < chunk_len: chunk_data = np.pad( chunk_data, (0, chunk_len - chunk_data.shape[0]), "constant" ) seg_res.append([chunk_st / fs + seg_st, chunk_ed / fs + seg_st, chunk_data]) return seg_res segs = [] for i, s in enumerate(vad_segments): segs.extend(seg_chunk(s)) return segs def extract_feature(audio): features = [] for au in audio: feature = Kaldi.fbank(au.unsqueeze(0), num_mel_bins=80) feature = feature - feature.mean(dim=0, keepdim=True) features.append(feature.unsqueeze(0)) features = torch.cat(features) return features def postprocess( segments: list, vad_segments: list, labels: np.ndarray, embeddings: np.ndarray ) -> list: assert len(segments) == len(labels) labels = correct_labels(labels) distribute_res = [] for i in range(len(segments)): distribute_res.append([segments[i][0], segments[i][1], labels[i]]) # merge the same speakers chronologically distribute_res = merge_seque(distribute_res) # accquire speaker center spk_embs = [] for i in range(labels.max() + 1): spk_emb = embeddings[labels == i].mean(0) spk_embs.append(spk_emb) spk_embs = np.stack(spk_embs) def is_overlapped(t1, t2): if t1 > t2 + 1e-4: return True return False # distribute the overlap region for i in range(1, len(distribute_res)): if is_overlapped(distribute_res[i - 1][1], distribute_res[i][0]): p = (distribute_res[i][0] + distribute_res[i - 1][1]) / 2 distribute_res[i][0] = p distribute_res[i - 1][1] = p # smooth the result distribute_res = smooth(distribute_res) return distribute_res def correct_labels(labels): labels_id = 0 id2id = {} new_labels = [] for i in labels: if i not in id2id: id2id[i] = labels_id labels_id += 1 new_labels.append(id2id[i]) return np.array(new_labels) def merge_seque(distribute_res): res = [distribute_res[0]] for i in range(1, len(distribute_res)): if distribute_res[i][2] != res[-1][2] or distribute_res[i][0] > res[-1][1]: res.append(distribute_res[i]) else: res[-1][1] = distribute_res[i][1] return res def smooth(res, mindur=1): # short segments are assigned to nearest speakers. for i in range(len(res)): res[i][0] = round(res[i][0], 2) res[i][1] = round(res[i][1], 2) if res[i][1] - res[i][0] < mindur: if i == 0: res[i][2] = res[i + 1][2] elif i == len(res) - 1: res[i][2] = res[i - 1][2] elif res[i][0] - res[i - 1][1] <= res[i + 1][0] - res[i][1]: res[i][2] = res[i - 1][2] else: res[i][2] = res[i + 1][2] # merge the speakers res = merge_seque(res) return res def distribute_spk(sentence_list, sd_time_list): sd_sentence_list = [] for d in sentence_list: sentence_start = d["ts_list"][0][0] sentence_end = d["ts_list"][-1][1] sentence_spk = 0 max_overlap = 0 for sd_time in sd_time_list: spk_st, spk_ed, spk = sd_time spk_st = spk_st * 1000 spk_ed = spk_ed * 1000 overlap = max(min(sentence_end, spk_ed) - max(sentence_start, spk_st), 0) if overlap > max_overlap: max_overlap = overlap sentence_spk = spk d["spk"] = sentence_spk sd_sentence_list.append(d) return sd_sentence_list