pronunciationTrainer.py

import time
from string import punctuation

import epitran
import numpy as np
import torch

import ModelInterfaces as mi
import RuleBasedModels
import WordMatching as wm
import WordMetrics
import models as mo
from constants import app_logger, MODEL_NAME_DEFAULT, sample_rate_resample


def preprocessAudioStandalone(audio: torch.tensor) -> torch.tensor:
    """
    Preprocess the audio by normalizing it.

    Args:
        audio (torch.tensor): The input audio tensor.

    Returns:
        torch.tensor: The normalized audio tensor.
    """
    audio = audio-torch.mean(audio)
    audio = audio/torch.max(torch.abs(audio))
    return audio


class PronunciationTrainer:
    """
    A class to train and evaluate pronunciation accuracy using ASR and phoneme conversion models.
    """
    current_transcript: str
    current_ipa: str

    current_recorded_audio: torch.Tensor
    current_recorded_transcript: str
    current_recorded_word_locations: list
    current_recorded_intonations: torch.tensor
    current_words_pronunciation_accuracy = []
    categories_thresholds = np.array([80, 60, 59])

    sampling_rate = sample_rate_resample

    def __init__(self, asr_model: mi.IASRModel, word_to_ipa_coverter: mi.ITextToPhonemModel) -> None:
        """
        Initialize the PronunciationTrainer with ASR and phoneme conversion models.

        Args:
            asr_model (mi.IASRModel): The ASR model to use.
            word_to_ipa_coverter (mi.ITextToPhonemModel): The phoneme conversion model to use.
        """
        self.asr_model = asr_model
        self.ipa_converter = word_to_ipa_coverter

    def getTranscriptAndWordsLocations(self, audio_length_in_samples: int) -> tuple[str, list]:
        """
        Get the transcript and word locations from the ASR model.

        Args:
            audio_length_in_samples (int): The length of the audio in samples.

        Returns:
            tuple: A tuple containing the audio transcript and word locations in samples.
        """
        audio_transcript = self.asr_model.getTranscript()
        word_locations_in_samples = self.asr_model.getWordLocations()

        fade_duration_in_samples = 0.05*self.sampling_rate
        word_locations_in_samples = [(int(np.maximum(0, word['start_ts']-fade_duration_in_samples)), int(np.minimum(
            audio_length_in_samples-1, word['end_ts']+fade_duration_in_samples))) for word in word_locations_in_samples]

        return audio_transcript, word_locations_in_samples

    # def getWordsRelativeIntonation(self, Audio: torch.tensor, word_locations: list):
    #     intonations = torch.zeros((len(word_locations), 1))
    #     intonation_fade_samples = 0.3*self.sampling_rate
    #     app_logger.info(f"intonations.shape: {intonations.shape}.")
    #     for word in range(len(word_locations)):
    #         intonation_start = int(np.maximum(
    #             0, word_locations[word][0]-intonation_fade_samples))
    #         intonation_end = int(np.minimum(
    #             Audio.shape[1]-1, word_locations[word][1]+intonation_fade_samples))
    #         intonations[word] = torch.sqrt(torch.mean(
    #             Audio[0][intonation_start:intonation_end]**2))
    #
    #     intonations = intonations/torch.mean(intonations)
    #     return intonations

    ##################### ASR Functions ###########################

    def processAudioForGivenText(self, recordedAudio: torch.Tensor = None, real_text=None) -> dict:
        """
        Process the recorded audio and evaluate pronunciation accuracy.

        Args:
            recordedAudio (torch.Tensor, optional): The recorded audio tensor. Defaults to None.
            real_text (str, optional): The real text to compare against. Defaults to None.

        Returns:
            dict: A dictionary containing the evaluation results.
        """
        start = time.time()
        recording_transcript, recording_ipa, word_locations = self.getAudioTranscript(
            recordedAudio)
        time_transcript_audio = time.time() - start
        app_logger.info(f'Time for NN to transcript audio: {time_transcript_audio:.2f}.')

        start = time.time()
        real_and_transcribed_words, real_and_transcribed_words_ipa, mapped_words_indices = self.matchSampleAndRecordedWords(
            real_text, recording_transcript)
        time_matching_transcripts = time.time() - start
        app_logger.info(f'Time for matching transcripts: {time_matching_transcripts:.3f}.')

        start_time, end_time = self.getWordLocationsFromRecordInSeconds(
            word_locations, mapped_words_indices)

        pronunciation_accuracy, current_words_pronunciation_accuracy = self.getPronunciationAccuracy(
            real_and_transcribed_words)  # _ipa

        pronunciation_categories = self.getWordsPronunciationCategory(
            current_words_pronunciation_accuracy)

        result = {'recording_transcript': recording_transcript,
                  'real_and_transcribed_words': real_and_transcribed_words,
                  'recording_ipa': recording_ipa, 'start_time': start_time, 'end_time': end_time,
                  'real_and_transcribed_words_ipa': real_and_transcribed_words_ipa, 'pronunciation_accuracy': pronunciation_accuracy,
                  'pronunciation_categories': pronunciation_categories}

        return result

    def getAudioTranscript(self, recordedAudio: torch.Tensor = None) -> tuple[str | list]:
        """
        Get the transcript and IPA representation of the recorded audio.

        Args:
            recordedAudio (torch.Tensor, optional): The recorded audio tensor. Defaults to None.

        Returns:
            tuple: A tuple containing the transcript, IPA representation, and word locations.
        """
        current_recorded_audio = recordedAudio

        current_recorded_audio = self.preprocessAudio(
            current_recorded_audio)
        self.asr_model.processAudio(current_recorded_audio)

        current_recorded_transcript, current_recorded_word_locations = self.getTranscriptAndWordsLocations(
            current_recorded_audio.shape[1])
        current_recorded_ipa = self.ipa_converter.convertToPhonem(
            current_recorded_transcript)

        # time.sleep(10000)
        return current_recorded_transcript, current_recorded_ipa, current_recorded_word_locations

    def getWordLocationsFromRecordInSeconds(self, word_locations, mapped_words_indices) -> list:
        """
        Get the start and end times of words in the recorded audio in seconds.

        Args:
            word_locations (list): The word locations in samples.
            mapped_words_indices (list): The indices of the mapped words.

        Returns:
            list: A list containing the start and end times of words in seconds.
        """
        app_logger.info(f"len_list: word_locations:{len(word_locations)},  mapped_words_indices:{len(mapped_words_indices)}, {len(word_locations) == len(mapped_words_indices)}...")
        start_time = []
        end_time = []
        for word_idx in range(len(mapped_words_indices)):
            start_time.append(float(word_locations[mapped_words_indices[word_idx]]
                                    [0])/self.sampling_rate)
            end_time.append(float(word_locations[mapped_words_indices[word_idx]]
                                  [1])/self.sampling_rate)
        return ' '.join([str(time) for time in start_time]), ' '.join([str(time) for time in end_time])

    ##################### END ASR Functions ###########################

    ##################### Evaluation Functions ###########################
    def matchSampleAndRecordedWords(self, real_text, recorded_transcript):
        """
        Match the real text with the recorded transcript and get the IPA representations.

        Args:
            real_text (str): The real text to compare against.
            recorded_transcript (str): The recorded transcript.

        Returns:
            tuple: A tuple containing the matched words, IPA representations, and mapped word indices.
        """
        words_estimated = recorded_transcript.split()

        try:
            words_real = real_text.split()
        except AttributeError:
            raise ValueError("Real text is None, but should be a string.")

        mapped_words, mapped_words_indices = wm.get_best_mapped_words(
            words_estimated, words_real)

        real_and_transcribed_words = []
        real_and_transcribed_words_ipa = []
        for word_idx in range(len(words_real)):
            if word_idx >= len(mapped_words)-1:
                mapped_words.append('-')
            real_and_transcribed_words.append(
                (words_real[word_idx],    mapped_words[word_idx]))
            real_and_transcribed_words_ipa.append((self.ipa_converter.convertToPhonem(words_real[word_idx]),
                                                   self.ipa_converter.convertToPhonem(mapped_words[word_idx])))
        return real_and_transcribed_words, real_and_transcribed_words_ipa, mapped_words_indices

    def getPronunciationAccuracy(self, real_and_transcribed_words_ipa) -> float:
        """
        Calculate the pronunciation accuracy based on the IPA representations.

        Args:
            real_and_transcribed_words_ipa (list): A list of tuples containing the real and transcribed IPA representations.

        Returns:
            float: The percentage of correct pronunciations.
        """
        total_mismatches = 0.
        number_of_phonemes = 0.
        current_words_pronunciation_accuracy = []
        for pair in real_and_transcribed_words_ipa:

            real_without_punctuation = self.removePunctuation(pair[0]).lower()
            number_of_word_mismatches = WordMetrics.edit_distance_python(
                real_without_punctuation, self.removePunctuation(pair[1]).lower())
            total_mismatches += number_of_word_mismatches
            number_of_phonemes_in_word = len(real_without_punctuation)
            number_of_phonemes += number_of_phonemes_in_word

            current_words_pronunciation_accuracy.append(float(
                number_of_phonemes_in_word-number_of_word_mismatches)/number_of_phonemes_in_word*100)

        percentage_of_correct_pronunciations = (
            number_of_phonemes-total_mismatches)/number_of_phonemes*100

        return np.round(percentage_of_correct_pronunciations), current_words_pronunciation_accuracy

    def removePunctuation(self, word: str) -> str:
        """
        Remove punctuation from a word.

        Args:
            word (str): The input word.

        Returns:
            str: The word without punctuation.
        """
        return ''.join([char for char in word if char not in punctuation])

    def getWordsPronunciationCategory(self, accuracies) -> list:
        """
        Get the pronunciation category for each word based on accuracy.

        Args:
            accuracies (list): A list of pronunciation accuracies.

        Returns:
            list: A list of pronunciation categories.
        """
        categories = []

        for accuracy in accuracies:
            categories.append(
                self.getPronunciationCategoryFromAccuracy(accuracy))

        return categories

    def getPronunciationCategoryFromAccuracy(self, accuracy) -> int:
        """
        Get the pronunciation category based on accuracy.

        Args:
            accuracy (float): The pronunciation accuracy.

        Returns:
            int: The pronunciation category.
        """
        return np.argmin(abs(self.categories_thresholds-accuracy))

    def preprocessAudio(self, audio: torch.tensor) -> torch.tensor:
        """
        Preprocess the audio by normalizing it.

        Args:
            audio (torch.tensor): The input audio tensor.

        Returns:
            torch.tensor: The normalized audio tensor.
        """
        return preprocessAudioStandalone(audio)


def getTrainer(language: str, model_name: str = MODEL_NAME_DEFAULT) -> PronunciationTrainer:
    """
    Get a PronunciationTrainer instance for the specified language and model.

    Args:
        language (str): The language of the model.
        model_name (str, optional): The name of the model. Defaults to MODEL_NAME_DEFAULT.

    Returns:
        PronunciationTrainer: An instance of PronunciationTrainer.
    """
    asr_model = mo.getASRModel(language, model_name=model_name)
    if language == 'de':
        phonem_converter = RuleBasedModels.EpitranPhonemConverter(epitran.Epitran('deu-Latn'))
    elif language == 'en':
        phonem_converter = RuleBasedModels.EngPhonemConverter()
    else:
        raise ValueError(f"Language '{language}' not implemented")
    trainer = PronunciationTrainer(asr_model, phonem_converter)

    return trainer