transformers/tests/models/clap/test_feature_extraction_clap.py

# coding=utf-8
# Copyright 2023 HuggingFace Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.


import itertools
import random
import unittest

import numpy as np

from transformers import ClapFeatureExtractor
from transformers.testing_utils import require_torch, require_torchaudio
from transformers.utils.import_utils import is_torch_available

from ...test_sequence_feature_extraction_common import SequenceFeatureExtractionTestMixin


if is_torch_available():
    import torch

global_rng = random.Random()


# Copied from tests.models.whisper.test_feature_extraction_whisper.floats_list
def floats_list(shape, scale=1.0, rng=None, name=None):
    """Creates a random float32 tensor"""
    if rng is None:
        rng = global_rng

    values = []
    for batch_idx in range(shape[0]):
        values.append([])
        for _ in range(shape[1]):
            values[-1].append(rng.random() * scale)

    return values


@require_torch
@require_torchaudio
# Copied from tests.models.whisper.test_feature_extraction_whisper.WhisperFeatureExtractionTester with Whisper->Clap
class ClapFeatureExtractionTester(unittest.TestCase):
    def __init__(
        self,
        parent,
        batch_size=7,
        min_seq_length=400,
        max_seq_length=2000,
        feature_size=10,
        hop_length=160,
        chunk_length=8,
        padding_value=0.0,
        sampling_rate=4_000,
        return_attention_mask=False,
        do_normalize=True,
    ):
        self.parent = parent
        self.batch_size = batch_size
        self.min_seq_length = min_seq_length
        self.max_seq_length = max_seq_length
        self.seq_length_diff = (self.max_seq_length - self.min_seq_length) // (self.batch_size - 1)
        self.padding_value = padding_value
        self.sampling_rate = sampling_rate
        self.return_attention_mask = return_attention_mask
        self.do_normalize = do_normalize
        self.feature_size = feature_size
        self.chunk_length = chunk_length
        self.hop_length = hop_length

    def prepare_feat_extract_dict(self):
        return {
            "feature_size": self.feature_size,
            "hop_length": self.hop_length,
            "chunk_length": self.chunk_length,
            "padding_value": self.padding_value,
            "sampling_rate": self.sampling_rate,
            "return_attention_mask": self.return_attention_mask,
            "do_normalize": self.do_normalize,
        }

    def prepare_inputs_for_common(self, equal_length=False, numpify=False):
        def _flatten(list_of_lists):
            return list(itertools.chain(*list_of_lists))

        if equal_length:
            speech_inputs = [floats_list((self.max_seq_length, self.feature_size)) for _ in range(self.batch_size)]
        else:
            # make sure that inputs increase in size
            speech_inputs = [
                floats_list((x, self.feature_size))
                for x in range(self.min_seq_length, self.max_seq_length, self.seq_length_diff)
            ]
        if numpify:
            speech_inputs = [np.asarray(x) for x in speech_inputs]
        return speech_inputs


@require_torch
@require_torchaudio
# Copied from tests.models.whisper.test_feature_extraction_whisper.WhisperFeatureExtractionTest with Whisper->Clap
class ClapFeatureExtractionTest(SequenceFeatureExtractionTestMixin, unittest.TestCase):
    feature_extraction_class = ClapFeatureExtractor

    def setUp(self):
        self.feat_extract_tester = ClapFeatureExtractionTester(self)

    def test_call(self):
        # Tests that all call wrap to encode_plus and batch_encode_plus
        feature_extractor = self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict())
        # create three inputs of length 800, 1000, and 1200
        speech_inputs = [floats_list((1, x))[0] for x in range(800, 1400, 200)]
        np_speech_inputs = [np.asarray(speech_input) for speech_input in speech_inputs]

        # Test feature size
        input_features = feature_extractor(np_speech_inputs, padding="max_length", return_tensors="np").input_features
        self.assertTrue(input_features.ndim == 4)

        # Test not batched input
        encoded_sequences_1 = feature_extractor(speech_inputs[0], return_tensors="np").input_features
        encoded_sequences_2 = feature_extractor(np_speech_inputs[0], return_tensors="np").input_features
        self.assertTrue(np.allclose(encoded_sequences_1, encoded_sequences_2, atol=1e-3))

        # Test batched
        encoded_sequences_1 = feature_extractor(speech_inputs, return_tensors="np").input_features
        encoded_sequences_2 = feature_extractor(np_speech_inputs, return_tensors="np").input_features
        for enc_seq_1, enc_seq_2 in zip(encoded_sequences_1, encoded_sequences_2):
            self.assertTrue(np.allclose(enc_seq_1, enc_seq_2, atol=1e-3))

    def test_double_precision_pad(self):
        import torch

        feature_extractor = self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict())
        np_speech_inputs = np.random.rand(100, 32).astype(np.float64)
        py_speech_inputs = np_speech_inputs.tolist()

        for inputs in [py_speech_inputs, np_speech_inputs]:
            np_processed = feature_extractor.pad([{"input_features": inputs}], return_tensors="np")
            self.assertTrue(np_processed.input_features.dtype == np.float32)
            pt_processed = feature_extractor.pad([{"input_features": inputs}], return_tensors="pt")
            self.assertTrue(pt_processed.input_features.dtype == torch.float32)

    def _load_datasamples(self, num_samples):
        from datasets import load_dataset

        ds = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation")
        # automatic decoding with librispeech
        speech_samples = ds.sort("id").select(range(num_samples))[:num_samples]["audio"]

        return [x["array"] for x in speech_samples]

    def integration_test_fusion(self):
        # fmt: off
        EXPECTED_INPUT_FEATURES = torch.tensor(
            [
                [
                    -30.2194, -22.4424, -18.6442, -17.2452, -22.7392, -32.2576, -36.1404,
                    -35.6120, -29.6229, -29.0454, -32.2157, -36.7664, -29.4436, -26.7825,
                    -31.1811, -38.3918, -38.8749, -43.4485, -47.6236, -38.7528, -31.8574,
                    -39.0591, -41.3190, -32.3319, -31.4699, -33.4502, -36.7412, -34.5265,
                    -35.1091, -40.4518, -42.7346, -44.5909, -44.9747, -45.8328, -47.0772,
                    -46.2723, -44.3613, -48.6253, -44.9551, -43.8700, -44.6104, -48.0146,
                    -42.7614, -47.3587, -47.4369, -45.5018, -47.0198, -42.8759, -47.5056,
                    -47.1567, -49.2621, -49.5643, -48.4330, -48.8495, -47.2512, -40.8439,
                    -48.1234, -49.1218, -48.7222, -50.2399, -46.8487, -41.9921, -50.4015,
                    -50.7827
                ],
                [
                    -89.0141, -89.1411, -88.8096, -88.5480, -88.3481, -88.2038,
                    -88.1105, -88.0647, -88.0636, -88.1051, -88.1877, -88.1110,
                    -87.8613, -88.6679, -88.2685, -88.9684, -88.7977, -89.6264,
                    -89.9299, -90.3184, -91.1446, -91.9265, -92.7267, -93.6099,
                    -94.6395, -95.3243, -95.5923, -95.5773, -95.0889, -94.3354,
                    -93.5746, -92.9287, -92.4525, -91.9798, -91.8852, -91.7500,
                    -91.7259, -91.7561, -91.7959, -91.7070, -91.6914, -91.5019,
                    -91.0640, -90.0807, -88.7102, -87.0826, -85.5956, -84.4441,
                    -83.8461, -83.8605, -84.6702, -86.3900, -89.3073, -93.2926,
                    -96.3813, -97.3529, -100.0000, -99.6942, -92.2851, -87.9588,
                    -85.7214, -84.6807, -84.1940, -84.2021
                ],
                [
                    -51.6882, -50.6852, -50.8198, -51.7428, -53.0325, -54.1619, -56.4903,
                    -59.0314, -60.7996, -60.5164, -59.9680, -60.5393, -62.5796, -65.4166,
                    -65.6149, -65.1409, -65.7226, -67.9057, -72.5089, -82.3530, -86.3189,
                    -83.4241, -79.1279, -79.3384, -82.7335, -79.8316, -80.2167, -74.3638,
                    -71.3930, -75.3849, -74.5381, -71.4504, -70.3791, -71.4547, -71.8820,
                    -67.3885, -69.5686, -71.9852, -71.0307, -73.0053, -80.8802, -72.9227,
                    -63.8526, -60.3260, -59.6012, -57.8316, -61.0603, -67.3403, -67.1709,
                    -60.4967, -60.5079, -68.3345, -67.5213, -70.6416, -79.6219, -78.2198,
                    -74.6851, -69.5718, -69.4968, -70.6882, -66.8175, -73.8558, -74.3855,
                    -72.9405
                ]
            ]
        )
        # fmt: on
        MEL_BIN = [963, 963, 161]
        input_speech = self._load_datasamples(1)
        feaure_extractor = ClapFeatureExtractor()
        for padding, EXPECTED_VALUES, idx_in_mel in zip(
            ["repeat", "repeatpad", None], EXPECTED_INPUT_FEATURES, MEL_BIN
        ):
            input_features = feaure_extractor(input_speech, return_tensors="pt", padding=padding).input_features
            self.assertTrue(torch.allclose(input_features[0, idx_in_mel], EXPECTED_VALUES, atol=1e-4))

    def integration_test_rand_trunc(self):
        # TODO in this case we should set the seed and use a longer audio to properly see the random truncation
        # fmt: off
        EXPECTED_INPUT_FEATURES = torch.tensor(
            [
                [
                    -42.3330, -36.2735, -35.9231, -43.5947, -48.4525, -46.5227, -42.6477,
                    -47.2740, -51.4336, -50.0846, -51.8711, -50.4232, -47.4736, -54.2275,
                    -53.3947, -55.4904, -54.8750, -54.5510, -55.4156, -57.4395, -51.7385,
                    -55.9118, -57.7800, -63.2064, -67.0651, -61.4379, -56.4268, -54.8667,
                    -52.3487, -56.4418, -57.1842, -55.1005, -55.6366, -59.4395, -56.8604,
                    -56.4949, -61.6573, -61.0826, -60.3250, -63.7876, -67.4882, -60.2323,
                    -54.6886, -50.5369, -47.7656, -45.8909, -49.1273, -57.4141, -58.3201,
                    -51.9862, -51.4897, -59.2561, -60.4730, -61.2203, -69.3174, -69.7464,
                    -65.5861, -58.9921, -59.5610, -61.0584, -58.1149, -64.4045, -66.2622,
                    -64.4610
                ],
                [
                    -41.2298, -38.4211, -39.8834, -45.9950, -47.3839, -43.9849, -46.0371,
                    -52.5490, -56.6912, -51.8794, -50.1284, -49.7506, -53.9422, -63.2854,
                    -56.5754, -55.0469, -55.3181, -55.8115, -56.0058, -57.9215, -58.7597,
                    -59.1994, -59.2141, -64.4198, -73.5138, -64.4647, -59.3351, -54.5626,
                    -54.7508, -65.0230, -60.0270, -54.7644, -56.0108, -60.1531, -57.6879,
                    -56.3766, -63.3395, -65.3032, -61.5202, -63.0677, -68.4217, -60.6868,
                    -54.4619, -50.8533, -47.7200, -45.9197, -49.0961, -57.7621, -59.0750,
                    -51.9122, -51.4332, -59.4132, -60.3415, -61.6558, -70.7049, -69.7905,
                    -66.9104, -59.0324, -59.6138, -61.2023, -58.2169, -65.3837, -66.4425,
                    -64.4142
                ],
                [
                    -51.6882, -50.6852, -50.8198, -51.7428, -53.0325, -54.1619, -56.4903,
                    -59.0314, -60.7996, -60.5164, -59.9680, -60.5393, -62.5796, -65.4166,
                    -65.6149, -65.1409, -65.7226, -67.9057, -72.5089, -82.3530, -86.3189,
                    -83.4241, -79.1279, -79.3384, -82.7335, -79.8316, -80.2167, -74.3638,
                    -71.3930, -75.3849, -74.5381, -71.4504, -70.3791, -71.4547, -71.8820,
                    -67.3885, -69.5686, -71.9852, -71.0307, -73.0053, -80.8802, -72.9227,
                    -63.8526, -60.3260, -59.6012, -57.8316, -61.0603, -67.3403, -67.1709,
                    -60.4967, -60.5079, -68.3345, -67.5213, -70.6416, -79.6219, -78.2198,
                    -74.6851, -69.5718, -69.4968, -70.6882, -66.8175, -73.8558, -74.3855,
                    -72.9405
                ]
            ]
        )
        # fmt: on

        input_speech = self._load_datasamples(1)
        feaure_extractor = ClapFeatureExtractor()
        for padding, EXPECTED_VALUES in zip(["repeat", "repeatpad", None], EXPECTED_INPUT_FEATURES):
            input_features = feaure_extractor(
                input_speech, return_tensors="pt", truncation="rand_trunc", padding=padding
            ).input_features
            self.assertTrue(torch.allclose(input_features[0, 0, :30], EXPECTED_VALUES, atol=1e-4))