utils/mert.py

import torch
import numpy as np
from transformers import Wav2Vec2FeatureExtractor, AutoModel

class FeatureExtractorMERT:
    def __init__(self, model_name="m-a-p/MERT-v1-95M", device = "None", sr=24000):
        self.model_name = model_name
        self.sr = sr
        if device == "None":
            use_cuda = torch.cuda.is_available()
            device = torch.device("cuda" if use_cuda else "cpu")
        else:
            self.device = device
                
        self.model = AutoModel.from_pretrained(self.model_name, trust_remote_code=True).to(self.device)
        self.processor = Wav2Vec2FeatureExtractor.from_pretrained(self.model_name, trust_remote_code=True)

    def extract_features_from_segment(self, segment, sample_rate, save_path):
        input_audio = segment.float()
        model_inputs = self.processor(input_audio, sampling_rate=sample_rate, return_tensors="pt")
        model_inputs = model_inputs.to(self.device)

        with torch.no_grad():
            model_outputs = self.model(**model_inputs, output_hidden_states=True)

        # Stack and process hidden states
        all_layer_hidden_states = torch.stack(model_outputs.hidden_states).squeeze()[1:, :, :].unsqueeze(0)
        all_layer_hidden_states = all_layer_hidden_states.mean(dim=2)
        features = all_layer_hidden_states.cpu().detach().numpy()

        # Save features
        np.save(save_path, features)