syncs

.gitattributes

@@ -1,35 +1,47 @@
 *.7z filter=lfs diff=lfs merge=lfs -text
 *.arrow filter=lfs diff=lfs merge=lfs -text
 *.bin filter=lfs diff=lfs merge=lfs -text
+*.bin.* filter=lfs diff=lfs merge=lfs -text
 *.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
 *.ftz filter=lfs diff=lfs merge=lfs -text
 *.gz filter=lfs diff=lfs merge=lfs -text
 *.h5 filter=lfs diff=lfs merge=lfs -text
 *.joblib filter=lfs diff=lfs merge=lfs -text
 *.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
 *.model filter=lfs diff=lfs merge=lfs -text
 *.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
 *.onnx filter=lfs diff=lfs merge=lfs -text
 *.ot filter=lfs diff=lfs merge=lfs -text
 *.parquet filter=lfs diff=lfs merge=lfs -text
 *.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
 *.pt filter=lfs diff=lfs merge=lfs -text
 *.pth filter=lfs diff=lfs merge=lfs -text
 *.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
 saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
 *.tflite filter=lfs diff=lfs merge=lfs -text
 *.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
 *.xz filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
+*.zstandard filter=lfs diff=lfs merge=lfs -text
+*.tfevents* filter=lfs diff=lfs merge=lfs -text
+*.db* filter=lfs diff=lfs merge=lfs -text
+*.ark* filter=lfs diff=lfs merge=lfs -text
+**/*ckpt*data* filter=lfs diff=lfs merge=lfs -text
+**/*ckpt*.meta filter=lfs diff=lfs merge=lfs -text
+**/*ckpt*.index filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.gguf* filter=lfs diff=lfs merge=lfs -text
+*.ggml filter=lfs diff=lfs merge=lfs -text
+*.llamafile* filter=lfs diff=lfs merge=lfs -text
+*.pt2 filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,6 @@
+*.pt
+*__pycache__*
+tmp/*
+flagged/*
+test.py
+rename.sh

app.py

@@ -0,0 +1,195 @@
+import os
+import torch
+import librosa
+import warnings
+import numpy as np
+import pandas as pd
+import gradio as gr
+import librosa.display
+from model import EvalNet
+from t_model import t_EvalNet
+from utils import get_modelist, find_files, embed, MODEL_DIR
+
+
+TRANSLATE = {
+    "chanyin": "Vibrato",  # 颤音
+    "boxian": "Plucks",  # 拨弦
+    "shanghua": "Upward Portamento",  # 上滑音
+    "xiahua": "Downward Portamento",  # 下滑音
+    "huazhi/guazou/lianmo/liantuo": "Glissando",  # 花指\刮奏\连抹\连托
+    "yaozhi": "Tremolo",  # 摇指
+    "dianyin": "Point Note",  # 点音
+}
+CLASSES = list(TRANSLATE.keys())
+TEMP_DIR = "./__pycache__/tmp"
+SAMPLE_RATE = 44100
+HOP_LENGTH = 512
+TIME_LENGTH = 3
+
+
+def logMel(y, sr=SAMPLE_RATE):
+    mel = librosa.feature.melspectrogram(
+        y=y,
+        sr=sr,
+        hop_length=HOP_LENGTH,
+        fmin=27.5,
+    )
+    return librosa.power_to_db(mel, ref=np.max)
+
+
+def logCqt(y, sr=SAMPLE_RATE):
+    cqt = librosa.cqt(
+        y,
+        sr=sr,
+        hop_length=HOP_LENGTH,
+        fmin=27.5,
+        n_bins=88,
+        bins_per_octave=12,
+    )
+    return ((1.0 / 80.0) * librosa.core.amplitude_to_db(np.abs(cqt), ref=np.max)) + 1.0
+
+
+def logChroma(y, sr=SAMPLE_RATE):
+    chroma = librosa.feature.chroma_stft(
+        y=y,
+        sr=sr,
+        hop_length=HOP_LENGTH,
+    )
+    return (
+        (1.0 / 80.0) * librosa.core.amplitude_to_db(np.abs(chroma), ref=np.max)
+    ) + 1.0
+
+
+def RoW_norm(data):
+    common_sum = 0
+    square_sum = 0
+    tfle = 0
+    for i in range(len(data)):
+        tfle += (data[i].sum(-1).sum(0) != 0).astype("float").sum()
+        common_sum += data[i].sum(-1).sum(-1)
+        square_sum += (data[i] ** 2).sum(-1).sum(-1)
+
+    common_avg = common_sum / tfle
+    square_avg = square_sum / tfle
+    std = np.sqrt(square_avg - common_avg**2)
+    return common_avg, std
+
+
+def norm(data):
+    size = data.shape
+    avg, std = RoW_norm(data)
+    avg = np.tile(avg.reshape((1, -1, 1, 1)), (size[0], 1, size[2], size[3]))
+    std = np.tile(std.reshape((1, -1, 1, 1)), (size[0], 1, size[2], size[3]))
+    return (data - avg) / std
+
+
+def chunk_data(f):
+    x = []
+    xdata = np.transpose(f)
+    s = SAMPLE_RATE * TIME_LENGTH // HOP_LENGTH
+    length = int(np.ceil((int(len(xdata) / s) + 1) * s))
+    app = np.zeros((length - xdata.shape[0], xdata.shape[1]))
+    xdata = np.concatenate((xdata, app), 0)
+    for i in range(int(length / s)):
+        data = xdata[int(i * s) : int(i * s + s)]
+        x.append(np.transpose(data[:s, :]))
+
+    return np.array(x)
+
+
+def load(audio_path: str, converto="mel"):
+    y, sr = librosa.load(audio_path, sr=SAMPLE_RATE)
+    spec = eval("log%s(y, sr)" % converto.capitalize())
+    x_spec = chunk_data(spec)
+    Xtr_spec = np.expand_dims(x_spec, axis=3)
+    return list(norm(Xtr_spec))
+
+
+def infer(audio_path: str, log_name: str):
+    if not audio_path:
+        return None, "Please input an audio!"
+
+    backbone = "_".join(log_name.split("_")[:-1])
+    spec = log_name.split("_")[-1]
+    try:
+        input = load(audio_path, converto=spec)
+        if "vit" in backbone or "swin" in backbone:
+            eval_net = t_EvalNet(
+                backbone,
+                len(TRANSLATE),
+                input[0].shape[1],
+                weight_path=f"{MODEL_DIR}/{log_name}.pt",
+            )
+
+        else:
+            eval_net = EvalNet(
+                backbone,
+                len(TRANSLATE),
+                input[0].shape[1],
+                weight_path=f"{MODEL_DIR}/{log_name}.pt",
+            )
+
+    except Exception as e:
+        return None, f"{e}"
+
+    input_size = eval_net.get_input_size()
+    embeded_input = embed(input, input_size)
+    output = list(eval_net.forward(embeded_input))
+    outputs = []
+    index = 0
+    for y in output:
+        preds = list(y.T)
+        for pred in preds:
+            outputs.append(
+                {
+                    "Frame": index,
+                    "Tech": TRANSLATE[CLASSES[torch.argmax(pred).item()]],
+                }
+            )
+            index += 1
+
+    return os.path.basename(audio_path), pd.DataFrame(outputs)
+
+
+if __name__ == "__main__":
+    warnings.filterwarnings("ignore")
+    models = get_modelist(assign_model="VGG19_mel")
+    examples = []
+    example_wavs = find_files()
+    for wav in example_wavs:
+        examples.append([wav, models[0]])
+
+    with gr.Blocks() as demo:
+        gr.Interface(
+            fn=infer,
+            inputs=[
+                gr.Audio(label="Upload audio", type="filepath"),
+                gr.Dropdown(choices=models, label="Select a model", value=models[0]),
+            ],
+            outputs=[
+                gr.Textbox(label="Audio filename", show_copy_button=True),
+                gr.Dataframe(label="Frame-level guzheng playing technique detection"),
+            ],
+            examples=examples,
+            cache_examples=False,
+            flagging_mode="never",
+            title="It is suggested that the recording time should not be too long",
+        )
+
+        gr.Markdown(
+            """
+# Cite
+```bibtex
+@dataset{zhaorui_liu_2021_5676893,
+  author       = {Monan Zhou, Shenyang Xu, Zhaorui Liu, Zhaowen Wang, Feng Yu, Wei Li and Baoqiang Han},
+  title        = {CCMusic: an Open and Diverse Database for Chinese Music Information Retrieval Research},
+  month        = {mar},
+  year         = {2024},
+  publisher    = {HuggingFace},
+  version      = {1.2},
+  url          = {https://huggingface.co/ccmusic-database}
+}
+```"""
+        )
+
+    demo.launch()

model.py

@@ -0,0 +1,183 @@
+import torch
+import torch.nn.functional as F
+import torch.nn as nn
+import numpy as np
+import torchvision.models as models
+from modelscope.msdatasets import MsDataset
+
+
+class Interpolate(nn.Module):
+    def __init__(
+        self,
+        size=None,
+        scale_factor=None,
+        mode="bilinear",
+        align_corners=False,
+    ):
+        super(Interpolate, self).__init__()
+        self.size = size
+        self.scale_factor = scale_factor
+        self.mode = mode
+        self.align_corners = align_corners
+
+    def forward(self, x):
+        return F.interpolate(
+            x,
+            size=self.size,
+            scale_factor=self.scale_factor,
+            mode=self.mode,
+            align_corners=self.align_corners,
+        )
+
+
+class EvalNet:
+    def __init__(
+        self,
+        backbone: str,
+        cls_num: int,
+        ori_T: int,
+        imgnet_ver="v1",
+        weight_path="",
+    ):
+        if not hasattr(models, backbone):
+            raise ValueError(f"Unsupported model {backbone}.")
+
+        self.imgnet_ver = imgnet_ver
+        self.training = bool(weight_path == "")
+        self.type, self.weight_url, self.input_size = self._model_info(backbone)
+        self.model: torch.nn.Module = eval("models.%s()" % backbone)
+        self.ori_T = ori_T
+        self.out_channel_before_classifier = 0
+        self._set_channel_outsize()  # set out channel size
+        self.cls_num = cls_num
+        self._set_classifier()
+        self._pseudo_foward()
+        checkpoint = (
+            torch.load(weight_path)
+            if torch.cuda.is_available()
+            else torch.load(weight_path, map_location="cpu")
+        )  # self.model.load_state_dict(checkpoint, False)
+        self.model.load_state_dict(checkpoint["model"], False)
+        self.classifier.load_state_dict(checkpoint["classifier"], False)
+        if torch.cuda.is_available():
+            self.model = self.model.cuda()
+            self.classifier = self.classifier.cuda()
+        self.model.eval()
+
+    def _get_backbone(self, backbone_ver, backbone_list):
+        for backbone_info in backbone_list:
+            if backbone_ver == backbone_info["ver"]:
+                return backbone_info
+
+        raise ValueError("[Backbone not found] Please check if --model is correct!")
+
+    def _model_info(self, backbone: str):
+        backbone_list = MsDataset.load(
+            "monetjoe/cv_backbones",
+            split=self.imgnet_ver,
+            cache_dir="./__pycache__",
+        )
+        backbone_info = self._get_backbone(backbone, backbone_list)
+        return (
+            str(backbone_info["type"]),
+            str(backbone_info["url"]),
+            int(backbone_info["input_size"]),
+        )
+
+    def _create_classifier(self):
+        original_T_size = self.ori_T
+        upsample_module = nn.Sequential(
+            nn.AdaptiveAvgPool2d((1, None)),  # F -> 1
+            nn.ConvTranspose2d(
+                self.out_channel_before_classifier,
+                256,
+                kernel_size=(1, 4),
+                stride=(1, 2),
+                padding=(0, 1),
+            ),
+            nn.ReLU(inplace=True),
+            nn.BatchNorm2d(256),
+            nn.ConvTranspose2d(
+                256, 128, kernel_size=(1, 4), stride=(1, 2), padding=(0, 1)
+            ),
+            nn.ReLU(inplace=True),
+            nn.BatchNorm2d(128),
+            nn.ConvTranspose2d(
+                128, 64, kernel_size=(1, 4), stride=(1, 2), padding=(0, 1)
+            ),
+            nn.ReLU(inplace=True),
+            nn.BatchNorm2d(64),
+            nn.ConvTranspose2d(
+                64, 32, kernel_size=(1, 4), stride=(1, 2), padding=(0, 1)
+            ),
+            nn.ReLU(inplace=True),
+            nn.BatchNorm2d(32),  # input for Interp: [bsz, C, 1, T]
+            Interpolate(
+                size=(1, original_T_size), mode="bilinear", align_corners=False
+            ),  # classifier
+            nn.Conv2d(32, 32, kernel_size=(1, 1)),
+            nn.ReLU(inplace=True),
+            nn.BatchNorm2d(32),
+            nn.Conv2d(32, self.cls_num, kernel_size=(1, 1)),
+        )
+
+        return upsample_module
+
+    def _set_channel_outsize(self):  #### get the output size before classifier ####
+        conv2d_out_ch = []
+        for name, module in self.model.named_modules():
+            if isinstance(module, torch.nn.Conv2d):
+                conv2d_out_ch.append(module.out_channels)
+
+            if (
+                str(name).__contains__("classifier")
+                or str(name).__eq__("fc")
+                or str(name).__contains__("head")
+            ):
+                if isinstance(module, torch.nn.Conv2d):
+                    conv2d_out_ch.append(module.in_channels)
+                    break
+
+        self.out_channel_before_classifier = conv2d_out_ch[-1]
+
+    def _set_classifier(self):  #### set custom classifier ####
+        if self.type == "resnet":
+            self.model.avgpool = nn.Identity()
+            self.model.fc = nn.Identity()
+            self.classifier = self._create_classifier()
+
+        elif (
+            self.type == "vgg" or self.type == "efficientnet" or self.type == "convnext"
+        ):
+            self.model.avgpool = nn.Identity()
+            self.model.classifier = nn.Identity()
+            self.classifier = self._create_classifier()
+
+        elif self.type == "squeezenet":
+            self.model.classifier = nn.Identity()
+            self.classifier = self._create_classifier()
+
+    def get_input_size(self):
+        return self.input_size
+
+    def _pseudo_foward(self):
+        temp = torch.randn(4, 3, self.input_size, self.input_size)
+        out = self.model(temp)
+        self.H = int(np.sqrt(out.size(1) / self.out_channel_before_classifier))
+
+    def forward(self, x):
+        if torch.cuda.is_available():
+            x = x.cuda()
+
+        if self.type == "convnext":
+            out = self.model(x)
+            out = self.classifier(out).squeeze()
+            return out
+
+        else:
+            out = self.model(x)
+            out = out.view(
+                out.size(0), self.out_channel_before_classifier, self.H, self.H
+            )
+            out = self.classifier(out).squeeze()
+            return out

requirements.txt

@@ -0,0 +1,5 @@
+torch
+pillow
+librosa
+matplotlib
+torchvision

t_model.py

@@ -0,0 +1,153 @@
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+import torchvision.models as models
+from modelscope.msdatasets import MsDataset
+
+
+class Interpolate(nn.Module):
+    def __init__(
+        self,
+        size=None,
+        scale_factor=None,
+        mode="bilinear",
+        align_corners=False,
+    ):
+        super(Interpolate, self).__init__()
+        self.size = size
+        self.scale_factor = scale_factor
+        self.mode = mode
+        self.align_corners = align_corners
+
+    def forward(self, x):
+        return F.interpolate(
+            x,
+            size=self.size,
+            scale_factor=self.scale_factor,
+            mode=self.mode,
+            align_corners=self.align_corners,
+        )
+
+
+class t_EvalNet:
+    def __init__(
+        self,
+        backbone: str,
+        cls_num: int,
+        ori_T: int,
+        imgnet_ver="v1",
+        weight_path="",
+    ):
+        if not hasattr(models, backbone):
+            raise ValueError(f"Unsupported model {backbone}.")
+
+        self.imgnet_ver = imgnet_ver
+        self.type, self.weight_url, self.input_size = self._model_info(backbone)
+        self.model: torch.nn.Module = eval("models.%s()" % backbone)
+        self.ori_T = ori_T
+        if self.type == "vit":
+            self.hidden_dim = self.model.hidden_dim
+            self.class_token = nn.Parameter(torch.zeros(1, 1, self.hidden_dim))
+
+        elif self.type == "swin_transformer":
+            self.hidden_dim = 768
+
+        self.cls_num = cls_num
+        self._set_classifier()
+        checkpoint = (
+            torch.load(weight_path)
+            if torch.cuda.is_available()
+            else torch.load(weight_path, map_location="cpu")
+        )
+        self.model.load_state_dict(checkpoint["model"], False)
+        self.classifier.load_state_dict(checkpoint["classifier"], False)
+        if torch.cuda.is_available():
+            self.model = self.model.cuda()
+            self.classifier = self.classifier.cuda()
+
+        self.model.eval()
+
+    def _get_backbone(self, backbone_ver, backbone_list):
+        for backbone_info in backbone_list:
+            if backbone_ver == backbone_info["ver"]:
+                return backbone_info
+
+        raise ValueError("[Backbone not found] Please check if --model is correct!")
+
+    def _model_info(self, backbone: str):
+        backbone_list = MsDataset.load(
+            "monetjoe/cv_backbones",
+            split=self.imgnet_ver,
+            cache_dir="./__pycache__",
+        )
+        backbone_info = self._get_backbone(backbone, backbone_list)
+        return (
+            str(backbone_info["type"]),
+            str(backbone_info["url"]),
+            int(backbone_info["input_size"]),
+        )
+
+    def _create_classifier(self):
+        original_T_size = self.ori_T
+        self.avgpool = nn.AdaptiveAvgPool2d((1, None))  # F -> 1
+        upsample_module = nn.Sequential(  # nn.AdaptiveAvgPool2d((1, None)), # F -> 1
+            nn.ConvTranspose2d(
+                self.hidden_dim, 256, kernel_size=(1, 4), stride=(1, 2), padding=(0, 1)
+            ),
+            nn.ReLU(inplace=True),
+            nn.BatchNorm2d(256),
+            nn.ConvTranspose2d(
+                256, 128, kernel_size=(1, 4), stride=(1, 2), padding=(0, 1)
+            ),
+            nn.ReLU(inplace=True),
+            nn.BatchNorm2d(128),
+            nn.ConvTranspose2d(
+                128, 64, kernel_size=(1, 4), stride=(1, 2), padding=(0, 1)
+            ),
+            nn.ReLU(inplace=True),
+            nn.BatchNorm2d(64),
+            nn.ConvTranspose2d(
+                64, 32, kernel_size=(1, 4), stride=(1, 2), padding=(0, 1)
+            ),
+            nn.ReLU(inplace=True),
+            nn.BatchNorm2d(32),  # input for Interp: [bsz, C, 1, T]
+            Interpolate(
+                size=(1, original_T_size), mode="bilinear", align_corners=False
+            ),  # classifier
+            nn.Conv2d(32, 32, kernel_size=(1, 1)),
+            nn.ReLU(inplace=True),
+            nn.BatchNorm2d(32),
+            nn.Conv2d(32, self.cls_num, kernel_size=(1, 1)),
+        )
+
+        return upsample_module
+
+    def _set_classifier(self):  #### set custom classifier ####
+        if self.type == "vit" or self.type == "swin_transformer":
+            self.classifier = self._create_classifier()
+
+    def get_input_size(self):
+        return self.input_size
+
+    def forward(self, x: torch.Tensor):
+        if torch.cuda.is_available():
+            x = x.cuda()
+
+        if self.type == "vit":
+            x = self.model._process_input(x)
+            batch_class_token = self.class_token.expand(x.size(0), -1, -1).cuda()
+            x = torch.cat([batch_class_token, x], dim=1)
+            x = self.model.encoder(x)
+            x = x[:, 1:].permute(0, 2, 1)
+            x = x.unsqueeze(2)
+            x = self.classifier(x).squeeze()  # x shape: [bsz, hidden_dim, 1, seq_len]
+            return x
+
+        elif self.type == "swin_transformer":
+            x = self.model.features(x)  # [B, H, W, C]
+            x = x.permute(0, 3, 1, 2)
+            x = self.avgpool(x)  # [B, C, 1, W]
+            x = self.classifier(x).squeeze()
+            return x
+
+        return None

utils.py

@@ -0,0 +1,59 @@
+import os
+import torch
+import numpy as np
+from torchvision.transforms import Compose, Resize, Normalize
+from modelscope import snapshot_download
+
+MODEL_DIR = snapshot_download(
+    "ccmusic-database/Guzheng_Tech99",
+    cache_dir="./__pycache__",
+)
+
+
+def toCUDA(x):
+    if hasattr(x, "cuda"):
+        if torch.cuda.is_available():
+            return x.cuda()
+
+    return x
+
+
+def find_files(folder_path=f"{MODEL_DIR}/examples", ext=".flac"):
+    audio_files = []
+    for root, _, files in os.walk(folder_path):
+        for file in files:
+            if file.endswith(ext):
+                file_path = os.path.join(root, file)
+                audio_files.append(file_path)
+
+    return audio_files
+
+
+def get_modelist(model_dir=MODEL_DIR, assign_model=""):
+    pt_files = []
+    for _, _, files in os.walk(model_dir):
+        for file in files:
+            if file.endswith(".pt"):
+                model = os.path.basename(file)[:-3]
+                if assign_model and assign_model in model:
+                    pt_files.append(model)
+                else:
+                    pt_files.insert(0, model)
+
+    return pt_files
+
+
+def embed(input: list, img_size: int):
+    compose = Compose(
+        [
+            Resize([img_size, img_size]),
+            Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)),
+        ]
+    )
+    inputs = []
+    for x in input:
+        x = np.array(x).transpose(2, 0, 1)
+        x = torch.from_numpy(x).repeat(3, 1, 1)
+        inputs.append(compose(x).float())
+
+    return toCUDA(torch.tensor(np.array(inputs)))