:tada: feat!: First commit

Pipfile

@@ -0,0 +1,22 @@
+[[source]]
+url = "https://pypi.org/simple"
+verify_ssl = true
+name = "pypi"
+
+[[source]]
+url = "https://download.pytorch.org/whl/cu118"
+verify_ssl = true
+name = "downloadpytorch"
+
+[packages]
+torch = {version = "*", index = "downloadpytorch"}
+transformers = "*"
+sentencepiece = "*"
+gradio = "*"
+pyopenjtalk-prebuilt = "*"
+
+[dev-packages]
+bandit = "*"
+
+[requires]
+python_version = "3.10"

app.py

@@ -0,0 +1,102 @@
+from typing import cast
+import gradio as gr
+import numpy as np
+import torch
+from transformers import SpeechT5ForTextToSpeech, SpeechT5Processor, SpeechT5HifiGan
+from speecht5_openjtalk_tokenizer import SpeechT5OpenjtalkTokenizer
+import pandas as pd
+
+import transformers
+
+setattr(transformers, SpeechT5OpenjtalkTokenizer.__name__, SpeechT5OpenjtalkTokenizer)
+
+
+class SpeechT5OpenjtalkProcessor(SpeechT5Processor):
+    tokenizer_class = SpeechT5OpenjtalkTokenizer.__name__
+
+
+model = SpeechT5ForTextToSpeech.from_pretrained("esnya/japanese_speecht5_tts")
+assert isinstance(model, SpeechT5ForTextToSpeech)
+
+processor = SpeechT5OpenjtalkProcessor.from_pretrained("esnya/japanese_speecht5_tts")
+assert isinstance(processor, SpeechT5OpenjtalkProcessor)
+
+vocoder = SpeechT5HifiGan.from_pretrained("microsoft/speecht5_hifigan")
+assert isinstance(vocoder, SpeechT5HifiGan)
+
+
+if torch.cuda.is_available():
+    model = model.cuda()
+    vocoder = vocoder.cuda()
+
+
+def convert_float32_to_int16(wav: np.ndarray) -> np.ndarray:
+    assert wav.dtype == np.float32
+    return np.clip(wav * 32768.0, -32768.0, 32767.0).astype(np.int16)
+
+
+@torch.inference_mode()
+def text_to_speech(
+    text: str,
+    threshold: float = 0.5,
+    minlenratio: float = 0.0,
+    maxlenratio: float = 10.0,
+):
+    speaker_embeddings = (
+        torch.rand(
+            (1, model.config.speaker_embedding_dim),
+            dtype=torch.float32,
+            device=model.device,
+        )
+        * 2
+        - 1
+    )
+
+    input_ids = processor(text=text, return_tensors="pt")
+    assert input_ids is not None
+    input_ids = input_ids.input_ids.to(model.device)
+
+    speaker_embeddings = cast(torch.FloatTensor, speaker_embeddings)
+
+    wav = model.generate_speech(
+        input_ids,
+        speaker_embeddings,
+        threshold=threshold,
+        minlenratio=minlenratio,
+        maxlenratio=maxlenratio,
+        vocoder=vocoder,
+    )
+    wav = cast(torch.FloatTensor, wav)
+
+    wav = convert_float32_to_int16(wav.reshape(-1).cpu().float().numpy())
+
+    return [
+        (vocoder.config.sampling_rate, wav),
+        pd.DataFrame(
+            {
+                "dim": range(speaker_embeddings.shape[-1]),
+                "value": speaker_embeddings[0].cpu().float().numpy(),
+            }
+        ),
+    ]
+
+
+demo = gr.Interface(
+    fn=text_to_speech,
+    inputs=[
+        "text",
+        gr.Slider(0, 0.5, 0.5, label="threshold"),
+        gr.Slider(0, 100, 0, label="minlenratio"),
+        gr.Slider(0, 100, 10, label="maxlenratio"),
+    ],
+    outputs=[
+        "audio",
+        gr.BarPlot(
+            label="speaker_embedding (random generated)",
+            x="dim",
+            y="value",
+            y_lim=[-1, 1],
+        ),
+    ],
+)
+demo.launch()

flagged/log.csv

@@ -0,0 +1,5 @@
+component 0,threshold,minlenratio,maxlenratio,output 0,speaker_embedding (random generated),flag,username,timestamp
+吾輩は猫である。名前はまだ無い。,0.5,0,10,C:\Users\esnya\Documents\space-japanese-tts\flagged\output 0\f3b115fb191a3158b81ce9c14ea8be68d6ccbb13\audio.wav,C:\Users\esnya\Documents\space-japanese-tts\flagged\speaker_embedding random generated\tmpi0ftk1a4.json,,,2023-08-09 19:10:28.404847
+吾輩は猫である。名前はまだ無い。,0.5,0,10,C:\Users\esnya\Documents\space-japanese-tts\flagged\output 0\a8f2aef3e7de612867c3cfc431eb23507893b8fb\audio.wav,C:\Users\esnya\Documents\space-japanese-tts\flagged\speaker_embedding random generated\tmpi_0pz0y_.json,,,2023-08-09 19:10:59.342286
+エリス帰りぬと答ふる間もなく。,0.5,0,10,C:\Users\esnya\Documents\space-japanese-tts\flagged\output 0\e83a2fdf4d1eea785fad09023d6fab00343e3329\audio.wav,C:\Users\esnya\Documents\space-japanese-tts\flagged\speaker_embedding random generated\tmpe5hqy2sj.json,,,2023-08-09 19:11:12.202799
+エリス帰りぬと答ふる間もなく。,0.5,0,10,C:\Users\esnya\Documents\space-japanese-tts\flagged\output 0\4ed6077f815534fa386a54ff81e6b16cd4e341d7\audio.wav,C:\Users\esnya\Documents\space-japanese-tts\flagged\speaker_embedding random generated\tmpvxdd17vj.json,,,2023-08-09 19:11:33.145021

flagged/speaker_embedding random generated/tmpe5hqy2sj.json

@@ -0,0 +1 @@
+{"type": "altair", "plot": "{\n  \"$schema\": \"https://vega.github.io/schema/vega-lite/v5.8.0.json\",\n  \"background\": \"transparent\",\n  \"config\": {\n    \"view\": {\n      \"continuousHeight\": 300,\n      \"continuousWidth\": 300\n    }\n  },\n  \"data\": {\n    \"name\": \"data-0ae9df14f2f8f47d301fa5647272660a\"\n  },\n  \"datasets\": {\n    \"data-0ae9df14f2f8f47d301fa5647272660a\": [\n      {\n        \"dim\": 0,\n        \"value\": 0.865778923034668\n      },\n      {\n        \"dim\": 1,\n        \"value\": 0.598870038986206\n      },\n      {\n        \"dim\": 2,\n        \"value\": 0.42185306549072266\n      },\n      {\n        \"dim\": 3,\n        \"value\": 0.2511407136917114\n      },\n      {\n        \"dim\": 4,\n        \"value\": -0.3122257590293884\n      },\n      {\n        \"dim\": 5,\n        \"value\": 0.3540457487106323\n      },\n      {\n        \"dim\": 6,\n        \"value\": 0.06121230125427246\n      },\n      {\n        \"dim\": 7,\n        \"value\": -0.6955808401107788\n      },\n      {\n        \"dim\": 8,\n        \"value\": 0.5161528587341309\n      },\n      {\n        \"dim\": 9,\n        \"value\": -0.34081584215164185\n      },\n      {\n        \"dim\": 10,\n        \"value\": 0.2803090810775757\n      },\n      {\n        \"dim\": 11,\n        \"value\": 0.4033799171447754\n      },\n      {\n        \"dim\": 12,\n        \"value\": -0.9365829825401306\n      },\n      {\n        \"dim\": 13,\n        \"value\": -0.2028934359550476\n      },\n      {\n        \"dim\": 14,\n        \"value\": -0.6850658655166626\n      },\n      {\n        \"dim\": 15,\n        \"value\": 0.9078857898712158\n      }\n    ]\n  },\n  \"encoding\": {\n    \"x\": {\n      \"field\": \"dim\",\n      \"title\": \"dim\",\n      \"type\": \"quantitative\"\n    },\n    \"y\": {\n      \"aggregate\": \"sum\",\n      \"field\": \"value\",\n      \"scale\": {\n        \"domain\": [\n          -1,\n          1\n        ]\n      },\n      \"title\": \"value\",\n      \"type\": \"quantitative\"\n    }\n  },\n  \"mark\": {\n    \"type\": \"bar\"\n  },\n  \"params\": [\n    {\n      \"bind\": \"scales\",\n      \"name\": \"param_46\",\n      \"select\": {\n        \"encodings\": [\n          \"x\",\n          \"y\"\n        ],\n        \"type\": \"interval\"\n      }\n    }\n  ]\n}", "chart": "bar"}

flagged/speaker_embedding random generated/tmpi0ftk1a4.json

@@ -0,0 +1 @@
+{"type": "altair", "plot": "{\n  \"$schema\": \"https://vega.github.io/schema/vega-lite/v5.8.0.json\",\n  \"background\": \"transparent\",\n  \"config\": {\n    \"view\": {\n      \"continuousHeight\": 300,\n      \"continuousWidth\": 300\n    }\n  },\n  \"data\": {\n    \"name\": \"data-9c5b21524e1befbb2ba9cb7a56a0869d\"\n  },\n  \"datasets\": {\n    \"data-9c5b21524e1befbb2ba9cb7a56a0869d\": [\n      {\n        \"dim\": 0,\n        \"value\": 0.13475024700164795\n      },\n      {\n        \"dim\": 1,\n        \"value\": -0.6327165961265564\n      },\n      {\n        \"dim\": 2,\n        \"value\": 0.142317533493042\n      },\n      {\n        \"dim\": 3,\n        \"value\": 0.21169781684875488\n      },\n      {\n        \"dim\": 4,\n        \"value\": -0.16417354345321655\n      },\n      {\n        \"dim\": 5,\n        \"value\": -0.8099290728569031\n      },\n      {\n        \"dim\": 6,\n        \"value\": 0.007316946983337402\n      },\n      {\n        \"dim\": 7,\n        \"value\": -0.07458484172821045\n      },\n      {\n        \"dim\": 8,\n        \"value\": 0.08054876327514648\n      },\n      {\n        \"dim\": 9,\n        \"value\": -0.21262741088867188\n      },\n      {\n        \"dim\": 10,\n        \"value\": 0.18876373767852783\n      },\n      {\n        \"dim\": 11,\n        \"value\": 0.9569865465164185\n      },\n      {\n        \"dim\": 12,\n        \"value\": -0.6338413953781128\n      },\n      {\n        \"dim\": 13,\n        \"value\": 0.4344193935394287\n      },\n      {\n        \"dim\": 14,\n        \"value\": 0.4842950105667114\n      },\n      {\n        \"dim\": 15,\n        \"value\": 0.8240410089492798\n      }\n    ]\n  },\n  \"encoding\": {\n    \"x\": {\n      \"field\": \"dim\",\n      \"title\": \"dim\",\n      \"type\": \"quantitative\"\n    },\n    \"y\": {\n      \"aggregate\": \"sum\",\n      \"field\": \"value\",\n      \"scale\": {\n        \"domain\": [\n          -1,\n          1\n        ]\n      },\n      \"title\": \"value\",\n      \"type\": \"quantitative\"\n    }\n  },\n  \"mark\": {\n    \"type\": \"bar\"\n  },\n  \"params\": [\n    {\n      \"bind\": \"scales\",\n      \"name\": \"param_39\",\n      \"select\": {\n        \"encodings\": [\n          \"x\",\n          \"y\"\n        ],\n        \"type\": \"interval\"\n      }\n    }\n  ]\n}", "chart": "bar"}

flagged/speaker_embedding random generated/tmpi_0pz0y_.json

@@ -0,0 +1 @@
+{"type": "altair", "plot": "{\n  \"$schema\": \"https://vega.github.io/schema/vega-lite/v5.8.0.json\",\n  \"background\": \"transparent\",\n  \"config\": {\n    \"view\": {\n      \"continuousHeight\": 300,\n      \"continuousWidth\": 300\n    }\n  },\n  \"data\": {\n    \"name\": \"data-6fb308b55e0eb4e1b35e48e0e7d97e92\"\n  },\n  \"datasets\": {\n    \"data-6fb308b55e0eb4e1b35e48e0e7d97e92\": [\n      {\n        \"dim\": 0,\n        \"value\": 0.29687047004699707\n      },\n      {\n        \"dim\": 1,\n        \"value\": 0.9556258916854858\n      },\n      {\n        \"dim\": 2,\n        \"value\": -0.6389535665512085\n      },\n      {\n        \"dim\": 3,\n        \"value\": -0.611727774143219\n      },\n      {\n        \"dim\": 4,\n        \"value\": -0.21317017078399658\n      },\n      {\n        \"dim\": 5,\n        \"value\": 0.7883336544036865\n      },\n      {\n        \"dim\": 6,\n        \"value\": -0.018283069133758545\n      },\n      {\n        \"dim\": 7,\n        \"value\": 0.35531842708587646\n      },\n      {\n        \"dim\": 8,\n        \"value\": 0.26885783672332764\n      },\n      {\n        \"dim\": 9,\n        \"value\": -0.4866262674331665\n      },\n      {\n        \"dim\": 10,\n        \"value\": -0.23639953136444092\n      },\n      {\n        \"dim\": 11,\n        \"value\": 0.7181340456008911\n      },\n      {\n        \"dim\": 12,\n        \"value\": -0.4331989884376526\n      },\n      {\n        \"dim\": 13,\n        \"value\": 0.2988170385360718\n      },\n      {\n        \"dim\": 14,\n        \"value\": -0.46783244609832764\n      },\n      {\n        \"dim\": 15,\n        \"value\": 0.12830126285552979\n      }\n    ]\n  },\n  \"encoding\": {\n    \"x\": {\n      \"field\": \"dim\",\n      \"title\": \"dim\",\n      \"type\": \"quantitative\"\n    },\n    \"y\": {\n      \"aggregate\": \"sum\",\n      \"field\": \"value\",\n      \"scale\": {\n        \"domain\": [\n          -1,\n          1\n        ]\n      },\n      \"title\": \"value\",\n      \"type\": \"quantitative\"\n    }\n  },\n  \"mark\": {\n    \"type\": \"bar\"\n  },\n  \"params\": [\n    {\n      \"bind\": \"scales\",\n      \"name\": \"param_45\",\n      \"select\": {\n        \"encodings\": [\n          \"x\",\n          \"y\"\n        ],\n        \"type\": \"interval\"\n      }\n    }\n  ]\n}", "chart": "bar"}

flagged/speaker_embedding random generated/tmpvxdd17vj.json

@@ -0,0 +1 @@
+{"type": "altair", "plot": "{\n  \"$schema\": \"https://vega.github.io/schema/vega-lite/v5.8.0.json\",\n  \"background\": \"transparent\",\n  \"config\": {\n    \"view\": {\n      \"continuousHeight\": 300,\n      \"continuousWidth\": 300\n    }\n  },\n  \"data\": {\n    \"name\": \"data-3d7bb59ef86b225300e3b1339e0be1fd\"\n  },\n  \"datasets\": {\n    \"data-3d7bb59ef86b225300e3b1339e0be1fd\": [\n      {\n        \"dim\": 0,\n        \"value\": -0.6320345401763916\n      },\n      {\n        \"dim\": 1,\n        \"value\": -0.45626479387283325\n      },\n      {\n        \"dim\": 2,\n        \"value\": 0.9872877597808838\n      },\n      {\n        \"dim\": 3,\n        \"value\": 0.5087642669677734\n      },\n      {\n        \"dim\": 4,\n        \"value\": -0.5172603726387024\n      },\n      {\n        \"dim\": 5,\n        \"value\": -0.5950517058372498\n      },\n      {\n        \"dim\": 6,\n        \"value\": -0.8185831308364868\n      },\n      {\n        \"dim\": 7,\n        \"value\": -0.6966900825500488\n      },\n      {\n        \"dim\": 8,\n        \"value\": 0.5786945819854736\n      },\n      {\n        \"dim\": 9,\n        \"value\": -0.9495899677276611\n      },\n      {\n        \"dim\": 10,\n        \"value\": 0.7928179502487183\n      },\n      {\n        \"dim\": 11,\n        \"value\": 0.0871659517288208\n      },\n      {\n        \"dim\": 12,\n        \"value\": 0.3648104667663574\n      },\n      {\n        \"dim\": 13,\n        \"value\": 0.451604962348938\n      },\n      {\n        \"dim\": 14,\n        \"value\": -0.3141704201698303\n      },\n      {\n        \"dim\": 15,\n        \"value\": -0.6407181024551392\n      }\n    ]\n  },\n  \"encoding\": {\n    \"x\": {\n      \"field\": \"dim\",\n      \"title\": \"dim\",\n      \"type\": \"quantitative\"\n    },\n    \"y\": {\n      \"aggregate\": \"sum\",\n      \"field\": \"value\",\n      \"scale\": {\n        \"domain\": [\n          -1,\n          1\n        ]\n      },\n      \"title\": \"value\",\n      \"type\": \"quantitative\"\n    }\n  },\n  \"mark\": {\n    \"type\": \"bar\"\n  },\n  \"params\": [\n    {\n      \"bind\": \"scales\",\n      \"name\": \"param_50\",\n      \"select\": {\n        \"encodings\": [\n          \"x\",\n          \"y\"\n        ],\n        \"type\": \"interval\"\n      }\n    }\n  ]\n}", "chart": "bar"}

requirements.txt

@@ -0,0 +1 @@
+pyopenjtalk-prebuilt==0.3.0

speecht5_openjtalk_tokenizer.py

@@ -0,0 +1,129 @@
+import json
+import logging
+from pathlib import Path
+import re
+from transformers import SpeechT5Tokenizer
+from transformers.models.speecht5.tokenization_speecht5 import (
+    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES,
+)
+from itertools import chain
+from typing import List, Optional
+
+
+logger = logging.getLogger(__name__)
+
+NP_CHARCTERS = " !\"#$%&'()=~|`{+*}<>?_-^\\@[;:],./　！”＃＄％＆’（）＝～｜｀｛＋＊｝＜＞？＿ー＾￥＠「；：」、。・`"
+
+
+def _g2p_with_np(text: str, np_lsit: str) -> List[str]:
+    from pyopenjtalk import g2p
+
+    np_pattern = re.compile(f"([{re.escape(np_lsit)}])")
+
+    return list(
+        chain.from_iterable(
+            [
+                (text,) if text in np_lsit else g2p(text, kana=False, join=False)
+                for text in np_pattern.split(text)
+                if len(text) > 0
+            ]
+        )
+    )
+
+
+VOCAB_FILES_NAMES = {
+    "vocab_file": "vocab.json",
+}
+
+PRETRAINED_VOCAB_FILES_MAP = {
+    "vocab_file": {
+        "esnya/japanese_speecht5_tts": "https://huggingface.co/esnya/japanese_speecht5_tts/resolve/main/vocab.json",
+    },
+}
+
+
+class SpeechT5OpenjtalkTokenizer(SpeechT5Tokenizer):
+    vocab_files_names = VOCAB_FILES_NAMES
+    pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
+    max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
+    model_input_names = ["input_ids", "attention_mask"]
+
+    def __init__(
+        self,
+        vocab_file,
+        bos_token: str = "<s>",
+        eos_token: str = "</s>",
+        unk_token: str = "<unk>",
+        pad_token: str = "<pad>",
+        non_phenome_characters: str = NP_CHARCTERS,
+        **kwargs,
+    ):
+        try:
+            super().__init__(
+                vocab_file=None,
+                bos_token=bos_token,
+                eos_token=eos_token,
+                unk_token=unk_token,
+                pad_token=pad_token,
+                **kwargs,
+            )
+        except TypeError:
+            pass
+
+        self.non_phenome_characters = non_phenome_characters
+        self.vocab_file = vocab_file
+
+        self._load_vocab()
+
+    def _load_vocab(self):
+        if isinstance(self.vocab_file, str) and self.vocab_file.endswith(".json"):
+            with open(self.vocab_file, encoding="utf-8") as f:
+                self.label2id = json.load(f)
+            self.id2label = {v: k for k, v in self.label2id.items()}
+
+    @property
+    def bos_token_id(self) -> int | None:
+        return super().bos_token_id
+
+    @property
+    def vocab_size(self):
+        return len(self.label2id)
+
+    def get_vocab(self):
+        return self.label2id
+
+    def __getstate__(self):
+        state = super().__getstate__()
+        del state["sp_model"]
+        return state
+
+    def __setstate__(self, d):
+        self.__dict__ = d
+        self._load_vocab()
+
+    def save_vocabulary(
+        self, save_directory: str, filename_prefix: Optional[str] = None
+    ):
+        if filename_prefix is None:
+            filename_prefix = ".json"
+
+        save_path = Path(save_directory)
+        if not save_path.is_dir():
+            logger.error(f"Vocabulary path ({save_directory}) should be a directory")
+            return
+
+        vocab_path = Path(save_directory) / Path(f"vocab{filename_prefix}")
+        vocab_path.parent.mkdir(parents=True, exist_ok=True)
+        with open(vocab_path, "w", encoding="utf-8") as f:
+            json.dump(self.label2id, f, ensure_ascii=False, indent=2)
+
+        return (str(vocab_path),)
+
+    def _tokenize(self, text: str) -> List[str]:
+        return _g2p_with_np(text, self.non_phenome_characters)
+
+    def _convert_token_to_id(self, token):
+        return self.label2id.get(token, self.label2id.get(self.unk_token))
+
+    def _convert_id_to_token(self, index):
+        return self.id2label.get(index, self.unk_token)