Added Model

README.md

@@ -1,3 +1,163 @@
 ---
+language: sw
 license: cc-by-sa-4.0
+tags:
+  - tensorflowtts
+  - audio
+  - text-to-speech
+  - text-to-mel
+inference: false
 ---
+
+# LightSpeech MFA SW v1
+
+LightSpeech MFA SW v1 is a text-to-mel-spectrogram model based on the [LightSpeech](https://arxiv.org/abs/2102.04040) architecture. This model was trained from scratch on a real audio dataset. The list of real speakers include:
+
+- sw-KE-OpenBible
+
+We trained an acoustic Swahili model on our speech corpus using [Montreal Forced Aligner v2.0.0](https://github.com/MontrealCorpusTools/Montreal-Forced-Aligner) and used it as the duration extractor. That model, and consequently our model, uses the IPA phone set for Swahili. We used [gruut](https://github.com/rhasspy/gruut) for phonemization purposes. We followed these [steps](https://github.com/TensorSpeech/TensorFlowTTS/tree/master/examples/mfa_extraction) to perform duration extraction.
+
+This model was trained using the [TensorFlowTTS](https://github.com/TensorSpeech/TensorFlowTTS) framework. All training was done on a Scaleway RENDER-S VM with a Tesla P100 GPU. All necessary scripts used for training could be found in this [Github Fork](https://github.com/bookbot-hive/TensorFlowTTS), as well as the [Training metrics](https://huggingface.co/bookbot/lightspeech-mfa-sw-v1/tensorboard) logged via Tensorboard.
+
+## Model
+
+| Model                   | Config                                                                            | SR (Hz) | Mel range (Hz) | FFT / Hop / Win (pt) | #steps |
+| ----------------------- | --------------------------------------------------------------------------------- | ------- | -------------- | -------------------- | ------ |
+| `lightspeech-mfa-sw-v1` | [Link](https://huggingface.co/bookbot/lightspeech-mfa-sw-v1/blob/main/config.yml) | 44.1K   | 20-11025       | 2048 / 512 / None    | 200K   |
+
+## Training Procedure
+
+<details>
+  <summary>Feature Extraction Setting</summary>
+
+    hop_size: 512 # Hop size.
+    format: "npy"
+
+</details>
+
+<details>
+  <summary>Network Architecture Setting</summary>
+
+    model_type: lightspeech
+    lightspeech_params:
+        dataset: "swahiliipa"
+        n_speakers: 1
+        encoder_hidden_size: 256
+        encoder_num_hidden_layers: 3
+        encoder_num_attention_heads: 2
+        encoder_attention_head_size: 16
+        encoder_intermediate_size: 1024
+        encoder_intermediate_kernel_size:
+            - 5
+            - 25
+            - 13
+            - 9
+        encoder_hidden_act: "mish"
+        decoder_hidden_size: 256
+        decoder_num_hidden_layers: 3
+        decoder_num_attention_heads: 2
+        decoder_attention_head_size: 16
+        decoder_intermediate_size: 1024
+        decoder_intermediate_kernel_size:
+            - 17
+            - 21
+            - 9
+            - 13
+        decoder_hidden_act: "mish"
+        variant_prediction_num_conv_layers: 2
+        variant_predictor_filter: 256
+        variant_predictor_kernel_size: 3
+        variant_predictor_dropout_rate: 0.5
+        num_mels: 80
+        hidden_dropout_prob: 0.2
+        attention_probs_dropout_prob: 0.1
+        max_position_embeddings: 2048
+        initializer_range: 0.02
+        output_attentions: False
+        output_hidden_states: False
+
+</details>
+
+<details>
+  <summary>Data Loader Setting</summary>
+
+    batch_size: 8 # Batch size for each GPU with assuming that gradient_accumulation_steps == 1.
+    eval_batch_size: 16
+    remove_short_samples: true # Whether to remove samples the length of which are less than batch_max_steps.
+    allow_cache: true # Whether to allow cache in dataset. If true, it requires cpu memory.
+    mel_length_threshold: 32 # remove all targets has mel_length <= 32
+    is_shuffle: true # shuffle dataset after each epoch.
+
+</details>
+
+<details>
+  <summary>Optimizer & Scheduler Setting</summary>
+
+    optimizer_params:
+        initial_learning_rate: 0.0001
+        end_learning_rate: 0.00005
+        decay_steps: 150000 # < train_max_steps is recommend.
+        warmup_proportion: 0.02
+        weight_decay: 0.001
+
+    gradient_accumulation_steps: 2
+    var_train_expr:
+        null # trainable variable expr (eg. 'embeddings|encoder|decoder' )
+        # must separate by |. if var_train_expr is null then we
+        # training all variable
+
+</details>
+
+<details>
+  <summary>Interval Setting</summary>
+
+    train_max_steps: 200000 # Number of training steps.
+    save_interval_steps: 5000 # Interval steps to save checkpoint.
+    eval_interval_steps: 5000 # Interval steps to evaluate the network.
+    log_interval_steps: 200 # Interval steps to record the training log.
+    delay_f0_energy_steps: 3 # 2 steps use LR outputs only then 1 steps LR + F0 + Energy.
+
+</details>
+
+<details>
+  <summary>Other Setting</summary>
+
+    num_save_intermediate_results: 1 # Number of batch to be saved as intermediate results.
+
+</details>
+
+## How to Use
+
+```py
+import tensorflow as tf
+from tensorflow_tts.inference import TFAutoModel, AutoProcessor
+
+lightspeech = TFAutoModel.from_pretrained("bookbot/lightspeech-mfa-sw-v1")
+processor = AutoProcessor.from_pretrained("bookbot/lightspeech-mfa-sw-v1")
+
+text, speaker_name = "Hello World", "sw-KE-OpenBible"
+input_ids = processor.text_to_sequence(text)
+
+mel, duration_outputs, _ = lightspeech.inference(
+    input_ids=tf.expand_dims(tf.convert_to_tensor(input_ids, dtype=tf.int32), 0),
+    speaker_ids=tf.convert_to_tensor(
+        [processor.speakers_map[speaker_name]], dtype=tf.int32
+    ),
+    speed_ratios=tf.convert_to_tensor([1.0], dtype=tf.float32),
+    f0_ratios=tf.convert_to_tensor([1.0], dtype=tf.float32),
+    energy_ratios=tf.convert_to_tensor([1.0], dtype=tf.float32),
+)
+```
+
+## Disclaimer
+
+Do consider the biases which came from pre-training datasets that may be carried over into the results of this model.
+
+## Authors
+
+LightSpeech MFA SW v1 was trained and evaluated by [David Samuel Setiawan](https://davidsamuell.github.io/), [Wilson Wongso](https://wilsonwongso.dev/). All computation and development are done on Scaleway.
+
+## Framework versions
+
+- TensorFlowTTS 1.8
+- TensorFlow 2.7.0

config.yml

@@ -0,0 +1,74 @@
+allow_cache: true
+batch_size: 8
+config: ./TensorFlowTTS/examples/lightspeech/conf/lightspeech_swahiliipa.yaml
+dataset_config: TensorFlowTTS/preprocess/swahiliipa_preprocess.yaml
+dataset_mapping: dump/swahiliipa_mapper.json
+dataset_stats: dump/stats.npy
+delay_f0_energy_steps: 3
+dev_dir: ./dump/valid/
+energy_stat: ./dump/stats_energy.npy
+eval_batch_size: 16
+eval_interval_steps: 5000
+f0_stat: ./dump/stats_f0.npy
+format: npy
+gradient_accumulation_steps: 2
+hop_size: 512
+is_shuffle: true
+lightspeech_params:
+  attention_probs_dropout_prob: 0.1
+  dataset: swahiliipa
+  decoder_attention_head_size: 16
+  decoder_hidden_act: mish
+  decoder_hidden_size: 256
+  decoder_intermediate_kernel_size:
+  - 17
+  - 21
+  - 9
+  - 13
+  decoder_intermediate_size: 1024
+  decoder_num_attention_heads: 2
+  decoder_num_hidden_layers: 3
+  encoder_attention_head_size: 16
+  encoder_hidden_act: mish
+  encoder_hidden_size: 256
+  encoder_intermediate_kernel_size:
+  - 5
+  - 25
+  - 13
+  - 9
+  encoder_intermediate_size: 1024
+  encoder_num_attention_heads: 2
+  encoder_num_hidden_layers: 3
+  hidden_dropout_prob: 0.2
+  initializer_range: 0.02
+  max_position_embeddings: 2048
+  n_speakers: 1
+  num_mels: 80
+  output_attentions: false
+  output_hidden_states: false
+  variant_prediction_num_conv_layers: 2
+  variant_predictor_dropout_rate: 0.5
+  variant_predictor_filter: 256
+  variant_predictor_kernel_size: 3
+log_interval_steps: 200
+mel_length_threshold: 32
+mixed_precision: true
+model_type: lightspeech
+num_save_intermediate_results: 1
+optimizer_params:
+  decay_steps: 150000
+  end_learning_rate: 5.0e-05
+  initial_learning_rate: 0.001
+  warmup_proportion: 0.02
+  weight_decay: 0.001
+outdir: ./lightspeech-openbible
+pretrained: ''
+remove_short_samples: true
+resume: ''
+save_interval_steps: 5000
+train_dir: ./dump/train/
+train_max_steps: 200000
+use_norm: true
+var_train_expr: null
+verbose: 1
+version: '0.0'

events.out.tfevents.1712388040.bookbot.343075.0.v2

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2a1dd5aa5acee3af9aadc9ec378fb480c882b7fae3030004ee60b46d56a8c692
+size 234708

model.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:063c3c6775c287e386c69b98dca7df41de2cff9846bebe1b7f3bd59b02ee24dc
+size 19484280

processor.json

@@ -0,0 +1 @@
+{"symbol_to_id": {"@PAD": 0, "@f": 1, "@h": 2, "@i": 3, "@j": 4, "@k": 5, "@l": 6, "@m": 7, "@n": 8, "@p": 9, "@s": 10, "@t": 11, "@t\u0361\u0283": 12, "@u": 13, "@v": 14, "@w": 15, "@x": 16, "@z": 17, "@\u00f0": 18, "@\u0251": 19, "@\u0253": 20, "@\u0254": 21, "@\u0257": 22, "@\u025b": 23, "@\u0260": 24, "@\u0263": 25, "@\u027e": 26, "@\u0283": 27, "@\u0284": 28, "@\u03b8": 29, "@\u1d50\u0253": 30, "@\u1d51g": 31, "@\u1dacv": 32, "@\u207fz": 33, "@\u207f\u0257": 34, "@\u207f\u0257\u0361\u0292": 35, "!": 36, ",": 37, ".": 38, "?": 39, ";": 40, ":": 41, "@SIL": 42, "@EOS": 43}, "id_to_symbol": {"0": "@PAD", "1": "@f", "2": "@h", "3": "@i", "4": "@j", "5": "@k", "6": "@l", "7": "@m", "8": "@n", "9": "@p", "10": "@s", "11": "@t", "12": "@t\u0361\u0283", "13": "@u", "14": "@v", "15": "@w", "16": "@x", "17": "@z", "18": "@\u00f0", "19": "@\u0251", "20": "@\u0253", "21": "@\u0254", "22": "@\u0257", "23": "@\u025b", "24": "@\u0260", "25": "@\u0263", "26": "@\u027e", "27": "@\u0283", "28": "@\u0284", "29": "@\u03b8", "30": "@\u1d50\u0253", "31": "@\u1d51g", "32": "@\u1dacv", "33": "@\u207fz", "34": "@\u207f\u0257", "35": "@\u207f\u0257\u0361\u0292", "36": "!", "37": ",", "38": ".", "39": "?", "40": ";", "41": ":", "42": "@SIL", "43": "@EOS"}, "speakers_map": {"sw-KE-OpenBible": 0}, "processor_name": "SwahiliIPAProcessor"}