Upload 9 files

.gitattributes

@@ -1,35 +1,35 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,62 @@
+
+---
+license: cc-by-nc-4.0
+tags:
+- tts
+- gpt2
+- vae
+pipeline_tag: text-to-speech
+---
+
+# Malayalam Text-to-Speech
+
+This repository contains the **Malayalam (mal)** language text-to-speech (TTS) model checkpoint.
+
+## Model Details
+
+Sura (**S**tochastic **U**nified **R**epresentation for **A**dversarial learning) is an advanced speech synthesis model that generates speech waveforms conditioned on input text sequences. It is based on a conditional variational autoencoder (VAE) architecture, consisting of a posterior encoder, a decoder, and a conditional prior.
+
+The model's text encoder is built on GPT-2, while the decoder is a VAE with 124M parameters. The flow-based module predicts spectrogram-based acoustic features, which is composed of the GPT-2-based encoder and cascaded dense layers. The spectrogram is then transformed into a speech waveform using a stack of transposed convolutional layers. To capture the one-to-many nature of TTS, where the same text can be spoken in multiple ways, the model also includes a stochastic duration predictor, allowing for varied speech rhythms from the same text input.
+
+Sura is trained end-to-end using a combination of losses from the variational lower bound and adversarial training techniques. During inference, the text encodings are up-sampled based on the predicted durations, and subsequently mapped into the waveform via the flow module and the VAE decoder. Due to the stochastic nature of the duration predictor, the model is non-deterministic and requires a fixed seed to produce identical speech outputs.
+## Usage
+
+```
+pip install --upgrade transformers accelerate
+```
+
+Then, run inference with the following code-snippet:
+
+```python
+from transformers import VitsModel, AutoTokenizer
+import torch
+
+model = VitsModel.from_pretrained("aoxo/gpt2-vae-tts-mal")
+tokenizer = AutoTokenizer.from_pretrained("facebook/gpt2-vae-tts-mal")
+
+text = "കള്ളാ കടയാടി മോനെ"
+inputs = tokenizer(text, return_tensors="pt")
+
+with torch.no_grad():
+    output = model(**inputs).waveform
+```
+
+The resulting waveform can be saved as a `.wav` file:
+
+```python
+import scipy
+
+scipy.io.wavfile.write("kadayadi_mone.wav", rate=model.config.sampling_rate, data=output)
+```
+
+Or displayed in a Jupyter Notebook / Google Colab:
+
+```python
+from IPython.display import Audio
+
+Audio(output, rate=model.config.sampling_rate)
+```
+
+## License
+
+The model is licensed as **CC-BY-NC 4.0**.

config.json

@@ -0,0 +1,82 @@
+{
+  "activation_dropout": 0.1,
+  "architectures": [
+    "VitsModel"
+  ],
+  "attention_dropout": 0.1,
+  "depth_separable_channels": 2,
+  "depth_separable_num_layers": 3,
+  "duration_predictor_dropout": 0.5,
+  "duration_predictor_filter_channels": 256,
+  "duration_predictor_flow_bins": 10,
+  "duration_predictor_kernel_size": 3,
+  "duration_predictor_num_flows": 4,
+  "duration_predictor_tail_bound": 5.0,
+  "ffn_dim": 768,
+  "ffn_kernel_size": 3,
+  "flow_size": 192,
+  "hidden_act": "relu",
+  "hidden_dropout": 0.1,
+  "hidden_size": 192,
+  "initializer_range": 0.02,
+  "layer_norm_eps": 1e-05,
+  "layerdrop": 0.1,
+  "leaky_relu_slope": 0.1,
+  "model_type": "vits",
+  "noise_scale": 0.667,
+  "noise_scale_duration": 0.8,
+  "num_attention_heads": 2,
+  "num_hidden_layers": 6,
+  "num_speakers": 1,
+  "posterior_encoder_num_wavenet_layers": 16,
+  "prior_encoder_num_flows": 4,
+  "prior_encoder_num_wavenet_layers": 4,
+  "resblock_dilation_sizes": [
+    [
+      1,
+      3,
+      5
+    ],
+    [
+      1,
+      3,
+      5
+    ],
+    [
+      1,
+      3,
+      5
+    ]
+  ],
+  "resblock_kernel_sizes": [
+    3,
+    7,
+    11
+  ],
+  "sampling_rate": 16000,
+  "speaker_embedding_size": 0,
+  "speaking_rate": 1.0,
+  "spectrogram_bins": 513,
+  "torch_dtype": "float32",
+  "transformers_version": "4.33.0.dev0",
+  "upsample_initial_channel": 512,
+  "upsample_kernel_sizes": [
+    16,
+    16,
+    4,
+    4
+  ],
+  "upsample_rates": [
+    8,
+    8,
+    2,
+    2
+  ],
+  "use_bias": true,
+  "use_stochastic_duration_prediction": true,
+  "vocab_size": 84,
+  "wavenet_dilation_rate": 1,
+  "wavenet_dropout": 0.0,
+  "wavenet_kernel_size": 5,
+  "window_size": 4
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a97a1e677ec67e05124b799dadd66630181fe9c29beb4e590454689ff8f698c5
+size 145262840

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:046f0de3c235f75b96b478bab98bebcfe8468a11add65a8cde3f88f2cb2c63b9
+size 145424050

special_tokens_map.json

@@ -0,0 +1,4 @@
+{
+  "pad_token": "—",
+  "unk_token": "<unk>"
+}

tokenizer_config.json

@@ -0,0 +1,12 @@
+{
+  "add_blank": true,
+  "clean_up_tokenization_spaces": true,
+  "is_uroman": false,
+  "language": "mal",
+  "model_max_length": 1000000000000000019884624838656,
+  "normalize": true,
+  "pad_token": "—",
+  "phonemize": false,
+  "tokenizer_class": "VitsTokenizer",
+  "unk_token": "<unk>"
+}

trainer.py

@@ -0,0 +1,270 @@
+import os
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from torch.utils.data import Dataset, DataLoader
+from transformers import GPT2Tokenizer, GPT2Model
+from torchaudio.transforms import MelSpectrogram, InverseMelScale, GriffinLim
+import torchaudio
+from sklearn.model_selection import train_test_split
+from tqdm import tqdm
+from torch.optim.lr_scheduler import CosineAnnealingLR
+from torch.amp import GradScaler, autocast
+
+class TextToSpeechDataset(Dataset):
+    def __init__(self, text_files, audio_files, tokenizer, mel_transform, max_length=512):
+        self.text_files = text_files
+        self.audio_files = audio_files
+        self.tokenizer = tokenizer
+        self.mel_transform = mel_transform
+        self.max_length = max_length
+
+    def __len__(self):
+        return len(self.text_files)
+
+    def __getitem__(self, idx):
+        # Load text
+        with open(self.text_files[idx], 'r') as f:
+            text = f.read().strip()
+        
+        # Tokenize text
+        text_tokens = self.tokenizer.encode(
+            text, 
+            truncation=True, 
+            padding='max_length', 
+            max_length=self.max_length, 
+            return_tensors="pt"
+        ).squeeze(0)
+        
+        # Load audio and convert to mel spectrogram
+        waveform, sample_rate = torchaudio.load(self.audio_files[idx])
+        mel_spec = self.mel_transform(waveform)
+        
+        return text_tokens, mel_spec.squeeze(0)
+
+def collate_fn(batch):
+    text_tokens, mel_specs = zip(*batch)
+    
+    # Pad text tokens
+    max_text_len = max(tokens.size(0) for tokens in text_tokens)
+    text_tokens_padded = torch.stack([
+        torch.cat([tokens, torch.zeros(max_text_len - tokens.size(0), dtype=tokens.dtype)], dim=0)
+        if tokens.size(0) < max_text_len
+        else tokens[:max_text_len]
+        for tokens in text_tokens
+    ])
+    
+    # Pad mel spectrograms
+    max_mel_len = max(spec.size(1) for spec in mel_specs)
+    mel_specs_padded = torch.stack([
+        F.pad(spec, (0, max_mel_len - spec.size(1)))
+        if spec.size(1) < max_mel_len
+        else spec[:, :max_mel_len]
+        for spec in mel_specs
+    ])
+    
+    return text_tokens_padded, mel_specs_padded
+
+class VAEDecoder(nn.Module):
+    def __init__(self, latent_dim, mel_channels=80):
+        super().__init__()
+        # Encoder part (probabilistic)
+        self.fc_mu = nn.Linear(latent_dim, latent_dim)
+        self.fc_var = nn.Linear(latent_dim, latent_dim)
+        
+        # Decoder part
+        self.decoder_layers = nn.Sequential(
+            nn.Linear(latent_dim, 512),
+            nn.ReLU(),
+            nn.Linear(512, 1024),
+            nn.ReLU(),
+            nn.Linear(1024, mel_channels * 80),  # Output mel spectrogram
+            nn.Unflatten(1, (mel_channels, 80))
+        )
+    
+    def reparameterize(self, mu, log_var):
+        std = torch.exp(0.5 * log_var)
+        eps = torch.randn_like(std)
+        return mu + eps * std
+    
+    def forward(self, z):
+        mu = self.fc_mu(z)
+        log_var = self.fc_var(z)
+        
+        # Reparameterization trick
+        z = self.reparameterize(mu, log_var)
+        
+        # Decode
+        mel_spec = self.decoder_layers(z)
+        
+        return mel_spec, mu, log_var
+
+class TextToSpeechModel(nn.Module):
+    def __init__(self, text_encoder, vae_decoder, latent_dim=256):
+        super().__init__()
+        self.text_encoder = text_encoder
+        self.vae_decoder = vae_decoder
+        
+        # Projection layer to map encoder output to latent space
+        self.projection = nn.Linear(text_encoder.config.hidden_size, latent_dim)
+    
+    def forward(self, text_tokens):
+        # Encode text
+        encoder_output = self.text_encoder(text_tokens).last_hidden_state
+        
+        # Mean pooling of encoder output
+        text_embedding = encoder_output.mean(dim=1)
+        
+        # Project to latent space
+        latent_z = self.projection(text_embedding)
+        
+        # Decode to mel spectrogram
+        mel_spec, mu, log_var = self.vae_decoder(latent_z)
+        
+        return mel_spec, mu, log_var
+
+def vae_loss(reconstruction, target, mu, log_var):
+    # Reconstruction loss (MSE)
+    recon_loss = F.mse_loss(reconstruction, target, reduction='mean')
+    
+    # KL Divergence loss
+    kl_loss = -0.5 * torch.sum(1 + log_var - mu.pow(2) - log_var.exp())
+    
+    return recon_loss + 0.001 * kl_loss
+
+def train_model(num_epochs=10, accumulation_steps=16):
+    # Tokenizer and mel spectrogram transform
+    tokenizer = GPT2Tokenizer.from_pretrained('gpt2')
+    tokenizer.pad_token = tokenizer.eos_token
+    
+    # Mel spectrogram configuration
+    mel_transform = MelSpectrogram(
+        sample_rate=16000, 
+        n_mels=80, 
+        n_fft=1024, 
+        hop_length=256
+    )
+    
+    # Data preparation
+    text_folder = './texts'
+    audio_folder = './audio'
+    
+    # Load text and audio files
+    text_files = [os.path.join(text_folder, f) for f in os.listdir(text_folder) if f.endswith('.txt')]
+    audio_files = [os.path.join(audio_folder, f) for f in os.listdir(audio_folder) if f.endswith('.wav')]
+    
+    # Split dataset
+    train_texts, val_texts, train_audios, val_audios = train_test_split(
+        text_files, audio_files, test_size=0.1, random_state=42
+    )
+    
+    # Create datasets and dataloaders
+    train_dataset = TextToSpeechDataset(train_texts, train_audios, tokenizer, mel_transform)
+    val_dataset = TextToSpeechDataset(val_texts, val_audios, tokenizer, mel_transform)
+    
+    train_loader = DataLoader(train_dataset, batch_size=8, shuffle=True, collate_fn=collate_fn)
+    val_loader = DataLoader(val_dataset, batch_size=8, shuffle=False, collate_fn=collate_fn)
+    
+    # Model components
+    text_encoder = GPT2Model.from_pretrained('gpt2')
+    vae_decoder = VAEDecoder(latent_dim=256)
+    
+    # Combine into full model
+    model = TextToSpeechModel(text_encoder, vae_decoder)
+    
+    # Device setup
+    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+    model = model.to(device)
+    
+    # Optimizer and scheduler
+    optimizer = torch.optim.Adam(model.parameters(), lr=1e-4)
+    scheduler = CosineAnnealingLR(optimizer, T_max=num_epochs, eta_min=1e-6)
+    
+    # Gradient scaler
+    scaler = GradScaler()
+    
+    best_val_loss = float('inf')
+    
+    # Training loop
+    for epoch in range(num_epochs):
+        model.train()
+        train_loss = 0
+        
+        for batch_idx, (text_tokens, mel_specs) in enumerate(tqdm(train_loader, desc=f"Epoch {epoch+1}")):
+            text_tokens = text_tokens.to(device)
+            mel_specs = mel_specs.to(device)
+            
+            with autocast(dtype=torch.float16, device_type='cuda'):
+                # Forward pass
+                reconstructed_mel, mu, log_var = model(text_tokens)
+                
+                # Compute loss
+                loss = vae_loss(reconstructed_mel, mel_specs, mu, log_var)
+            
+            # Scaled loss and backpropagation
+            loss = loss / accumulation_steps
+            scaler.scale(loss).backward()
+            
+            if (batch_idx + 1) % accumulation_steps == 0:
+                scaler.step(optimizer)
+                scaler.update()
+                optimizer.zero_grad()
+            
+            train_loss += loss.item()
+        
+        # Validation
+        model.eval()
+        val_loss = 0
+        with torch.no_grad():
+            for text_tokens, mel_specs in tqdm(val_loader, desc=f"Validation {epoch+1}"):
+                text_tokens = text_tokens.to(device)
+                mel_specs = mel_specs.to(device)
+                
+                reconstructed_mel, mu, log_var = model(text_tokens)
+                loss = vae_loss(reconstructed_mel, mel_specs, mu, log_var)
+                val_loss += loss.item()
+        
+        # Scheduler step
+        scheduler.step()
+        
+        # Print epoch summary
+        print(f'Epoch {epoch+1}: Train Loss: {train_loss/len(train_loader)}, Val Loss: {val_loss/len(val_loader)}')
+        
+        # Model saving
+        if val_loss < best_val_loss:
+            best_val_loss = val_loss
+            torch.save(model.state_dict(), 'best_tts_model.pth')
+    
+    return model
+
+# Run training
+trained_model = train_model()
+
+# Optional: Inference function for generating mel spectrograms
+def generate_mel_spectrogram(text, model, tokenizer, device):
+    model.eval()
+    with torch.no_grad():
+        # Tokenize input text
+        text_tokens = tokenizer.encode(
+            text, 
+            return_tensors="pt", 
+            truncation=True, 
+            padding='max_length', 
+            max_length=512
+        ).to(device)
+        
+        # Generate mel spectrogram
+        mel_spec, _, _ = model(text_tokens)
+        
+        return mel_spec
+
+# Optional: Convert mel spectrogram back to audio
+def mel_to_audio(mel_spec, sample_rate=16000):
+    # Use griffin-lim for mel spectrogram inversion
+    inverse_mel = InverseMelScale(sample_rate=sample_rate)
+    griffin_lim = GriffinLim(sample_rate=sample_rate)
+    
+    # Convert mel spectrogram back to waveform
+    waveform = griffin_lim(inverse_mel(mel_spec))
+    
+    return waveform

vocab.json

@@ -0,0 +1,86 @@
+{
+  " ": 61,
+  "'": 81,
+  "-": 33,
+  "0": 6,
+  "1": 57,
+  "2": 54,
+  "3": 66,
+  "4": 23,
+  "5": 79,
+  "6": 46,
+  "_": 43,
+  "c": 75,
+  "i": 18,
+  "m": 30,
+  "o": 73,
+  "q": 82,
+  "ം": 29,
+  "ഃ": 38,
+  "അ": 63,
+  "ആ": 26,
+  "ഇ": 10,
+  "ഈ": 67,
+  "ഉ": 19,
+  "ഊ": 13,
+  "എ": 47,
+  "ഏ": 45,
+  "ഐ": 80,
+  "ഒ": 5,
+  "ഓ": 51,
+  "ഔ": 68,
+  "ക": 56,
+  "ഖ": 8,
+  "ഗ": 39,
+  "ഘ": 50,
+  "ങ": 59,
+  "ച": 14,
+  "ഛ": 35,
+  "ജ": 31,
+  "ഞ": 9,
+  "ട": 37,
+  "ഠ": 22,
+  "ഡ": 11,
+  "ഢ": 62,
+  "ണ": 40,
+  "ത": 1,
+  "ഥ": 49,
+  "ദ": 78,
+  "ധ": 32,
+  "ന": 71,
+  "പ": 69,
+  "ഫ": 20,
+  "ബ": 65,
+  "ഭ": 60,
+  "മ": 64,
+  "യ": 16,
+  "ര": 2,
+  "റ": 44,
+  "ല": 21,
+  "ള": 3,
+  "ഴ": 4,
+  "വ": 76,
+  "ശ": 83,
+  "ഷ": 17,
+  "സ": 27,
+  "ഹ": 34,
+  "ാ": 15,
+  "ി": 12,
+  "ീ": 52,
+  "ു": 28,
+  "ൂ": 7,
+  "ൃ": 74,
+  "െ": 24,
+  "േ": 55,
+  "ൈ": 53,
+  "ൊ": 72,
+  "ോ": 42,
+  "്": 36,
+  "ൗ": 25,
+  "ൺ": 70,
+  "ൻ": 77,
+  "ർ": 48,
+  "ൽ": 41,
+  "ൾ": 58,
+  "—": 0
+}