Add files using upload-large-folder tool

best_model.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:23aa1fbbef3ec860e559fcb94a9830930ea7111c3bd73ba3d3e3957c5aeca369
+size 5656611329

best_model_80180.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:23aa1fbbef3ec860e559fcb94a9830930ea7111c3bd73ba3d3e3957c5aeca369
+size 5656611329

checkpoint_80000.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d0ec02d5d1a4a668edc2a4bb01dfbb7d78842aa1bc64fa63f456a40bf7637a7a
+size 5656611265

config.json

@@ -0,0 +1,193 @@
+{
+    "output_path": "checkpoints/",
+    "logger_uri": null,
+    "run_name": "GPT_XTTS_FT",
+    "project_name": "XTTS_trainer",
+    "run_description": [
+        "\n        GPT XTTS training\n        "
+    ],
+    "print_step": 50,
+    "plot_step": 100,
+    "model_param_stats": false,
+    "wandb_entity": null,
+    "dashboard_logger": "tensorboard",
+    "save_on_interrupt": true,
+    "log_model_step": 100,
+    "save_step": 1000,
+    "save_n_checkpoints": 1,
+    "save_checkpoints": true,
+    "save_all_best": false,
+    "save_best_after": 10000,
+    "target_loss": null,
+    "print_eval": false,
+    "test_delay_epochs": 0,
+    "run_eval": true,
+    "run_eval_steps": null,
+    "distributed_backend": "nccl",
+    "distributed_url": "tcp://localhost:54321",
+    "mixed_precision": false,
+    "precision": "fp16",
+    "epochs": 10,
+    "batch_size": 8,
+    "eval_batch_size": 16,
+    "grad_clip": 0.0,
+    "scheduler_after_epoch": true,
+    "lr": 5e-06,
+    "optimizer": "AdamW",
+    "optimizer_params": {
+        "betas": [
+            0.9,
+            0.96
+        ],
+        "eps": 1e-08,
+        "weight_decay": 0.01
+    },
+    "lr_scheduler": "MultiStepLR",
+    "lr_scheduler_params": {
+        "milestones": [
+            900000,
+            2700000,
+            5400000
+        ],
+        "gamma": 0.5,
+        "last_epoch": -1
+    },
+    "use_grad_scaler": false,
+    "allow_tf32": false,
+    "cudnn_enable": true,
+    "cudnn_deterministic": false,
+    "cudnn_benchmark": false,
+    "training_seed": 54321,
+    "model": "xtts",
+    "num_loader_workers": 8,
+    "num_eval_loader_workers": 0,
+    "use_noise_augment": false,
+    "audio": {
+        "sample_rate": 22050,
+        "output_sample_rate": 24000,
+        "dvae_sample_rate": 22050
+    },
+    "use_phonemes": false,
+    "phonemizer": null,
+    "phoneme_language": null,
+    "compute_input_seq_cache": false,
+    "text_cleaner": null,
+    "enable_eos_bos_chars": false,
+    "test_sentences_file": "",
+    "phoneme_cache_path": null,
+    "characters": null,
+    "add_blank": false,
+    "batch_group_size": 0,
+    "loss_masking": null,
+    "min_audio_len": 1,
+    "max_audio_len": Infinity,
+    "min_text_len": 1,
+    "max_text_len": Infinity,
+    "compute_f0": false,
+    "compute_energy": false,
+    "compute_linear_spec": false,
+    "precompute_num_workers": 0,
+    "start_by_longest": false,
+    "shuffle": false,
+    "drop_last": false,
+    "datasets": [
+        {
+            "formatter": "",
+            "dataset_name": "",
+            "path": "",
+            "meta_file_train": "",
+            "ignored_speakers": null,
+            "language": "",
+            "phonemizer": "",
+            "meta_file_val": "",
+            "meta_file_attn_mask": ""
+        }
+    ],
+    "test_sentences": [],
+    "eval_split_max_size": 256,
+    "eval_split_size": 0.01,
+    "use_speaker_weighted_sampler": false,
+    "speaker_weighted_sampler_alpha": 1.0,
+    "use_language_weighted_sampler": false,
+    "language_weighted_sampler_alpha": 1.0,
+    "use_length_weighted_sampler": false,
+    "length_weighted_sampler_alpha": 1.0,
+    "model_args": {
+        "gpt_batch_size": 1,
+        "enable_redaction": false,
+        "kv_cache": true,
+        "gpt_checkpoint": "",
+        "clvp_checkpoint": null,
+        "decoder_checkpoint": null,
+        "num_chars": 255,
+        "tokenizer_file": "checkpoints/XTTS_v2.0_original_model_files/vocab.json",
+        "gpt_max_audio_tokens": 605,
+        "gpt_max_text_tokens": 402,
+        "gpt_max_prompt_tokens": 70,
+        "gpt_layers": 30,
+        "gpt_n_model_channels": 1024,
+        "gpt_n_heads": 16,
+        "gpt_number_text_tokens": 8661,
+        "gpt_start_text_token": 261,
+        "gpt_stop_text_token": 0,
+        "gpt_num_audio_tokens": 1026,
+        "gpt_start_audio_token": 1024,
+        "gpt_stop_audio_token": 1025,
+        "gpt_code_stride_len": 1024,
+        "gpt_use_masking_gt_prompt_approach": true,
+        "gpt_use_perceiver_resampler": true,
+        "input_sample_rate": 22050,
+        "output_sample_rate": 24000,
+        "output_hop_length": 256,
+        "decoder_input_dim": 1024,
+        "d_vector_dim": 512,
+        "cond_d_vector_in_each_upsampling_layer": true,
+        "duration_const": 102400,
+        "min_conditioning_length": 11025,
+        "max_conditioning_length": 132300,
+        "gpt_loss_text_ce_weight": 0.01,
+        "gpt_loss_mel_ce_weight": 1.0,
+        "debug_loading_failures": false,
+        "max_wav_length": 330750,
+        "max_text_length": 400,
+        "mel_norm_file": "checkpoints/XTTS_v2.0_original_model_files/mel_stats.pth",
+        "dvae_checkpoint": "checkpoints/XTTS_v2.0_original_model_files/dvae.pth",
+        "xtts_checkpoint": "checkpoints/XTTS_v2.0_original_model_files/model.pth",
+        "vocoder": ""
+    },
+    "model_dir": null,
+    "languages": [
+        "en",
+        "es",
+        "fr",
+        "de",
+        "it",
+        "pt",
+        "pl",
+        "tr",
+        "ru",
+        "nl",
+        "cs",
+        "ar",
+        "zh-cn",
+        "hu",
+        "ko",
+        "ja",
+        "hi",
+        "te"
+    ],
+    "temperature": 0.75,
+    "length_penalty": 1.0,
+    "repetition_penalty": 5.0,
+    "top_k": 50,
+    "top_p": 0.85,
+    "num_gpt_outputs": 1,
+    "gpt_cond_len": 30,
+    "gpt_cond_chunk_len": 4,
+    "max_ref_len": 30,
+    "sound_norm_refs": false,
+    "optimizer_wd_only_on_weights": true,
+    "weighted_loss_attrs": null,
+    "weighted_loss_multipliers": null,
+    "github_branch": "* main"
+}

events.out.tfevents.1739298885.node2.188457.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3ea042bd25d446dae787dfc01d9ec0249466eb72bea5d5a8f0eeb957501a8fd4
+size 331915

inference.py

@@ -0,0 +1,81 @@
+import torch
+import torchaudio
+from tqdm import tqdm
+from underthesea import sent_tokenize
+import os
+
+from TTS.tts.configs.xtts_config import XttsConfig
+from TTS.tts.models.xtts import Xtts
+from TTS.tts.layers.xtts.tokenizer import VoiceBpeTokenizer  # Import the tokenizer
+
+# Device configuration
+device = "cuda:0" if torch.cuda.is_available() else "cpu"
+
+# Model paths
+checkpoint_dir = "/export/home/vivian/svarah/XTTSv2-Finetuning-for-New-Languages/checkpoints/GPT_XTTS_FT-February-12-2025_12+04AM-8e59ec3"
+xtts_checkpoint = os.path.join(checkpoint_dir, "best_model.pth")
+xtts_config = os.path.join(checkpoint_dir, "config.json")
+xtts_vocab = "/export/home/vivian/svarah/XTTSv2-Finetuning-for-New-Languages/checkpoints/XTTS_v2.0_original_model_files/vocab.json"  # Path to vocab file
+speaker_file_path = os.path.join(checkpoint_dir, "speakers_xtts.pth")  # Path to speaker file
+
+# Load model
+config = XttsConfig()
+config.load_json(xtts_config)
+
+# Initialize the tokenizer
+tokenizer = VoiceBpeTokenizer(xtts_vocab)  # Manually initialize the tokenizer
+
+# Initialize the model
+XTTS_MODEL = Xtts.init_from_config(config)
+
+# Load checkpoint
+XTTS_MODEL.load_checkpoint(
+    config,
+    checkpoint_path=xtts_checkpoint,
+    checkpoint_dir=checkpoint_dir,  # Explicitly provide checkpoint_dir
+    vocab_path=xtts_vocab,
+    speaker_file_path=speaker_file_path,  # Explicitly provide speaker file path
+    use_deepspeed=False,
+)
+XTTS_MODEL.to(device)
+# Manually set the tokenizer
+
+print("Model loaded successfully!")
+
+# Inference
+tts_text = "పల్నాడు ప్రాంతంలోని చిన్న గ్రామంలో అపరాజితుడు అనే బాలుడు తన తల్లిదండ్రులతో కలిసి ఉండేవాడు. అతని కుటుంబం గొప్ప ధనవంతం కాకపోయినా, అతనికి గొప్ప లక్ష్యసాధన కోరిక ఉండేది. అతనికి చదువుపై అపారమైన ఆసక్తి, కానీ కుటుంబ స్థితిగతుల వల్ల మంచి పాఠశాలకు వెళ్లే అవకాశం లేకపోయింది. అపరాజితుడు ఉదయాన్నే లేచి మేతెల్లో పనిచేసి తల్లికి సహాయపడుతూ, మిగిలిన సమయాన్ని తన చదువుకు అంకితం చేసేవాడు. ఊరిలోని చిన్న గ్రంథాలయంలో ఉన్న పుస్తకాలను చదివి కొత్త విషయాలు నేర్చుకునే ప్రయత్నం చేసేవాడు. అతని పట్టుదల చూసిన గ్రామంలోని ఓ పండితుడు, బాలుడా నీ ప్రయత్నం అమోఘం, నీ లక్ష్యాన్ని సాధించేందుకు నేను సహాయపడతాను అని చెప్పి, అతనికి కావాల్సిన పుస్తకాలను అందించేవాడు. కొన్నేళ్ల తర్వాత, అపరాజితుడు తన కృషి, పట్టుదలతో పరీక్షల్లో రాష్ట్ర స్థాయిలో ఉత్తమ ర్యాంకు సాధించి, ప్రభుత్వ స్కాలర్‌షిప్‌తో పెద్ద నగరానికి వెళ్లాడు. అక్కడ ఉన్నత చదువులు పూర్తి చేసి, ఒక గొప్ప శాస్త్రవేత్తగా ఎదిగాడు. తన ఊరిని మర్చిపోకుండా, అక్కడ ఒక పాఠశాలను స్థాపించి, చాలా మంది పేద పిల్లలకు ఉచిత విద్య అందించాడు. అతని విజయాన్ని చూసి గ్రామస్థులు గర్వంతో ఆనందంతో మన ఊరి బాలుడు ప్రపంచానికి వెలుగునిచ్చాడు అని ప్రశంసించారు. కష్టపడే మనసు, పట్టుదల ఉంటే, ఏ లక్ష్యాన్నైనా సాధించవచ్చు."
+speaker_audio_file = "/export/home/vivian/svarah/telugu_tts/Telugu/wavs/4503599627574654_chunk_10_enhanced.wav"
+
+lang = "te"
+
+gpt_cond_latent, speaker_embedding = XTTS_MODEL.get_conditioning_latents(
+    audio_path=speaker_audio_file,
+    gpt_cond_len=XTTS_MODEL.config.gpt_cond_len,
+    max_ref_length=XTTS_MODEL.config.max_ref_len,
+    sound_norm_refs=XTTS_MODEL.config.sound_norm_refs,
+)
+
+tts_texts = sent_tokenize(tts_text)
+
+wav_chunks = []
+for text in tqdm(tts_texts):
+    wav_chunk = XTTS_MODEL.inference(
+        text=text,
+        language=lang,
+        gpt_cond_latent=gpt_cond_latent,
+        speaker_embedding=speaker_embedding,
+        temperature=0.1,
+        length_penalty=1.0,
+        repetition_penalty=10.0,
+        top_k=10,
+        top_p=0.3,
+    )
+    wav_chunks.append(torch.tensor(wav_chunk["wav"]))
+
+out_wav = torch.cat(wav_chunks, dim=0).unsqueeze(0).cpu()
+
+# Play audio (for Jupyter Notebook)
+output_audio_path = "/export/home/vivian/svarah/XTTSv2-Finetuning-for-New-Languages/checkpoints/output_audio2.wav"
+torchaudio.save(output_audio_path, out_wav, sample_rate=24000)
+
+print(f"Audio saved to {output_audio_path}")

train_gpt_xtts.py

@@ -0,0 +1,237 @@
+import os
+import gc
+
+from trainer import Trainer, TrainerArgs
+
+from TTS.config.shared_configs import BaseDatasetConfig
+from TTS.tts.datasets import load_tts_samples
+from TTS.tts.layers.xtts.trainer.gpt_trainer import GPTArgs, GPTTrainer, GPTTrainerConfig, XttsAudioConfig
+from TTS.utils.manage import ModelManager
+
+from dataclasses import dataclass, field
+from typing import Optional
+from transformers import HfArgumentParser
+
+import argparse
+
+def create_xtts_trainer_parser():
+    parser = argparse.ArgumentParser(description="Arguments for XTTS Trainer")
+
+    parser.add_argument("--output_path", type=str, required=True,
+                        help="Path to pretrained + checkpoint model")
+    parser.add_argument("--metadatas", nargs='+', type=str, required=True,
+                        help="train_csv_path,eval_csv_path,language")
+    parser.add_argument("--num_epochs", type=int, default=1,
+                        help="Number of epochs")
+    parser.add_argument("--batch_size", type=int, default=1,
+                        help="Mini batch size")
+    parser.add_argument("--grad_acumm", type=int, default=1,
+                        help="Grad accumulation steps")
+    parser.add_argument("--max_audio_length", type=int, default=255995,
+                        help="Max audio length")
+    parser.add_argument("--max_text_length", type=int, default=200,
+                        help="Max text length")
+    parser.add_argument("--weight_decay", type=float, default=1e-2,
+                        help="Weight decay")
+    parser.add_argument("--lr", type=float, default=5e-6,
+                        help="Learning rate")
+    parser.add_argument("--save_step", type=int, default=5000,
+                        help="Save step")
+
+    return parser
+
+
+
+def train_gpt(metadatas, num_epochs, batch_size, grad_acumm, output_path, max_audio_length, max_text_length, lr, weight_decay, save_step):
+    #  Logging parameters
+    RUN_NAME = "GPT_XTTS_FT"
+    PROJECT_NAME = "XTTS_trainer"
+    DASHBOARD_LOGGER = "tensorboard"
+    LOGGER_URI = None
+
+    # Set here the path that the checkpoints will be saved. Default: ./run/training/
+    # OUT_PATH = os.path.join(output_path, "run", "training")
+    OUT_PATH = output_path
+
+    # Training Parameters
+    OPTIMIZER_WD_ONLY_ON_WEIGHTS = True  # for multi-gpu training please make it False
+    START_WITH_EVAL = False  # if True it will star with evaluation
+    BATCH_SIZE = batch_size  # set here the batch size
+    GRAD_ACUMM_STEPS = grad_acumm  # set here the grad accumulation steps
+
+
+    # Define here the dataset that you want to use for the fine-tuning on.
+    DATASETS_CONFIG_LIST = []
+    for metadata in metadatas:
+        train_csv, eval_csv, language = metadata.split(",")
+        print(train_csv, eval_csv, language)
+
+        config_dataset = BaseDatasetConfig(
+            formatter="coqui",
+            dataset_name="ft_dataset",
+            path=os.path.dirname(train_csv),
+            meta_file_train=os.path.basename(train_csv),
+            meta_file_val=os.path.basename(eval_csv),
+            language=language,
+        )
+
+        DATASETS_CONFIG_LIST.append(config_dataset)
+
+    # Define the path where XTTS v2.0.1 files will be downloaded
+    CHECKPOINTS_OUT_PATH = os.path.join(OUT_PATH, "XTTS_v2.0_original_model_files/")
+    os.makedirs(CHECKPOINTS_OUT_PATH, exist_ok=True)
+
+
+    # DVAE files
+    DVAE_CHECKPOINT_LINK = "https://coqui.gateway.scarf.sh/hf-coqui/XTTS-v2/main/dvae.pth"
+    MEL_NORM_LINK = "https://coqui.gateway.scarf.sh/hf-coqui/XTTS-v2/main/mel_stats.pth"
+
+    # Set the path to the downloaded files
+    DVAE_CHECKPOINT = os.path.join(CHECKPOINTS_OUT_PATH, os.path.basename(DVAE_CHECKPOINT_LINK))
+    MEL_NORM_FILE = os.path.join(CHECKPOINTS_OUT_PATH, os.path.basename(MEL_NORM_LINK))
+
+    # download DVAE files if needed
+    if not os.path.isfile(DVAE_CHECKPOINT) or not os.path.isfile(MEL_NORM_FILE):
+        print(" > Downloading DVAE files!")
+        ModelManager._download_model_files([MEL_NORM_LINK, DVAE_CHECKPOINT_LINK], CHECKPOINTS_OUT_PATH, progress_bar=True)
+
+
+    # Download XTTS v2.0 checkpoint if needed
+    TOKENIZER_FILE_LINK = "https://coqui.gateway.scarf.sh/hf-coqui/XTTS-v2/main/vocab.json"
+    XTTS_CHECKPOINT_LINK = "https://coqui.gateway.scarf.sh/hf-coqui/XTTS-v2/main/model.pth"
+    XTTS_CONFIG_LINK = "https://coqui.gateway.scarf.sh/hf-coqui/XTTS-v2/main/config.json"
+
+    # XTTS transfer learning parameters: You we need to provide the paths of XTTS model checkpoint that you want to do the fine tuning.
+    TOKENIZER_FILE = os.path.join(CHECKPOINTS_OUT_PATH, os.path.basename(TOKENIZER_FILE_LINK))  # vocab.json file
+    XTTS_CHECKPOINT = os.path.join(CHECKPOINTS_OUT_PATH, os.path.basename(XTTS_CHECKPOINT_LINK))  # model.pth file
+    XTTS_CONFIG_FILE = os.path.join(CHECKPOINTS_OUT_PATH, os.path.basename(XTTS_CONFIG_LINK))  # config.json file
+
+    # download XTTS v2.0 files if needed
+    if not os.path.isfile(TOKENIZER_FILE):
+        print(" > Downloading XTTS v2.0 tokenizer!")
+        ModelManager._download_model_files(
+            [TOKENIZER_FILE_LINK], CHECKPOINTS_OUT_PATH, progress_bar=True
+        )
+    if not os.path.isfile(XTTS_CHECKPOINT):
+        print(" > Downloading XTTS v2.0 checkpoint!")
+        ModelManager._download_model_files(
+            [XTTS_CHECKPOINT_LINK], CHECKPOINTS_OUT_PATH, progress_bar=True
+        )
+    if not os.path.isfile(XTTS_CONFIG_FILE):
+        print(" > Downloading XTTS v2.0 config!")
+        ModelManager._download_model_files(
+            [XTTS_CONFIG_LINK], CHECKPOINTS_OUT_PATH, progress_bar=True
+        )
+
+    # init args and config
+    model_args = GPTArgs(
+        max_conditioning_length=132300,  # 6 secs
+        min_conditioning_length=11025,  # 0.5 secs
+        debug_loading_failures=False,
+        max_wav_length=max_audio_length,  # ~11.6 seconds
+        max_text_length=max_text_length,
+        mel_norm_file=MEL_NORM_FILE,
+        dvae_checkpoint=DVAE_CHECKPOINT,
+        xtts_checkpoint=XTTS_CHECKPOINT,  # checkpoint path of the model that you want to fine-tune
+        tokenizer_file=TOKENIZER_FILE,
+        gpt_num_audio_tokens=1026,
+        gpt_start_audio_token=1024,
+        gpt_stop_audio_token=1025,
+        gpt_use_masking_gt_prompt_approach=True,
+        gpt_use_perceiver_resampler=True,
+    )
+    # define audio config
+    audio_config = XttsAudioConfig(sample_rate=22050, dvae_sample_rate=22050, output_sample_rate=24000)
+    # training parameters config
+
+    config = GPTTrainerConfig()
+
+    config.load_json(XTTS_CONFIG_FILE)
+
+    config.epochs = num_epochs
+    config.output_path = OUT_PATH
+    config.model_args = model_args
+    config.run_name = RUN_NAME
+    config.project_name = PROJECT_NAME
+    config.run_description = """
+        GPT XTTS training
+        """,
+    config.dashboard_logger = DASHBOARD_LOGGER
+    config.logger_uri = LOGGER_URI
+    config.audio = audio_config
+    config.batch_size = BATCH_SIZE
+    config.num_loader_workers = 8
+    config.eval_split_max_size = 256
+    config.print_step = 50
+    config.plot_step = 100
+    config.log_model_step = 100
+    config.save_step = save_step
+    config.save_n_checkpoints = 1
+    config.save_checkpoints = True
+    config.print_eval = False
+    config.optimizer = "AdamW"
+    config.optimizer_wd_only_on_weights = OPTIMIZER_WD_ONLY_ON_WEIGHTS
+    config.optimizer_params = {"betas": [0.9, 0.96], "eps": 1e-8, "weight_decay": weight_decay}
+    config.lr = lr
+    config.lr_scheduler = "MultiStepLR"
+    config.lr_scheduler_params = {"milestones": [50000 * 18, 150000 * 18, 300000 * 18], "gamma": 0.5, "last_epoch": -1}
+    config.test_sentences = []
+
+    # init the model from config
+    model = GPTTrainer.init_from_config(config)
+
+    # load training samples
+    train_samples, eval_samples = load_tts_samples(
+        DATASETS_CONFIG_LIST,
+        eval_split=True,
+        eval_split_max_size=config.eval_split_max_size,
+        eval_split_size=config.eval_split_size,
+    )
+
+    # init the trainer and 🚀
+    trainer = Trainer(
+        TrainerArgs(
+            restore_path=None,  # xtts checkpoint is restored via xtts_checkpoint key so no need of restore it using Trainer restore_path parameter
+            skip_train_epoch=False,
+            start_with_eval=START_WITH_EVAL,
+            grad_accum_steps=GRAD_ACUMM_STEPS
+        ),
+        config,
+        output_path=os.path.join(output_path, "run", "training"),
+        model=model,
+        train_samples=train_samples,
+        eval_samples=eval_samples,
+    )
+    trainer.fit()
+
+    # get the longest text audio file to use as speaker reference
+    samples_len = [len(item["text"].split(" ")) for item in train_samples]
+    longest_text_idx =  samples_len.index(max(samples_len))
+    speaker_ref = train_samples[longest_text_idx]["audio_file"]
+
+    trainer_out_path = trainer.output_path
+
+    # deallocate VRAM and RAM
+    del model, trainer, train_samples, eval_samples
+    gc.collect()
+
+    return trainer_out_path
+
+if __name__ == "__main__":
+    parser = create_xtts_trainer_parser()
+    args = parser.parse_args()
+
+    trainer_out_path = train_gpt(
+        metadatas=args.metadatas,
+        output_path=args.output_path,
+        num_epochs=args.num_epochs,
+        batch_size=args.batch_size,
+        grad_acumm=args.grad_acumm,
+        weight_decay=args.weight_decay,
+        lr=args.lr,
+        max_text_length=args.max_text_length,
+        max_audio_length=args.max_audio_length,
+        save_step=args.save_step
+    )
+
+    print(f"Checkpoint saved in dir: {trainer_out_path}")