Update app.py

app.py

@@ -180,17 +180,12 @@ footer {
 
 @torch.no_grad()
 @torch.inference_mode()
-def voice_conversion(input, reference, steps, guidance, speed, use_conditioned, use_auto_adjustment, pitch):
+def voice_conversion(input, reference, steps, guidance, speed):
     print("[INFO] | Voice conversion started.")
     
-    inference_module = model if not use_conditioned else model_f0
-    mel_fn = to_mel if not use_conditioned else to_mel_f0
-    bigvgan_fn = bigvgan_model if not use_conditioned else bigvgan_44k_model
-    sr_current = 22050 if not use_conditioned else 44100
-    hop_length_current = 256 if not use_conditioned else 512
-    max_context_window = sr_current // hop_length_current * 30
-    overlap_wave_len = 16 * hop_length_current
-    bitrate = "320k"
+    inference_module, mel_fn, bigvgan_fn = model, to_mel, bigvgan_model
+    bitrate, sampling_rate, sr_current, hop_length_current = "320k", 16000, 22050, 256
+    max_context_window, overlap_wave_len = sr_current // hop_length_current * 30, 16 * hop_length_current
     
     # Load audio using librosa
     print("[INFO] | Loading source and reference audio.")
@@ -206,13 +201,13 @@ def voice_conversion(input, reference, steps, guidance, speed, use_conditioned,
     ref_audio_tensor = torch.tensor(ref_audio).unsqueeze(0).float().to(device)
     
     # Resample to 16kHz
-    ref_waves_16k = torchaudio.functional.resample(ref_audio_tensor, sr_current, 16000)
-    converted_waves_16k = torchaudio.functional.resample(source_audio_tensor, sr_current, 16000)
+    ref_waves_16k = torchaudio.functional.resample(ref_audio_tensor, sr_current, sampling_rate)
+    converted_waves_16k = torchaudio.functional.resample(source_audio_tensor, sr_current, sampling_rate)
     
     # Generate Whisper features
     print("[INFO] | Generating Whisper features for source audio.")
-    if converted_waves_16k.size(-1) <= 16000 * 30:
-        alt_inputs = whisper_feature_extractor([converted_waves_16k.squeeze(0).cpu().numpy()], return_tensors="pt", return_attention_mask=True, sampling_rate=16000)
+    if converted_waves_16k.size(-1) <= sampling_rate * 30:
+        alt_inputs = whisper_feature_extractor([converted_waves_16k.squeeze(0).cpu().numpy()], return_tensors="pt", return_attention_mask=True, sampling_rate)
         alt_input_features = whisper_model._mask_input_features(alt_inputs.input_features, attention_mask=alt_inputs.attention_mask).to(device)
         alt_outputs = whisper_model.encoder(alt_input_features.to(torch.float32), head_mask=None, output_attentions=False, output_hidden_states=False, return_dict=True)
         S_alt = alt_outputs.last_hidden_state.to(torch.float32)
@@ -222,8 +217,8 @@ def voice_conversion(input, reference, steps, guidance, speed, use_conditioned,
         # Process in chunks
         print("[INFO] | Processing source audio in chunks.")
         overlapping_time = 5  # seconds
-        chunk_size = 16000 * 30  # 30 seconds
-        overlap_size = 16000 * overlapping_time
+        chunk_size = sampling_rate * 30  # 30 seconds
+        overlap_size = sampling_rate * overlapping_time
         S_alt_list = []
         buffer = None
         traversed_time = 0
@@ -234,7 +229,7 @@ def voice_conversion(input, reference, steps, guidance, speed, use_conditioned,
                 chunk = converted_waves_16k[:, traversed_time:traversed_time + chunk_size]
             else:
                 chunk = torch.cat([buffer, converted_waves_16k[:, traversed_time:traversed_time + chunk_size - overlap_size]], dim=-1)
-            alt_inputs = whisper_feature_extractor([chunk.squeeze(0).cpu().numpy()],return_tensors="pt", return_attention_mask=True, sampling_rate=16000)
+            alt_inputs = whisper_feature_extractor([chunk.squeeze(0).cpu().numpy()], return_tensors="pt", return_attention_mask=True, sampling_rate)
             alt_input_features = whisper_model._mask_input_features(alt_inputs.input_features, attention_mask=alt_inputs.attention_mask).to(device)
             alt_outputs = whisper_model.encoder(alt_input_features.to(torch.float32), head_mask=None, output_attentions=False, output_hidden_states=False, return_dict=True)
             S_chunk = alt_outputs.last_hidden_state.to(torch.float32)
@@ -255,8 +250,8 @@ def voice_conversion(input, reference, steps, guidance, speed, use_conditioned,
     
     # Original Whisper features
     print("[INFO] | Generating Whisper features for reference audio.")
-    ori_waves_16k = torchaudio.functional.resample(ref_audio_tensor, sr_current, 16000)
-    ori_inputs = whisper_feature_extractor([ori_waves_16k.squeeze(0).cpu().numpy()], return_tensors="pt", return_attention_mask=True, sampling_rate=16000)
+    ori_waves_16k = torchaudio.functional.resample(ref_audio_tensor, sr_current, sampling_rate)
+    ori_inputs = whisper_feature_extractor([ori_waves_16k.squeeze(0).cpu().numpy()], return_tensors="pt", return_attention_mask=True, sampling_rate)
     ori_input_features = whisper_model._mask_input_features(ori_inputs.input_features, attention_mask=ori_inputs.attention_mask).to(device)
     ori_outputs = whisper_model.encoder(ori_input_features.to(torch.float32), head_mask=None, output_attentions=False, output_hidden_states=False, return_dict=True)
     S_ori = ori_outputs.last_hidden_state.to(torch.float32)
@@ -276,48 +271,15 @@ def voice_conversion(input, reference, steps, guidance, speed, use_conditioned,
     
     # Extract style features
     print("[INFO] | Extracting style features from reference audio.")
-    feat2 = torchaudio.compliance.kaldi.fbank(ref_waves_16k, num_mel_bins=80, dither=0, sample_frequency=16000)
+    feat2 = torchaudio.compliance.kaldi.fbank(ref_waves_16k, num_mel_bins=80, dither=0, sample_frequency=sampling_rate)
     feat2 = feat2 - feat2.mean(dim=0, keepdim=True)
     style2 = campplus_model(feat2.unsqueeze(0))
     print(f"[INFO] | Style2 shape: {style2.shape}")
     
-    # F0 Conditioning
-    if use_conditioned:
-        print("[INFO] | Performing F0 conditioning.")
-        F0_ori = rmvpe.infer_from_audio(ref_waves_16k[0], thred=0.5)
-        F0_alt = rmvpe.infer_from_audio(converted_waves_16k[0], thred=0.5)
-        
-        F0_ori = torch.from_numpy(F0_ori).to(device)[None].float()
-        F0_alt = torch.from_numpy(F0_alt).to(device)[None].float()
-        
-        voiced_F0_ori = F0_ori[F0_ori > 1]
-        voiced_F0_alt = F0_alt[F0_alt > 1]
-        
-        log_f0_alt = torch.log(F0_alt + 1e-5)
-        voiced_log_f0_ori = torch.log(voiced_F0_ori + 1e-5)
-        voiced_log_f0_alt = torch.log(voiced_F0_alt + 1e-5)
-        
-        median_log_f0_ori = torch.median(voiced_log_f0_ori)
-        median_log_f0_alt = torch.median(voiced_log_f0_alt)
-        
-        # Shift F0 levels
-        shifted_log_f0_alt = log_f0_alt.clone()
-        if auto_f0_adjust:
-            shifted_log_f0_alt[F0_alt > 1] = (log_f0_alt[F0_alt > 1] - median_log_f0_alt + median_log_f0_ori)
-        shifted_f0_alt = torch.exp(shifted_log_f0_alt)
-        if pitch != 0:
-            shifted_f0_alt[F0_alt > 1] = adjust_f0_semitones(shifted_f0_alt[F0_alt > 1], pitch)
-        print("[INFO] | F0 conditioning completed.")
-    else:
-        F0_ori = None
-        F0_alt = None
-        shifted_f0_alt = None
-        print("[INFO] | F0 conditioning not applied.")
-    
     # Length Regulation
     print("[INFO] | Applying length regulation.")
-    cond, _, _, _, _ = inference_module.length_regulator(S_alt, ylens=target_lengths, n_quantizers=3, f0=shifted_f0_alt)
-    prompt_condition, _, _, _, _ = inference_module.length_regulator(S_ori, ylens=target2_lengths, n_quantizers=3, f0=F0_ori)
+    cond, _, _, _, _ = inference_module.length_regulator(S_alt, ylens=target_lengths, n_quantizers=3, f0=None)
+    prompt_condition, _, _, _, _ = inference_module.length_regulator(S_ori, ylens=target2_lengths, n_quantizers=3, f0=None)
     print(f"[INFO] | Cond shape: {cond.shape}, Prompt condition shape: {prompt_condition.shape}")
     
     # Initialize variables for audio generation
@@ -345,8 +307,8 @@ def voice_conversion(input, reference, steps, guidance, speed, use_conditioned,
         output_wave = vc_wave[0].cpu().numpy()
         generated_wave_chunks.append(output_wave)
         
-        # Fix: Ensure processed_frames increments correctly to avoid infinite loop
-        processed_frames += vc_target.size(2)  # Changed from 'vc_target.size(2) - 16' to 'vc_target.size(2)'
+        # Ensure processed_frames increments correctly to avoid infinite loop
+        processed_frames += vc_target.size(2)
         print(f"[INFO] | Processed frames updated to: {processed_frames}")
     
     # Concatenate all generated wave chunks
@@ -392,11 +354,6 @@ with gr.Blocks(css=css) as main:
         guidance = gr.Slider(label="Guidance", value=0.7, minimum=0.0, maximum=1.0, step=0.1)
         speed = gr.Slider(label="Speed", value=1.0, minimum=0.5, maximum=2.0, step=0.1)
 
-    with gr.Column():
-        use_conditioned = gr.Checkbox(label="Use 'F0 Conditioned Model'", value=False)
-        use_auto_adjustment = gr.Checkbox(label="Use 'Auto F0 Adjustment' with 'F0 Conditioned Model'", value=True)
-        pitch = gr.Slider(label="Pitch with 'F0 Conditioned Model'", value=0, minimum=-12, maximum=12, step=1)
-
     with gr.Column():
         submit = gr.Button("▶")
         maintain = gr.Button("☁️")
@@ -404,7 +361,7 @@ with gr.Blocks(css=css) as main:
     with gr.Column():
         output = gr.Audio(label="Output", type="filepath")
 
-    submit.click(voice_conversion, inputs=[input, reference_input, steps, guidance, speed, use_conditioned, use_auto_adjustment, pitch], outputs=output, queue=False)
+    submit.click(voice_conversion, inputs=[input, reference_input, steps, guidance, speed], outputs=output, queue=False)
     maintain.click(cloud, inputs=[], outputs=[], queue=False)
 
 main.launch(show_api=True)