Finaly workaround chrome bug where audio is opus encoded but gets .wav extension

app.py

@@ -1,5 +1,6 @@
 import math
 import tempfile
+from typing import Tuple
 
 import gradio
 import gradio.inputs
@@ -13,6 +14,7 @@ from loguru import logger
 from df import config
 from df.enhance import enhance, init_df, load_audio, save_audio
 from df.utils import resample
+from torchaudio.backend.common import AudioMetaData
 
 device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 model, df, _ = init_df()
@@ -39,52 +41,46 @@ def mix_at_snr(clean, noise, snr, eps=1e-10):
         noise = noise.repeat((1, int(math.ceil(clean.shape[1] / noise.shape[1]))))
     max_start = int(noise.shape[1] - clean.shape[1])
     start = torch.randint(0, max_start, ()).item()
-    print("start:", start, clean.shape)
+    logger.debug(f"start: {start}, {clean.shape}")
     noise = noise[:, start : start + clean.shape[1]]
     E_speech = torch.mean(clean.pow(2)) + eps
     E_noise = torch.mean(noise.pow(2))
     K = torch.sqrt((E_noise / E_speech) * 10 ** (snr / 10) + eps)
     noise = noise / K
     mixture = clean + noise
-    print("mixture:", mixture.shape)
+    logger.debug("mixture: {mixture.shape}")
     assert torch.isfinite(mixture).all()
     max_m = mixture.abs().max()
     if max_m > 1:
-        print(f"Clipping detected during mixing. Reducing gain by {1/max_m}")
+        logger.warning(f"Clipping detected during mixing. Reducing gain by {1/max_m}")
         clean, noise, mixture = clean / max_m, noise / max_m, mixture / max_m
     return clean, noise, mixture
 
 
-def mix_and_denoise(speech_rec, speech_upl, noise, snr):
+def mix_and_denoise(speech_rec: Tuple[int, np.ndarray], speech_upl: str, noise_fn: str, snr: int):
     sr = config("sr", 48000, int, section="df")
     logger.info(
-        f"Got parameters speech_rec: {speech_rec}, speech_upl: {speech_upl}, noise: {noise}, snr: {snr}"
+        f"Got parameters speech_rec: {speech_rec}, speech_upl: {speech_upl}, noise: {noise_fn}, snr: {snr}"
     )
-    if noise is None:
-        noise = "samples/dkitchen.wav"
-    sp_kwargs = {}
-    if speech_rec is None or "none" in speech_rec:
+    if noise_fn is None:
+        noise_fn = "samples/dkitchen.wav"
+    meta = AudioMetaData(-1, -1, -1, -1, "")
+    if speech_upl is not None and "none" not in speech_upl:
         speech_file = "samples/p232_013_clean.wav"
         if speech_upl is not None and "none" not in speech_upl:
             speech_file = speech_upl
+        speech, meta = load_audio(speech_file, sr)
     else:
-        speech_file = speech_rec
-        sp_kwargs = {"frame_offset": 4800}
-    meta = None
-    # Apperently chrome uses mp3 or opus?
-    for f in ("wav", "mp3", "flac", "vorbis", "opus"):
-        sp_kwargs["format"] = f
-        logger.info(f"Trying to load speech: {speech_file}, with codec {f}")
-        try:
-            speech, meta = load_audio(speech_file, sr, **sp_kwargs)
-        except RuntimeError:
-            if meta is not None:
-                print(meta)
-                break
-    if meta is None:
-        raise ValueError("Could not load recorded speech")
+        meta.sample_rate, speech_rec_a = speech_rec
+        # Gradio documentation says, the shape is [samples, 2], but apparently sometimes its not.
+        speech_rec_a = speech_rec_a.reshape(speech_rec_a.shape[0], -1).T
+        if speech_rec_a.dtype == np.int16:
+            speech_rec_a = (speech_rec_a / (1 << 15)).astype(np.float32)
+        elif speech_rec_a.dtype == np.int32:
+            speech_rec_a = (speech_rec_a / (1 << 31)).astype(np.float32)
+        speech = resample(torch.from_numpy(speech_rec_a), meta.sample_rate, sr)
     logger.info(f"Loaded speech with shape {speech.shape}")
-    noise, _ = load_audio(noise, sr)
+    noise, _ = load_audio(noise_fn, sr)  # type: ignore
     if meta.sample_rate != sr:
         # Low pass filter by resampling
         noise = resample(resample(noise, sr, meta.sample_rate), meta.sample_rate, sr)
@@ -215,7 +211,7 @@ def spec_figure(
 inputs = [
     gradio.inputs.Audio(
         source="microphone",
-        type="filepath",
+        type="numpy",
         optional=True,
         label="Record your own voice",
     ),
@@ -262,4 +258,4 @@ iface = gradio.Interface(
     allow_flagging="never",
     article=markdown.markdown(open("usage.md").read()),
 )
-iface.launch(cache_examples=False)
+iface.launch(cache_examples=False, debug=True)