Try use GGML to improve inference speed

app.py

@@ -18,9 +18,9 @@ import spaces
 MODELS = {v.value: v for _, v in outetts.Models.__members__.items()}
 
 MODEL_QUANTIZATION = {
-    outetts.Models.VERSION_0_1_SIZE_350M: outetts.LlamaCppQuantization.Q8_0,
-    outetts.Models.VERSION_0_2_SIZE_500M: outetts.LlamaCppQuantization.Q8_0,
-    outetts.Models.VERSION_0_3_SIZE_500M: outetts.LlamaCppQuantization.Q8_0,
+    outetts.Models.VERSION_0_1_SIZE_350M: outetts.LlamaCppQuantization.FP16,
+    outetts.Models.VERSION_0_2_SIZE_500M: outetts.LlamaCppQuantization.FP16,
+    outetts.Models.VERSION_0_3_SIZE_500M: outetts.LlamaCppQuantization.FP16,
 }
 
 # Cache for speaker profiles to avoid re-transcribing the same audio
@@ -61,8 +61,11 @@ def get_interface(model_name: str):
     """Get interface instance for the model (no caching to avoid CUDA memory issues)."""
     model = MODELS[model_name]
 
-    has_cuda = torch.cuda.is_available()
-    if has_cuda:
+    try:
+        quantization = MODEL_QUANTIZATION.get(model, outetts.LlamaCppQuantization.Q8_0)
+        config = try_ggml_model(model, outetts.Backend.LLAMACPP, quantization)
+    except:
+        has_cuda = torch.cuda.is_available()
         model_config = MODEL_INFO[model]
         config = outetts.ModelConfig(
             model_path=f"OuteAI/{model_name}",
@@ -77,9 +80,6 @@ def get_interface(model_name: str):
             },
             **model_config
         )
-    else:
-        quantization = MODEL_QUANTIZATION.get(model, outetts.LlamaCppQuantization.Q6_K)
-        config = try_ggml_model(model, outetts.Backend.LLAMACPP, quantization)
     
     # Initialize the interface
     interface = outetts.Interface(config=config)