Update app.py

change bnb config

app.py

@@ -18,7 +18,7 @@ from transformers import (
 )
 from transformers.pipelines.audio_utils import ffmpeg_read
 
-# import torch  # If you're using PyTorch
+import torch  # If you're using PyTorch
 import spaces
 
 os.environ["HF_HUB_ENABLE_HF_TRANSFER"] = "1"
@@ -28,12 +28,21 @@ BATCH_SIZE = 8
 YT_LENGTH_LIMIT_S = 4800  # 1 hour 20 minutes
 
 # Quantization
-bnb_config = BitsAndBytesConfig(load_in_4bit=True)
+
+bnb_config = BitsAndBytesConfig(
+    load_in_4bit=True,
+    bnb_4bit_use_double_quant=True,
+    bnb_4bit_quant_type="nf4",
+    bnb_4bit_compute_dtype=torch.bfloat16
+)
+
 model = AutoModelForSpeechSeq2Seq.from_pretrained(
     MODEL_NAME,
     quantization_config=bnb_config,
+    use_cache=False,
     device_map="auto"
 )
+
 tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)
 feature_extractor = AutoFeatureExtractor.from_pretrained(MODEL_NAME)