Update app.py

app.py

@@ -384,15 +384,13 @@ else:
     index = faiss.read_index('./storage/faiss_index.faiss')
 
 # Load the model
-quantization_config = BitsAndBytesConfig(
-    load_in_4bit=True,
-    bnb_4bit_use_double_quant=True,
-    bnb_4bit_quant_type="nf4",
-    bnb_4bit_compute_dtype=torch.bfloat16
-)
+nf4_config = BitsAndBytesConfig(
+        load_in_4bit=True,
+        bnb_4bit_quant_type="nf4",
+    )
 
 tokenizer = AutoTokenizer.from_pretrained("google/gemma-2b-it", token=HF_TOKEN)
-model = AutoModelForCausalLM.from_pretrained("google/gemma-2b-it", quantization_config=quantization_config, torch_dtype=torch.float16, low_cpu_mem_usage=True, token=HF_TOKEN)
+model = AutoModelForCausalLM.from_pretrained("google/gemma-2b-it", quantization_config = nf4_config, token=HF_TOKEN)
 
 
 def make_inference(query, hist):