Update app.py

app.py

@@ -12,14 +12,15 @@ import streamlit as st
 #     low_cpu_mem_usage=True,
 # )
 
-
+gptq_config = GPTQConfig(bits=4, disable_exllama=True)
 model = AutoPeftModelForCausalLM.from_pretrained(
     "Aneeth/zephyr_10k",
     low_cpu_mem_usage=True,
     return_dict=True,
     torch_dtype=torch.float16,
     device_map="cpu",
-    trust_remote_code=True
+    trust_remote_code=True,
+    quantization_config = gptq_config
     )
 
 tokenizer = AutoTokenizer.from_pretrained("Aneeth/zephyr_10k")