Update app.py

app.py

@@ -43,8 +43,8 @@ from transformers import AutoTokenizer, AutoModelForCausalLM
 
 
 READER_MODEL = "HuggingFaceH4/zephyr-7b-beta"
-#bnb_config = BitsAndBytesConfig(load_in_4bit=True, bnb_4bit_use_double_quant=True, bnb_4bit_quant_type="nf4")
-model = AutoModelForCausalLM.from_pretrained(READER_MODEL)
+#bnb_config = BitsAndBytesConfig(load_in_4bit=True, bnb_4bit_use_double_quant=True, bnb_4bit_quant_type="nf4", bnb_4bit_compute_dtype=torch.float16)
+model = AutoModelForCausalLM.from_pretrained(READER_MODEL,quantization_config=bnb_config)
 tokenizer = AutoTokenizer.from_pretrained(READER_MODEL)
 
 from langchain.llms import HuggingFacePipeline