requirement

app.py

@@ -78,10 +78,11 @@ def main():
     if st.button("Get Answer"):
     # Get the answer and relevant documents
         bnb_config = BitsAndBytesConfig(
-            load_in_4bit=True,
-            bnb_4bit_use_double_quant=True,
-            bnb_4bit_quant_type="nf4",
-            bnb_4bit_compute_dtype=torch.bfloat16,
+            load_in_8bit=True
+            #load_in_4bit=True,
+            #bnb_4bit_use_double_quant=True,
+            #bnb_4bit_quant_type="nf4",
+            #bnb_4bit_compute_dtype=torch.bfloat16,
         )   
         model = AutoModelForCausalLM.from_pretrained(READER_MODEL_NAME, quantization_config=bnb_config,
             device_map = 'auto')

requirements.txt

@@ -13,7 +13,9 @@ sentence_transformers==2.7.0
 torch
 #==2.5.0+cu121
 tqdm==4.66.6
-#transformers==4.44.2
-transformers==4.30
-bitsandbytes
-accelerate
+transformers==4.44.2
+#transformers==4.30
+bitsandbytes>=0.39.0
+accelerate
+faiss-cpu==1.9.0
+faiss-gpu==1.7.2