changes

Signed-off-by: Suvaditya Mukherjee <[email protected]>

app.py

@@ -7,7 +7,7 @@ import pymupdf
 import gradio as gr
 from qdrant_client import QdrantClient
 from utils import download_pdf_from_gdrive, merge_strings_with_prefix
-from transformers import AutoModelForCausalLM, AutoTokenizer
+from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig
 
 def rag_query(query: str):
     """
@@ -150,7 +150,7 @@ if __name__ == "__main__":
         model_name,
         torch_dtype=torch.bfloat16,
         device_map="auto",
-        load_in_4bit=True,
+        quantization_config = BitsAndBytesConfig(load_in_8bit=True)
     ).to_bettertransformer().to('cuda')
 
     tokenizer = AutoTokenizer.from_pretrained(model_name)