Update app.py

app.py

@@ -1,11 +1,16 @@
 import gradio as gr
 from huggingface_hub import InferenceClient
 import transformers
-from transformers import AutoTokenizer,GenerationConfig
+from transformers import AutoTokenizer,GenerationConfig, BitsAndBytesConfig
 import torch
 from peft import PeftModel
 import spaces
 import torch
+import bitsandbytes, accelerate
+
+print(transformers.__version__)  # Should be >= 4.26.0
+print(bitsandbytes.__version__)  # Should be >= 0.37.0
+print(accelerate.__version__)    # Should be >= 0.12.0
 
 num_gpus = torch.cuda.device_count()
 print(f"Number of available GPUs: {num_gpus}")
@@ -20,6 +25,12 @@ for i in range(num_gpus):
     print(f"  Allocated Memory: {torch.cuda.memory_allocated(i) / 1e9:.2f} GB")
     print(f"  Cached Memory: {torch.cuda.memory_reserved(i) / 1e9:.2f} GB")
     print(f"  Free Memory: {torch.cuda.get_device_properties(i).total_memory / 1e9 - torch.cuda.memory_reserved(i) / 1e9:.2f} GB")
+
+quantization_config = BitsAndBytesConfig(
+    load_in_8bit=True,  # Enable 8-bit quantization
+    llm_int8_enable_fp32_cpu_offload=True  # Enable FP32 CPU offloading
+)
+
 """
 For more information on `huggingface_hub` Inference API support, please check the docs: https://huggingface.co/docs/huggingface_hub/v0.22.2/en/guides/inference
 """
@@ -88,8 +99,7 @@ def loadModel():
                     torch_dtype=torch.float16,
                     cache_dir=cache_dir,
                     device_map="auto",
-                    load_in_8bit=True,
-                    load_in_8bit_fp32_cpu_offload=True
+                    quantization_config=quantization_config
                 )
         #model = PeftModel.from_pretrained(
         #            model,