Update app.py

app.py

@@ -16,10 +16,19 @@ from ultralytics import YOLO
 import gradio as gr
 
 # # default: Load the model on the available device(s)
+# model = Qwen2VLForConditionalGeneration.from_pretrained(
+#     "Qwen/Qwen2-VL-7B-Instruct", torch_dtype="auto", device_map="auto"
+# )
+
+#8-bit quantization
 model = Qwen2VLForConditionalGeneration.from_pretrained(
-    "Qwen/Qwen2-VL-7B-Instruct", torch_dtype="auto", device_map="auto"
+    "Qwen/Qwen2-VL-7B-Instruct",
+    torch_dtype=torch.float16,
+    load_in_8bit=True,  # Use 8-bit quantization
+    device_map="auto",
 )
 
+
 # We recommend enabling flash_attention_2 for better acceleration and memory saving, especially in multi-image and video scenarios.
 # model = Qwen2VLForConditionalGeneration.from_pretrained(
 #     "Qwen/Qwen2-VL-7B-Instruct",