Update app.py

app.py

@@ -3,7 +3,7 @@ app.py
 
 A unified Gradio chat application for Multimodal OCR Granite Vision.
 Commands (enter these as a prefix in the text input):
-  - @rag: For retrieval‐augmented generation (e.g. PDF or text-based queries).
+  - @rag: For retrieval‐augmented generation (e.g. PDFs or text-based queries).
   - @granite: For image understanding.
   - @video-infer: For video understanding (video is downsampled into frames).
 
@@ -14,13 +14,13 @@ import os
 import time
 import uuid
 import random
+import spaces
 import logging
 from threading import Thread
 from pathlib import Path
 from datetime import datetime, timezone
 
 import torch
-import spaces
 import numpy as np
 import cv2
 from PIL import Image
@@ -165,7 +165,8 @@ def load_granite_model():
     global granite_processor, granite_model
     if granite_processor is None or granite_model is None:
         granite_processor = AutoProcessor.from_pretrained(GRANITE_MODEL_ID)
-        granite_model = AutoModelForVision2Seq.from_pretrained(GRANITE_MODEL_ID, device_map="auto").to(device)
+        # Remove the .to(device) call to avoid moving a model already offloaded via accelerate.
+        granite_model = AutoModelForVision2Seq.from_pretrained(GRANITE_MODEL_ID, device_map="auto")
     return granite_processor, granite_model
 
 def create_single_turn(image, text):
@@ -328,4 +329,4 @@ demo = gr.ChatInterface(
 )
 
 if __name__ == "__main__":
-    demo.queue(max_size=20).launch()
+    demo.queue(max_size=20).launch()