Update app.py

app.py

@@ -1,42 +1,59 @@
-from huggingface_hub import login
 import os
+import base64
+import io
+import cv2
+import torch
+import gradio as gr
 from peft import PeftModel, PeftConfig
 from transformers import AutoProcessor, PaliGemmaForConditionalGeneration
 from PIL import Image
-import requests
-import torch
-import io
-import base64
-import cv2
-
-access_token = os.environ["HF_TOKEN"]
+from huggingface_hub import login
+ 
+# Step 1: Log in to Hugging Face
+access_token = os.environ["HF_TOKEN"]  # Ensure your Hugging Face token is stored in an environment variable
 login(token=access_token)
-
+ 
+# Step 2: Setup device and load model
 device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 dtype = torch.bfloat16
-
+ 
+# Load configuration and model
 config = PeftConfig.from_pretrained("anushettypsl/paligemma_vqav2")
-# base_model = AutoModelForCausalLM.from_pretrained("google/paligemma-3b-pt-448")
 base_model = PaliGemmaForConditionalGeneration.from_pretrained("google/paligemma-3b-pt-448")
 model = PeftModel.from_pretrained(base_model, "anushettypsl/paligemma_vqav2", device_map=device)
 processor = AutoProcessor.from_pretrained("google/paligemma-3b-pt-448", device_map=device)
+ 
 model.to(device)
+ 
+# Step 3: Define prediction function
+def predict(input_image, input_text):
+    # Convert the uploaded image to RGB format
+    input_image = input_image.convert('RGB')
+ 
+    # Prepare the model inputs
+    model_inputs = processor(text=input_text, images=input_image, return_tensors="pt").to(device)
+ 
+    # Perform inference
+    with torch.inference_mode():
+        generation = model.generate(**model_inputs, max_new_tokens=100, do_sample=False)
+ 
+    # Decode the output
+    decoded_output = processor.decode(generation[0], skip_special_tokens=True)
+    return decoded_output
+ 
+# Step 4: Create the Gradio interface
+interface = gr.Interface(
+    fn=predict,
+    inputs=[
+        gr.Image(type="pil", label="Upload Image"),  # Image input
+        gr.Textbox(label="Input Prompt", placeholder="Enter your prompt here...")  # Text input
+    ],
+    outputs="text",  # Text output
+    title="Image and Prompt to Text Model",
+    description="Upload an image and provide a prompt to generate a descriptive text."
+)
+ 
+# Step 5: Launch the Gradio app
+interface.launch()
+has context menu
 
-image = cv2.imread('/content/15_BC_G2_6358_40x_2_jpg.rf.97595fa4965f66ad45be8fd055331933.jpg')
-
-# Convert the image to base64 encoding
-image_bytes = cv2.imencode('.jpg', image)[1]
-base64_string = base64.b64encode(image_bytes).decode('utf-8')
-
-input_image = Image.open(io.BytesIO(image_bytes)).convert('RGB')
-
-model_inputs = processor(
-    text=input_text, images=input_image, return_tensors="pt").to(device)
-input_len = model_inputs["input_ids"].shape[-1]
-model.to(device)
-with torch.inference_mode():
-    generation = model.generate(
-        **model_inputs, max_new_tokens=100, do_sample=False)
-    generation = generation[0][input_len:]
-    decoded = processor.decode(generation, skip_special_tokens=True)
-    print(decoded)