Spaces:
Running
Running
File size: 2,531 Bytes
93c21cd |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 |
import gradio as gr
import torch
import numpy as np
import cv2
from PIL import Image
from transformers import BlipProcessor, BlipForConditionalGeneration, SamModel, SamProcessor
import time
# Set device to GPU if available
device = "cuda" if torch.cuda.is_available() else "cpu"
# Load Florence BLIP model (Public Model - No Authentication Required)
processor = BlipProcessor.from_pretrained("Salesforce/blip-image-captioning-base")
model = BlipForConditionalGeneration.from_pretrained("Salesforce/blip-image-captioning-base").to(device)
# Load SAM model (Public Model - No Authentication Required)
sam_processor = SamProcessor.from_pretrained("facebook/sam-vit-base")
sam_model = SamModel.from_pretrained("facebook/sam-vit-base").to(device)
def process_image(image):
start_time = time.time()
# Convert and resize image
pil_image = Image.fromarray(image).resize((512, 512)) # Resize to optimize processing
print("✅ Image loaded and resized.")
# Generate caption using Florence
try:
inputs = processor(pil_image, return_tensors="pt").to(device)
with torch.no_grad():
out = model.generate(**inputs)
description = processor.decode(out[0], skip_special_tokens=True)
print(f"📝 Florence Captioning done in {time.time() - start_time:.2f} sec")
except Exception as e:
print(f"❌ Error in Florence: {e}")
return "Failed to generate description.", image
# Process Image for SAM
try:
encoding = sam_processor(images=pil_image, return_tensors="pt").to(device)
with torch.no_grad():
outputs = sam_model(**encoding)
# Extract segmentation mask
mask = outputs.pred_masks[0, 0].cpu().numpy()
mask_overlay = image.copy()
mask_overlay[mask > 0.5] = [0, 255, 0] # Green overlay for segmentation
print(f"🎨 SAM Segmentation done in {time.time() - start_time:.2f} sec")
except Exception as e:
print(f"❌ Error in SAM: {e}")
return description, image
return description, mask_overlay
# Gradio Interface
demo = gr.Interface(
fn=process_image,
inputs=gr.Image(type="numpy"),
outputs=[gr.Textbox(label="Image Description"), gr.Image(label="Segmented Image")],
title="Florence + SAM Image Processing",
description="Upload an image to get its description using Florence and segmentation using SAM (loaded from Hugging Face)."
)
if __name__ == "__main__":
demo.launch(server_name="0.0.0.0", server_port=7860)
|