app.py

import os
import cv2
import gradio as gr
from PIL import Image
import numpy as np
from sam_extension.utils import add_points_tag, add_boxes_tag, mask2greyimg
from sam_extension.pipeline import SAMEncoderPipeline, SAMDecoderPipeline, GroundingDinoPipeline
point_coords = []
point_labels = []
boxes = []
boxes_point = []
texts = []
sam_encoder_pipeline = None
sam_decoder_pipeline = None
result_list = []
result_index_list = []
mask_result_list = []
mask_result_index_list = []
def resize(image, des_max=512):
    h, w = image.shape[:2]
    if h >= w:
        new_h = des_max
        new_w = int(des_max * w / h)
    else:
        new_w = des_max
        new_h = int(des_max * h / w)
    return cv2.resize(image, (new_w, new_h))
def show_prompt(img, prompt_mode, pos_point, evt: gr.SelectData):  # SelectData is a subclass of EventData
    global point_coords, point_labels, boxes_point, boxes
    if prompt_mode == 'point':
        point_coords.append([evt.index[0], evt.index[1]])
        point_labels.append(1 if pos_point else 0)
        result_img = add_points_tag(img, np.array(point_labels), np.array(point_coords))
    elif prompt_mode == 'box':
        boxes_point.append(evt.index[0])
        boxes_point.append(evt.index[1])
        if len(boxes_point) == 4:
            boxes.append(boxes_point)
            boxes_point = []
        result_img = add_boxes_tag(img, np.array(boxes))
    else:
        result_img = img
    return result_img, point_coords, point_labels, boxes_point, boxes

def reset_points(img):
    global point_coords, point_labels
    point_coords = []
    point_labels = []
    return img, point_coords, point_labels


def reset_boxes(img):
    global boxes_point, boxes
    boxes_point = []
    boxes = []
    return img, boxes_point, boxes

def load_sam(sam_ckpt_path, sam_version):
    global sam_encoder_pipeline, sam_decoder_pipeline
    sam_encoder_pipeline = SAMEncoderPipeline.from_pretrained(ckpt_path=sam_ckpt_path, sam_version=sam_version, device='cpu')
    sam_decoder_pipeline = SAMDecoderPipeline.from_pretrained(ckpt_path=sam_ckpt_path, sam_version=sam_version, device='cpu')
    return 'sam loaded!'


def generate_mask(img, prompt_mode, text_prompt):
    global result_list, mask_result_list, result_index_list, mask_result_index_list
    image = Image.fromarray(img)
    img_size = sam_decoder_pipeline.img_size
    des_img = image.resize((img_size, img_size))
    sam_encoder_output = sam_encoder_pipeline(des_img)
    if prompt_mode == 'point':
        point_coords_ = np.array(point_coords)
        point_labels_ = np.array(point_labels)
        boxes_ = None
        texts_ = None
        grounding_dino_pipeline = None
    elif prompt_mode == 'box':
        point_coords_ = None
        point_labels_ = None
        boxes_ = np.array(boxes)
        texts_ = None
        grounding_dino_pipeline = None
    else:
        point_coords_ = None
        point_labels_ = None
        boxes_ = None
        texts_ = text_prompt.split(',')
        grounding_dino_pipeline = GroundingDinoPipeline.from_pretrained(
            'GroundingDINO/groundingdino/config/GroundingDINO_SwinT_OGC.py',
            'weights/groundingdino/groundingdino_swint_ogc.pth',
            device='cpu')
    result_list, mask_result_list, masks_list = sam_decoder_pipeline.visualize_results(
        image,
        des_img,
        sam_encoder_output,
        point_coords=point_coords_,
        point_labels=point_labels_,
        boxes=boxes_,
        texts=texts_,
        grounding_dino_pipeline=grounding_dino_pipeline,
        multimask_output=True,
        visualize_promts=True,
        pil=False)
    # result_index_list = [f'result_{i}' for i in range(len(result_list))]
    # mask_result_index_list = [f'mask_result_{i}' for i in range(len(mask_result_list))]
    return 'mask generated!', f'result_num : {len(result_list)}', f'mask_result_num : {len(masks_list)}'
    # mask_grey_result_list = mask2greyimg(masks_list, False)


def show_result(result_index):
    return result_list[int(result_index)]


def show_mask_result(mask_result_index):
    return mask_result_list[int(mask_result_index)]


with gr.Blocks() as demo:
    with gr.Row():
        img = gr.Image(None, width=400, height=400, label='input_image', type='numpy')
        result_img = gr.Image(None, width=400, height=400, label='output_image', type='numpy')
    with gr.Row():
        pos_point = gr.Checkbox(value=True, label='pos_point')
        prompt_mode = gr.Dropdown(choices=['point', 'box', 'text'], value='point', label='prompt_mode')
    with gr.Row():
        point_coords_text = gr.Textbox(value=str(point_coords), interactive=True, label='point_coords')
        point_labels_text = gr.Textbox(value=str(point_labels), interactive=True, label='point_labels')
        reset_points_bu = gr.Button(value='reset_points')
        reset_points_bu.click(fn=reset_points, inputs=[img], outputs=[result_img, point_coords_text, point_labels_text])
    with gr.Row():
        boxes_point_text = gr.Textbox(value=str(boxes_point), interactive=True, label='boxes_point')
        boxes_text = gr.Textbox(value=str(boxes), interactive=True, label='boxes')
        reset_boxes_bu = gr.Button(value='reset_boxes')
        reset_boxes_bu.click(fn=reset_boxes, inputs=[img], outputs=[result_img, boxes_point_text, boxes_text])
    with gr.Row():
        text_prompt = gr.Textbox(value='', interactive=True, label='text_prompt')
    with gr.Row():
        sam_ckpt_path = gr.Dropdown(choices=['weights/sam/mobile_sam.pt'],
                                    value='weights/sam/mobile_sam.pt',
                                    label='SAM ckpt_path')
        sam_version = gr.Dropdown(choices=['mobile_sam'],
                                  value='mobile_sam',
                                  label='SAM version')
        load_sam_bu = gr.Button(value='load SAM')
        sam_load_text = gr.Textbox(value='', interactive=True, label='sam_load')
        load_sam_bu.click(fn=load_sam, inputs=[sam_ckpt_path, sam_version], outputs=sam_load_text)
    with gr.Row():
        result_num_text = gr.Textbox(value='', interactive=True, label='result_num')
        result_index = gr.Number(value=0, label='result_index')
        show_result_bu = gr.Button(value='show_result')
        show_result_bu.click(fn=show_result, inputs=[result_index], outputs=[result_img])
    with gr.Row():
        mask_result_num_text = gr.Textbox(value='', interactive=True, label='mask_result_num')
        mask_result_index = gr.Number(value=0, label='mask_result_index')
        show_mask_result_bu = gr.Button(value='show_mask_result')
        show_mask_result_bu.click(fn=show_mask_result, inputs=[mask_result_index], outputs=[result_img])
    with gr.Row():
        generate_masks_bu = gr.Button(value='SAM generate masks')
        sam_text = gr.Textbox(value='', interactive=True, label='SAM')
        generate_masks_bu.click(fn=generate_mask, inputs=[img, prompt_mode, text_prompt], outputs=[sam_text, result_num_text, mask_result_num_text])
    img.select(show_prompt, [img, prompt_mode, pos_point], [result_img, point_coords_text, point_labels_text, boxes_point_text, boxes_text])
if __name__ == '__main__':
    os.system('cd/GroundingDINO && pip install -e. && cd.. && cd segment_anything && pip install -e.')
    demo.launch()