import re import torch import json_repair from transformers import Qwen2_5_VLForConditionalGeneration, AutoProcessor from PIL import Image, ImageDraw def draw_bbox(image, annotation): x1, y1, x2, y2 = annotation["bbox_2d"] label = annotation["label"] draw = ImageDraw.Draw(image) # 绘制边界框 draw.rectangle((x1, y1, x2, y2), outline="red", width=5) # 绘制标签文本 font_size = 20 text_position = (x1, y1 - font_size - 5) if y1 > font_size + 5 else (x1, y2 + 5) try: draw.text(text_position, label, fill="red", font_size = font_size) except Exception as e: print(f"文本绘制错误: {e}") # 如果默认绘制失败,使用简单的方式绘制文本 draw.text(text_position, label, fill="red") return image def draw_bboxes(image, annotations): """绘制多个边界框和标签""" result_image = image.copy() for annotation in annotations: result_image = draw_bbox(result_image, annotation) return result_image def extract_bbox_answer(content): # Extract content between and if present answer_matches = re.findall(r'(.*?)', content, re.DOTALL) if answer_matches: # Use the last match text = answer_matches[-1] else: text = content # 使用json_repair修复JSON try: data = json_repair.loads(text) if isinstance(data, list) and len(data) > 0: return data else: return [] except Exception as e: print(f"JSON解析错误: {e}") return [] import spaces @spaces.GPU def process_image_and_text(image, text): """Process image and text input, return thinking process and bbox""" labels = text.split(",") question = f"First thinks about the reasoning process in the mind and then provides the user with the answer. The reasoning process and answer are enclosed within and tags, respectively, i.e., reasoning process here answer here . Please carefully check the image and detect the following objects: {labels}. " question = question + "Output the bbox coordinates of detected objects in . The bbox coordinates in Markdown format should be: \n```json\n[{\"bbox_2d\": [x1, y1, x2, y2], \"label\": \"object name\"}]\n```\n If no targets are detected in the image, simply respond with \"None\"." print("question: ", question) messages = [ { "role": "user", "content": [ {"type": "image"}, {"type": "text", "text": question}, ], } ] text = processor.apply_chat_template( messages, tokenize=False, add_generation_prompt=True ) inputs = processor( text=[text], images=image, return_tensors="pt", padding=True, padding_side="left", add_special_tokens=False, ) inputs = inputs.to("cuda") with torch.no_grad(): generated_ids = model.generate(**inputs, use_cache=True, max_new_tokens=1024, do_sample=False) generated_ids_trimmed = [ out_ids[len(inputs.input_ids[0]):] for out_ids in generated_ids ] output_text = processor.batch_decode( generated_ids_trimmed, skip_special_tokens=True )[0] print("output_text: ", output_text) # Extract thinking process think_match = re.search(r'(.*?)', output_text, re.DOTALL) thinking_process = think_match.group(1).strip() if think_match else "No thinking process found" answer_match = re.search(r'(.*?)', output_text, re.DOTALL) answer_output = answer_match.group(1).strip() if answer_match else "No answer extracted" # Get bbox and draw bbox = extract_bbox_answer(output_text) # Draw bbox on the image result_image = image.copy() result_image = draw_bboxes(result_image, bbox) return thinking_process, answer_output,result_image if __name__ == "__main__": import gradio as gr model_path = "omlab/VLM-R1-Qwen2.5VL-3B-OVD-0321" # device = "cuda" if torch.cuda.is_available() else "cpu" device = "cuda" model = Qwen2_5_VLForConditionalGeneration.from_pretrained(model_path, torch_dtype=torch.float16) model.to(device) processor = AutoProcessor.from_pretrained(model_path) def gradio_interface(image, text): thinking, output,result_image = process_image_and_text(image, text) return thinking, output, result_image demo = gr.Interface( fn=gradio_interface, inputs=[ gr.Image(type="pil", label="Input Image"), gr.Textbox(label="Objects to detect (separated by ,)") ], outputs=[ gr.Textbox(label="Thinking Process"), gr.Textbox(label="Response"), gr.Image(type="pil", label="Result with Bbox") ], title="Open-Vocabulary Object Detection Demo", description="Upload an image and input description text, the system will return the thinking process and region annotation. \n\nOur GitHub: [VLM-R1](https://github.com/om-ai-lab/VLM-R1/tree/main)", examples=[ ["examples/image1.jpg", "person"], ["examples/image2.jpg", "drink,fruit"], ["examples/image3.png", "keyboard,white cup,laptop"], ], cache_examples=False, examples_per_page=10 ) demo.launch(server_name="0.0.0.0", server_port=7860, share=True)