import os os.chdir(os.path.dirname(os.path.abspath(__file__))) import numpy as np import torch import onnxruntime from PIL import Image import requests from io import BytesIO import matplotlib.pyplot as plt from sam2.build_sam import build_sam2 from sam2.sam2_image_predictor import SAM2ImagePredictor def load_image(url): """加载并预处理图片""" response = requests.get(url) image = Image.open(BytesIO(response.content)).convert("RGB") print(f"Original image size: {image.size}") # 计算resize后的尺寸,保持长宽比 target_size = (1024, 1024) w, h = image.size scale = min(target_size[0] / w, target_size[1] / h) new_w = int(w * scale) new_h = int(h * scale) print(f"Scale factor: {scale}") print(f"Resized dimensions: {new_w}x{new_h}") # resize图片 resized_image = image.resize((new_w, new_h), Image.Resampling.LANCZOS) # 创建1024x1024的黑色背景 processed_image = Image.new("RGB", target_size, (0, 0, 0)) # 将resized图片粘贴到中心位置 paste_x = (target_size[0] - new_w) // 2 paste_y = (target_size[1] - new_h) // 2 print(f"Paste position: ({paste_x}, {paste_y})") processed_image.paste(resized_image, (paste_x, paste_y)) # 保存处理后的图片用于检查 processed_image.save("debug_processed_image.png") # 转换为numpy数组并归一化到[0,1] img_np = np.array(processed_image).astype(np.float32) / 255.0 # 调整维度顺序从HWC到CHW img_np = img_np.transpose(2, 0, 1) # 添加batch维度 img_np = np.expand_dims(img_np, axis=0) print(f"Final input tensor shape: {img_np.shape}") return image, img_np, (scale, paste_x, paste_y) def prepare_point_input(point_coords, point_labels, image_size=(1024, 1024)): """准备点击输入数据""" point_coords = np.array(point_coords, dtype=np.float32) point_labels = np.array(point_labels, dtype=np.float32) # 添加batch维度 point_coords = np.expand_dims(point_coords, axis=0) point_labels = np.expand_dims(point_labels, axis=0) # 准备mask输入 mask_input = np.zeros((1, 1, 256, 256), dtype=np.float32) has_mask_input = np.zeros(1, dtype=np.float32) orig_im_size = np.array(image_size, dtype=np.int32) return point_coords, point_labels, mask_input, has_mask_input, orig_im_size def main(): # 1. 加载原始图片 url = "https://raw.githubusercontent.com/facebookresearch/segment-anything/main/notebooks/images/dog.jpg" orig_image, input_image, (scale, offset_x, offset_y) = load_image(url) # 2. 准备输入点 - 需要根据scale和offset调整点击坐标 input_point_orig = [[750, 400]] input_point = [[ int(x * scale + offset_x), int(y * scale + offset_y) ] for x, y in input_point_orig] print(f"Original point: {input_point_orig}") print(f"Transformed point: {input_point}") input_label = [1] # 3. 运行PyTorch模型 print("Running PyTorch model...") checkpoint = "sam2.1_hiera_large.pt" model_cfg = "configs/sam2.1/sam2.1_hiera_l.yaml" predictor = SAM2ImagePredictor(build_sam2(model_cfg, checkpoint)) with torch.inference_mode(): predictor.set_image(orig_image) masks_pt, iou_scores_pt, low_res_masks_pt = predictor.predict( point_coords=np.array(input_point), point_labels=np.array(input_label), multimask_output=True ) # 4. 运行ONNX模型 print("Running ONNX model...") encoder_path = "sam2.1_hiera_tiny_encoder.s.onnx" decoder_path = "sam2.1_hiera_tiny_decoder.onnx" # 创建ONNX Runtime会话 encoder_session = onnxruntime.InferenceSession(encoder_path) decoder_session = onnxruntime.InferenceSession(decoder_path) # 运行encoder encoder_inputs = {'image': input_image} high_res_feats_0, high_res_feats_1, image_embed = encoder_session.run(None, encoder_inputs) # 准备decoder输入 point_coords, point_labels, mask_input, has_mask_input, orig_im_size = prepare_point_input( input_point, input_label, orig_image.size[::-1] ) # 运行decoder decoder_inputs = { 'image_embed': image_embed, 'high_res_feats_0': high_res_feats_0, 'high_res_feats_1': high_res_feats_1, 'point_coords': point_coords, 'point_labels': point_labels, # 'orig_im_size': orig_im_size, 'mask_input': mask_input, 'has_mask_input': has_mask_input, } low_res_masks, iou_predictions = decoder_session.run(None, decoder_inputs) # 后处理: 将low_res_masks缩放到原始图片尺寸 w, h = orig_image.size # 1. 首先将mask缩放到1024x1024 masks_1024 = torch.nn.functional.interpolate( torch.from_numpy(low_res_masks), size=(1024, 1024), mode="bilinear", align_corners=False ) # 2. 去除padding new_h = int(h * scale) new_w = int(w * scale) start_h = (1024 - new_h) // 2 start_w = (1024 - new_w) // 2 masks_no_pad = masks_1024[..., start_h:start_h+new_h, start_w:start_w+new_w] # 3. 缩放到原始图片尺寸 masks_onnx = torch.nn.functional.interpolate( masks_no_pad, size=(h, w), mode="bilinear", align_corners=False ) # 4. 二值化 masks_onnx = masks_onnx > 0.0 masks_onnx = masks_onnx.numpy() # 在运行ONNX模型后,打印输出的shape print(f"\nOutput shapes:") print(f"PyTorch masks shape: {masks_pt.shape}") print(f"ONNX masks shape: {masks_onnx.shape}") # 修改可视化部分,暂时注释掉差异图 plt.figure(figsize=(10, 5)) # PyTorch结果 plt.subplot(121) plt.imshow(orig_image) plt.imshow(masks_pt[0], alpha=0.5) plt.plot(input_point_orig[0][0], input_point_orig[0][1], 'rx') plt.title('PyTorch Output') plt.axis('off') # ONNX结果 plt.subplot(122) plt.imshow(orig_image) plt.imshow(masks_onnx[0,0], alpha=0.5) plt.plot(input_point_orig[0][0], input_point_orig[0][1], 'rx') plt.title('ONNX Output') plt.axis('off') plt.tight_layout() plt.show() # 6. 打印一些统计信息 print("\nStatistics:") print(f"PyTorch IoU scores: {iou_scores_pt}") print(f"ONNX IoU predictions: {iou_predictions}") if __name__ == "__main__": main()