# This is a small gradio interface to access our RIPE keypoint extractor. # You can either upload two images or use one of the example image pairs. import os import gradio as gr from PIL import Image from ripe import vgg_hyper SEED = 32000 os.environ["PYTHONHASHSEED"] = str(SEED) import random from pathlib import Path import numpy as np import torch torch.manual_seed(SEED) np.random.seed(SEED) random.seed(SEED) import cv2 import kornia.feature as KF import kornia.geometry as KG from ripe.utils.utils import cv2_matches_from_kornia, to_cv_kpts MIN_SIZE = 512 MAX_SIZE = 768 description_text = """

🌊🌺 ICCV 2025 🌺🌊

Johannes Künzel · Anna Hilsmann · Peter Eisert

Arxiv | Project Page | Code


### This demo showcases our new keypoint extractor model, RIPE (Reinforcement Learning on Unlabeled Image Pairs for Robust Keypoint Extraction). ### RIPE is trained without requiring pose or depth supervision or artificial augmentations. By leveraging reinforcement learning, it learns to extract keypoints solely based on whether an image pair depicts the same scene or not. ### For more detailed information, please refer to our [paper](link to be added). The demo code extracts the 2048-top keypoints from the two input images. It uses the mutual nearest neighbor (MNN) descriptor matcher from kornia to find matches between the two images. If the number of matches is greater than 8, it applies RANSAC to filter out outliers based on the inlier threshold provided by the user. Images are resized to fit within a maximum size of 2048x2048 pixels with maintained aspect ratio.
""" path_weights = Path( "/media/jwkuenzel/work/projects/CVG_Reinforced_Keypoints/output/train/ablation_iccv/inlier_threshold/1571243/2025-02-19/14-00-10_789013/model_inlier_threshold_best.pth" ) model = vgg_hyper(path_weights) def get_new_image_size(image, min_size=1600, max_size=2048): """ Get a new size for the image that is scaled to fit between min_size and max_size while maintaining the aspect ratio. Args: image (PIL.Image): Input image. min_size (int): Minimum allowed size for width and height. max_size (int): Maximum allowed size for width and height. Returns: tuple: New size (width, height) for the image. """ width, height = image.size aspect_ratio = width / height if width > height: new_width = max(min_size, min(max_size, width)) new_height = int(new_width / aspect_ratio) else: new_height = max(min_size, min(max_size, height)) new_width = int(new_height * aspect_ratio) new_size = (new_width, new_height) return new_size def extract_keypoints(image1, image2, inl_th): """ Extract keypoints from two input images using the RIPE model. Args: image1 (PIL.Image): First input image. image2 (PIL.Image): Second input image. inl_th (float): RANSAC inlier threshold. Returns: dict: A dictionary containing keypoints and matches. """ log_text = "Extracting keypoints and matches with RIPE\n" log_text += f"Image 1 size: {image1.size}\n" log_text += f"Image 2 size: {image2.size}\n" # check not larger than 2048x2048 new_size = get_new_image_size(image1, min_size=MIN_SIZE, max_size=MAX_SIZE) image1 = image1.resize(new_size) new_size = get_new_image_size(image2, min_size=MIN_SIZE, max_size=MAX_SIZE) image2 = image2.resize(new_size) log_text += f"Resized Image 1 size: {image1.size}\n" log_text += f"Resized Image 2 size: {image2.size}\n" dev = torch.device("cuda" if torch.cuda.is_available() else "cpu") model.to(dev) image1 = image1.convert("RGB") image2 = image2.convert("RGB") image1_original = image1.copy() image2_original = image2.copy() # convert PIL images to numpy arrays image1_original = np.array(image1_original) image2_original = np.array(image2_original) # convert PIL images to tensors image1 = torch.tensor(np.array(image1)).permute(2, 0, 1).float() / 255.0 image2 = torch.tensor(np.array(image2)).permute(2, 0, 1).float() / 255.0 image1 = image1.to(dev).unsqueeze(0) # Add batch dimension image2 = image2.to(dev).unsqueeze(0) # Add batch dimension kpts_1, desc_1, score_1 = model.detectAndCompute(image1, threshold=0.5, top_k=2048) kpts_2, desc_2, score_2 = model.detectAndCompute(image2, threshold=0.5, top_k=2048) log_text += f"Number of keypoints in image 1: {kpts_1.shape[0]}\n" log_text += f"Number of keypoints in image 2: {kpts_2.shape[0]}\n" matcher = KF.DescriptorMatcher("mnn") # threshold is not used with mnn match_dists, match_idxs = matcher(desc_1, desc_2) log_text += f"Number of MNN matches: {match_idxs.shape[0]}\n" cv2_matches = cv2_matches_from_kornia(match_dists, match_idxs) do_ransac = match_idxs.shape[0] > 8 if do_ransac: matched_pts_1 = kpts_1[match_idxs[:, 0]] matched_pts_2 = kpts_2[match_idxs[:, 1]] H, mask = KG.ransac.RANSAC(model_type="fundamental", inl_th=inl_th)(matched_pts_1, matched_pts_2) matchesMask = mask.int().ravel().tolist() log_text += f"RANSAC found {mask.sum().item()} inliers out of {mask.shape[0]} matches with an inlier threshold of {inl_th}.\n" else: log_text += "Not enough matches for RANSAC, skipping RANSAC step.\n" kpts_1 = to_cv_kpts(kpts_1, score_1) kpts_2 = to_cv_kpts(kpts_2, score_2) keypoints_raw_1 = cv2.drawKeypoints(image1_original, kpts_1, image1_original, color=(0, 255, 0)) keypoints_raw_2 = cv2.drawKeypoints(image2_original, kpts_2, image2_original, color=(0, 255, 0)) # pad height smaller image to match the height of the larger image if keypoints_raw_1.shape[0] < keypoints_raw_2.shape[0]: pad_height = keypoints_raw_2.shape[0] - keypoints_raw_1.shape[0] keypoints_raw_1 = np.pad( keypoints_raw_1, ((0, pad_height), (0, 0), (0, 0)), mode="constant", constant_values=255 ) elif keypoints_raw_1.shape[0] > keypoints_raw_2.shape[0]: pad_height = keypoints_raw_1.shape[0] - keypoints_raw_2.shape[0] keypoints_raw_2 = np.pad( keypoints_raw_2, ((0, pad_height), (0, 0), (0, 0)), mode="constant", constant_values=255 ) # concatenate keypoints images horizontally keypoints_raw = np.concatenate((keypoints_raw_1, keypoints_raw_2), axis=1) keypoints_raw_pil = Image.fromarray(keypoints_raw) result_raw = cv2.drawMatches( image1_original, kpts_1, image2_original, kpts_2, cv2_matches, None, matchColor=(0, 255, 0), matchesMask=None, # matchesMask=None, flags=cv2.DrawMatchesFlags_NOT_DRAW_SINGLE_POINTS, ) if not do_ransac: result_ransac = None else: result_ransac = cv2.drawMatches( image1_original, kpts_1, image2_original, kpts_2, cv2_matches, None, matchColor=(0, 255, 0), matchesMask=matchesMask, singlePointColor=(0, 0, 255), flags=cv2.DrawMatchesFlags_DEFAULT, ) # result = cv2.cvtColor(result, cv2.COLOR_BGR2RGB) # Convert BGR to RGB for display # convert to PIL Image result_raw_pil = Image.fromarray(result_raw) if result_ransac is not None: result_ransac_pil = Image.fromarray(result_ransac) else: result_ransac_pil = None return log_text, result_ransac_pil, result_raw_pil, keypoints_raw_pil demo = gr.Interface( fn=extract_keypoints, inputs=[ gr.Image(type="pil", label="Image 1"), gr.Image(type="pil", label="Image 2"), gr.Slider( minimum=0.1, maximum=3.0, step=0.1, value=0.5, label="RANSAC inlier threshold", info="Threshold for RANSAC inlier detection. Lower values may yield fewer inliers but more robust matches.", ), ], outputs=[ gr.Textbox(type="text", label="Log"), gr.Image(type="pil", label="Keypoints and Matches (RANSAC)"), gr.Image(type="pil", label="Keypoints and Matches"), gr.Image(type="pil", label="Keypoint Detection Results"), ], title="RIPE: Reinforcement Learning on Unlabeled Image Pairs for Robust Keypoint Extraction", description=description_text, examples=[ [ "assets_gradio/all_souls_000013.jpg", "assets_gradio/all_souls_000055.jpg", ], [ "assets_gradio/167170681_0e5c42fd21_o.jpg", "assets_gradio/170804731_6bf4fbecd4_o.jpg", ], [ "assets_gradio/4171014767_0fe879b783_o.jpg", "assets_gradio/4174108353_20422632d6_o.jpg", ], ], flagging_mode="never", theme="default", ) demo.launch()