import ast from matplotlib.patches import Polygon from matplotlib.collections import PatchCollection import matplotlib.pyplot as plt import numpy as np import warnings import inflect import gradio as gr p = inflect.engine() # user_error = ValueError user_error = gr.Error img_dir = "imgs" objects_text = "Objects: " bg_prompt_text = "Background prompt: " bg_prompt_text_no_trailing_space = bg_prompt_text.rstrip() neg_prompt_text = "Negative prompt: " neg_prompt_text_no_trailing_space = neg_prompt_text.rstrip() # h, w box_scale = (512, 512) size = box_scale size_h, size_w = size print(f"Using box scale: {box_scale}") def parse_input(text=None, no_input=False): warnings.warn("Parsing input without negative prompt is deprecated.") if not text: if no_input: raise user_error(f"No input parsed in \"{text}\".") text = input("Enter the response: ") if objects_text in text: text = text.split(objects_text)[1] text_split = text.split(bg_prompt_text_no_trailing_space) if len(text_split) == 2: gen_boxes, bg_prompt = text_split elif len(text_split) == 1: if no_input: raise user_error(f"Invalid input (no background prompt): {text}") gen_boxes = text bg_prompt = "" while not bg_prompt: # Ignore the empty lines in the response bg_prompt = input("Enter the background prompt: ").strip() if bg_prompt_text_no_trailing_space in bg_prompt: bg_prompt = bg_prompt.split(bg_prompt_text_no_trailing_space)[1] else: raise user_error(f"Invalid input (possibly multiple background prompts): {text}") try: gen_boxes = ast.literal_eval(gen_boxes) except SyntaxError as e: # Sometimes the response is in plain text if "No objects" in gen_boxes: gen_boxes = [] else: raise e bg_prompt = bg_prompt.strip() return gen_boxes, bg_prompt def parse_input_with_negative(text=None, no_input=False): # no_input: should not request interactive input if not text: if no_input: raise user_error(f"No input parsed in \"{text}\".") text = input("Enter the response: ") if objects_text in text: text = text.split(objects_text)[1] text_split = text.split(bg_prompt_text_no_trailing_space) if len(text_split) == 2: gen_boxes, text_rem = text_split elif len(text_split) == 1: if no_input: raise user_error(f"Invalid input (no background prompt): {text}") gen_boxes = text text_rem = "" while not text_rem: # Ignore the empty lines in the response text_rem = input("Enter the background prompt: ").strip() if bg_prompt_text_no_trailing_space in text_rem: text_rem = text_rem.split(bg_prompt_text_no_trailing_space)[1] else: raise user_error(f"Invalid input (possibly multiple background prompts): {text}") text_split = text_rem.split(neg_prompt_text_no_trailing_space) if len(text_split) == 2: bg_prompt, neg_prompt = text_split elif len(text_split) == 1: bg_prompt = text_rem # Negative prompt is optional: if it's not provided, we default to empty string neg_prompt = "" if not no_input: # Ignore the empty lines in the response neg_prompt = input("Enter the negative prompt: ").strip() if neg_prompt_text_no_trailing_space in neg_prompt: neg_prompt = neg_prompt.split(neg_prompt_text_no_trailing_space)[1] else: raise user_error(f"Invalid input (possibly multiple negative prompts): {text}") try: gen_boxes = ast.literal_eval(gen_boxes) except SyntaxError as e: # Sometimes the response is in plain text if "No objects" in gen_boxes or gen_boxes.strip() == "": gen_boxes = [] else: raise e bg_prompt = bg_prompt.strip() neg_prompt = neg_prompt.strip() # LLM may return "None" to mean no negative prompt provided. if neg_prompt == "None": neg_prompt = "" return gen_boxes, bg_prompt, neg_prompt def filter_boxes(gen_boxes, scale_boxes=True, ignore_background=True, max_scale=3): if gen_boxes is None: return [] if len(gen_boxes) == 0: return [] box_dict_format = False gen_boxes_new = [] for gen_box in gen_boxes: if isinstance(gen_box, dict): if not gen_box['bounding_box']: continue name, [bbox_x, bbox_y, bbox_w, bbox_h] = gen_box['name'], gen_box['bounding_box'] box_dict_format = True else: if not gen_box[1]: continue name, [bbox_x, bbox_y, bbox_w, bbox_h] = gen_box if bbox_w <= 0 or bbox_h <= 0: # Empty boxes continue if ignore_background: if (bbox_w >= size[1] and bbox_h >= size[0]) or bbox_x > size[1] or bbox_y > size[0]: # Ignore the background boxes continue if bbox_x < 0 or bbox_y < 0 or bbox_x + bbox_w > size[1] or bbox_y + bbox_h > size[0]: # Out of bounds boxes exist: we need to scale and shift all the boxes print(f"**Some boxes are out of bounds: {gen_box}, scaling all the boxes to fit**") scale_boxes = True gen_boxes_new.append(gen_box) gen_boxes = gen_boxes_new if len(gen_boxes) == 0: return [] filtered_gen_boxes = [] if box_dict_format: # For compatibility bbox_left_x_min = min([gen_box['bounding_box'][0] for gen_box in gen_boxes]) bbox_right_x_max = max([gen_box['bounding_box'][0] + gen_box['bounding_box'][2] for gen_box in gen_boxes]) bbox_top_y_min = min([gen_box['bounding_box'][1] for gen_box in gen_boxes]) bbox_bottom_y_max = max([gen_box['bounding_box'][1] + gen_box['bounding_box'][3] for gen_box in gen_boxes]) else: bbox_left_x_min = min([gen_box[1][0] for gen_box in gen_boxes]) bbox_right_x_max = max([gen_box[1][0] + gen_box[1][2] for gen_box in gen_boxes]) bbox_top_y_min = min([gen_box[1][1] for gen_box in gen_boxes]) bbox_bottom_y_max = max([gen_box[1][1] + gen_box[1][3] for gen_box in gen_boxes]) # All boxes are empty if (bbox_right_x_max - bbox_left_x_min) == 0: return [] # Used if scale_boxes is True shift = -bbox_left_x_min # Make sure the boxes fit horizontally and vertically scale_w = size_w / (bbox_right_x_max - bbox_left_x_min) scale_h = size_h / (bbox_bottom_y_max - bbox_top_y_min) scale = min(scale_w, scale_h, max_scale) for gen_box in gen_boxes: if box_dict_format: name, [bbox_x, bbox_y, bbox_w, bbox_h] = gen_box['name'], gen_box['bounding_box'] else: name, [bbox_x, bbox_y, bbox_w, bbox_h] = gen_box if scale_boxes: # Vertical: move the boxes if out of bound # Horizontal: move and scale the boxes so it spans the horizontal line bbox_x = (bbox_x + shift) * scale bbox_y = bbox_y * scale bbox_w, bbox_h = bbox_w * scale, bbox_h * scale # TODO: verify this makes the y center not moving bbox_y_offset = 0 if bbox_top_y_min * scale + bbox_y_offset < 0: bbox_y_offset -= bbox_top_y_min * scale if bbox_bottom_y_max * scale + bbox_y_offset >= size_h: bbox_y_offset -= bbox_bottom_y_max * scale - size_h bbox_y += bbox_y_offset if bbox_y < 0: bbox_y, bbox_h = 0, bbox_h - bbox_y name = name.rstrip(".") bounding_box = (int(np.round(bbox_x)), int(np.round(bbox_y)), int(np.round(bbox_w)), int(np.round(bbox_h))) if box_dict_format: gen_box = { 'name': name, 'bounding_box': bounding_box } else: gen_box = (name, bounding_box) filtered_gen_boxes.append(gen_box) return filtered_gen_boxes def draw_boxes(anns): ax = plt.gca() ax.set_autoscale_on(False) polygons = [] color = [] for ann in anns: c = (np.random.random((1, 3))*0.6+0.4) [bbox_x, bbox_y, bbox_w, bbox_h] = ann['bbox'] poly = [[bbox_x, bbox_y], [bbox_x, bbox_y+bbox_h], [bbox_x+bbox_w, bbox_y+bbox_h], [bbox_x+bbox_w, bbox_y]] np_poly = np.array(poly).reshape((4, 2)) polygons.append(Polygon(np_poly)) color.append(c) # print(ann) name = ann['name'] if 'name' in ann else str(ann['category_id']) ax.text(bbox_x, bbox_y, name, style='italic', bbox={'facecolor': 'white', 'alpha': 0.7, 'pad': 5}) p = PatchCollection(polygons, facecolor='none', edgecolors=color, linewidths=2) ax.add_collection(p) def show_boxes(gen_boxes, bg_prompt=None, neg_prompt=None, ind=None, show=False): if len(gen_boxes) == 0: return if isinstance(gen_boxes[0], dict): anns = [{'name': gen_box['name'], 'bbox': gen_box['bounding_box']} for gen_box in gen_boxes] else: anns = [{'name': gen_box[0], 'bbox': gen_box[1]} for gen_box in gen_boxes] # White background (to allow line to show on the edge) I = np.ones((size[0]+4, size[1]+4, 3), dtype=np.uint8) * 255 plt.imshow(I) plt.axis('off') if bg_prompt is not None: ax = plt.gca() ax.text(0, 0, bg_prompt + f"(Neg: {neg_prompt})" if neg_prompt else bg_prompt, style='italic', bbox={'facecolor': 'white', 'alpha': 0.7, 'pad': 5}) c = (np.zeros((1, 3))) [bbox_x, bbox_y, bbox_w, bbox_h] = (0, 0, size[1], size[0]) poly = [[bbox_x, bbox_y], [bbox_x, bbox_y+bbox_h], [bbox_x+bbox_w, bbox_y+bbox_h], [bbox_x+bbox_w, bbox_y]] np_poly = np.array(poly).reshape((4, 2)) polygons = [Polygon(np_poly)] color = [c] p = PatchCollection(polygons, facecolor='none', edgecolors=color, linewidths=2) ax.add_collection(p) draw_boxes(anns) if show: plt.show() def show_masks(masks): masks_to_show = np.zeros((*size, 3), dtype=np.float32) for mask in masks: c = (np.random.random((3,))*0.6+0.4) masks_to_show += mask[..., None] * c[None, None, :] plt.imshow(masks_to_show) plt.savefig(f"{img_dir}/masks.png") plt.show() plt.clf() def convert_box(box, height, width): # box: x, y, w, h (in 512 format) -> x_min, y_min, x_max, y_max x_min, y_min = box[0] / width, box[1] / height w_box, h_box = box[2] / width, box[3] / height x_max, y_max = x_min + w_box, y_min + h_box return x_min, y_min, x_max, y_max def convert_spec(spec, height, width, include_counts=True, verbose=False): # Infer from spec prompt, gen_boxes, bg_prompt = spec['prompt'], spec['gen_boxes'], spec['bg_prompt'] # This ensures the same objects appear together because flattened `overall_phrases_bboxes` should EXACTLY correspond to `so_prompt_phrase_box_list`. gen_boxes = sorted(gen_boxes, key=lambda gen_box: gen_box[0]) gen_boxes = [(name, convert_box(box, height=height, width=width)) for name, box in gen_boxes] # NOTE: so phrase should include all the words associated to the object (otherwise "an orange dog" may be recognized as "an orange" by the model generating the background). # so word should have one token that includes the word to transfer cross attention (the object name). # Currently using the last word of the object name as word. if bg_prompt: so_prompt_phrase_word_box_list = [(f"{bg_prompt} with {name}", name, name.split(" ")[-1], box) for name, box in gen_boxes] else: so_prompt_phrase_word_box_list = [(f"{name}", name, name.split(" ")[-1], box) for name, box in gen_boxes] objects = [gen_box[0] for gen_box in gen_boxes] objects_unique, objects_count = np.unique(objects, return_counts=True) num_total_matched_boxes = 0 overall_phrases_words_bboxes = [] for ind, object_name in enumerate(objects_unique): bboxes = [box for name, box in gen_boxes if name == object_name] if objects_count[ind] > 1: phrase = p.plural_noun(object_name.replace("an ", "").replace("a ", "")) if include_counts: phrase = p.number_to_words(objects_count[ind]) + " " + phrase else: phrase = object_name # Currently using the last word of the phrase as word. word = phrase.split(' ')[-1] num_total_matched_boxes += len(bboxes) overall_phrases_words_bboxes.append((phrase, word, bboxes)) assert num_total_matched_boxes == len(gen_boxes), f"{num_total_matched_boxes} != {len(gen_boxes)}" objects_str = ", ".join([phrase for phrase, _, _ in overall_phrases_words_bboxes]) if objects_str: if bg_prompt: overall_prompt = f"{bg_prompt} with {objects_str}" else: overall_prompt = objects_str else: overall_prompt = bg_prompt if verbose: print("so_prompt_phrase_word_box_list:", so_prompt_phrase_word_box_list) print("overall_prompt:", overall_prompt) print("overall_phrases_words_bboxes:", overall_phrases_words_bboxes) return so_prompt_phrase_word_box_list, overall_prompt, overall_phrases_words_bboxes