# Copyright (C) 2021-2024, Mindee. # This program is licensed under the Apache License 2.0. # See LICENSE or go to for full license details. import colorsys from copy import deepcopy from typing import Any, Dict, List, Optional, Tuple, Union import cv2 import matplotlib.patches as patches import matplotlib.pyplot as plt import numpy as np from matplotlib.figure import Figure from .common_types import BoundingBox, Polygon4P __all__ = ["visualize_page", "visualize_kie_page", "draw_boxes"] def rect_patch( geometry: BoundingBox, page_dimensions: Tuple[int, int], label: Optional[str] = None, color: Tuple[float, float, float] = (0, 0, 0), alpha: float = 0.3, linewidth: int = 2, fill: bool = True, preserve_aspect_ratio: bool = False, ) -> patches.Rectangle: """Create a matplotlib rectangular patch for the element Args: ---- geometry: bounding box of the element page_dimensions: dimensions of the Page in format (height, width) label: label to display when hovered color: color to draw box alpha: opacity parameter to fill the boxes, 0 = transparent linewidth: line width fill: whether the patch should be filled preserve_aspect_ratio: pass True if you passed True to the predictor Returns: ------- a rectangular Patch """ if len(geometry) != 2 or any(not isinstance(elt, tuple) or len(elt) != 2 for elt in geometry): raise ValueError("invalid geometry format") # Unpack height, width = page_dimensions (xmin, ymin), (xmax, ymax) = geometry # Switch to absolute coords if preserve_aspect_ratio: width = height = max(height, width) xmin, w = xmin * width, (xmax - xmin) * width ymin, h = ymin * height, (ymax - ymin) * height return patches.Rectangle( (xmin, ymin), w, h, fill=fill, linewidth=linewidth, edgecolor=(*color, alpha), facecolor=(*color, alpha), label=label, ) def polygon_patch( geometry: np.ndarray, page_dimensions: Tuple[int, int], label: Optional[str] = None, color: Tuple[float, float, float] = (0, 0, 0), alpha: float = 0.3, linewidth: int = 2, fill: bool = True, preserve_aspect_ratio: bool = False, ) -> patches.Polygon: """Create a matplotlib polygon patch for the element Args: ---- geometry: bounding box of the element page_dimensions: dimensions of the Page in format (height, width) label: label to display when hovered color: color to draw box alpha: opacity parameter to fill the boxes, 0 = transparent linewidth: line width fill: whether the patch should be filled preserve_aspect_ratio: pass True if you passed True to the predictor Returns: ------- a polygon Patch """ if not geometry.shape == (4, 2): raise ValueError("invalid geometry format") # Unpack height, width = page_dimensions geometry[:, 0] = geometry[:, 0] * (max(width, height) if preserve_aspect_ratio else width) geometry[:, 1] = geometry[:, 1] * (max(width, height) if preserve_aspect_ratio else height) return patches.Polygon( geometry, fill=fill, linewidth=linewidth, edgecolor=(*color, alpha), facecolor=(*color, alpha), label=label, ) def create_obj_patch( geometry: Union[BoundingBox, Polygon4P, np.ndarray], page_dimensions: Tuple[int, int], **kwargs: Any, ) -> patches.Patch: """Create a matplotlib patch for the element Args: ---- geometry: bounding box (straight or rotated) of the element page_dimensions: dimensions of the page in format (height, width) **kwargs: keyword arguments for the patch Returns: ------- a matplotlib Patch """ if isinstance(geometry, tuple): if len(geometry) == 2: # straight word BB (2 pts) return rect_patch(geometry, page_dimensions, **kwargs) elif len(geometry) == 4: # rotated word BB (4 pts) return polygon_patch(np.asarray(geometry), page_dimensions, **kwargs) elif isinstance(geometry, np.ndarray) and geometry.shape == (4, 2): # rotated line return polygon_patch(geometry, page_dimensions, **kwargs) raise ValueError("invalid geometry format") def get_colors(num_colors: int) -> List[Tuple[float, float, float]]: """Generate num_colors color for matplotlib Args: ---- num_colors: number of colors to generate Returns: ------- colors: list of generated colors """ colors = [] for i in np.arange(0.0, 360.0, 360.0 / num_colors): hue = i / 360.0 lightness = (50 + np.random.rand() * 10) / 100.0 saturation = (90 + np.random.rand() * 10) / 100.0 colors.append(colorsys.hls_to_rgb(hue, lightness, saturation)) return colors def visualize_page( page: Dict[str, Any], image: np.ndarray, words_only: bool = True, display_artefacts: bool = True, scale: float = 10, interactive: bool = True, add_labels: bool = True, **kwargs: Any, ) -> Figure: """Visualize a full page with predicted blocks, lines and words >>> import numpy as np >>> import matplotlib.pyplot as plt >>> from doctr.utils.visualization import visualize_page >>> from doctr.models import ocr_db_crnn >>> model = ocr_db_crnn(pretrained=True) >>> input_page = (255 * np.random.rand(600, 800, 3)).astype(np.uint8) >>> out = model([[input_page]]) >>> visualize_page(out[0].pages[0].export(), input_page) >>> plt.show() Args: ---- page: the exported Page of a Document image: np array of the page, needs to have the same shape than page['dimensions'] words_only: whether only words should be displayed display_artefacts: whether artefacts should be displayed scale: figsize of the largest windows side interactive: whether the plot should be interactive add_labels: for static plot, adds text labels on top of bounding box **kwargs: keyword arguments for the polygon patch Returns: ------- the matplotlib figure """ # Get proper scale and aspect ratio h, w = image.shape[:2] size = (scale * w / h, scale) if h > w else (scale, h / w * scale) fig, ax = plt.subplots(figsize=size) # Display the image ax.imshow(image) # hide both axis ax.axis("off") if interactive: artists: List[patches.Patch] = [] # instantiate an empty list of patches (to be drawn on the page) for block in page["blocks"]: if not words_only: rect = create_obj_patch( block["geometry"], page["dimensions"], label="block", color=(0, 1, 0), linewidth=1, **kwargs ) # add patch on figure ax.add_patch(rect) if interactive: # add patch to cursor's artists artists.append(rect) for line in block["lines"]: if not words_only: rect = create_obj_patch( line["geometry"], page["dimensions"], label="line", color=(1, 0, 0), linewidth=1, **kwargs ) ax.add_patch(rect) if interactive: artists.append(rect) for word in line["words"]: rect = create_obj_patch( word["geometry"], page["dimensions"], label=f"{word['value']} (confidence: {word['confidence']:.2%})", color=(0, 0, 1), **kwargs, ) ax.add_patch(rect) if interactive: artists.append(rect) elif add_labels: if len(word["geometry"]) == 5: text_loc = ( int(page["dimensions"][1] * (word["geometry"][0] - word["geometry"][2] / 2)), int(page["dimensions"][0] * (word["geometry"][1] - word["geometry"][3] / 2)), ) else: text_loc = ( int(page["dimensions"][1] * word["geometry"][0][0]), int(page["dimensions"][0] * word["geometry"][0][1]), ) if len(word["geometry"]) == 2: # We draw only if boxes are in straight format ax.text( *text_loc, word["value"], size=10, alpha=0.5, color=(0, 0, 1), ) if display_artefacts: for artefact in block["artefacts"]: rect = create_obj_patch( artefact["geometry"], page["dimensions"], label="artefact", color=(0.5, 0.5, 0.5), linewidth=1, **kwargs, ) ax.add_patch(rect) if interactive: artists.append(rect) if interactive: import mplcursors # Create mlp Cursor to hover patches in artists mplcursors.Cursor(artists, hover=2).connect("add", lambda sel: sel.annotation.set_text(sel.artist.get_label())) fig.tight_layout(pad=0.0) return fig def visualize_kie_page( page: Dict[str, Any], image: np.ndarray, words_only: bool = False, display_artefacts: bool = True, scale: float = 10, interactive: bool = True, add_labels: bool = True, **kwargs: Any, ) -> Figure: """Visualize a full page with predicted blocks, lines and words >>> import numpy as np >>> import matplotlib.pyplot as plt >>> from doctr.utils.visualization import visualize_page >>> from doctr.models import ocr_db_crnn >>> model = ocr_db_crnn(pretrained=True) >>> input_page = (255 * np.random.rand(600, 800, 3)).astype(np.uint8) >>> out = model([[input_page]]) >>> visualize_kie_page(out[0].pages[0].export(), input_page) >>> plt.show() Args: ---- page: the exported Page of a Document image: np array of the page, needs to have the same shape than page['dimensions'] words_only: whether only words should be displayed display_artefacts: whether artefacts should be displayed scale: figsize of the largest windows side interactive: whether the plot should be interactive add_labels: for static plot, adds text labels on top of bounding box **kwargs: keyword arguments for the polygon patch Returns: ------- the matplotlib figure """ # Get proper scale and aspect ratio h, w = image.shape[:2] size = (scale * w / h, scale) if h > w else (scale, h / w * scale) fig, ax = plt.subplots(figsize=size) # Display the image ax.imshow(image) # hide both axis ax.axis("off") if interactive: artists: List[patches.Patch] = [] # instantiate an empty list of patches (to be drawn on the page) colors = {k: color for color, k in zip(get_colors(len(page["predictions"])), page["predictions"])} for key, value in page["predictions"].items(): for prediction in value: if not words_only: rect = create_obj_patch( prediction["geometry"], page["dimensions"], label=f"{key} \n {prediction['value']} (confidence: {prediction['confidence']:.2%}", color=colors[key], linewidth=1, **kwargs, ) # add patch on figure ax.add_patch(rect) if interactive: # add patch to cursor's artists artists.append(rect) if interactive: import mplcursors # Create mlp Cursor to hover patches in artists mplcursors.Cursor(artists, hover=2).connect("add", lambda sel: sel.annotation.set_text(sel.artist.get_label())) fig.tight_layout(pad=0.0) return fig def draw_boxes(boxes: np.ndarray, image: np.ndarray, color: Optional[Tuple[int, int, int]] = None, **kwargs) -> None: """Draw an array of relative straight boxes on an image Args: ---- boxes: array of relative boxes, of shape (*, 4) image: np array, float32 or uint8 color: color to use for bounding box edges **kwargs: keyword arguments from `matplotlib.pyplot.plot` """ h, w = image.shape[:2] # Convert boxes to absolute coords _boxes = deepcopy(boxes) _boxes[:, [0, 2]] *= w _boxes[:, [1, 3]] *= h _boxes = _boxes.astype(np.int32) for box in _boxes.tolist(): xmin, ymin, xmax, ymax = box image = cv2.rectangle( image, (xmin, ymin), (xmax, ymax), color=color if isinstance(color, tuple) else (0, 0, 255), thickness=2 ) plt.imshow(image) plt.plot(**kwargs)