Upload mean_iou.py

mean_iou.py

@@ -0,0 +1,314 @@
+# Copyright 2022 The HuggingFace Evaluate Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Mean IoU (Intersection-over-Union) metric."""
+
+from typing import Dict, Optional
+
+import datasets
+import numpy as np
+
+import evaluate
+
+
+_DESCRIPTION = """
+IoU is the area of overlap between the predicted segmentation and the ground truth divided by the area of union
+between the predicted segmentation and the ground truth. For binary (two classes) or multi-class segmentation,
+the mean IoU of the image is calculated by taking the IoU of each class and averaging them.
+"""
+
+_KWARGS_DESCRIPTION = """
+Args:
+    predictions (`List[ndarray]`):
+        List of predicted segmentation maps, each of shape (height, width). Each segmentation map can be of a different size.
+    references (`List[ndarray]`):
+        List of ground truth segmentation maps, each of shape (height, width). Each segmentation map can be of a different size.
+    num_labels (`int`):
+        Number of classes (categories).
+    ignore_index (`int`):
+        Index that will be ignored during evaluation.
+    nan_to_num (`int`, *optional*):
+        If specified, NaN values will be replaced by the number defined by the user.
+    label_map (`dict`, *optional*):
+        If specified, dictionary mapping old label indices to new label indices.
+    reduce_labels (`bool`, *optional*, defaults to `False`):
+        Whether or not to reduce all label values of segmentation maps by 1. Usually used for datasets where 0 is used for background,
+        and background itself is not included in all classes of a dataset (e.g. ADE20k). The background label will be replaced by 255.
+
+Returns:
+    `Dict[str, float | ndarray]` comprising various elements:
+    - *mean_iou* (`float`):
+        Mean Intersection-over-Union (IoU averaged over all categories).
+    - *mean_accuracy* (`float`):
+        Mean accuracy (averaged over all categories).
+    - *overall_accuracy* (`float`):
+        Overall accuracy on all images.
+    - *per_category_accuracy* (`ndarray` of shape `(num_labels,)`):
+        Per category accuracy.
+    - *per_category_iou* (`ndarray` of shape `(num_labels,)`):
+        Per category IoU.
+
+Examples:
+
+    >>> import numpy as np
+
+    >>> mean_iou = evaluate.load("mean_iou")
+
+    >>> # suppose one has 3 different segmentation maps predicted
+    >>> predicted_1 = np.array([[1, 2], [3, 4], [5, 255]])
+    >>> actual_1 = np.array([[0, 3], [5, 4], [6, 255]])
+
+    >>> predicted_2 = np.array([[2, 7], [9, 2], [3, 6]])
+    >>> actual_2 = np.array([[1, 7], [9, 2], [3, 6]])
+
+    >>> predicted_3 = np.array([[2, 2, 3], [8, 2, 4], [3, 255, 2]])
+    >>> actual_3 = np.array([[1, 2, 2], [8, 2, 1], [3, 255, 1]])
+
+    >>> predicted = [predicted_1, predicted_2, predicted_3]
+    >>> ground_truth = [actual_1, actual_2, actual_3]
+
+    >>> results = mean_iou.compute(predictions=predicted, references=ground_truth, num_labels=10, ignore_index=255, reduce_labels=False)
+    >>> print(results) # doctest: +NORMALIZE_WHITESPACE
+    {'mean_iou': 0.47750000000000004, 'mean_accuracy': 0.5916666666666666, 'overall_accuracy': 0.5263157894736842, 'per_category_iou': array([0.   , 0.   , 0.375, 0.4  , 0.5  , 0.   , 0.5  , 1.   , 1.   , 1.   ]), 'per_category_accuracy': array([0.        , 0.        , 0.75      , 0.66666667, 1.        , 0.        , 0.5       , 1.        , 1.        , 1.        ])}
+"""
+
+_CITATION = """\
+@software{MMSegmentation_Contributors_OpenMMLab_Semantic_Segmentation_2020,
+author = {{MMSegmentation Contributors}},
+license = {Apache-2.0},
+month = {7},
+title = {{OpenMMLab Semantic Segmentation Toolbox and Benchmark}},
+url = {https://github.com/open-mmlab/mmsegmentation},
+year = {2020}
+}"""
+
+
+def intersect_and_union(
+    pred_label,
+    label,
+    num_labels,
+    ignore_index: bool,
+    label_map: Optional[Dict[int, int]] = None,
+    reduce_labels: bool = False,
+):
+    """Calculate intersection and Union.
+
+    Args:
+        pred_label (`ndarray`):
+            Prediction segmentation map of shape (height, width).
+        label (`ndarray`):
+            Ground truth segmentation map of shape (height, width).
+        num_labels (`int`):
+            Number of categories.
+        ignore_index (`int`):
+            Index that will be ignored during evaluation.
+        label_map (`dict`, *optional*):
+            Mapping old labels to new labels. The parameter will work only when label is str.
+        reduce_labels (`bool`, *optional*, defaults to `False`):
+            Whether or not to reduce all label values of segmentation maps by 1. Usually used for datasets where 0 is used for background,
+            and background itself is not included in all classes of a dataset (e.g. ADE20k). The background label will be replaced by 255.
+
+     Returns:
+         area_intersect (`ndarray`):
+            The intersection of prediction and ground truth histogram on all classes.
+         area_union (`ndarray`):
+            The union of prediction and ground truth histogram on all classes.
+         area_pred_label (`ndarray`):
+            The prediction histogram on all classes.
+         area_label (`ndarray`):
+            The ground truth histogram on all classes.
+    """
+    if label_map is not None:
+        for old_id, new_id in label_map.items():
+            label[label == old_id] = new_id
+
+    # turn into Numpy arrays
+    pred_label = np.array(pred_label)
+    label = np.array(label)
+
+    if reduce_labels:
+        label[label == 0] = 255
+        label = label - 1
+        label[label == 254] = 255
+
+    mask = label != ignore_index
+    mask = np.not_equal(label, ignore_index)
+    pred_label = pred_label[mask]
+    label = np.array(label)[mask]
+
+    intersect = pred_label[pred_label == label]
+
+    area_intersect = np.histogram(intersect, bins=num_labels, range=(0, num_labels - 1))[0]
+    area_pred_label = np.histogram(pred_label, bins=num_labels, range=(0, num_labels - 1))[0]
+    area_label = np.histogram(label, bins=num_labels, range=(0, num_labels - 1))[0]
+
+    area_union = area_pred_label + area_label - area_intersect
+
+    return area_intersect, area_union, area_pred_label, area_label
+
+
+def total_intersect_and_union(
+    results,
+    gt_seg_maps,
+    num_labels,
+    ignore_index: bool,
+    label_map: Optional[Dict[int, int]] = None,
+    reduce_labels: bool = False,
+):
+    """Calculate Total Intersection and Union, by calculating `intersect_and_union` for each (predicted, ground truth) pair.
+
+    Args:
+        results (`ndarray`):
+            List of prediction segmentation maps, each of shape (height, width).
+        gt_seg_maps (`ndarray`):
+            List of ground truth segmentation maps, each of shape (height, width).
+        num_labels (`int`):
+            Number of categories.
+        ignore_index (`int`):
+            Index that will be ignored during evaluation.
+        label_map (`dict`, *optional*):
+            Mapping old labels to new labels. The parameter will work only when label is str.
+        reduce_labels (`bool`, *optional*, defaults to `False`):
+            Whether or not to reduce all label values of segmentation maps by 1. Usually used for datasets where 0 is used for background,
+            and background itself is not included in all classes of a dataset (e.g. ADE20k). The background label will be replaced by 255.
+
+     Returns:
+         total_area_intersect (`ndarray`):
+            The intersection of prediction and ground truth histogram on all classes.
+         total_area_union (`ndarray`):
+            The union of prediction and ground truth histogram on all classes.
+         total_area_pred_label (`ndarray`):
+            The prediction histogram on all classes.
+         total_area_label (`ndarray`):
+            The ground truth histogram on all classes.
+    """
+    total_area_intersect = np.zeros((num_labels,), dtype=np.float64)
+    total_area_union = np.zeros((num_labels,), dtype=np.float64)
+    total_area_pred_label = np.zeros((num_labels,), dtype=np.float64)
+    total_area_label = np.zeros((num_labels,), dtype=np.float64)
+    for result, gt_seg_map in zip(results, gt_seg_maps):
+        area_intersect, area_union, area_pred_label, area_label = intersect_and_union(
+            result, gt_seg_map, num_labels, ignore_index, label_map, reduce_labels
+        )
+        total_area_intersect += area_intersect
+        total_area_union += area_union
+        total_area_pred_label += area_pred_label
+        total_area_label += area_label
+    return total_area_intersect, total_area_union, total_area_pred_label, total_area_label
+
+
+def mean_iou(
+    results,
+    gt_seg_maps,
+    num_labels,
+    ignore_index: bool,
+    nan_to_num: Optional[int] = None,
+    label_map: Optional[Dict[int, int]] = None,
+    reduce_labels: bool = False,
+):
+    """Calculate Mean Intersection and Union (mIoU).
+
+    Args:
+        results (`ndarray`):
+            List of prediction segmentation maps, each of shape (height, width).
+        gt_seg_maps (`ndarray`):
+            List of ground truth segmentation maps, each of shape (height, width).
+        num_labels (`int`):
+            Number of categories.
+        ignore_index (`int`):
+            Index that will be ignored during evaluation.
+        nan_to_num (`int`, *optional*):
+            If specified, NaN values will be replaced by the number defined by the user.
+        label_map (`dict`, *optional*):
+            Mapping old labels to new labels. The parameter will work only when label is str.
+        reduce_labels (`bool`, *optional*, defaults to `False`):
+            Whether or not to reduce all label values of segmentation maps by 1. Usually used for datasets where 0 is used for background,
+            and background itself is not included in all classes of a dataset (e.g. ADE20k). The background label will be replaced by 255.
+
+    Returns:
+        `Dict[str, float | ndarray]` comprising various elements:
+        - *mean_iou* (`float`):
+            Mean Intersection-over-Union (IoU averaged over all categories).
+        - *mean_accuracy* (`float`):
+            Mean accuracy (averaged over all categories).
+        - *overall_accuracy* (`float`):
+            Overall accuracy on all images.
+        - *per_category_accuracy* (`ndarray` of shape `(num_labels,)`):
+            Per category accuracy.
+        - *per_category_iou* (`ndarray` of shape `(num_labels,)`):
+            Per category IoU.
+    """
+    total_area_intersect, total_area_union, total_area_pred_label, total_area_label = total_intersect_and_union(
+        results, gt_seg_maps, num_labels, ignore_index, label_map, reduce_labels
+    )
+
+    # compute metrics
+    metrics = dict()
+
+    all_acc = total_area_intersect.sum() / total_area_label.sum()
+    iou = total_area_intersect / total_area_union
+    acc = total_area_intersect / total_area_label
+
+    metrics["mean_iou"] = np.nanmean(iou)
+    metrics["mean_accuracy"] = np.nanmean(acc)
+    metrics["overall_accuracy"] = all_acc
+    metrics["per_category_iou"] = iou
+    metrics["per_category_accuracy"] = acc
+
+    if nan_to_num is not None:
+        metrics = dict(
+            {metric: np.nan_to_num(metric_value, nan=nan_to_num) for metric, metric_value in metrics.items()}
+        )
+
+    return metrics
+
+
+@evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION)
+class MeanIoU(evaluate.Metric):
+    def _info(self):
+        return evaluate.MetricInfo(
+            description=_DESCRIPTION,
+            citation=_CITATION,
+            inputs_description=_KWARGS_DESCRIPTION,
+            features=datasets.Features(
+                # 1st Seq - height dim, 2nd - width dim
+                {
+                    "predictions": datasets.Sequence(datasets.Sequence(datasets.Value("uint16"))),
+                    "references": datasets.Sequence(datasets.Sequence(datasets.Value("uint16"))),
+                }
+            ),
+            reference_urls=[
+                "https://github.com/open-mmlab/mmsegmentation/blob/71c201b1813267d78764f306a297ca717827c4bf/mmseg/core/evaluation/metrics.py"
+            ],
+        )
+
+    def _compute(
+        self,
+        predictions,
+        references,
+        num_labels: int,
+        ignore_index: bool,
+        nan_to_num: Optional[int] = None,
+        label_map: Optional[Dict[int, int]] = None,
+        reduce_labels: bool = False,
+    ):
+        iou_result = mean_iou(
+            results=predictions,
+            gt_seg_maps=references,
+            num_labels=num_labels,
+            ignore_index=ignore_index,
+            nan_to_num=nan_to_num,
+            label_map=label_map,
+            reduce_labels=reduce_labels,
+        )
+        return iou_result