Spaces:
Sleeping
Sleeping
File size: 10,375 Bytes
9bf4bd7 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 |
# Copyright (c) OpenMMLab. All rights reserved.
from typing import Dict, List, Optional, Sequence
import numpy as np
import torch
from mmengine.evaluator import BaseMetric
from mmengine.logging import MMLogger
from scipy.sparse import csr_matrix
from scipy.sparse.csgraph import maximum_bipartite_matching
from shapely.geometry import Polygon
from mmocr.evaluation.functional import compute_hmean
from mmocr.registry import METRICS
from mmocr.utils import poly_intersection, poly_iou, polys2shapely
@METRICS.register_module()
class HmeanIOUMetric(BaseMetric):
"""HmeanIOU metric.
This method computes the hmean iou metric, which is done in the
following steps:
- Filter the prediction polygon:
- Scores is smaller than minimum prediction score threshold.
- The proportion of the area that intersects with gt ignored polygon is
greater than ignore_precision_thr.
- Computing an M x N IoU matrix, where each element indexing
E_mn represents the IoU between the m-th valid GT and n-th valid
prediction.
- Based on different prediction score threshold:
- Obtain the ignored predictions according to prediction score.
The filtered predictions will not be involved in the later metric
computations.
- Based on the IoU matrix, get the match metric according to
``match_iou_thr``.
- Based on different `strategy`, accumulate the match number.
- calculate H-mean under different prediction score threshold.
Args:
match_iou_thr (float): IoU threshold for a match. Defaults to 0.5.
ignore_precision_thr (float): Precision threshold when prediction and\
gt ignored polygons are matched. Defaults to 0.5.
pred_score_thrs (dict): Best prediction score threshold searching
space. Defaults to dict(start=0.3, stop=0.9, step=0.1).
strategy (str): Polygon matching strategy. Options are 'max_matching'
and 'vanilla'. 'max_matching' refers to the optimum strategy that
maximizes the number of matches. Vanilla strategy matches gt and
pred polygons if both of them are never matched before. It was used
in MMOCR 0.x and and academia. Defaults to 'vanilla'.
collect_device (str): Device name used for collecting results from
different ranks during distributed training. Must be 'cpu' or
'gpu'. Defaults to 'cpu'.
prefix (str, optional): The prefix that will be added in the metric
names to disambiguate homonymous metrics of different evaluators.
If prefix is not provided in the argument, self.default_prefix
will be used instead. Defaults to None
"""
default_prefix: Optional[str] = 'icdar'
def __init__(self,
match_iou_thr: float = 0.5,
ignore_precision_thr: float = 0.5,
pred_score_thrs: Dict = dict(start=0.3, stop=0.9, step=0.1),
strategy: str = 'vanilla',
collect_device: str = 'cpu',
prefix: Optional[str] = None) -> None:
super().__init__(collect_device=collect_device, prefix=prefix)
self.match_iou_thr = match_iou_thr
self.ignore_precision_thr = ignore_precision_thr
self.pred_score_thrs = np.arange(**pred_score_thrs)
assert strategy in ['max_matching', 'vanilla']
self.strategy = strategy
def process(self, data_batch: Sequence[Dict],
data_samples: Sequence[Dict]) -> None:
"""Process one batch of data samples and predictions. The processed
results should be stored in ``self.results``, which will be used to
compute the metrics when all batches have been processed.
Args:
data_batch (Sequence[Dict]): A batch of data from dataloader.
data_samples (Sequence[Dict]): A batch of outputs from
the model.
"""
for data_sample in data_samples:
pred_instances = data_sample.get('pred_instances')
pred_polygons = pred_instances.get('polygons')
pred_scores = pred_instances.get('scores')
if isinstance(pred_scores, torch.Tensor):
pred_scores = pred_scores.cpu().numpy()
pred_scores = np.array(pred_scores, dtype=np.float32)
gt_instances = data_sample.get('gt_instances')
gt_polys = gt_instances.get('polygons')
gt_ignore_flags = gt_instances.get('ignored')
if isinstance(gt_ignore_flags, torch.Tensor):
gt_ignore_flags = gt_ignore_flags.cpu().numpy()
gt_polys = polys2shapely(gt_polys)
pred_polys = polys2shapely(pred_polygons)
pred_ignore_flags = self._filter_preds(pred_polys, gt_polys,
pred_scores,
gt_ignore_flags)
gt_num = np.sum(~gt_ignore_flags)
pred_num = np.sum(~pred_ignore_flags)
iou_metric = np.zeros([gt_num, pred_num])
# Compute IoU scores amongst kept pred and gt polygons
for pred_mat_id, pred_poly_id in enumerate(
self._true_indexes(~pred_ignore_flags)):
for gt_mat_id, gt_poly_id in enumerate(
self._true_indexes(~gt_ignore_flags)):
iou_metric[gt_mat_id, pred_mat_id] = poly_iou(
gt_polys[gt_poly_id], pred_polys[pred_poly_id])
result = dict(
iou_metric=iou_metric,
pred_scores=pred_scores[~pred_ignore_flags])
self.results.append(result)
def compute_metrics(self, results: List[Dict]) -> Dict:
"""Compute the metrics from processed results.
Args:
results (list[dict]): The processed results of each batch.
Returns:
dict: The computed metrics. The keys are the names of the metrics,
and the values are corresponding results.
"""
logger: MMLogger = MMLogger.get_current_instance()
best_eval_results = dict(hmean=-1)
logger.info('Evaluating hmean-iou...')
dataset_pred_num = np.zeros_like(self.pred_score_thrs)
dataset_hit_num = np.zeros_like(self.pred_score_thrs)
dataset_gt_num = 0
for result in results:
iou_metric = result['iou_metric'] # (gt_num, pred_num)
pred_scores = result['pred_scores'] # (pred_num)
dataset_gt_num += iou_metric.shape[0]
# Filter out predictions by IoU threshold
for i, pred_score_thr in enumerate(self.pred_score_thrs):
pred_ignore_flags = pred_scores < pred_score_thr
# get the number of matched boxes
matched_metric = iou_metric[:, ~pred_ignore_flags] \
> self.match_iou_thr
if self.strategy == 'max_matching':
csr_matched_metric = csr_matrix(matched_metric)
matched_preds = maximum_bipartite_matching(
csr_matched_metric, perm_type='row')
# -1 denotes unmatched pred polygons
dataset_hit_num[i] += np.sum(matched_preds != -1)
else:
# first come first matched
matched_gt_indexes = set()
matched_pred_indexes = set()
for gt_idx, pred_idx in zip(*np.nonzero(matched_metric)):
if gt_idx in matched_gt_indexes or \
pred_idx in matched_pred_indexes:
continue
matched_gt_indexes.add(gt_idx)
matched_pred_indexes.add(pred_idx)
dataset_hit_num[i] += len(matched_gt_indexes)
dataset_pred_num[i] += np.sum(~pred_ignore_flags)
for i, pred_score_thr in enumerate(self.pred_score_thrs):
recall, precision, hmean = compute_hmean(
int(dataset_hit_num[i]), int(dataset_hit_num[i]),
int(dataset_gt_num), int(dataset_pred_num[i]))
eval_results = dict(
precision=precision, recall=recall, hmean=hmean)
logger.info(f'prediction score threshold: {pred_score_thr:.2f}, '
f'recall: {eval_results["recall"]:.4f}, '
f'precision: {eval_results["precision"]:.4f}, '
f'hmean: {eval_results["hmean"]:.4f}\n')
if eval_results['hmean'] > best_eval_results['hmean']:
best_eval_results = eval_results
return best_eval_results
def _filter_preds(self, pred_polys: List[Polygon], gt_polys: List[Polygon],
pred_scores: List[float],
gt_ignore_flags: np.ndarray) -> np.ndarray:
"""Filter out the predictions by score threshold and whether it
overlaps ignored gt polygons.
Args:
pred_polys (list[Polygon]): Pred polygons.
gt_polys (list[Polygon]): GT polygons.
pred_scores (list[float]): Pred scores of polygons.
gt_ignore_flags (np.ndarray): 1D boolean array indicating
the positions of ignored gt polygons.
Returns:
np.ndarray: 1D boolean array indicating the positions of ignored
pred polygons.
"""
# Filter out predictions based on the minimum score threshold
pred_ignore_flags = pred_scores < self.pred_score_thrs.min()
# Filter out pred polygons which overlaps any ignored gt polygons
for pred_id in self._true_indexes(~pred_ignore_flags):
for gt_id in self._true_indexes(gt_ignore_flags):
# Match pred with ignored gt
precision = poly_intersection(
gt_polys[gt_id], pred_polys[pred_id]) / (
pred_polys[pred_id].area + 1e-5)
if precision > self.ignore_precision_thr:
pred_ignore_flags[pred_id] = True
break
return pred_ignore_flags
def _true_indexes(self, array: np.ndarray) -> np.ndarray:
"""Get indexes of True elements from a 1D boolean array."""
return np.where(array)[0]
|