Spaces:
Sleeping
Sleeping
File size: 4,375 Bytes
9bf4bd7 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 |
# Copyright (c) OpenMMLab. All rights reserved.
import warnings
from collections import OrderedDict
from typing import Sequence, Union
from mmengine.dist import (broadcast_object_list, collect_results,
is_main_process)
from mmengine.evaluator import BaseMetric, Evaluator
from mmengine.evaluator.metric import _to_cpu
from mmocr.registry import EVALUATOR
from mmocr.utils.typing_utils import ConfigType
@EVALUATOR.register_module()
class MultiDatasetsEvaluator(Evaluator):
"""Wrapper class to compose class: `ConcatDataset` and multiple
:class:`BaseMetric` instances.
The metrics will be evaluated on each dataset slice separately. The name of
the each metric is the concatenation of the dataset prefix, the metric
prefix and the key of metric - e.g.
`dataset_prefix/metric_prefix/accuracy`.
Args:
metrics (dict or BaseMetric or Sequence): The config of metrics.
dataset_prefixes (Sequence[str]): The prefix of each dataset. The
length of this sequence should be the same as the length of the
datasets.
"""
def __init__(self, metrics: Union[ConfigType, BaseMetric, Sequence],
dataset_prefixes: Sequence[str]) -> None:
super().__init__(metrics)
self.dataset_prefixes = dataset_prefixes
def evaluate(self, size: int) -> dict:
"""Invoke ``evaluate`` method of each metric and collect the metrics
dictionary.
Args:
size (int): Length of the entire validation dataset. When batch
size > 1, the dataloader may pad some data samples to make
sure all ranks have the same length of dataset slice. The
``collect_results`` function will drop the padded data based on
this size.
Returns:
dict: Evaluation results of all metrics. The keys are the names
of the metrics, and the values are corresponding results.
"""
metrics_results = OrderedDict()
dataset_slices = self.dataset_meta.get('cumulative_sizes', [size])
assert len(dataset_slices) == len(self.dataset_prefixes)
for metric in self.metrics:
if len(metric.results) == 0:
warnings.warn(
f'{metric.__class__.__name__} got empty `self.results`.'
'Please ensure that the processed results are properly '
'added into `self.results` in `process` method.')
results = collect_results(metric.results, size,
metric.collect_device)
if is_main_process():
# cast all tensors in results list to cpu
results = _to_cpu(results)
for start, end, dataset_prefix in zip([0] +
dataset_slices[:-1],
dataset_slices,
self.dataset_prefixes):
metric_results = metric.compute_metrics(
results[start:end]) # type: ignore
# Add prefix to metric names
if metric.prefix:
final_prefix = '/'.join(
(dataset_prefix, metric.prefix))
else:
final_prefix = dataset_prefix
metric_results = {
'/'.join((final_prefix, k)): v
for k, v in metric_results.items()
}
# Check metric name conflicts
for name in metric_results.keys():
if name in metrics_results:
raise ValueError(
'There are multiple evaluation results with '
f'the same metric name {name}. Please make '
'sure all metrics have different prefixes.')
metrics_results.update(metric_results)
metric.results.clear()
if is_main_process():
metrics_results = [metrics_results]
else:
metrics_results = [None] # type: ignore
broadcast_object_list(metrics_results)
return metrics_results[0]
|