# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """TODO: Add a description here.""" from typing import List, Tuple, Dict, Literal import evaluate import datasets import numpy as np from seametrics.detection import PrecisionRecallF1Support _CITATION = """\ @InProceedings{coco:2020, title = {Microsoft {COCO:} Common Objects in Context}, authors={Tsung{-}Yi Lin and Michael Maire and Serge J. Belongie and James Hays and Pietro Perona and Deva Ramanan and Piotr Dollar and C. Lawrence Zitnick}, booktitle = {Computer Vision - {ECCV} 2014 - 13th European Conference, Zurich, Switzerland, September 6-12, 2014, Proceedings, Part {V}}, series = {Lecture Notes in Computer Science}, volume = {8693}, pages = {740--755}, publisher = {Springer}, year={2014} } """ _DESCRIPTION = """\ This evaluation metric is designed to give provide object detection metrics at different object size levels. It is based on a modified version of the commonly used COCO-evaluation metrics. """ _KWARGS_DESCRIPTION = """ Calculates object detection metrics given predicted and ground truth bounding boxes for a single image. Args: predictions: list of predictions to score. Each prediction should be a list containing the four co-ordinates that specify the bounding box. Co-ordinate format is as defined when instantiating the metric (parameter: bbox_type, defaults to xywh). references: list of reference for each prediction. Each prediction should be a list containing the four co-ordinates that specify the bounding box. Bounding box format should be the same as for the predictions. Returns: dict containing dicts for each specified area range with following items: 'range': specified area with [max_px_area, max_px_area] 'iouThr': min. IOU-threshold of a prediction with a ground truth box to be considered a correct prediction 'maxDets': maximum number of detections 'tp': number of true positive (correct) predictions 'fp': number of false positive (incorrect) predictions 'fn': number of false negative (missed) predictions 'duplicates': number of duplicate predictions 'precision': best possible score = 1, worst possible score = 0 large if few false positive predictions formula: tp/(fp+tp) 'recall' best possible score = 1, worst possible score = 0 large if few missed predictions formula: tp/(tp+fn) 'f1': best possible score = 1, worst possible score = 0 trades off precision and recall formula: 2*(precision*recall)/(precision+recall) 'support': number of ground truth bounding boxes considered in the evaluation, 'fpi': number of images with no ground truth but false positive predictions, 'nImgs': number of images considered in evaluation Examples: >>> import evaluate >>> from seametrics.fo_to_payload.utils import fo_to_payload >>> payload = fo_to_payload(..., models=model_list) >>> for model in payload["models"]: >>> module = evaluate.load("./detection_metric.py", iou_thresholds=0.9) >>> module.add_batch(payload) >>> result = module.compute() >>> print(result) {'all': { 'range': [0, 10000000000.0], 'iouThr': '0.00', 'maxDets': 100, 'tp': 1, 'fp': 3, 'fn': 1, 'duplicates': 0, 'precision': 0.25, 'recall': 0.5, 'f1': 0.3333333333333333, 'support': 2, 'fpi': 0, 'nImgs': 2 } } """ @evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION) class DetectionMetric(evaluate.Metric): def __init__( self, area_ranges_tuples: List[Tuple[str, List[int]]] = [("all", [0, 1e5 ** 2])], iou_threshold: float = 1e-10, class_agnostic: bool = True, bbox_format: str = "xywh", iou_type: Literal["bbox", "segm"] = "bbox", **kwargs ): super().__init__(**kwargs) area_ranges = [v for _, v in area_ranges_tuples] area_ranges_labels = [k for k, _ in area_ranges_tuples] metric_params = dict( iou_thresholds=[iou_threshold], area_ranges=area_ranges, area_ranges_labels=area_ranges_labels, class_agnostic=class_agnostic, iou_type=iou_type, box_format=bbox_format ) self.coco_metric = PrecisionRecallF1Support(**metric_params) def _info(self): return evaluate.MetricInfo( # This is the description that will appear on the modules page. module_type="metric", description=_DESCRIPTION, citation=_CITATION, inputs_description=_KWARGS_DESCRIPTION, # This defines the format of each prediction and reference features=datasets.Features( { 'predictions': datasets.Sequence(feature=datasets.Sequence(datasets.Value("float"))), 'references': datasets.Sequence(feature=datasets.Sequence(datasets.Value("float"))), } ), # Additional links to the codebase or references codebase_urls=["https://github.com/SEA-AI/metrics/tree/main", "https://github.com/cocodataset/cocoapi/tree/master"] ) def add_batch( self, data: dict, model: str = None ): """Add predictions and ground truths of a single image to update the metric. Args: data (dict): containing standard payload of data that should be evaluated format should be as returned by function `fo_to_payload()` in seametrics library model (str): should be one out of values given in data["models"] if not defined, defaults to data["models"][0], as only one model can be evaluated a time. """ # populate two empty lists in format suitable for hugging face metric # nothing is computed based on them but prevents huggingface error predictions, references = [], [] if model is None: model = data["models"][0] for sequence in data["sequence_list"]: seq_data = data["sequences"][sequence] gt_normalized = seq_data[data["gt_field_name"]] # shape: (n_frames, m_gts) pred_normalized = seq_data[model] # shape: (n_frames, l_preds) img_res = seq_data["resolution"] # (h, w) for gt_frame, pred_frame in zip(gt_normalized, pred_normalized): # iterate over all frame processed_pred = self._fo_dets_to_metrics_dict( fo_dets=pred_frame, w=img_res[1], h=img_res[0], include_scores=True ) processed_gt = self._fo_dets_to_metrics_dict( fo_dets=gt_frame, w=img_res[1], h=img_res[0], include_scores=False ) predictions.append(processed_pred[0]["boxes"].tolist()) references.append(processed_gt[0]["boxes"].tolist()) # where the magic happens: update metric with data from current frame self.coco_metric.update(processed_pred, processed_gt) # prevents hugging face error, doesn't do a lot super(evaluate.Metric, self).add_batch( predictions=predictions, references=references ) def _compute( self, predictions, references ): """Returns the scores""" result = self.coco_metric.compute()["metrics"] return result @staticmethod def _fo_dets_to_metrics_dict(fo_dets: list, w: int, h: int, include_scores: bool = False) -> List[Dict[str, np.ndarray]]: """Convert list of fiftyone detections to format that is required by PrecisionRecallF1Support() function of seametrics library Args: fo_dets (list): list containing fiftyone detections (or empty if frame without any detections) note: bounding boxes in fo-detections are in format xywhn w (int): width in pixel of image h (int): height in pixel of image Returns: List[Dict[str, np.ndarray]]: list holding single dict with items: "boxes": denormalized bounding boxes of whole frame in numpy array (shape: n_bboxes, 4) "scores": confidence scores in numpy array (shape: n_bboxes) "labels": labels in numpy array (shape: n_bboxes) """ detections = [] scores = [] labels = [] #TODO: map to numbers if len(fo_dets) == 0: return [ dict( boxes=np.array([]), scores=np.array([]), labels=np.array([]) ) ] for det in fo_dets: bbox = det["bounding_box"] detections.append( [bbox[0]*w, bbox[1]*h, bbox[2]*w, bbox[3]*h] ) scores.append(det["confidence"] if det["confidence"] is not None else 1.0) # None for gt labels.append(1) if include_scores: return [ dict( boxes=np.array(detections), scores=np.array(scores), labels=np.array(labels) ) ] else: return [ dict( boxes=np.array(detections), labels=np.array(labels) ) ]