Merge branch 'main' of https://huggingface.co/spaces/SEA-AI/det-metrics

README.md

@@ -61,10 +61,11 @@ results = module.compute()
 print(results)
 ```
 
-This will output the following dictionary containing metrics for the detection model. The key of the dictionary will be the model name or "custom" if no model names are available like in this case.
+This will output the following dictionary containing metrics for the detection model. The key of the dictionary will be the model name or "custom" if no model names are available like in this case. Additionally, there is a single key "classes" which maps the labels to the respective indices of the results. If the results are class agnostic, the value of "classes" is None.
 
 ```json
 {
+    "classes": ...
     "custom": {
         "metrics": ...,
         "eval": ...,
@@ -83,6 +84,8 @@ See [Output Values](#output-values) for more detailed information about the retu
 
 Integrate SEA-AI/det-metrics with FiftyOne datasets for enhanced analysis and visualization:
 
+### Class-agnostic Example
+
 ```python
 import evaluate
 import logging
@@ -97,6 +100,7 @@ processor = PayloadProcessor(
     models=["yolov5n6_RGB_D2304-v1_9C", "tf1zoo_ssd-mobilenet-v2_agnostic_D2207"],
     sequence_list=["Trip_14_Seq_1"],
     data_type="rgb",
+    slices=["rgb"]
 )
 
 # Evaluate using SEA-AI/det-metrics
@@ -127,6 +131,54 @@ This will output the following dictionary containing metrics for the detection m
 
 See [Output Values](#output-values) for more detailed information about the returned results structure, which includes metrics, eval, and params fields for each model passed as input.
 
+### Class-specific example
+```python
+import evaluate
+import logging
+from seametrics.payload.processor import PayloadProcessor
+
+logging.basicConfig(level=logging.WARNING)
+
+# Configure your dataset and model details
+processor = PayloadProcessor(
+    dataset_name="SAILING_DATASET_QA",
+    gt_field="ground_truth_det",
+    models=["yolov5n6_RGB_D2304-v1_9C", "tf1zoo_ssd-mobilenet-v2_agnostic_D2207"],
+    sequence_list=["Trip_14_Seq_1"],
+    data_type="rgb",
+    slices=["rgb"]
+)
+
+# Evaluate using SEA-AI/det-metrics
+module = evaluate.load("SEA-AI/det-metrics", payload=processor.payload, class_agnostic=False)
+print("Used labels: \n", module.label_mapping)
+results = module.compute()
+
+print("Results: \n", results)
+```
+
+```json
+Used labels:
+{
+    "SHIP": 0,
+    "FISHING_SHIP": 0,
+    "BOAT_WITHOUT_SAILS": 1,
+    ... 
+}
+Results:
+{
+    "yolov5n6_RGB_D2304-v1_9C": {
+        "metrics": ..., # metrics are arrays instead of single numbers, where the indices represent class 0, 1, etc. from the label mapping
+        "eval": ...,
+        "params": ...
+    },
+    "tf1zoo_ssd-mobilenet-v2_agnostic_D2207": {
+        "metrics": ...,
+        "eval": ...,
+        "params": ...
+    }
+}
+```
 
 ## Metric Settings
 
@@ -136,6 +188,7 @@ Customize your evaluation by specifying various parameters when loading SEA-AI/d
 - **bbox_format**: Set the bounding box format (e.g., `"xywh"`).
 - **iou_threshold**: Choose the IOU threshold for determining correct detections.
 - **class_agnostic**: Specify whether to calculate metrics disregarding class labels.
+- **label_mapping**: Provide an optional mapping of string labels to numeric labels in the form of a dictionary (e.g., `{"SHIP": 0, "BOAT": 1}`). Defaults to a label mapping defined by the SEA.AI label merging map.
 
 ```python
 area_ranges_tuples = [
@@ -190,6 +243,8 @@ SEA-AI/det-metrics metrics dictionary provides a detailed breakdown of performan
 - **fpi**: Number of images with predictions but no ground truths.
 - **nImgs**: Total number of images evaluated.
 
+If the det-metrics is computed with `class_agnostic=False`, all counts (`tp/fp/fn/duplicates/support/fpi`) and scores (`precision/recall/f1`) are arrays instead of single numbers. For a label mapping of `{"SHIP": 0, "BOAT": 1}`, a exemplary array could be `tp=np.array([10, 4])`, which means there are 10 true positive ships and 4 true positive boats.
+
 ### Eval
 
 The SEA-AI/det-metrics evaluation dictionary provides details about evaluation metrics and results. Below is a description of each field:
@@ -243,7 +298,8 @@ The params return value of the COCO evaluation parameters in PyCOCO represents a
 - **areaRng**: Object area ranges for evaluation. This parameter defines the sizes of objects to evaluate. It is specified as a list of tuples, where each tuple represents a range of area in square pixels.
 - **maxDets**: List of thresholds on maximum detections per image for evaluation. By default, it evaluates with thresholds of 1, 10, and 100 detections per image.
 - **iouType**: Type of IoU calculation used for evaluation. It can be ‘segm’ (segmentation), ‘bbox’ (bounding box), or ‘keypoints’.
-- **useCats**: Boolean flag indicating whether to use category labels for evaluation (default is 1, meaning true).
+- **class_agnostic**: Boolean flag indicating whether to use category labels for evaluation (default is 1, meaning true).
+- **label_mapping**: Dict of str: int pairs, mapping string labels to numeric labels, so that the payload labels can be mapped to numeric labels (default is a label mapping defined by the class merging structure). Should be provided only if `class_agnostic=False`.
 
 > Note:
 > If useCats=0 category labels are ignored as in proposal scoring.

det-metrics.py

@@ -13,7 +13,7 @@
 # limitations under the License.
 """TODO: Add a description here."""
 
-from typing import List, Literal, Tuple
+from typing import List, Literal, Tuple, Dict
 
 import datasets
 import evaluate
@@ -23,6 +23,43 @@ from seametrics.detection import PrecisionRecallF1Support
 from seametrics.detection.utils import payload_to_det_metric
 from seametrics.payload import Payload
 
+LABEL_MAPPING = {
+    'SHIP': 0,
+    'BATTLE_SHIP': 0,
+    'FISHING_SHIP': 0,
+    'CONTAINER_SHIP': 0,
+    'CRUISE_SHIP': 0,
+    'BOAT_WITHOUT_SAILS': 1,
+    'MOTORBOAT': 1,
+    'MARITIME_VEHICLE': 1,
+    'BOAT': 1,
+    'SAILING_BOAT': 2, 
+    'SAILING_BOAT_WITH_CLOSED_SAILS': 2,
+    'SAILING_BOAT_WITH_OPEN_SAILS': 2,
+    'LEISURE_VEHICLE': 3, 
+    'WATER_SKI': 3,
+    'BUOY': 4, 
+    'CONSTRUCTION': 4,
+    'FISHING_BUOY': 4,
+    'HARBOUR_BUOY': 4,
+    'FLOTSAM': 5, 
+    'CONTAINER': 5,
+    'SEA_MINE': 5,
+    'WOODEN_LOG': 5,
+    'UNKNOWN': 5,      
+    'HUMAN_IN_WATER': 5,        
+    'FAR_AWAY_OBJECT': 6, 
+    'MARITIME_ANIMAL': 7, 
+    'ANIMAL': 7,
+    'FISH': 7,
+    'DOLPHIN': 7,
+    'MAMMAL': 7,
+    'WHALE': 7,
+    'AERIAL_ANIMAL': 8,
+    'SEAGULL': 8,
+    'BIRD': 8,
+}
+
 _CITATION = """\
 @InProceedings{coco:2020,
 title = {Microsoft {COCO:} Common Objects in Context},
@@ -124,6 +161,7 @@ class DetectionMetric(evaluate.Metric):
         bbox_format: str = "xywh",
         iou_type: Literal["bbox", "segm"] = "bbox",
         payload: Payload = None,
+        label_mapping: Dict[str, int] = None,
         **kwargs,
     ):
         super().__init__(**kwargs)
@@ -131,15 +169,25 @@ class DetectionMetric(evaluate.Metric):
         # save parameters for later
         self.payload = payload
         self.model_names = payload.models if payload else ["custom"]
+        self.iou_threshold = iou_threshold
+        self.area_ranges_tuples = area_ranges_tuples
+        self.class_agnostic = class_agnostic
+        self.iou_type = iou_type
+        self.bbox_format = bbox_format
+        self.label_mapping = LABEL_MAPPING if not self.class_agnostic else None
+        if not class_agnostic:
+            if label_mapping:
+                print("WARNING: overwritting the default label mapping with the \
+                      custom label mapping provided via `label_mapping`.")
+                self.label_mapping = label_mapping
+
+        # postprocess parameters
         self.iou_thresholds = (
             iou_threshold if isinstance(iou_threshold, list) else [iou_threshold]
         )
         self.area_ranges = [v for _, v in area_ranges_tuples]
         self.area_ranges_labels = [k for k, _ in area_ranges_tuples]
-        self.class_agnostic = class_agnostic
-        self.iou_type = iou_type
-        self.box_format = bbox_format
-
+        
         # initialize coco_metrics
         self.coco_metric = PrecisionRecallF1Support(
             iou_thresholds=self.iou_thresholds,
@@ -147,7 +195,8 @@ class DetectionMetric(evaluate.Metric):
             area_ranges_labels=self.area_ranges_labels,
             class_agnostic=self.class_agnostic,
             iou_type=self.iou_type,
-            box_format=self.box_format,
+            box_format=self.bbox_format,
+            labels=sorted(list(set(list(self.label_mapping.values())))) if self.label_mapping else None,
         )
 
         # initialize evaluation metric
@@ -233,6 +282,7 @@ class DetectionMetric(evaluate.Metric):
         """Called within the evaluate.Metric.compute() method"""
 
         results = {}
+        results["classes"] = self.label_mapping
         for model_name in self.model_names:
             print(f"\n##### {model_name} #####")
             # add payload if available (otherwise predictions and references must be added with add function)
@@ -248,7 +298,7 @@ class DetectionMetric(evaluate.Metric):
                 area_ranges_labels=self.area_ranges_labels,
                 class_agnostic=self.class_agnostic,
                 iou_type=self.iou_type,
-                box_format=self.box_format,
+                box_format=self.bbox_format,
             )
         return results
 
@@ -256,7 +306,7 @@ class DetectionMetric(evaluate.Metric):
         """Converts the payload to the format expected by the metric"""
         # import only if needed since fiftyone is not a direct dependency
 
-        predictions, references = payload_to_det_metric(payload, model_name)
+        predictions, references = payload_to_det_metric(payload, model_name, class_agnostic=self.class_agnostic, label_mapping=self.label_mapping)
         self.add(prediction=predictions, reference=references)
 
         return self
@@ -308,6 +358,11 @@ class DetectionMetric(evaluate.Metric):
         import plotly.graph_objects as go
         from seametrics.detection.utils import get_confidence_metric_vals
 
+        if not self.class_agnostic:
+            raise ValueError(
+                "This method is not yet implemented for `self.class_agnostic=False`."
+            )
+
         # Create traces
         fig = go.Figure()
         metrics = ["precision", "recall", "f1"]
@@ -373,6 +428,11 @@ class DetectionMetric(evaluate.Metric):
             wandb: To interact with the Weights and Biases platform.
             datetime: To generate a timestamp for run names.
         """
+        if not self.class_agnostic:
+            raise ValueError(
+                "This method is not yet implemented for `self.class_agnostic=False`."
+            )
+        
         import os
         import wandb
         import datetime
@@ -414,3 +474,76 @@ class DetectionMetric(evaluate.Metric):
         references = [{"boxes": [[1.0, 2.0, 3.0, 4.0]], "labels": [0], "area": [1.0]}]
 
         return predictions, references
+
+
+    def compute_from_payload(self, payload: Payload, **kwargs):
+        """
+        Compute the metric from the payload.
+        Args:
+            payload (Payload): The payload to compute the metric from.
+            **kwargs: Additional keyword arguments.
+        Returns:
+            dict: The computed metric results with the following format:
+            {
+              "model_name": {
+                "overall": {
+                  "all": {"tp": ..., "fp": ..., "fn": ..., "f1": ...},
+                  ...  # more area ranges
+                },
+                "per_sequence": {
+                  "sequence_name": {
+                    "all": {...},
+                    ...  # more area ranges
+                  },
+                  ...  # more sequences
+                }
+              },
+              ...  # more models
+            }
+        
+        Note:
+        - If the metric does not support area ranges, the metric should store the results under the `all` key.
+        - If a range area is provided it will be displayed in the output. if area_ranges_tuples is None, then all the area ranges will be displayed
+        """        
+        results = {}
+
+        for model_name in payload.models:
+            results[model_name] = {"overall": {}, "per_sequence": {}}
+            
+            # per-sequence loop
+            for seq_name, sequence in payload.sequences.items():
+                print(f"\n##### {seq_name} #####")
+                # create new payload only with specific sequence and model
+                sequence_payload = Payload(
+                    dataset=payload.dataset,
+                    gt_field_name=payload.gt_field_name,
+                    models=[model_name],
+                    sequences={seq_name: sequence}
+                )
+                module = DetectionMetric(
+                    area_ranges_tuples=kwargs["area_ranges_tuples"],
+                    iou_threshold=self.iou_threshold,
+                    class_agnostic=self.class_agnostic,
+                    bbox_format=self.bbox_format,
+                    iou_type=self.iou_type,
+                    payload=sequence_payload
+                )
+                results[model_name]["per_sequence"][seq_name] = module.compute()[model_name]["metrics"]
+
+            # overall per-model loop
+            model_payload = Payload(
+                    dataset=payload.dataset,
+                    gt_field_name=payload.gt_field_name,
+                    models=[model_name],
+                    sequences=payload.sequences
+                )
+            module = DetectionMetric(
+                area_ranges_tuples=kwargs["area_ranges_tuples"],
+                iou_threshold=self.iou_threshold,
+                class_agnostic=self.class_agnostic,
+                bbox_format=self.bbox_format,
+                iou_type=self.iou_type,
+                payload=model_payload
+            )
+            results[model_name]["overall"] = module.compute()[model_name]["metrics"]
+        return results