implement confidence curve feature

det-metrics.py

@@ -20,6 +20,7 @@ import evaluate
 import numpy as np
 from deprecated import deprecated
 from seametrics.detection import PrecisionRecallF1Support
+from seametrics.detection.utils import payload_to_det_metric
 from seametrics.payload import Payload
 
 _CITATION = """\
@@ -91,7 +92,7 @@ Examples:
     >>> from seametrics.payload.processor import PayloadProcessor
     >>> payload = PayloadProcessor(...).payload
     >>> module = evaluate.load("SEA-AI/det-metrics", ...)
-    >>> module.add_payload(payload)
+    >>> module._add_payload(payload)
     >>> result = module.compute()
     >>> print(result)
         {'all': {
@@ -122,20 +123,36 @@ class DetectionMetric(evaluate.Metric):
         class_agnostic: bool = True,
         bbox_format: str = "xywh",
         iou_type: Literal["bbox", "segm"] = "bbox",
-        **kwargs
+        payload: Payload = None,
+        **kwargs,
     ):
         super().__init__(**kwargs)
+
+        # save parameters for later
+        self.payload = payload
+        self.model_names = payload.models if payload else ["custom"]
+        self.iou_thresholds = (
+            iou_threshold if isinstance(iou_threshold, list) else [iou_threshold]
+        )
+        self.area_ranges = [v for _, v in area_ranges_tuples]
+        self.area_ranges_labels = [k for k, _ in area_ranges_tuples]
+        self.class_agnostic = class_agnostic
+        self.iou_type = iou_type
+        self.box_format = bbox_format
+
+        # initialize coco_metrics
         self.coco_metric = PrecisionRecallF1Support(
-            iou_thresholds=(
-                iou_threshold if isinstance(iou_threshold, list) else [iou_threshold]
-            ),
-            area_ranges=[v for _, v in area_ranges_tuples],
-            area_ranges_labels=[k for k, _ in area_ranges_tuples],
-            class_agnostic=class_agnostic,
-            iou_type=iou_type,
-            box_format=bbox_format,
+            iou_thresholds=self.iou_thresholds,
+            area_ranges=self.area_ranges,
+            area_ranges_labels=self.area_ranges_labels,
+            class_agnostic=self.class_agnostic,
+            iou_type=self.iou_type,
+            box_format=self.box_format,
         )
 
+        # initialize evaluation metric
+        self._init_evaluation_metric()
+
     def _info(self):
         return evaluate.MetricInfo(
             # This is the description that will appear on the modules page.
@@ -185,29 +202,63 @@ class DetectionMetric(evaluate.Metric):
 
         self.coco_metric.update(prediction, reference)
 
+    def _init_evaluation_metric(self, **kwargs):
+        """
+        Initializes the evaluation metric by generating sample data, preprocessing predictions and references,
+        and then adding the processed data to the metric using the super class method with additional keyword arguments.
+
+        Parameters:
+            **kwargs: Additional keyword arguments for the super class method.
+
+        Returns:
+            None
+        """
+        predictions, references = self._generate_sample_data()
+        predictions = self._preprocess(predictions)
+        references = self._preprocess(references)
+
         # does not impact the metric, but is required for the interface x_x
         super(evaluate.Metric, self).add(
-            prediction=self._postprocess(prediction),
-            references=self._postprocess(reference),
-            **kwargs
+            prediction=self._postprocess(predictions),
+            references=self._postprocess(references),
+            **kwargs,
         )
 
-    @deprecated(reason="Use `module.add_payload` instead")
+    @deprecated(reason="Use `module._add_payload` instead")
     def add_batch(self, payload: Payload, model_name: str = None):
         """Takes as input a payload and adds the batch to the metric"""
-        self.add_payload(payload, model_name)
+        self._add_payload(payload, model_name)
 
     def _compute(self, *, predictions, references, **kwargs):
         """Called within the evaluate.Metric.compute() method"""
-        return self.coco_metric.compute()
 
-    def add_payload(self, payload: Payload, model_name: str = None):
+        results = {}
+        for model_name in self.model_names:
+            print(f"\n##### {model_name} #####")
+            # add payload if available (otherwise predictions and references must be added with add function)
+            if self.payload:
+                self._add_payload(self.payload, model_name)
+
+            results[model_name] = self.coco_metric.compute()
+
+            # reset coco_metrics for next model
+            self.coco_metric = PrecisionRecallF1Support(
+                iou_thresholds=self.iou_thresholds,
+                area_ranges=self.area_ranges,
+                area_ranges_labels=self.area_ranges_labels,
+                class_agnostic=self.class_agnostic,
+                iou_type=self.iou_type,
+                box_format=self.box_format,
+            )
+        return results
+
+    def _add_payload(self, payload: Payload, model_name: str = None):
         """Converts the payload to the format expected by the metric"""
         # import only if needed since fiftyone is not a direct dependency
-        from seametrics.detection.utils import payload_to_det_metric
 
         predictions, references = payload_to_det_metric(payload, model_name)
         self.add(prediction=predictions, reference=references)
+
         return self
 
     def _preprocess(self, list_of_dicts):
@@ -235,55 +286,79 @@ class DetectionMetric(evaluate.Metric):
             elif isinstance(v, list):
                 d[k] = np.array(v)
         return d
-    
-    def compute_for_multiple_models(self, payload):
-        results = {}
-        for model_name in payload.models:
-            self.add_payload(payload, model_name)
-            results[model_name] = self._compute()
 
-        return results
+    def generate_confidence_curves(
+        self, results, confidence_config={"T": 0, "R": 0, "K": 0, "A": 0, "M": 0}
+    ):
+        """
+        Generate confidence curves based on results and confidence configuration.
 
+        Parameters:
+            results (dict): Results of the evaluation for different models.
+            confidence_config (dict): Configuration for confidence values. Defaults to {"T": 0, "R": 0, "K": 0, "A": 0, "M": 0}.
+                                      T: [1e-10] iou threshold
+                                      R: recall threshold (not used)
+                                      K: class index (class-agnostic mAP, so only 0)
+                                      A: 0=all, 1=small, 2=medium, 3=large, ... (depending on area ranges)
+                                      M: [100] maxDets default in precision_recall_f1_support
 
-    def generate_confidence_curves(self, results, models, confidence_config = {"T":0,
-                                                                       "R":0,
-                                                                       "K":0,
-                                                                       "A":0,
-                                                                       "M":0}):
-        
+        Returns:
+            fig (plotly.graph_objects.Figure): The plotly figure showing the confidence curves.
+        """
         import plotly.graph_objects as go
         from seametrics.detection.utils import get_confidence_metric_vals
 
         # Create traces
         fig = go.Figure()
-        metrics = ['precision', 'recall', 'f1']
-        for model in models:
+        metrics = ["precision", "recall", "f1"]
+        for model_name in self.model_names:
+            print(f"##### {model_name} #####")
             plot_data = get_confidence_metric_vals(
-                cocoeval=results[model['name']]['eval'],
-                T=confidence_config['T'], 
-                R=confidence_config['R'], 
-                K=confidence_config['K'], 
-                A=confidence_config['A'], 
-                M=confidence_config['M']
+                cocoeval=results[model_name]["eval"],
+                T=confidence_config["T"],
+                R=confidence_config["R"],
+                K=confidence_config["K"],
+                A=confidence_config["A"],
+                M=confidence_config["M"],
             )
 
             for metric in metrics:
                 fig.add_trace(
                     go.Scatter(
-                        x=plot_data['conf'],
+                        x=plot_data["conf"],
                         y=plot_data[metric],
-                        mode='lines',
-                        name=f"{model['name'].split('_')[0]} {metric}",
-                        line=dict(dash=None if metric == 'f1' else 'dash'),
+                        mode="lines",
+                        name=f"{model_name} {metric}",
+                        line=dict(dash=None if metric == "f1" else "dash"),
                     )
                 )
 
         fig.update_layout(
             title="Metric vs Confidence",
-            hovermode='x unified',
+            hovermode="x unified",
             xaxis_title="Confidence",
-            yaxis_title="Metric value")
-        fig.show()
-
+            yaxis_title="Metric value",
+        )
         return fig
-        
+
+    def _generate_sample_data(self):
+        """
+        Generates dummy sample data for predictions and references used for initialization.
+
+        Returns:
+            Tuple[List[Dict[str, List[Union[float, int]]]], List[Dict[str, List[Union[float, int]]]]]:
+                - predictions (List[Dict[str, List[Union[float, int]]]]): A list of dictionaries representing the predictions. Each dictionary contains the following keys:
+                    - boxes (List[List[float]]): A list of bounding boxes in the format [x, y, w, h].
+                    - labels (List[int]): A list of labels.
+                    - scores (List[float]): A list of scores.
+                - references (List[Dict[str, List[Union[float, int]]]]): A list of dictionaries representing the references. Each dictionary contains the following keys:
+                    - boxes (List[List[float]]): A list of bounding boxes in the format [x, y, w, h].
+                    - labels (List[int]): A list of labels.
+                    - area (List[float]): A list of areas.
+        """
+        predictions = [
+            {"boxes": [[1.0, 2.0, 3.0, 4.0]], "labels": [0], "scores": [1.0]}
+        ]
+        references = [{"boxes": [[1.0, 2.0, 3.0, 4.0]], "labels": [0], "area": [1.0]}]
+
+        return predictions, references