ERR initial version

eer.py

@@ -11,11 +11,13 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-"""TODO: Add a description here."""
+"""EER compute script"""
 
 import evaluate
 import datasets
 
+import numpy as np 
+import sklearn.metrics
 
 # TODO: Add BibTeX citation
 _CITATION = """\
@@ -26,70 +28,50 @@ year={2020}
 }
 """
 
-# TODO: Add description of the module here
 _DESCRIPTION = """\
-This new module is designed to solve this great ML task and is crafted with a lot of care.
+This module is designed to compute Equal Error Rate metric, which is used a lot in the Automatic Speaker Verification task.
 """
 
 
-# TODO: Add description of the arguments of the module here
 _KWARGS_DESCRIPTION = """
-Calculates how good are predictions given some references, using certain scores
-Args:
-    predictions: list of predictions to score. Each predictions
-        should be a string with tokens separated by spaces.
-    references: list of reference for each prediction. Each
-        reference should be a string with tokens separated by spaces.
-Returns:
-    accuracy: description of the first score,
-    another_score: description of the second score,
-Examples:
-    Examples should be written in doctest format, and should illustrate how
-    to use the function.
-
-    >>> my_new_module = evaluate.load("my_new_module")
-    >>> results = my_new_module.compute(references=[0, 1], predictions=[0, 1])
-    >>> print(results)
-    {'accuracy': 1.0}
+The EER is the location on a ROC or DET curve where the false acceptance rate and false rejection rate are equal. In general, the lower the equal error rate value, the higher the accuracy of the biometric system.
 """
 
-# TODO: Define external resources urls if needed
-BAD_WORDS_URL = "http://url/to/external/resource/bad_words.txt"
-
 
 @evaluate.utils.file_utils.add_start_docstrings(_DESCRIPTION, _KWARGS_DESCRIPTION)
 class EER(evaluate.Metric):
-    """TODO: Short description of my evaluation module."""
+    """Compute Equal error rate metrics"""
 
     def _info(self):
-        # TODO: Specifies the evaluate.EvaluationModuleInfo object
         return evaluate.MetricInfo(
-            # This is the description that will appear on the modules page.
             module_type="metric",
             description=_DESCRIPTION,
             citation=_CITATION,
             inputs_description=_KWARGS_DESCRIPTION,
-            # This defines the format of each prediction and reference
             features=datasets.Features({
                 'predictions': datasets.Value('int64'),
                 'references': datasets.Value('int64'),
+                'pos_label': datasets.Value('int64')
             }),
-            # Homepage of the module for documentation
-            homepage="http://module.homepage",
-            # Additional links to the codebase or references
-            codebase_urls=["http://github.com/path/to/codebase/of/new_module"],
-            reference_urls=["http://path.to.reference.url/new_module"]
+            reference_urls=["https://github.com/YuanGongND/python-compute-eer"]
         )
 
-    def _download_and_prepare(self, dl_manager):
-        """Optional: download external resources useful to compute the scores"""
-        # TODO: Download external resources if needed
-        pass
+    def _compute(self, predictions, references, pos_label=1):
+        """Returns EER the scores"""
+        eer_score_list = []
+  
+        fpr, tpr, threshold = sklearn.metrics.roc_curve(references, predictions, pos_label=pos_label)
+        fnr = 1 - tpr
+
+        # the threshold of fnr == fpr
+        eer_threshold = threshold[np.nanargmin(np.absolute((fnr - fpr)))]
+
+        # theoretically eer from fpr and eer from fnr should be identical but they can be slightly differ in reality
+        eer_1 = fpr[np.nanargmin(np.absolute((fnr - fpr)))]
+        eer_2 = fnr[np.nanargmin(np.absolute((fnr - fpr)))]
 
-    def _compute(self, predictions, references):
-        """Returns the scores"""
-        # TODO: Compute the different scores of the module
-        accuracy = sum(i == j for i, j in zip(predictions, references)) / len(predictions)
-        return {
-            "accuracy": accuracy,
-        }
+        # return the mean of eer from fpr and from fnr
+        eer = (eer_1 + eer_2) / 2
+  
+        
+        return {"eer": eer}