add overlapped DER + tests

Author: FrenchKrab

src/pyannote/metrics/diarization.py

@@ -41,12 +41,24 @@
 from .types import Details, MetricComponents
 from .utils import UEMSupportMixin
 
+from .matcher import (
+    LabelMatcher,
+    MATCH_TOTAL,
+    MATCH_CORRECT,
+    MATCH_CONFUSION,
+    MATCH_MISSED_DETECTION,
+    MATCH_FALSE_ALARM,
+)
+
 if TYPE_CHECKING:
     pass
 
 # TODO: can't we put these as class attributes?
 DER_NAME = "diarization error rate"
 
+OVLDER_PREFIX_OVL = "ovl"
+OVLDER_PREFIX_NONOVL = "nonovl"
+
 
 class DiarizationErrorRate(IdentificationErrorRate):
     """Diarization error rate
@@ -646,3 +658,96 @@ def compute_components(
         return super(DiarizationCompleteness, self).compute_components(
             hypothesis, reference, uem=uem, **kwargs
         )
+
+
+class OverlappedDiarizationErrorRate(BaseMetric):
+    """Diarization error rate with details for overlap and non-overlap errors.
+    Error components will be prefixed with 'ovl' or 'nonovl' (e.g. 'ovl false alarm')
+
+    Parameters
+    ----------
+    collar : float, optional
+        Duration (in seconds) of collars removed from evaluation around
+        boundaries of reference segments.
+    """
+
+    OVDER_NAME = "diarization error rate"
+
+    def __init__(self, collar: float = 0.0):
+        super().__init__()
+
+        self.der_ovl = DiarizationErrorRate(collar=collar, skip_overlap=False)
+        self.der_nonovl = DiarizationErrorRate(collar=collar, skip_overlap=False)
+
+    @classmethod
+    def metric_components(cls) -> MetricComponents:
+        comps = []
+        for ovl in [OVLDER_PREFIX_NONOVL, OVLDER_PREFIX_OVL]:
+            for comp in [
+                MATCH_TOTAL,
+                MATCH_CORRECT,
+                MATCH_CONFUSION,
+                MATCH_MISSED_DETECTION,
+                MATCH_FALSE_ALARM,
+            ]:
+                comps.append(f"{ovl} {comp}")
+        return comps
+
+    @classmethod
+    def metric_name(cls) -> str:
+        return cls.OVDER_NAME
+
+    def compute_components(
+        self, reference: Annotation, hypothesis: Annotation, uem: Timeline | None = None
+    ) -> Details:
+
+        # map 'hypothesis' labels to 'reference' labels
+        mapping: dict = DiarizationErrorRate().optimal_mapping(
+            reference, hypothesis, uem=uem
+        )
+        hypothesis = hypothesis.rename_labels(mapping)
+
+        # split uem into non-overlapping and overlapping regions
+        overlap: Timeline = reference.get_overlap()
+        if uem is None:
+            uem = (
+                reference.support()
+                .get_timeline()
+                .union(hypothesis.support().get_timeline())
+            )
+        nonovl_regions: Timeline = uem.extrude(overlap)
+        ovl_regions: Timeline = uem.crop(overlap)
+
+        # update internal metrics for (non-)overlapping errors
+        comps_nonovl = self.der_nonovl.compute_components(
+            reference, hypothesis, uem=nonovl_regions
+        )
+        comps_ovl = self.der_ovl.compute_components(
+            reference, hypothesis, uem=ovl_regions
+        )
+
+        components = {}
+        components.update(
+            {f"{OVLDER_PREFIX_NONOVL} {k}": v for k, v in comps_nonovl.items()}
+        )
+        components.update({f"{OVLDER_PREFIX_OVL} {k}": v for k, v in comps_ovl.items()})
+
+        return components
+
+    def compute_metric(self, detail: Details) -> float:
+        numerator = 0.0
+        denominator = 0.0
+        for ovl in [OVLDER_PREFIX_NONOVL, OVLDER_PREFIX_OVL]:
+            numerator += (
+                detail[f"{ovl} {MATCH_FALSE_ALARM}"]
+                + detail[f"{ovl} {MATCH_MISSED_DETECTION}"]
+                + detail[f"{ovl} {MATCH_CONFUSION}"]
+            )
+            denominator += detail[f"{ovl} {MATCH_TOTAL}"]
+        if denominator == 0.0:
+            if numerator == 0:
+                return 0.0
+            else:
+                return 1.0
+        else:
+            return numerator / denominator

tests/test_diarization.py

@@ -1,3 +1,4 @@
+from calendar import c
 import pytest
 
 import pyannote.core
@@ -7,9 +8,17 @@
 from pyannote.metrics.diarization import DiarizationErrorRate
 from pyannote.metrics.diarization import DiarizationPurity
 from pyannote.metrics.diarization import DiarizationCoverage
+from pyannote.metrics.diarization import (
+    OverlappedDiarizationErrorRate,
+    OVLDER_PREFIX_NONOVL,
+    OVLDER_PREFIX_OVL,
+)
 
 import numpy.testing as npt
 
+from pyannote.metrics.matcher import MATCH_TOTAL
+from pyannote.metrics.types import Details
+
 
 @pytest.fixture
 def reference():
@@ -42,6 +51,40 @@ def hypothesis():
     return hypothesis
 
 
+def test_ovl_der(reference_with_overlap, hypothesis):
+    der_ovl = OverlappedDiarizationErrorRate()
+    der_regular = DiarizationErrorRate()
+
+    error_rate_ovl = der_ovl(reference_with_overlap, hypothesis)
+    error_rate_regular = der_regular(reference_with_overlap, hypothesis)
+
+    npt.assert_almost_equal(error_rate_ovl, error_rate_regular, decimal=7)
+
+
+def test_ovl_der_components(reference_with_overlap, hypothesis):
+    for collar in [0.0, 0.1, 0.5]:
+        der_ovl = OverlappedDiarizationErrorRate(collar=collar)
+        der_regular = DiarizationErrorRate(collar=collar)
+
+        comp_ovl: Details = der_ovl(reference_with_overlap, hypothesis, detailed=True)
+        comp_regular: Details = der_regular(
+            reference_with_overlap, hypothesis, detailed=True
+        )
+
+        print(comp_ovl)
+        print(comp_regular)
+
+        # test that for each component, the sum of non-overlapped and overlapped components is equal to the regular component
+        # eg check that ovl confusion+nonovl confusion = confusion
+        for component in der_regular.metric_components():
+            ovl_compsum = comp_ovl["nonovl " + component] + comp_ovl["ovl " + component]
+            reg_compsum = comp_regular[component]
+            npt.assert_almost_equal(ovl_compsum, reg_compsum, decimal=7)
+        # check there is overlapped and nonoverlapped speech
+        assert comp_ovl[f"{OVLDER_PREFIX_NONOVL} {MATCH_TOTAL}"] > 0.0
+        assert comp_ovl[f"{OVLDER_PREFIX_OVL} {MATCH_TOTAL}"] > 0.0
+
+
 def test_error_rate(reference, hypothesis):
     diarizationErrorRate = DiarizationErrorRate()
     error_rate = diarizationErrorRate(reference, hypothesis)