Merge pull request #998 from scap3yvt/997_panoptica_integration

Added instance wise segmentation metrics

Author: sarthakpati

.devcontainer/Dockerfile

@@ -1,4 +1,4 @@
-FROM mcr.microsoft.com/devcontainers/python:3.9-bullseye
+FROM mcr.microsoft.com/devcontainers/python:3.11-bullseye
 
 # Copy environment.yml (if found) to a temp location so we update the environment. Also
 # copy "noop.txt" so the COPY instruction does not fail if no environment.yml exists.

.gitignore

@@ -4,6 +4,7 @@ __pycache__
 *.egg-info*
 */__pycache__/*
 .vscode
+.vscode/*
 *.py.*
 *.pkl
 *.swp

.spelling/.spelling/expect.txt

@@ -732,3 +732,7 @@ kwonly
 torchscript
 hann
 numcodecs
+ASSD
+listmetric
+panoptica
+RVAE

GANDLF/cli/generate_metrics.py

@@ -20,6 +20,7 @@
     mean_squared_log_error,
     mean_absolute_error,
     ncc_metrics,
+    generate_instance_segmentation,
 )
 from GANDLF.losses.segmentation import dice
 from GANDLF.metrics.segmentation import (
@@ -259,6 +260,26 @@ def generate_metrics_dict(
                     "volumeSimilarity_" + str(class_index)
                 ] = label_overlap_filter.GetVolumeSimilarity()
 
+    elif problem_type == "segmentation_brats":
+        for _, row in tqdm(input_df.iterrows(), total=input_df.shape[0]):
+            current_subject_id = row["SubjectID"]
+            overall_stats_dict[current_subject_id] = {}
+            label_image = torchio.LabelMap(row["Target"])
+            pred_image = torchio.LabelMap(row["Prediction"])
+            label_tensor = label_image.data
+            pred_tensor = pred_image.data
+            spacing = label_image.spacing
+            if label_tensor.data.shape[-1] == 1:
+                spacing = spacing[0:2]
+            # add dimension for batch
+            parameters["subject_spacing"] = torch.Tensor(spacing).unsqueeze(0)
+            label_array = label_tensor.unsqueeze(0).numpy()
+            pred_array = pred_tensor.unsqueeze(0).numpy()
+
+            overall_stats_dict[current_subject_id] = generate_instance_segmentation(
+                prediction=pred_array, target=label_array
+            )
+
     elif problem_type == "synthesis":
 
         def __fix_2d_tensor(input_tensor):

GANDLF/metrics/__init__.py

@@ -40,6 +40,7 @@
 )
 import GANDLF.metrics.classification as classification
 import GANDLF.metrics.regression as regression
+from .segmentation_panoptica import generate_instance_segmentation
 
 
 # global defines for the metrics

GANDLF/metrics/panoptica_config_brats.yaml

@@ -0,0 +1,50 @@
+!Panoptica_Evaluator
+decision_metric: null
+decision_threshold: null
+edge_case_handler: !EdgeCaseHandler
+  empty_list_std: !EdgeCaseResult NAN
+  listmetric_zeroTP_handling:
+    !Metric DSC: !MetricZeroTPEdgeCaseHandling {empty_prediction_result: !EdgeCaseResult ZERO,
+      empty_reference_result: !EdgeCaseResult ZERO, no_instances_result: !EdgeCaseResult NAN,
+      normal: !EdgeCaseResult ZERO}
+    !Metric clDSC: !MetricZeroTPEdgeCaseHandling {empty_prediction_result: !EdgeCaseResult ZERO,
+      empty_reference_result: !EdgeCaseResult ZERO, no_instances_result: !EdgeCaseResult NAN,
+      normal: !EdgeCaseResult ZERO}
+    !Metric IOU: !MetricZeroTPEdgeCaseHandling {empty_prediction_result: !EdgeCaseResult ZERO,
+      empty_reference_result: !EdgeCaseResult ZERO, no_instances_result: !EdgeCaseResult NAN,
+      normal: !EdgeCaseResult ZERO}
+    !Metric ASSD: !MetricZeroTPEdgeCaseHandling {empty_prediction_result: !EdgeCaseResult INF,
+      empty_reference_result: !EdgeCaseResult INF, no_instances_result: !EdgeCaseResult NAN,
+      normal: !EdgeCaseResult INF}
+    !Metric RVD: !MetricZeroTPEdgeCaseHandling {empty_prediction_result: !EdgeCaseResult NAN,
+      empty_reference_result: !EdgeCaseResult NAN, no_instances_result: !EdgeCaseResult NAN,
+      normal: !EdgeCaseResult NAN}
+    !Metric RVAE: !MetricZeroTPEdgeCaseHandling {empty_prediction_result: !EdgeCaseResult NAN,
+      empty_reference_result: !EdgeCaseResult NAN, no_instances_result: !EdgeCaseResult NAN,
+      normal: !EdgeCaseResult NAN}
+expected_input: !InputType SEMANTIC
+global_metrics: [!Metric DSC]
+instance_approximator: !ConnectedComponentsInstanceApproximator {cca_backend: null}
+instance_matcher: !NaiveThresholdMatching {allow_many_to_one: false, matching_metric: !Metric IOU,
+  matching_threshold: 0.5}
+instance_metrics: [!Metric DSC, !Metric IOU, !Metric ASSD, !Metric RVD]
+log_times: false
+save_group_times: false
+segmentation_class_groups: !SegmentationClassGroups
+  groups:
+    ed: !LabelGroup
+      single_instance: false
+      value_labels: [2]
+    et: !LabelGroup
+      single_instance: false
+      value_labels: [3]
+    net: !LabelGroup
+      single_instance: false
+      value_labels: [1]
+    tc: !LabelMergeGroup
+      single_instance: false
+      value_labels: [1, 3]
+    wt: !LabelMergeGroup
+      single_instance: false
+      value_labels: [1, 2, 3]
+verbose: false

GANDLF/metrics/segmentation_panoptica.py

@@ -0,0 +1,35 @@
+from pathlib import Path
+
+import numpy as np
+
+from panoptica import Panoptica_Evaluator
+
+
+def generate_instance_segmentation(
+    prediction: np.ndarray, target: np.ndarray, panoptica_config_path: str = None
+) -> dict:
+    """
+    Evaluate a single exam using Panoptica.
+
+    Args:
+        prediction (np.ndarray): The input prediction containing objects.
+        label_path (str): The path to the reference label.
+        panoptica_config_path (str): The path to the Panoptica configuration file.
+
+    Returns:
+        dict: The evaluation results.
+    """
+
+    cwd = Path(__file__).parent.absolute()
+    panoptica_config_path = (
+        cwd / "panoptica_config_path.yaml"
+        if panoptica_config_path is None
+        else panoptica_config_path
+    )
+    evaluator = Panoptica_Evaluator.load_from_config(panoptica_config_path)
+
+    # call evaluate
+    group2result = evaluator.evaluate(prediction_arr=prediction, reference_arr=target)
+
+    results = {k: r.to_dict() for k, r in group2result.items()}
+    return results

docs/usage.md

@@ -280,14 +280,29 @@ SubjectID,Target,Prediction
 ...
 ```
 
-To generate image to image metrics for synthesis tasks (including for the BraTS synthesis tasks [[1](https://www.synapse.org/#!Synapse:syn51156910/wiki/622356), [2](https://www.synapse.org/#!Synapse:syn51156910/wiki/622357)]), ensure that the config has `problem_type: synthesis`, and the CSV can be in the same format as segmentation (note that the `Mask` column is optional):
+### Special cases 
 
-```csv
-SubjectID,Target,Prediction,Mask
-001,/path/to/001/target_image.nii.gz,/path/to/001/prediction_image.nii.gz,/path/to/001/brain_mask.nii.gz
-002,/path/to/002/target_image.nii.gz,/path/to/002/prediction_image.nii.gz,/path/to/002/brain_mask.nii.gz
-...
-```
+  1. BraTS Segmentation Metrics
+
+  To generate annotation to annotation metrics for BraTS segmentation tasks [[ref](https://www.synapse.org/brats)], ensure that the config has `problem_type: segmentation_brats`, and the CSV can be in the same format as segmentation:
+
+  ```csv
+  SubjectID,Target,Prediction
+  001,/path/to/001/target_image.nii.gz,/path/to/001/prediction_image.nii.gz
+  002,/path/to/002/target_image.nii.gz,/path/to/002/prediction_image.nii.gz
+  ...
+  ```
+
+  2. BraTS Synthesis Metrics
+
+  To generate image to image metrics for synthesis tasks (including for the BraTS synthesis tasks [[1](https://www.synapse.org/#!Synapse:syn51156910/wiki/622356), [2](https://www.synapse.org/#!Synapse:syn51156910/wiki/622357)]), ensure that the config has `problem_type: synthesis`, and the CSV can be in the same format as segmentation (note that the `Mask` column is optional):
+
+  ```csv
+  SubjectID,Target,Prediction,Mask
+  001,/path/to/001/target_image.nii.gz,/path/to/001/prediction_image.nii.gz,/path/to/001/brain_mask.nii.gz
+  002,/path/to/002/target_image.nii.gz,/path/to/002/prediction_image.nii.gz,/path/to/002/brain_mask.nii.gz
+  ...
+  ```
 
 
 ## Parallelize the Training

setup.py

@@ -4,8 +4,6 @@
 
 
 import sys, re, os
-
-
 from setuptools import setup, find_packages
 
 
@@ -33,7 +31,8 @@
 ]
 
 # Any extra files should be located at `GANDLF` module folder (not in repo root)
-extra_files = ["logging_config.yaml"]
+extra_files_root = ["logging_config.yaml"]
+extra_files_metrics = ["panoptica_config_brats.yaml"]
 toplevel_package_excludes = ["testing*"]
 
 # specifying version for `black` separately because it is also used to [check for lint](https://github.com/mlcommons/GaNDLF/blob/master/.github/workflows/black.yml)
@@ -88,6 +87,7 @@
     "openslide-python==1.4.1",
     "lion-pytorch==0.2.2",
     "pydantic==2.10.6",
+    "panoptica>=1.3.3",
 ]
 
 if __name__ == "__main__":
@@ -140,7 +140,10 @@
         long_description=readme,
         long_description_content_type="text/markdown",
         include_package_data=True,
-        package_data={"GANDLF": extra_files},
+        package_data={
+            "GANDLF": extra_files_root,
+            "GANDLF.metrics": extra_files_metrics,
+        },
         keywords="semantic, segmentation, regression, classification, data-augmentation, medical-imaging, clinical-workflows, deep-learning, pytorch",
         zip_safe=False,
     )

testing/test_full.py

@@ -3143,6 +3143,45 @@ def test_generic_cli_function_metrics_cli_rad_nd():
 
             sanitize_outputDir()
 
+    # this is for the brats segmentation metrics test
+    problem_type = "segmentation_brats"
+    reference_image_file = os.path.join(
+        inputDir, "metrics", "brats", "reference.nii.gz"
+    )
+    prediction_image_file = os.path.join(
+        inputDir, "metrics", "brats", "prediction.nii.gz"
+    )
+    subject_id = "brats_subject_1"
+    # write to a temporary CSV file
+    df = pd.DataFrame(
+        {
+            "SubjectID": [subject_id],
+            "Prediction": [prediction_image_file],
+            "Target": [reference_image_file],
+        }
+    )
+    temp_infer_csv = os.path.join(outputDir, "temp_csv.csv")
+    df.to_csv(temp_infer_csv, index=False)
+
+    # read and initialize parameters for specific data dimension
+    parameters = ConfigManager(
+        testingDir + "/config_segmentation.yaml", version_check_flag=False
+    )
+    parameters["modality"] = "rad"
+    parameters["patch_size"] = patch_size["3D"]
+    parameters["model"]["dimension"] = 3
+    parameters["verbose"] = False
+    temp_config = write_temp_config_path(parameters)
+
+    output_file = os.path.join(outputDir, "output_single-csv.json")
+    generate_metrics_dict(temp_infer_csv, temp_config, output_file)
+
+    assert os.path.isfile(
+        output_file
+    ), "Metrics output file was not generated for single-csv input"
+
+    sanitize_outputDir()
+
 
 # def test_generic_deploy_metrics_docker():
 #     print("50: Testing deployment of a metrics generator to Docker")