[CVAT] Add polygons support (#2722)

* [Exchange Oracle] Adapt generate_jwt_token to be reused during development

* [Exchange Oracle] Add polygon task type support

* [Recording Oracle] Add initial polygon task type support

* [Exchange Oracle] Fix get_auth_header in tests

* [Exchange Oracle] Fix test_cannot_register_400_with_duplicated_address

* [Recording Oracle] Fix linting issues

* [Recording Oracle] Update development patches for recording oracle

* [Recording Oracle] Add simple PolygonsDatasetComparator

Currently, it just computes bounding box from the polyhon shape and uses it for comparison.
This should be enough until we move to CVAT for validation.

* [Exchange Oracle] Update local manifests on each request

* [Recording Oracle] Fix linting

* [Recording Oracle] Apply ruff format

* [Exchange Oracle] Remove redundant session.commit()

* Delete debug.py

* [Exchange Oracle] Revert changes to `generate_jwt_token`

* [Exchange Oracle] Revert changes to .gitignore

Author: Bobronium

packages/examples/cvat/exchange-oracle/.gitignore

@@ -5,4 +5,3 @@ __pycache__/
 # Env file
 .env
 .python-version
-

packages/examples/cvat/exchange-oracle/src/core/types.py

@@ -44,12 +44,14 @@ class TaskTypes(str, Enum, metaclass=BetterEnumMeta):
     image_boxes = "image_boxes"
     image_boxes_from_points = "image_boxes_from_points"
     image_skeletons_from_boxes = "image_skeletons_from_boxes"
+    image_polygons = "image_polygons"
 
 
 class CvatLabelTypes(str, Enum, metaclass=BetterEnumMeta):
     tag = "tag"
     points = "points"
     rectangle = "rectangle"
+    polygon = "polygon"
 
 
 class OracleWebhookTypes(str, Enum, metaclass=BetterEnumMeta):

packages/examples/cvat/exchange-oracle/src/handlers/job_creation.py

@@ -48,6 +48,7 @@
     TaskTypes.image_label_binary: CvatLabelTypes.tag,
     TaskTypes.image_points: CvatLabelTypes.points,
     TaskTypes.image_boxes: CvatLabelTypes.rectangle,
+    TaskTypes.image_polygons: CvatLabelTypes.polygon,
     TaskTypes.image_boxes_from_points: CvatLabelTypes.rectangle,
     TaskTypes.image_skeletons_from_boxes: CvatLabelTypes.points,
 }
@@ -56,6 +57,7 @@
     TaskTypes.image_label_binary: "cvat_images",
     TaskTypes.image_points: "coco_person_keypoints",
     TaskTypes.image_boxes: "coco_instances",
+    TaskTypes.image_polygons: "coco_instances",
     TaskTypes.image_boxes_from_points: "coco_instances",
     TaskTypes.image_skeletons_from_boxes: "coco_person_keypoints",
 }
@@ -65,6 +67,7 @@
     TaskTypes.image_label_binary: "cvat_images",
     TaskTypes.image_points: "coco_instances",
     TaskTypes.image_boxes: "coco_instances",
+    TaskTypes.image_polygons: "coco_instances",
     TaskTypes.image_boxes_from_points: "coco_instances",
     TaskTypes.image_skeletons_from_boxes: "coco_person_keypoints",
 }
@@ -2499,6 +2502,7 @@ def create_task(escrow_address: str, chain_id: int) -> None:
         TaskTypes.image_boxes,
         TaskTypes.image_points,
         TaskTypes.image_label_binary,
+        TaskTypes.image_polygons,
     ]:
         builder_type = SimpleTaskBuilder
     elif manifest.annotation.type in [TaskTypes.image_boxes_from_points]:

packages/examples/cvat/exchange-oracle/src/handlers/job_export.py

@@ -26,6 +26,7 @@
     TaskTypes.image_label_binary: "CVAT for images 1.1",
     TaskTypes.image_points: "CVAT for images 1.1",
     TaskTypes.image_boxes: "COCO 1.0",
+    TaskTypes.image_polygons: "COCO 1.0",
     TaskTypes.image_boxes_from_points: "COCO 1.0",
     TaskTypes.image_skeletons_from_boxes: "CVAT for images 1.1",
 }
@@ -160,6 +161,10 @@ class _BoxesTaskProcessor(_TaskProcessor):
     pass
 
 
+class _PolygonsTaskProcessor(_TaskProcessor):
+    pass
+
+
 class _PointsTaskProcessor(_TaskProcessor):
     def _parse_dataset(self, ann_descriptor: FileDescriptor, dataset_dir: str) -> Dataset:
         annotation_utils.prepare_cvat_annotations_for_dm(dataset_dir)
@@ -586,6 +591,7 @@ def postprocess_annotations(
     processor_classes: dict[TaskTypes, type[_TaskProcessor]] = {
         TaskTypes.image_label_binary: _LabelsTaskProcessor,
         TaskTypes.image_boxes: _BoxesTaskProcessor,
+        TaskTypes.image_polygons: _PolygonsTaskProcessor,
         TaskTypes.image_points: _PointsTaskProcessor,
         TaskTypes.image_boxes_from_points: _BoxesFromPointsTaskProcessor,
         TaskTypes.image_skeletons_from_boxes: _SkeletonsFromBoxesTaskProcessor,

packages/examples/cvat/recording-oracle/src/core/types.py

@@ -13,6 +13,7 @@ class Networks(int, Enum):
 class TaskTypes(str, Enum, metaclass=BetterEnumMeta):
     image_label_binary = "image_label_binary"
     image_points = "image_points"
+    image_polygons = "image_polygons"
     image_boxes = "image_boxes"
     image_boxes_from_points = "image_boxes_from_points"
     image_skeletons_from_boxes = "image_skeletons_from_boxes"

packages/examples/cvat/recording-oracle/src/handlers/process_intermediate_results.py

@@ -31,6 +31,7 @@
     BboxDatasetComparator,
     DatasetComparator,
     PointsDatasetComparator,
+    PolygonsDatasetComparator,
     SkeletonDatasetComparator,
     TooFewGtError,
 )
@@ -42,6 +43,7 @@
 
 DM_DATASET_FORMAT_MAPPING = {
     TaskTypes.image_label_binary: "cvat_images",
+    TaskTypes.image_polygons: "coco_instances",
     TaskTypes.image_points: "coco_person_keypoints",
     TaskTypes.image_boxes: "coco_instances",
     TaskTypes.image_boxes_from_points: "coco_instances",
@@ -51,6 +53,7 @@
 DM_GT_DATASET_FORMAT_MAPPING = {
     TaskTypes.image_label_binary: "cvat_images",
     TaskTypes.image_points: "coco_instances",  # we compare points against boxes
+    TaskTypes.image_polygons: "coco_instances",
     TaskTypes.image_boxes: "coco_instances",
     TaskTypes.image_boxes_from_points: "coco_instances",
     TaskTypes.image_skeletons_from_boxes: "coco_person_keypoints",
@@ -60,6 +63,7 @@
 DATASET_COMPARATOR_TYPE_MAP: dict[TaskTypes, type[DatasetComparator]] = {
     # TaskType.image_label_binary: TagDatasetComparator, # TODO: implement if support is needed
     TaskTypes.image_boxes: BboxDatasetComparator,
+    TaskTypes.image_polygons: PolygonsDatasetComparator,
     TaskTypes.image_points: PointsDatasetComparator,
     TaskTypes.image_boxes_from_points: BboxDatasetComparator,
     TaskTypes.image_skeletons_from_boxes: SkeletonDatasetComparator,
@@ -199,7 +203,7 @@ def _validate_jobs(self):
             try:
                 job_mean_accuracy = comparator.compare(gt_dataset, job_dataset)
             except TooFewGtError as e:
-                job_results[job_cvat_id] = self.self.UNKNOWN_QUALITY
+                job_results[job_cvat_id] = self.UNKNOWN_QUALITY
                 rejected_jobs[job_cvat_id] = e
                 continue
 
@@ -271,7 +275,7 @@ def _put_gt_into_merged_dataset(
         """
 
         match manifest.annotation.type:
-            case TaskTypes.image_boxes.value:
+            case TaskTypes.image_boxes.value | TaskTypes.image_polygons.value:
                 merged_dataset.update(gt_dataset)
             case TaskTypes.image_points.value:
                 merged_label_cat: dm.LabelCategories = merged_dataset.categories()[
@@ -929,7 +933,12 @@ def process_intermediate_results(  # noqa: PLR0912
     # actually validate jobs
 
     task_type = manifest.annotation.type
-    if task_type in [TaskTypes.image_label_binary, TaskTypes.image_boxes, TaskTypes.image_points]:
+    if task_type in [
+        TaskTypes.image_label_binary,
+        TaskTypes.image_boxes,
+        TaskTypes.image_polygons,
+        TaskTypes.image_points,
+    ]:
         validator_type = _TaskValidator
     elif task_type == TaskTypes.image_boxes_from_points:
         validator_type = _BoxesFromPointsValidator

packages/examples/cvat/recording-oracle/src/validation/dataset_comparison.py

@@ -183,6 +183,29 @@ def compare_sample_annotations(
         return matching_result, similarity_fn
 
 
+class PolygonsDatasetComparator(DatasetComparator):
+    def compare_sample_annotations(
+        self, gt_sample: dm.DatasetItem, ds_sample: dm.DatasetItem, *, similarity_threshold: float
+    ) -> tuple[MatchResult, SimilarityFunction]:
+        similarity_fn = CachedSimilarityFunction(bbox_iou)
+
+        ds_boxes = [
+            Bbox(*a.get_bbox(), a.label) for a in ds_sample.annotations if isinstance(a, dm.Polygon)
+        ]
+        gt_boxes = [
+            Bbox(*a.get_bbox(), a.label) for a in gt_sample.annotations if isinstance(a, dm.Polygon)
+        ]
+
+        matching_result = match_annotations(
+            gt_boxes,
+            ds_boxes,
+            similarity=similarity_fn,
+            min_similarity=similarity_threshold,
+        )
+
+        return matching_result, similarity_fn
+
+
 _SkeletonInfo = list[str]