Update recording oracle

Author: Marishka17

packages/examples/cvat/recording-oracle/alembic/versions/9d4367899f90_recreate_gt_stats.py

@@ -21,19 +21,19 @@ def upgrade() -> None:
     # ### commands auto generated by Alembic - please adjust! ###
     op.drop_index("ix_gt_stats_gt_key", table_name="gt_stats")
     op.drop_table("gt_stats")
-    # ### end Alembic commands ###
 
     op.create_table(
         "gt_stats",
-        sa.Column("task_id", sa.VARCHAR(), autoincrement=False, nullable=False),
-        sa.Column("cvat_task_id", sa.INTEGER(), autoincrement=False, nullable=False),
-        sa.Column("gt_frame_id", sa.INTEGER(), autoincrement=False, nullable=False),
-        sa.Column("failed_attempts", sa.INTEGER(), autoincrement=False, nullable=False),
-        sa.ForeignKeyConstraint(
-            ["task_id"], ["tasks.id"], name="gt_stats_task_id_fkey", ondelete="CASCADE"
-        ),
-        sa.PrimaryKeyConstraint("task_id", "gt_frame_id", name="gt_stats_pkey"),
+        sa.Column("task_id", sa.String(), nullable=False),
+        sa.Column("cvat_task_id", sa.Integer(), nullable=False),
+        sa.Column("gt_frame_id", sa.Integer(), nullable=False),
+        sa.Column("failed_attempts", sa.Integer(), nullable=False),
+        sa.Column("accepted_attempts", sa.Integer(), nullable=False),
+        sa.Column("accumulated_quality", sa.Float(), nullable=False),
+        sa.ForeignKeyConstraint(["task_id"], ["tasks.id"], ondelete="CASCADE"),
+        sa.PrimaryKeyConstraint("task_id", "gt_frame_id"),
     )
+    # ### end Alembic commands ###
 
 
 def downgrade() -> None:

packages/examples/cvat/recording-oracle/src/.env.template

@@ -78,5 +78,7 @@ PGP_PRIVATE_KEY=
 PGP_PASSPHRASE=
 PGP_PUBLIC_KEY_URL=
 
+CVAT_URL=
 CVAT_ADMIN=
-CVAT_ADMIN_PASS=
+CVAT_ADMIN_PASS=
+CVAT_ORG_SLUG=

packages/examples/cvat/recording-oracle/src/core/annotation_meta.py

@@ -1,3 +1,4 @@
+from collections.abc import Iterator
 from pathlib import Path
 
 from pydantic import BaseModel
@@ -12,7 +13,19 @@ class JobMeta(BaseModel):
     annotation_filename: Path
     annotator_wallet_address: str
     assignment_id: str
+    start_frame: int
+    stop_frame: int
+
+    @property
+    def job_frame_range(self) -> Iterator[int]:
+        return range(self.start_frame, self.stop_frame + 1)
 
 
 class AnnotationMeta(BaseModel):
     jobs: list[JobMeta]
+
+    def skip_jobs(self, job_ids: list[int]):
+        # self.jobs = [
+        #     job for job in self.jobs if job.job_id not in job_ids
+        # ]
+        return AnnotationMeta(jobs=[job for job in self.jobs if job.job_id not in job_ids])

packages/examples/cvat/recording-oracle/src/core/gt_stats.py

@@ -0,0 +1,17 @@
+from attrs import define
+
+
+@define(kw_only=True)
+class ValidationFrameStats:
+    accumulated_quality: float = 0.0
+    failed_attempts: int = 0
+    accepted_attempts: int = 0
+
+    @property
+    def average_quality(self) -> float:
+        return self.accumulated_quality / ((self.failed_attempts + self.accepted_attempts) or 1)
+
+
+_TaskIdValFrameIdPair = tuple[int, int]
+
+GtStats = dict[_TaskIdValFrameIdPair, ValidationFrameStats]

packages/examples/cvat/recording-oracle/src/cvat/api_calls.py

@@ -127,14 +127,13 @@ def get_task_validation_layout(task_id: int) -> models.TaskValidationLayoutRead:
             raise
 
 
-def get_jobs_quality_reports(parent_id: int) -> dict[int, models.QualityReport]:
+def get_jobs_quality_reports(parent_id: int) -> list[models.QualityReport]:
     logger = logging.getLogger("app")
     with get_api_client() as api_client:
         try:
-            reports: list[models.QualityReport] = get_paginated_collection(
+            return get_paginated_collection(
                 api_client.quality_api.list_reports_endpoint, parent_id=parent_id, target="job"
             )
-            return {report.job_id: report for report in reports}
 
         except exceptions.ApiException as e:
             logger.exception(f"Exception when calling QualityApi.list_reports: {e}\n")
@@ -145,21 +144,18 @@ def update_task_validation_layout(
     task_id: int,
     *,
     disabled_frames: list[int],
-    shuffle_honeypots: bool = True,
+    honeypot_real_frames: list[int],
 ) -> None:
     logger = logging.getLogger("app")
-    params = {
-        "disabled_frames": disabled_frames,
-    }
-    if shuffle_honeypots:
-        params["frame_selection_method"] = models.FrameSelectionMethod("random_uniform")
 
     with get_api_client() as api_client:
         try:
             validation_layout, _ = api_client.tasks_api.partial_update_validation_layout(
                 task_id,
                 patched_task_validation_layout_write_request=models.PatchedTaskValidationLayoutWriteRequest(
-                    **params
+                    disabled_frames=disabled_frames,
+                    honeypot_real_frames=honeypot_real_frames,
+                    frame_selection_method=models.FrameSelectionMethod("manual"),
                 ),
             )
         except exceptions.ApiException as ex: