[CVAT] Update cvat api uses (#3348)

* Fix possible no assignments error when there are

* Update readme

* Update cvat sdk versions

* Address api changes

* Improve performance of some cvat calls

* Improve description

Author: zhiltsov-max

packages/examples/cvat/exchange-oracle/poetry.lock

@@ -16,7 +16,7 @@ sqlalchemy-utils = "^0.41.1"
 alembic = "^1.11.1"
 httpx = "^0.24.1"
 pytest = "^7.2.2"
-cvat-sdk = "2.31.0"
+cvat-sdk = "2.37.0"
 sqlalchemy = "^2.0.16"
 apscheduler = "^3.10.1"
 xmltodict = "^0.13.0"

packages/examples/cvat/exchange-oracle/pyproject.toml

@@ -77,6 +77,8 @@ CVAT_IOU_THRESHOLD=
 CVAT_OKS_SIGMA=
 CVAT_EXPORT_TIMEOUT=
 CVAT_IMPORT_TIMEOUT=
+CVAT_PROJECTS_PAGE_SIZE=
+CVAT_JOBS_PAGE_SIZE=
 
 # Storage Config (S3/GCS)
 

packages/examples/cvat/exchange-oracle/src/.env.template

@@ -146,7 +146,7 @@ class CronConfig:
     "Maximum number of downloading attempts per job or project during results downloading"
 
     track_completed_escrows_jobs_downloading_batch_size = int(
-        getenv("TRACK_COMPLETED_ESCROWS_JOBS_DOWNLOADING_BATCH_SIZE", 500)
+        getenv("TRACK_COMPLETED_ESCROWS_JOBS_DOWNLOADING_BATCH_SIZE", 10)
     )
     "Maximum number of parallel downloading requests during results downloading"
 
@@ -183,6 +183,9 @@ class CvatConfig:
     incoming_webhooks_url = getenv("CVAT_INCOMING_WEBHOOKS_URL")
     webhook_secret = getenv("CVAT_WEBHOOK_SECRET", "thisisasamplesecret")
 
+    projects_page_size = int(getenv("CVAT_PROJECTS_PAGE_SIZE", 100))
+    jobs_page_size = int(getenv("CVAT_JOBS_PAGE_SIZE", 100))
+
 
 class StorageConfig:
     provider: ClassVar[str] = os.environ["STORAGE_PROVIDER"].lower()

packages/examples/cvat/exchange-oracle/src/core/config.py

@@ -47,15 +47,23 @@ def _request_annotations(endpoint: Endpoint, cvat_id: int, format_name: str) ->
     _get_annotations(request_id, ...)
     """
 
-    (_, response) = endpoint.call_with_http_info(
-        id=cvat_id,
-        format=format_name,
-        save_images=False,
-        _parse_response=False,
-    )
+    try:
+        (_, response) = endpoint.call_with_http_info(
+            id=cvat_id,
+            format=format_name,
+            save_images=False,
+            _parse_response=False,
+        )
+
+        assert response.status in [HTTPStatus.ACCEPTED, HTTPStatus.CREATED]
+        rq_id = response.json()["rq_id"]
+    except exceptions.ApiException as e:
+        if e.status == HTTPStatus.CONFLICT:
+            rq_id = json.loads(e.body)["rq_id"]
+        else:
+            raise
 
-    assert response.status in [HTTPStatus.ACCEPTED, HTTPStatus.CREATED]
-    return response.json()["rq_id"]
+    return rq_id
 
 
 def _get_annotations(
@@ -462,6 +470,7 @@ def fetch_task_jobs(task_id: int) -> list[models.JobRead]:
                 api_client.jobs_api.list_endpoint,
                 task_id=task_id,
                 type="annotation",
+                page_size=Config.cvat_config.jobs_page_size,
             )
         except exceptions.ApiException as e:
             logger.exception(f"Exception when calling JobsApi.list: {e}\n")
@@ -535,6 +544,7 @@ def fetch_projects(assignee: str = "") -> list[models.ProjectRead]:
             return get_paginated_collection(
                 api_client.projects_api.list_endpoint,
                 **({"assignee": assignee} if assignee else {}),
+                page_size=Config.cvat_config.projects_page_size,
             )
         except exceptions.ApiException as e:
             logger.exception(f"Exception when calling ProjectsApi.list(): {e}\n")
@@ -711,6 +721,7 @@ def update_quality_control_settings(
     logger = logging.getLogger("app")
 
     params = {
+        "inherit": False,
         "max_validations_per_job": max_validations_per_job,
         "target_metric": target_metric,
         "target_metric_threshold": target_metric_threshold,

packages/examples/cvat/exchange-oracle/src/cvat/api_calls.py

@@ -24,7 +24,7 @@ google-cloud-storage = "^2.14.0"
 datumaro = {git = "https://github.com/cvat-ai/datumaro.git", rev = "ff83c00c2c1bc4b8fdfcc55067fcab0a9b5b6b11"}
 hexbytes = ">=1.2.0" # required for to_0x_hex() function
 starlette = ">=0.40.0" # avoid the vulnerability with multipart/form-data
-cvat-sdk = "2.31.0"
+cvat-sdk = "2.37.0"
 cryptography = "<44.0.0" # human-protocol-sdk -> pgpy dep requires cryptography < 45
 human-protocol-sdk = "^4.0.3"
 

packages/examples/cvat/recording-oracle/poetry.lock

@@ -61,6 +61,7 @@ CVAT_ADMIN_PASS=
 CVAT_ORG_SLUG=
 CVAT_QUALITY_RETRIEVAL_TIMEOUT=
 CVAT_QUALITY_CHECK_INTERVAL=
+CVAT_QUALITY_REPORTS_PAGE_SIZE=
 
 # Localhost
 

packages/examples/cvat/recording-oracle/pyproject.toml

@@ -195,6 +195,7 @@ class ValidationConfig:
     warmup_iterations = int(getenv("WARMUP_ITERATIONS", "1"))
     """
     The first escrow iterations where the annotation speed is checked to be big enough.
+    Set to 0 to disable.
     """
 
     min_warmup_progress = float(getenv("MIN_WARMUP_PROGRESS", "10"))
@@ -234,6 +235,7 @@ class CvatConfig:
 
     quality_retrieval_timeout = int(getenv("CVAT_QUALITY_RETRIEVAL_TIMEOUT", 60 * 60))
     quality_check_interval = int(getenv("CVAT_QUALITY_CHECK_INTERVAL", 5))
+    quality_reports_page_size = int(getenv("CVAT_QUALITY_REPORTS_PAGE_SIZE", 100))
 
 
 class Config:

packages/examples/cvat/recording-oracle/src/.env.template

@@ -134,7 +134,10 @@ def get_jobs_quality_reports(parent_id: int) -> list[models.QualityReport]:
     with get_api_client() as api_client:
         try:
             return get_paginated_collection(
-                api_client.quality_api.list_reports_endpoint, parent_id=parent_id, target="job"
+                api_client.quality_api.list_reports_endpoint,
+                parent_id=parent_id,
+                target="job",
+                page_size=Config.cvat_config.quality_reports_page_size,
             )
 
         except exceptions.ApiException as e: