Full cycle E2E test (#957)

Author: JBWilkie

e2e_tests/cli/test_full_cycle.py

@@ -0,0 +1,105 @@
+import shutil
+from pathlib import Path
+
+from e2e_tests.helpers import assert_cli, run_cli_command, export_release
+from e2e_tests.objects import E2EDataset, ConfigValues
+from e2e_tests.cli.test_import import compare_annotations_export
+
+
+def test_full_cycle(
+    local_dataset: E2EDataset,
+    config_values: ConfigValues,
+):
+    """
+    This test performs the following steps:
+    - 1: Registers a set of files from external storage to a dataset
+    - 2: Imports some annotations
+    - 3: Creates and pulls a release of the dataset
+    - 4: Deletes all items from the dataset
+    - 5: Pushes and imports the pulled files & annotations to the dataset
+    - 6: Deletes locally pulled copies of the dataset files
+    - 7: Creates and pulls a new release of the dataset
+    - 8: Assert that the pulled data is as expected
+
+    It is designed to catch errors that may arise from changes to exported Darwin JSON
+    """
+    item_type = "single_slotted"
+    annotation_format = "darwin"
+    first_release_name = "first_release"
+    second_release_name = "second_release"
+    pull_dir = Path(
+        f"{Path.home()}/.darwin/datasets/{config_values.team_slug}/{local_dataset.slug}"
+    )
+    annotations_import_dir = (
+        Path(__file__).parents[1]
+        / "data"
+        / "import"
+        / "image_annotations_with_item_level_properties"
+    )
+    expected_filepaths = [
+        f"{pull_dir}/images/image_1.jpg",
+        f"{pull_dir}/images/image_2.jpg",
+        f"{pull_dir}/images/dir1/image_3.jpg",
+        f"{pull_dir}/images/dir1/image_4.jpg",
+        f"{pull_dir}/images/dir2/image_5.jpg",
+        f"{pull_dir}/images/dir2/image_6.jpg",
+        f"{pull_dir}/images/dir1/dir3/image_7.jpg",
+        f"{pull_dir}/images/dir1/dir3/image_8.jpg",
+    ]
+
+    # Populate the dataset with items and annotations
+    local_dataset.register_read_only_items(config_values, item_type)
+    result = run_cli_command(
+        f"darwin dataset import {local_dataset.name} {annotation_format} {annotations_import_dir}"
+    )
+    assert_cli(result, 0)
+
+    # Pull a first release of the dataset
+    original_release = export_release(
+        annotation_format, local_dataset, config_values, release_name=first_release_name
+    )
+    result = run_cli_command(
+        f"darwin dataset pull {local_dataset.name}:{original_release.name}"
+    )
+    assert_cli(result, 0)
+
+    # Delete all items in the dataset
+    local_dataset.delete_items(config_values)
+
+    # Push and import the pulled files and annotations to the dataset
+    result = run_cli_command(
+        f"darwin dataset push {local_dataset.name} {pull_dir}/images --preserve-folders"
+    )
+    assert_cli(result, 0)
+    result = run_cli_command(
+        f"darwin dataset import {local_dataset.name} {annotation_format} {pull_dir}/releases/{first_release_name}/annotations"
+    )
+    assert_cli(result, 0)
+
+    # Delete local copies of the dataset files for the dataset
+    shutil.rmtree(f"{pull_dir}/images")
+
+    # Pull a second release of the dataset
+    new_release = export_release(
+        annotation_format,
+        local_dataset,
+        config_values,
+        release_name=second_release_name,
+    )
+    result = run_cli_command(
+        f"darwin dataset pull {local_dataset.name}:{new_release.name}"
+    )
+    assert_cli(result, 0)
+
+    # Check that all expected files have been downloaded
+    all_filepaths = list(pull_dir.rglob("*"))
+    for expected_file in expected_filepaths:
+        assert Path(expected_file) in all_filepaths
+
+    # Check that all downloaded annotations are as expected
+    compare_annotations_export(
+        Path(f"{pull_dir}/releases/{first_release_name}/annotations"),
+        Path(f"{pull_dir}/releases/{second_release_name}/annotations"),
+        item_type,
+        unzip=False,
+    )

e2e_tests/cli/test_import.py

@@ -226,13 +226,15 @@ def compare_annotations_export(
     item_type: str,
     base_slot: Optional[str] = "0",
     annotation_format: str = "darwin",
+    unzip: Optional[bool] = True,
 ):
     """
     Compares a set of downloaded annotation files with the imported files that resulted
     in those annotations. Ensures equality
     """
-    with zipfile.ZipFile(actual_annotations_dir / "dataset.zip") as z:
-        z.extractall(actual_annotations_dir)
+    if unzip:
+        with zipfile.ZipFile(actual_annotations_dir / "dataset.zip") as z:
+            z.extractall(actual_annotations_dir)
 
     file_prefixes_to_ignore = [".", "metadata.json"]
     expected_annotation_files = {

e2e_tests/cli/test_pull.py

@@ -16,14 +16,14 @@ def test_pull_with_remote_folder_structure(
         f"{Path.home()}/.darwin/datasets/{config_values.team_slug}/{local_dataset.slug}/images"
     )
     expected_filepaths = [
-        f"{pull_dir}/image_1",
-        f"{pull_dir}/image_2",
-        f"{pull_dir}/dir1/image_3",
-        f"{pull_dir}/dir1/image_4",
-        f"{pull_dir}/dir2/image_5",
-        f"{pull_dir}/dir2/image_6",
-        f"{pull_dir}/dir1/dir3/image_7",
-        f"{pull_dir}/dir1/dir3/image_8",
+        f"{pull_dir}/image_1.jpg",
+        f"{pull_dir}/image_2.jpg",
+        f"{pull_dir}/dir1/image_3.jpg",
+        f"{pull_dir}/dir1/image_4.jpg",
+        f"{pull_dir}/dir2/image_5.jpg",
+        f"{pull_dir}/dir2/image_6.jpg",
+        f"{pull_dir}/dir1/dir3/image_7.jpg",
+        f"{pull_dir}/dir1/dir3/image_8.jpg",
     ]
     item_type = "single_slotted"
     annotation_format = "darwin"

e2e_tests/data/import/coco_annotations/output.json

@@ -17,7 +17,7 @@
   "images": [
     {
       "license": 0,
-      "file_name": "image_1",
+      "file_name": "image_1.jpg",
       "coco_url": "n/a",
       "height": 1080,
       "width": 1920,
@@ -30,7 +30,7 @@
     },
     {
       "license": 0,
-      "file_name": "image_2",
+      "file_name": "image_2.jpg",
       "coco_url": "n/a",
       "height": 1080,
       "width": 1920,
@@ -43,7 +43,7 @@
     },
     {
       "license": 0,
-      "file_name": "image_3",
+      "file_name": "image_3.jpg",
       "coco_url": "n/a",
       "height": 1080,
       "width": 1920,
@@ -56,7 +56,7 @@
     },
     {
       "license": 0,
-      "file_name": "image_4",
+      "file_name": "image_4.jpg",
       "coco_url": "n/a",
       "height": 1080,
       "width": 1920,
@@ -69,7 +69,7 @@
     },
     {
       "license": 0,
-      "file_name": "image_5",
+      "file_name": "image_5.jpg",
       "coco_url": "n/a",
       "height": 1080,
       "width": 1920,
@@ -82,7 +82,7 @@
     },
     {
       "license": 0,
-      "file_name": "image_6",
+      "file_name": "image_6.jpg",
       "coco_url": "n/a",
       "height": 1080,
       "width": 1920,
@@ -95,7 +95,7 @@
     },
     {
       "license": 0,
-      "file_name": "image_7",
+      "file_name": "image_7.jpg",
       "coco_url": "n/a",
       "height": 1080,
       "width": 1920,
@@ -108,7 +108,7 @@
     },
     {
       "license": 0,
-      "file_name": "image_8",
+      "file_name": "image_8.jpg",
       "coco_url": "n/a",
       "height": 1080,
       "width": 1920,

e2e_tests/data/import/csv_tag_annotations/csv_tags.csv

@@ -1,8 +1,8 @@
-image_1, test_tag_basic
-image_2, test_tag_basic
-dir1/image_3, test_tag_basic
-dir1/image_4, test_tag_basic
-dir2/image_5, test_tag_basic
-dir2/image_6, test_tag_basic
-dir1/dir3/image_7, test_tag_basic
-dir1/dir3/image_8, test_tag_basic
+image_1.jpg, test_tag_basic
+image_2.jpg, test_tag_basic
+dir1/image_3.jpg, test_tag_basic
+dir1/image_4.jpg, test_tag_basic
+dir2/image_5.jpg, test_tag_basic
+dir2/image_6.jpg, test_tag_basic
+dir1/dir3/image_7.jpg, test_tag_basic
+dir1/dir3/image_8.jpg, test_tag_basic

e2e_tests/data/import/image_annotations_item_level_properties_no_annotations/image_1.json

@@ -2,7 +2,7 @@
   "version": "2.0",
   "schema_ref": "https://darwin-public.s3.eu-west-1.amazonaws.com/darwin_json/2.0/schema.json",
   "item": {
-    "name": "image_1",
+    "name": "image_1.jpg",
     "path": "/",
     "source_info": {
       "item_id": "01920b92-1d5d-94a4-6fbe-8a4d7f9fa15d",
@@ -26,7 +26,7 @@
         "thumbnail_url": "https://staging.v7labs.com/api/v2/teams/e2e-testing/files/2ec69e41-91b2-4155-9b05-6ed995677b1e/thumbnail",
         "source_files": [
           {
-            "file_name": "image_1",
+            "file_name": "image_1.jpg",
             "storage_key": "darwin-py/images/image_1.jpg",
             "url": "https://staging.v7labs.com/api/v2/teams/e2e-testing/uploads/9dfc5eac-bf16-4380-a148-9fff6e63b9f0"
           }

e2e_tests/data/import/image_annotations_item_level_properties_no_annotations/image_2.json

@@ -2,7 +2,7 @@
   "version": "2.0",
   "schema_ref": "https://darwin-public.s3.eu-west-1.amazonaws.com/darwin_json/2.0/schema.json",
   "item": {
-    "name": "image_2",
+    "name": "image_2.jpg",
     "path": "/",
     "source_info": {
       "item_id": "01920b92-1d5d-ea77-8fa4-16378bafedb3",
@@ -26,7 +26,7 @@
         "thumbnail_url": "https://staging.v7labs.com/api/v2/teams/e2e-testing/files/5e0b3d9d-9bf8-4166-8949-6ab7392161ad/thumbnail",
         "source_files": [
           {
-            "file_name": "image_2",
+            "file_name": "image_2.jpg",
             "storage_key": "darwin-py/images/image_2.jpg",
             "url": "https://staging.v7labs.com/api/v2/teams/e2e-testing/uploads/4920b12a-1706-47f1-b084-2d2234ed1151"
           }

e2e_tests/data/import/image_annotations_item_level_properties_no_annotations/image_3.json

@@ -2,7 +2,7 @@
   "version": "2.0",
   "schema_ref": "https://darwin-public.s3.eu-west-1.amazonaws.com/darwin_json/2.0/schema.json",
   "item": {
-    "name": "image_3",
+    "name": "image_3.jpg",
     "path": "/dir1",
     "source_info": {
       "item_id": "01920b92-1d5d-e8ad-986f-ad4942f1bbfc",
@@ -26,7 +26,7 @@
         "thumbnail_url": "https://staging.v7labs.com/api/v2/teams/e2e-testing/files/ddd13905-9bbb-4fab-9642-bf4604686fda/thumbnail",
         "source_files": [
           {
-            "file_name": "image_3",
+            "file_name": "image_3.jpg",
             "storage_key": "darwin-py/images/image_3.jpg",
             "url": "https://staging.v7labs.com/api/v2/teams/e2e-testing/uploads/30ec0f13-caaa-4374-be5a-e90b3493fb73"
           }

e2e_tests/data/import/image_annotations_item_level_properties_no_annotations/image_4.json

@@ -2,7 +2,7 @@
   "version": "2.0",
   "schema_ref": "https://darwin-public.s3.eu-west-1.amazonaws.com/darwin_json/2.0/schema.json",
   "item": {
-    "name": "image_4",
+    "name": "image_4.jpg",
     "path": "/dir1",
     "source_info": {
       "item_id": "01920b92-1d5d-8b50-17e9-c0f178e6eee6",
@@ -26,7 +26,7 @@
         "thumbnail_url": "https://staging.v7labs.com/api/v2/teams/e2e-testing/files/3c731d84-7d7f-4ac8-bbd9-0d53f1d47195/thumbnail",
         "source_files": [
           {
-            "file_name": "image_4",
+            "file_name": "image_4.jpg",
             "storage_key": "darwin-py/images/image_4.jpg",
             "url": "https://staging.v7labs.com/api/v2/teams/e2e-testing/uploads/609ba1a4-79da-4743-b331-e57ccd9ee518"
           }

e2e_tests/data/import/image_annotations_item_level_properties_no_annotations/image_5.json

@@ -2,7 +2,7 @@
   "version": "2.0",
   "schema_ref": "https://darwin-public.s3.eu-west-1.amazonaws.com/darwin_json/2.0/schema.json",
   "item": {
-    "name": "image_5",
+    "name": "image_5.jpg",
     "path": "/dir2",
     "source_info": {
       "item_id": "01920b92-1d5d-55bf-d705-8b39dea7fde6",
@@ -26,7 +26,7 @@
         "thumbnail_url": "https://staging.v7labs.com/api/v2/teams/e2e-testing/files/8f95e81c-def7-4973-9152-6d0fc39e1473/thumbnail",
         "source_files": [
           {
-            "file_name": "image_5",
+            "file_name": "image_5.jpg",
             "storage_key": "darwin-py/images/image_5.jpg",
             "url": "https://staging.v7labs.com/api/v2/teams/e2e-testing/uploads/08448a07-4e23-41f9-abbd-0dc149ef2be4"
           }