Add 2nd layer of SDK (#19)

Author: zhiltsov-max

.coveragerc

@@ -4,6 +4,7 @@ branch = true
 
 source =
     cvat/apps/
+    cvat-sdk/
     cvat-cli/
     utils/dataset_manifest
 

.github/workflows/black.yml

@@ -14,16 +14,47 @@ jobs:
           PR_FILES_AM: ${{ steps.files.outputs.added_modified }}
           PR_FILES_RENAMED: ${{ steps.files.outputs.renamed }}
         run: |
+          # If different modules use different Black configs,
+          # we need to run Black for each python component group separately.
+          # Otherwise, they all will use the same config.
+          ENABLED_DIRS=("cvat-sdk" "cvat-cli" "tests/python/sdk" "tests/python/cli")
+
+          isValueIn () {
+            # Checks if a value is in an array
+            # https://stackoverflow.com/a/8574392
+            # args: value, array
+            local e match="$1"
+            shift
+            for e; do
+              [[ "$e" == "$match" ]] && return 0;
+            done
+            return 1
+          }
+
+          startswith () {
+            # Inspired by https://stackoverflow.com/a/2172367
+            # Checks if the first arg starts with the second one
+            local value="$1"
+            local beginning="$2"
+            return $([[ $value == ${beginning}* ]])
+          }
+
           PR_FILES="$PR_FILES_AM $PR_FILES_RENAMED"
+          UPDATED_DIRS=""
           for FILE in $PR_FILES; do
               EXTENSION="${FILE##*.}"
-              DIRECTORY="${FILE%%/*}"
-              if [[ "$EXTENSION" == "py" && "$DIRECTORY" == "cvat-cli" ]]; then
-                  CHANGED_FILES+=" $FILE"
+              DIRECTORY="$(dirname $FILE)"
+              if [[ "$EXTENSION" == "py" ]]; then
+                  for EDIR in ${ENABLED_DIRS[@]}; do
+                      if startswith "${DIRECTORY}/" "${EDIR}/" && ! isValueIn "${EDIR}" ${UPDATED_DIRS[@]};
+                      then
+                          UPDATED_DIRS+=" ${EDIR}"
+                      fi
+                  done
               fi
           done
 
-          if [[ ! -z $CHANGED_FILES ]]; then
+          if [[ ! -z $UPDATED_DIRS ]]; then
             sudo apt-get --no-install-recommends install -y build-essential curl python3-dev python3-pip python3-venv
             python3 -m venv .env
             . .env/bin/activate
@@ -32,8 +63,11 @@ jobs:
             mkdir -p black_report
 
             echo "Black version: "$(black --version)
-            echo "The files will be checked: "$(echo $CHANGED_FILES)
-            black --check --config ./cvat-cli/pyproject.toml $CHANGED_FILES > ./black_report/black_checks.txt || EXIT_CODE=$(echo $?) || true
+            echo "The dirs will be checked: $UPDATED_DIRS"
+            EXIT_CODE=0
+            for DIR in $UPDATED_DIRS; do
+              black --check $DIR >> ./black_report/black_checks.txt || EXIT_CODE=$(($? | $EXIT_CODE)) || true
+            done
             deactivate
             exit $EXIT_CODE
           else

.github/workflows/full.yml

@@ -164,11 +164,20 @@ jobs:
           docker load --input /tmp/cvat_ui/image.tar
           docker image ls -a
 
-      - name: Running REST API tests
+      - name: Running REST API and SDK tests
         run: |
+          docker run --rm -v ${PWD}/cvat-sdk/schema/:/transfer \
+            --entrypoint /bin/bash -u root cvat/server \
+            -c 'python manage.py spectacular --file /transfer/schema.yml'
+          pip3 install --user -r cvat-sdk/gen/requirements.txt
+          cd cvat-sdk/
+          gen/generate.sh
+          cd ..
+
           pip3 install --user cvat-sdk/
-          pip3 install --user -r tests/rest_api/requirements.txt
-          pytest tests/rest_api/ -s -v
+          pip3 install --user cvat-cli/
+          pip3 install --user -r tests/python/requirements.txt
+          pytest tests/python -s -v
 
       - name: Creating a log file from cvat containers
         if: failure()
@@ -226,7 +235,7 @@ jobs:
           while [[  $(curl -s -o /dev/null -w "%{http_code}" localhost:8181/health) != "200" && max_tries -gt 0 ]]; do (( max_tries-- )); sleep 5; done
 
           docker-compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.ci.yml run cvat_ci /bin/bash \
-            -c 'python manage.py test cvat/apps cvat-cli -v 2'
+            -c 'python manage.py test cvat/apps -v 2'
 
           docker-compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.ci.yml run cvat_ci /bin/bash \
             -c 'yarn --frozen-lockfile --ignore-scripts && yarn workspace cvat-core run test'

.github/workflows/isort.yml

@@ -14,16 +14,47 @@ jobs:
           PR_FILES_AM: ${{ steps.files.outputs.added_modified }}
           PR_FILES_RENAMED: ${{ steps.files.outputs.renamed }}
         run: |
+          # If different modules use different isort configs,
+          # we need to run isort for each python component group separately.
+          # Otherwise, they all will use the same config.
+          ENABLED_DIRS=("cvat-sdk" "cvat-cli" "tests/python/sdk" "tests/python/cli")
+
+          isValueIn () {
+            # Checks if a value is in an array
+            # https://stackoverflow.com/a/8574392
+            # args: value, array
+            local e match="$1"
+            shift
+            for e; do
+              [[ "$e" == "$match" ]] && return 0;
+            done
+            return 1
+          }
+
+          startswith () {
+            # Inspired by https://stackoverflow.com/a/2172367
+            # Checks if the first arg starts with the second one
+            local value="$1"
+            local beginning="$2"
+            return $([[ $value == ${beginning}* ]])
+          }
+
           PR_FILES="$PR_FILES_AM $PR_FILES_RENAMED"
+          UPDATED_DIRS=""
           for FILE in $PR_FILES; do
               EXTENSION="${FILE##*.}"
-              DIRECTORY="${FILE%%/*}"
-              if [[ "$EXTENSION" == "py" && "$DIRECTORY" == "cvat-cli" ]]; then
-                  CHANGED_FILES+=" $FILE"
+              DIRECTORY="$(dirname $FILE)"
+              if [[ "$EXTENSION" == "py" ]]; then
+                  for EDIR in ${ENABLED_DIRS[@]}; do
+                      if startswith "${DIRECTORY}/" "${EDIR}/" && ! isValueIn "${EDIR}" ${UPDATED_DIRS[@]};
+                      then
+                          UPDATED_DIRS+=" ${EDIR}"
+                      fi
+                  done
               fi
           done
 
-          if [[ ! -z $CHANGED_FILES ]]; then
+          if [[ ! -z $UPDATED_DIRS ]]; then
             sudo apt-get --no-install-recommends install -y build-essential curl python3-dev python3-pip python3-venv
             python3 -m venv .env
             . .env/bin/activate
@@ -32,8 +63,11 @@ jobs:
             mkdir -p isort_report
 
             echo "isort version: "$(isort --version)
-            echo "The files will be checked: "$(echo $CHANGED_FILES)
-            isort --check --sp ./cvat-cli/pyproject.toml $CHANGED_FILES > ./isort_report/isort_checks.txt || EXIT_CODE=$(echo $?) || true
+            echo "The dirs will be checked: $UPDATED_DIRS"
+            EXIT_CODE=0
+            for DIR in $UPDATED_DIRS; do
+              isort --check $DIR >> ./isort_report/isort_checks.txt || EXIT_CODE=$(($? | $EXIT_CODE)) || true
+            done
             deactivate
             exit $EXIT_CODE
           else

.github/workflows/main.yml

@@ -133,9 +133,17 @@ jobs:
 
       - name: Running REST API tests
         run: |
+          docker run --rm -v ${PWD}/cvat-sdk/schema/:/transfer \
+            --entrypoint /bin/bash -u root cvat/server \
+            -c 'python manage.py spectacular --file /transfer/schema.yml'
+          pip3 install --user -r cvat-sdk/gen/requirements.txt
+          cd cvat-sdk/
+          gen/generate.sh
+          cd ..
+
           pip3 install --user cvat-sdk/
-          pip3 install --user -r tests/rest_api/requirements.txt
-          pytest tests/rest_api/ -k 'GET' -s
+          pip3 install --user -r tests/python/requirements.txt
+          pytest tests/python/rest_api -k 'GET' -s
 
       - name: Creating a log file from cvat containers
         if: failure()
@@ -192,7 +200,7 @@ jobs:
           while [[  $(curl -s -o /dev/null -w "%{http_code}" localhost:8181/health) != "200" && max_tries -gt 0 ]]; do (( max_tries-- )); sleep 5; done
 
           docker-compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.ci.yml run cvat_ci /bin/bash \
-          -c 'python manage.py test cvat/apps cvat-cli -k tasks_id -k lambda -k share -v 2'
+          -c 'python manage.py test cvat/apps -k tasks_id -k lambda -k share -v 2'
 
           docker-compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.ci.yml run cvat_ci /bin/bash \
           -c 'yarn --frozen-lockfile --ignore-scripts && yarn workspace cvat-core run test'

.github/workflows/pylint.yml

@@ -17,8 +17,7 @@ jobs:
           PR_FILES="$PR_FILES_AM $PR_FILES_RENAMED"
           for FILE in $PR_FILES; do
               EXTENSION="${FILE##*.}"
-              DIRECTORY="${FILE%%/*}"
-              if [[ "$EXTENSION" == 'py' && "$DIRECTORY" != 'cvat-sdk' ]]; then
+              if [[ "$EXTENSION" == 'py' ]]; then
                   CHANGED_FILES+=" $FILE"
               fi
           done

.github/workflows/schedule.yml

@@ -222,12 +222,21 @@ jobs:
           chmod +x ./opa
           ./opa test cvat/apps/iam/rules
 
-      - name: REST API tests
+      - name: REST API and SDK tests
         run: |
+          docker run --rm -v ${PWD}/cvat-sdk/schema/:/transfer \
+            --entrypoint /bin/bash -u root cvat/server \
+            -c 'python manage.py spectacular --file /transfer/schema.yml'
+          pip3 install --user -r cvat-sdk/gen/requirements.txt
+          cd cvat-sdk/
+          gen/generate.sh
+          cd ..
+
           pip3 install --user cvat-sdk/
-          pip3 install --user -r tests/rest_api/requirements.txt
-          pytest tests/rest_api/
-          pytest tests/rest_api/ --stop-services
+          pip3 install --user cvat-cli/
+          pip3 install --user -r tests/python/requirements.txt
+          pytest tests/python/
+          pytest tests/python/ --stop-services
 
       - name: Unit tests
         env:
@@ -239,7 +248,7 @@ jobs:
           while [[  $(curl -s -o /dev/null -w "%{http_code}" localhost:8181/health) != "200" && max_tries -gt 0 ]]; do (( max_tries-- )); sleep 5; done
 
           docker-compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.ci.yml run cvat_ci /bin/bash \
-          -c 'coverage run -a manage.py test cvat/apps cvat-cli && mv .coverage ${CONTAINER_COVERAGE_DATA_DIR}'
+          -c 'coverage run -a manage.py test cvat/apps && mv .coverage ${CONTAINER_COVERAGE_DATA_DIR}'
 
           docker-compose -f docker-compose.yml -f docker-compose.dev.yml -f docker-compose.ci.yml run cvat_ci /bin/bash \
           -c 'yarn --frozen-lockfile --ignore-scripts && yarn workspace cvat-core run test'