Merge pull request #99 from buildkite-plugins/toote_compression_refactoring

pzeballos · web-flow · commit f2b4ab00cc85 · 2025-02-17T16:25:00.000-03:00
Compression refactoring
diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
@@ -10,6 +10,7 @@ steps:
             - hooks/**
             - lib/**
             - backends/**
+            - compression/**
           options: "-x"
 
   - label: ":sparkles: Lint"
diff --git a/README.md b/README.md
@@ -69,7 +69,11 @@ Setting the `BUILDKITE_PLUGIN_S3_CACHE_ONLY_SHOW_ERRORS` environment variable wi
 
 Allows for the cached file/folder to be saved/restored as a single file. You will need to make sure to use the same compression when saving and restoring or it will cause a cache miss.
 
-Assuming the underlying executables are available, the allowed values are:
+The value `none` is equivalent to not specifying a compression. Any other value (`X`) will assume that an executable wrapper script exists in the agent's `PATH` to handle both compression and decompression with the following protocol:
+* `X_wrapper compress SOURCE TARGET`: compress the file/folder `SOURCE` into the `TARGET` file
+* `X_wrapper decompress SOURCE TARGET`: decompress the artifact `SOURCE` into `TARGET` destination
+
+The plugin includes wrappers to provide both examples and backwards-compatibility:
 * `tgz`: `tar` with gzip compression
 * `zip`: `(un)zip` compression
 
diff --git a/compression/tgz_wrapper b/compression/tgz_wrapper
@@ -0,0 +1,28 @@
+#!/bin/bash
+
+OPERATION=${1?Operation not specified}
+SOURCE=${2?Source not specified}
+TARGET=${3?Target not specified}
+
+is_absolute_path() {
+  [ "${1:0:1}" = "/" ]
+}
+
+if [ "${OPERATION}" = "compress" ]; then
+    TAR_OPTS='cz'
+    if is_absolute_path "${SOURCE}"; then
+      TAR_OPTS="${TAR_OPTS}"P
+    fi
+
+    tar "${TAR_OPTS}"f "${TARGET}" "${SOURCE}"
+elif [ "${OPERATION}" = "decompress" ]; then
+    TAR_OPTS='xz'
+    if is_absolute_path "${TARGET}"; then
+      TAR_OPTS="${TAR_OPTS}"P
+    fi
+
+    tar "${TAR_OPTS}"f "${SOURCE}" "${TARGET}"
+else
+    echo "Invalid operation"
+    exit 1
+fi
diff --git a/compression/zip_wrapper b/compression/zip_wrapper
@@ -0,0 +1,45 @@
+#!/bin/bash
+
+OPERATION=${1?Operation not specified}
+SOURCE=${2?Source not specified}
+TARGET=${3?Target not specified}
+
+is_absolute_path() {
+  [ "${1:0:1}" = "/" ]
+}
+
+if [ "${OPERATION}" = "compress" ]; then
+    if is_absolute_path "${SOURCE}"; then
+      COMPRESS_DIR="$(dirname "${SOURCE}")"
+      ( # subshell to avoid changing the working directory
+        # shellcheck disable=SC2164 # we will exit anyway
+        cd "${COMPRESS_DIR}"
+        # because ZIP complains if the file does not end with .zip
+        zip -r "${TARGET}.zip" "${SOURCE}"
+        mv "${TARGET}.zip" "${TARGET}"
+      )
+    else
+      # because ZIP complains if the file does not end with .zip
+      zip -r "${TARGET}.zip" "${SOURCE}"
+      mv "${TARGET}.zip" "${TARGET}"
+    fi
+elif [ "${OPERATION}" = "decompress" ]; then
+    if is_absolute_path "${TARGET}"; then
+      RESTORE_DIR="$(dirname "${TARGET}")"
+      ( # subshell to avoid changing the working directory
+        mkdir -p "${RESTORE_DIR}"
+        # shellcheck disable=SC2164 # we will exit anyway
+        cd "${RESTORE_DIR}"
+        mv "${SOURCE}" "${RESTORE_DIR}/compressed.zip"
+        unzip -o "compressed.zip"
+        rm "compressed.zip"
+      )
+    else
+      # because ZIP complains if the file does not end with .zip
+      mv "${SOURCE}" "${SOURCE}.zip"
+      unzip -o "${SOURCE}.zip"
+    fi
+else
+    echo "Invalid operation"
+    exit 1
+fi
diff --git a/hooks/post-command b/hooks/post-command
@@ -22,10 +22,6 @@ if [ -z "${CACHE_PATH}" ] ; then
 fi
 
 COMPRESS=$(plugin_read_config COMPRESSION 'none')
-if ! validate_compression "${COMPRESS}"; then
-  echo "+++ 🚨 Invalid value for compression option"
-  exit 1
-fi
 
 if ! plugin_read_list_into_result SAVE; then
   echo 'Cache not setup for saving'
diff --git a/hooks/pre-command b/hooks/pre-command
@@ -25,10 +25,6 @@ elif [ "${MAX_LEVEL}" = 'file' ] && [ -z "$(plugin_read_config MANIFEST)" ]; the
 fi
 
 COMPRESS=$(plugin_read_config COMPRESSION 'none')
-if ! validate_compression "${COMPRESS}"; then
-  echo "+++ 🚨 Invalid value for compression option"
-  exit 1
-fi
 
 build_key "${MAX_LEVEL}" "${RESTORE_PATH}" >/dev/null # to validate the level
 
@@ -46,7 +42,7 @@ for CURRENT_LEVEL in "${SORTED_LEVELS[@]}"; do
     if compression_active; then
       TEMP_PATH=$(mktemp)
       backend_exec get "${KEY}" "${TEMP_PATH}"
-      uncompress "${TEMP_PATH}" "${RESTORE_PATH}"
+      decompress "${TEMP_PATH}" "${RESTORE_PATH}"
       if [ "$(plugin_read_config KEEP_COMPRESSED_ARTIFACTS 'false')" = 'false' ]; then
         rm -rf "${TEMP_PATH}"
       fi
diff --git a/lib/compression.bash b/lib/compression.bash
@@ -5,19 +5,6 @@ DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 # shellcheck source=lib/plugin.bash
 . "${DIR}/plugin.bash"
 
-validate_compression() {
-  local COMPRESSION="$1"
-
-  VALID_COMPRESSIONS=(none tgz zip)
-  for VALID in "${VALID_COMPRESSIONS[@]}"; do
-    if [ "${COMPRESSION}" = "${VALID}" ]; then
-      return 0
-    fi
-  done
-
-  return 1
-}
-
 compression_active() {
   local COMPRESSION=''
   COMPRESSION="$(plugin_read_config COMPRESSION 'none')"
@@ -32,71 +19,21 @@ compress() {
   local COMPRESSION=''
   COMPRESSION="$(plugin_read_config COMPRESSION 'none')"
 
-  echo "Compressing ${COMPRESSED_FILE} with ${COMPRESSION}..."
-
-  if [ "${COMPRESSION}" = 'tgz' ]; then
-    TAR_OPTS='cz'
-    if is_absolute_path "${COMPRESSED_FILE}"; then
-      TAR_OPTS="${TAR_OPTS}"P
-    fi
-
-    tar "${TAR_OPTS}"f "${FILE}" "${COMPRESSED_FILE}"
-  elif [ "${COMPRESSION}" = 'zip' ]; then
-    if is_absolute_path "${COMPRESSED_FILE}"; then
-      local COMPRESS_DIR
-      COMPRESS_DIR="$(dirname "${COMPRESSED_FILE}")"
-      ( # subshell to avoid changing the working directory
-        # shellcheck disable=SC2164 # we will exit anyway
-        cd "${COMPRESS_DIR}"
-        # because ZIP complains if the file does not end with .zip
-        zip -r "${FILE}.zip" "${COMPRESSED_FILE}"
-        mv "${FILE}.zip" "${FILE}"
-      )
-    else
-      # because ZIP complains if the file does not end with .zip
-      zip -r "${FILE}.zip" "${COMPRESSED_FILE}"
-      mv "${FILE}.zip" "${FILE}"
-    fi
+  if [ "${COMPRESSION}" != 'none' ]; then
+    echo "Compressing ${COMPRESSED_FILE} with ${COMPRESSION}..."
+    PATH="${PATH}:${DIR}/../compression" "${COMPRESSION}_wrapper" "compress" "${COMPRESSED_FILE}" "${FILE}"
   fi
 }
 
-uncompress() {
+decompress() {
   local FILE="$1"
   local RESTORE_PATH="$2"
 
   local COMPRESSION=''
   COMPRESSION="$(plugin_read_config COMPRESSION 'none')"
 
-  echo "Cache is compressed, uncompressing with ${COMPRESSION}..."
-
-  if [ "${COMPRESSION}" = 'tgz' ]; then
-    TAR_OPTS='xz'
-    if is_absolute_path "${RESTORE_PATH}"; then
-      TAR_OPTS="${TAR_OPTS}"P
-    fi
-
-    tar "${TAR_OPTS}"f "${FILE}" "${RESTORE_PATH}"
-  elif [ "${COMPRESSION}" = 'zip' ]; then
-    if is_absolute_path "${RESTORE_PATH}"; then
-      local RESTORE_DIR
-      RESTORE_DIR="$(dirname "${RESTORE_PATH}")"
-      ( # subshell to avoid changing the working directory
-        mkdir -p "${RESTORE_DIR}"
-        # shellcheck disable=SC2164 # we will exit anyway
-        cd "${RESTORE_DIR}"
-        mv "${FILE}" "${RESTORE_DIR}/compressed.zip"
-        unzip -o "compressed.zip"
-        rm "compressed.zip"
-      )
-    else
-      # because ZIP complains if the file does not end with .zip
-      mv "${FILE}" "${FILE}.zip"
-      unzip -o "${FILE}.zip"
-    fi
+  if [ "${COMPRESSION}" != 'none' ]; then
+    echo "Cache is compressed, decompressing with ${COMPRESSION}..."
+    PATH="${PATH}:${DIR}/../compression" "${COMPRESSION}_wrapper" "decompress" "${FILE}" "${RESTORE_PATH}"
   fi
 }
-
-is_absolute_path() {
-  local FILEPATH="${1}"
-  [ "${FILEPATH:0:1}" = "/" ]
-}
diff --git a/plugin.yml b/plugin.yml
@@ -17,10 +17,6 @@ configuration:
       type: string
     compression:
       type: string
-      enum:
-        - zip
-        - tar
-        - tgz
     force:
       type: boolean
     keep-compressed-artifacts:
diff --git a/tests/compression.bats b/tests/compression.bats
@@ -1,25 +1,83 @@
 #!/usr/bin/env bats
 
 # To debug stubs, uncomment these lines:
-# export HASH_STUB_DEBUG=/dev/tty
+# export DUMMY_COMPRESS_WRAPPER_STUB_DEBUG=/dev/tty
+# export DUMMY_DECOMPRESS_WRAPPER_STUB_DEBUG=/dev/tty
 
 setup() {
   load "${BATS_PLUGIN_PATH}/load.bash"
 
+  # prevent a warning when using flags on run
+  bats_require_minimum_version 1.5.0
+
   source "${PWD}/lib/compression.bash"
 }
 
-@test 'validate_compression works' {
-  run validate_compression none
+# TODO: Add tests for wrappers
+#TODO: change plugin.yml and README
+
+@test 'compression_active works' {
+  # default value
+  run compression_active
+  assert_failure
+
+  # turned off
+  export BUILDKITE_PLUGIN_CACHE_COMPRESSION=none
+  run compression_active
+  assert_failure
 
+  # any other value
+  export BUILDKITE_PLUGIN_CACHE_COMPRESSION=something_else
+  run compression_active
   assert_success
+}
 
-  run validate_compression tgz
+@test 'compress works' {
+  # default value does nothing
+  run compress SOURCE TARGET
   assert_success
+  refute_output 'Compressiong SOURCE with'
+
+  # define a compression
+  export BUILDKITE_PLUGIN_CACHE_COMPRESSION=dummy_compress
+  # -127 prevents a warning due to command not found
+  run -127 compress SOURCE TARGET
+  assert_failure
+  assert_output --partial 'Compressing SOURCE with dummy_compress'
+  assert_output --partial 'dummy_compress_wrapper: command not found'
+
+  # using an existing compression
+  stub dummy_compress_wrapper \
+    "compress \* \* : echo Compressed wrapper \$2 with dummy_compress"
 
-  run validate_compression zip
+  run compress SOURCE TARGET
   assert_success
+  assert_output --partial 'Compressing SOURCE with dummy_compress'
 
-  run validate_compression invalid
+  unstub dummy_compress_wrapper
+}
+
+@test 'decompress works' {
+  # default value does nothing
+  run decompress SOURCE TARGET
+  assert_success
+  refute_output 'uncompressing with'
+
+  # define a compression
+  export BUILDKITE_PLUGIN_CACHE_COMPRESSION=dummy_decompress
+  # -127 prevents a warning due to command not found
+  run -127 decompress SOURCE TARGET
   assert_failure
+  assert_output --partial 'decompressing with dummy_decompress'
+  assert_output --partial 'dummy_decompress_wrapper: command not found'
+
+  # using an existing compression
+  stub dummy_decompress_wrapper \
+    "decompress \* \* : echo Decompressing wrapper \$2 with dummy_decompress"
+
+  run decompress SOURCE TARGET
+  assert_success
+  assert_output --partial 'decompressing with dummy_decompress'
+
+  unstub dummy_decompress_wrapper
 }
diff --git a/tests/post-command.bats b/tests/post-command.bats
@@ -63,16 +63,6 @@ teardown() {
   assert_output --partial 'Invalid levels in the save list'
 }
 
-@test "Invalid compression level fails" {
-  export BUILDKITE_PLUGIN_CACHE_SAVE=all
-  export BUILDKITE_PLUGIN_CACHE_COMPRESSION=invalid
-
-  run "$PWD/hooks/post-command"
-
-  assert_failure
-  assert_output --partial 'Invalid value for compression option'
-}
-
 @test "File-based cache with no manifest fails" {
   export BUILDKITE_PLUGIN_CACHE_SAVE=file
 
diff --git a/tests/pre-command-tgz.bats b/tests/pre-command-tgz.bats
diff --git a/tests/pre-command-zip.bats b/tests/pre-command-zip.bats
diff --git a/tests/pre-command.bats b/tests/pre-command.bats
