Merge pull request #103 from sp-aaflalo/patch-1

Improve zstd compression implementation by relying on tar directly

Author: pzeballos

compression/zstd_wrapper

@@ -8,31 +8,26 @@ is_absolute_path() {
   [ "${1:0:1}" = "/" ]
 }
 
-if ! command -v zstd &> /dev/null; then
-  echo "zstd is not installed"
-  if [[ "$OSTYPE" == "darwin"* ]]; then
-    echo "Try 'brew install zstd'"
-  else
-    echo "Try 'apt-get install zstd'"
-  fi
+if ! tar --help | grep -q -- "--zstd"; then
+  echo "tar does not support zstd compression"
+  echo "You need a version of tar with zstd support (tar 1.31 or higher)"
   exit 1
 fi
 
 if [ "${OPERATION}" = "compress" ]; then
-  TAR_OPTS=('-c' '-f' '-')
-  if is_absolute_path "${SOURCE}"; then
-    TAR_OPTS+=('-P')
-  fi
-  tar "${TAR_OPTS[@]}" "${SOURCE}" | zstd -o "${TARGET}"
+    TAR_OPTS=('-c' '--zstd')
+    if is_absolute_path "${SOURCE}"; then
+      TAR_OPTS+=('-P')
+    fi
+
+    tar "${TAR_OPTS[@]}" -f "${TARGET}" "${SOURCE}"
 elif [ "${OPERATION}" = "decompress" ]; then
-  mkdir -p "${TARGET}"
-  TAR_OPTS=('-x' '-f' '-')
-  if is_absolute_path "${TARGET}"; then
-    TAR_OPTS+=('-P')
-    zstd -d -c "${SOURCE}" | tar "${TAR_OPTS[@]}"
-  else
-    zstd -d -c "${SOURCE}" | tar "${TAR_OPTS[@]}" -C "${TARGET}"
-  fi
+    TAR_OPTS=('-x' '--zstd')
+    if is_absolute_path "${TARGET}"; then
+      TAR_OPTS+=('-P')
+    fi
+
+    tar "${TAR_OPTS[@]}" -f "${SOURCE}" "${TARGET}"
 else
     echo "Invalid operation"
     exit 1

tests/post-command-zstd.bats

@@ -2,7 +2,7 @@
 
 # To debug stubs, uncomment these lines:
 # export CACHE_DUMMY_STUB_DEBUG=/dev/tty
-# export ZSTD_STUB_DEBUG=/dev/tty
+# export TAR_STUB_DEBUG=/dev/tty
 
 setup() {
   load "${BATS_PLUGIN_PATH}/load.bash"
@@ -26,19 +26,19 @@ setup() {
   export BUILDKITE_ORGANIZATION_SLUG="bk-cache-test"
   export BUILDKITE_PIPELINE_SLUG="cache-pipeline"
 
-  # stubs are the same for every test
-  stub zstd \
-    "\* \* : echo compressed stdin into \$2 with options \$1"
-
   stub cache_dummy \
     "save \* \* : echo saving \$3 in \$2"
+
+  stub tar \
+    "--help : echo '--zstd'" \
+    "echo called tar with options \$@"
 }
 
 teardown() {
   rm -rf tests/data
 
   unstub cache_dummy
-  unstub zstd
+  unstub tar
 }
 
 @test "File-level saving with compression" {
@@ -50,7 +50,7 @@ teardown() {
   assert_success
   assert_output --partial 'Saving file-level cache'
   assert_output --partial 'Compressing tests/data/my_files with zstd'
-  assert_output --partial "with options -o"
+  assert_output --partial "with options -c --zstd"
 }
 
 @test "Step-level saving" {
@@ -119,7 +119,7 @@ teardown() {
   assert_success
   assert_output --partial 'Saving pipeline-level cache'
   assert_output --partial "Compressing ${BUILDKITE_PLUGIN_CACHE_PATH} with zstd..."
-  assert_output --partial "with options -o"
+  assert_output --partial "with options -c --zstd -P"
 
   rm -rf "${BUILDKITE_PLUGIN_CACHE_PATH}"
 }

tests/pre-command-zstd.bats

@@ -3,7 +3,6 @@
 # To debug stubs, uncomment these lines:
 # export CACHE_DUMMY_STUB_DEBUG=/dev/tty
 # export TAR_STUB_DEBUG=/dev/tty
-# export ZSTD_STUB_DEBUG=/dev/tty
 
 setup() {
   load "${BATS_PLUGIN_PATH}/load.bash"
@@ -23,18 +22,14 @@ setup() {
   export BUILDKITE_ORGANIZATION_SLUG="bk-cache-test"
   export BUILDKITE_PIPELINE_SLUG="cache-pipeline"
 
-  # stub is the same for all tests
-  stub zstd \
-    "-d \* \* : echo uncompressed \$2 into \$3"
-
-  stub tar "echo called tar with options \$@ and input; cat"
-
+  stub tar \
+    "--help : echo '--zstd'" \
+    "echo called tar with options \$@"
 }
 
 teardown() {
   rm -rf tests/data
 
-  unstub zstd
   unstub tar
 }