Skip to content

Commit fbfb8ef

Browse files
pzeballospzeballos
authored
Merge pull request #67 from buildkite-plugins/toote_zip_issues
Compression issues
2 parents bae0376 + 9a6d3b9 commit fbfb8ef

File tree

9 files changed

+121
-72
lines changed

9 files changed

+121
-72
lines changed

‎hooks/post-checkout‎

Lines changed: 8 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@ DIR="$(cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd)"
55

66
# shellcheck source=lib/shared.bash
77
. "${DIR}/../lib/shared.bash"
8+
# shellcheck source=lib/compression.bash
9+
. "${DIR}/../lib/compression.bash"
810

911
RESTORE_PATH=$(plugin_read_config PATH)
1012

@@ -30,16 +32,6 @@ fi
3032

3133
build_key "${MAX_LEVEL}" "${RESTORE_PATH}" >/dev/null # to validate the level
3234

33-
ACTUAL_PATH=$(mktemp)
34-
35-
if [ "${COMPRESS}" = 'tgz' ]; then
36-
UNCOMPRESS_COMMAND=(tar xzf)
37-
elif [ "${COMPRESS}" = 'zip' ]; then
38-
UNCOMPRESS_COMMAND=(unzip)
39-
else
40-
ACTUAL_PATH="${RESTORE_PATH}"
41-
fi
42-
4335
SORTED_LEVELS=(file step branch pipeline all)
4436

4537
for CURRENT_LEVEL in "${SORTED_LEVELS[@]}"; do
@@ -50,11 +42,13 @@ for CURRENT_LEVEL in "${SORTED_LEVELS[@]}"; do
5042
KEY=$(build_key "${CURRENT_LEVEL}" "${RESTORE_PATH}" "${COMPRESS}")
5143
if backend_exec exists "${KEY}"; then
5244
echo "Cache hit at ${CURRENT_LEVEL} level, restoring ${RESTORE_PATH}..."
53-
backend_exec get "${KEY}" "${ACTUAL_PATH}"
5445

55-
if [ "${COMPRESS}" != 'none' ]; then
56-
echo "Cache is compressed, uncompressing..."
57-
"${UNCOMPRESS_COMMAND[@]}" "${ACTUAL_PATH}" "${RESTORE_PATH}"
46+
if compression_active; then
47+
TEMP_PATH=$(mktemp)
48+
backend_exec get "${KEY}" "${TEMP_PATH}"
49+
uncompress "${TEMP_PATH}" "${RESTORE_PATH}"
50+
else
51+
backend_exec get "${KEY}" "${RESTORE_PATH}"
5852
fi
5953

6054
exit 0

‎hooks/post-command‎

Lines changed: 5 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@ DIR="$(cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd)"
55

66
# shellcheck source=lib/shared.bash
77
. "${DIR}/../lib/shared.bash"
8+
# shellcheck source=lib/compression.bash
9+
. "${DIR}/../lib/compression.bash"
810

911
CACHE_PATH=$(plugin_read_config PATH)
1012

@@ -30,19 +32,12 @@ fi
3032

3133
KEY=$(build_key "${LEVEL}" "${CACHE_PATH}" "${COMPRESS}")
3234

33-
if [ "${COMPRESS}" = 'tgz' ]; then
34-
COMPRESS_COMMAND=(tar czf)
35-
elif [ "${COMPRESS}" = 'zip' ]; then
36-
COMPRESS_COMMAND=(zip)
37-
fi
38-
39-
if [ "${COMPRESS}" != 'none' ]; then
40-
echo "Compressing ${CACHE_PATH} with ${COMPRESS}..."
35+
if compression_active; then
4136
ACTUAL_PATH=$(mktemp)
42-
"${COMPRESS_COMMAND[@]}" "${ACTUAL_PATH}" "${CACHE_PATH}"
37+
compress "${CACHE_PATH}" "${ACTUAL_PATH}"
4338
else
4439
ACTUAL_PATH="${CACHE_PATH}"
4540
fi
4641

4742
echo "Saving ${LEVEL}-level cache of ${CACHE_PATH}"
48-
backend_exec save "${KEY}" "${ACTUAL_PATH}"
43+
backend_exec save "${KEY}" "${ACTUAL_PATH}"

‎lib/compression.bash‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
#!/bin/bash
2+
3+
DIR="$(cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd)"
4+
5+
# shellcheck source=lib/plugin.bash
6+
. "${DIR}/plugin.bash"
7+
8+
validate_compression() {
9+
local COMPRESSION="$1"
10+
11+
VALID_COMPRESSIONS=(none tgz zip)
12+
for VALID in "${VALID_COMPRESSIONS[@]}"; do
13+
if [ "${COMPRESSION}" = "${VALID}" ]; then
14+
return 0
15+
fi
16+
done
17+
18+
return 1
19+
}
20+
21+
compression_active() {
22+
local COMPRESSION=''
23+
COMPRESSION="$(plugin_read_config COMPRESSION 'none')"
24+
25+
[ "${COMPRESSION}" != 'none' ]
26+
}
27+
28+
compress() {
29+
local COMPRESSED_FILE="$1"
30+
local FILE="$2"
31+
32+
local COMPRESSION=''
33+
COMPRESSION="$(plugin_read_config COMPRESSION 'none')"
34+
35+
echo "Compressing ${COMPRESSED_FILE} with ${COMPRESSION}..."
36+
37+
if [ "${COMPRESSION}" = 'tgz' ]; then
38+
tar czf "${FILE}" "${COMPRESSED_FILE}"
39+
elif [ "${COMPRESSION}" = 'zip' ]; then
40+
# because ZIP complains if the file does not end with .zip
41+
zip -r "${FILE}.zip" "${COMPRESSED_FILE}"
42+
mv "${FILE}.zip" "${FILE}"
43+
fi
44+
}
45+
46+
uncompress() {
47+
local FILE="$1"
48+
local _RESTORE_PATH="$2" # pretty sure this is not necessary
49+
50+
local COMPRESSION=''
51+
COMPRESSION="$(plugin_read_config COMPRESSION 'none')"
52+
53+
echo "Cache is compressed, uncompressing with ${COMPRESSION}..."
54+
55+
if [ "${COMPRESSION}" = 'tgz' ]; then
56+
tar xzf "${FILE}"
57+
elif [ "${COMPRESSION}" = 'zip' ]; then
58+
# because ZIP complains if the file does not end with .zip
59+
mv "${FILE}" "${FILE}.zip"
60+
unzip -o "${FILE}.zip"
61+
fi
62+
}

‎lib/shared.bash‎

Lines changed: 0 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -62,16 +62,3 @@ backend_exec() {
6262

6363
PATH="${PATH}:${DIR}/../backends" "cache_${BACKEND_NAME}" "$@"
6464
}
65-
66-
validate_compression() {
67-
local COMPRESSION="$1"
68-
69-
VALID_COMPRESSIONS=(none tgz zip)
70-
for VALID in "${VALID_COMPRESSIONS[@]}"; do
71-
if [ "${COMPRESSION}" = "${VALID}" ]; then
72-
return 0
73-
fi
74-
done
75-
76-
return 1
77-
}

‎tests/compression.bats‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
#!/usr/bin/env bats
2+
3+
# To debug stubs, uncomment these lines:
4+
# export HASH_STUB_DEBUG=/dev/tty
5+
6+
setup() {
7+
load "${BATS_PLUGIN_PATH}/load.bash"
8+
9+
source "${PWD}/lib/compression.bash"
10+
}
11+
12+
@test 'validate_compression works' {
13+
run validate_compression none
14+
15+
assert_success
16+
17+
run validate_compression tgz
18+
assert_success
19+
20+
run validate_compression zip
21+
assert_success
22+
23+
run validate_compression invalid
24+
assert_failure
25+
}

‎tests/post-checkout-tgz.bats‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ teardown() {
4444

4545
assert_success
4646
assert_output --partial 'Cache hit at file level'
47-
assert_output --partial 'Cache is compressed, uncompressing...'
47+
assert_output --partial 'Cache is compressed, uncompressing with tgz'
4848

4949
unstub cache_dummy
5050
}
@@ -60,7 +60,7 @@ teardown() {
6060

6161
assert_success
6262
assert_output --partial 'Cache hit at file level'
63-
assert_output --partial 'Cache is compressed, uncompressing...'
63+
assert_output --partial 'Cache is compressed, uncompressing with tgz'
6464

6565
unstub cache_dummy
6666
}
@@ -77,7 +77,7 @@ teardown() {
7777

7878
assert_success
7979
assert_output --partial 'Cache hit at step level'
80-
assert_output --partial 'Cache is compressed, uncompressing...'
80+
assert_output --partial 'Cache is compressed, uncompressing with tgz'
8181

8282
unstub cache_dummy
8383
}
@@ -95,7 +95,7 @@ teardown() {
9595

9696
assert_success
9797
assert_output --partial 'Cache hit at branch level'
98-
assert_output --partial 'Cache is compressed, uncompressing...'
98+
assert_output --partial 'Cache is compressed, uncompressing with tgz'
9999

100100
unstub cache_dummy
101101
}
@@ -113,7 +113,7 @@ teardown() {
113113

114114
assert_success
115115
assert_output --partial 'Cache hit at pipeline level'
116-
assert_output --partial 'Cache is compressed, uncompressing...'
116+
assert_output --partial 'Cache is compressed, uncompressing with tgz'
117117

118118
unstub cache_dummy
119119
}
@@ -133,7 +133,7 @@ teardown() {
133133

134134
assert_success
135135
assert_output --partial 'Cache hit at all level'
136-
assert_output --partial 'Cache is compressed, uncompressing...'
136+
assert_output --partial 'Cache is compressed, uncompressing with tgz'
137137

138138
unstub cache_dummy
139139
}
@@ -151,7 +151,7 @@ teardown() {
151151

152152
assert_success
153153
assert_output --partial 'Cache hit at branch level'
154-
assert_output --partial 'Cache is compressed, uncompressing...'
154+
assert_output --partial 'Cache is compressed, uncompressing with tgz'
155155

156156
unstub cache_dummy
157-
}
157+
}

‎tests/post-checkout-zip.bats‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ setup() {
2424

2525
# stub is the same for all tests
2626
stub unzip \
27-
"\* \* : echo uncompressed \$2 into \$3"
27+
"-o \* \* : echo uncompressed \$3 into \$4"
2828
}
2929

3030
teardown() {
@@ -44,7 +44,7 @@ teardown() {
4444

4545
assert_success
4646
assert_output --partial 'Cache hit at file level'
47-
assert_output --partial "Cache is compressed, uncompressing..."
47+
assert_output --partial "Cache is compressed, uncompressing with zip"
4848

4949
unstub cache_dummy
5050
}
@@ -60,7 +60,7 @@ teardown() {
6060

6161
assert_success
6262
assert_output --partial 'Cache hit at file level'
63-
assert_output --partial "Cache is compressed, uncompressing..."
63+
assert_output --partial "Cache is compressed, uncompressing with zip"
6464

6565
unstub cache_dummy
6666
}
@@ -77,7 +77,7 @@ teardown() {
7777

7878
assert_success
7979
assert_output --partial 'Cache hit at step level'
80-
assert_output --partial "Cache is compressed, uncompressing..."
80+
assert_output --partial "Cache is compressed, uncompressing with zip"
8181

8282
unstub cache_dummy
8383
}
@@ -95,7 +95,7 @@ teardown() {
9595

9696
assert_success
9797
assert_output --partial 'Cache hit at branch level'
98-
assert_output --partial "Cache is compressed, uncompressing..."
98+
assert_output --partial "Cache is compressed, uncompressing with zip"
9999

100100
unstub cache_dummy
101101
}
@@ -113,7 +113,7 @@ teardown() {
113113

114114
assert_success
115115
assert_output --partial 'Cache hit at pipeline level'
116-
assert_output --partial "Cache is compressed, uncompressing..."
116+
assert_output --partial "Cache is compressed, uncompressing with zip"
117117

118118
unstub cache_dummy
119119
}
@@ -133,7 +133,7 @@ teardown() {
133133

134134
assert_success
135135
assert_output --partial 'Cache hit at all level'
136-
assert_output --partial "Cache is compressed, uncompressing..."
136+
assert_output --partial "Cache is compressed, uncompressing with zip"
137137

138138
unstub cache_dummy
139139
}
@@ -152,7 +152,7 @@ teardown() {
152152

153153
assert_success
154154
assert_output --partial 'Cache hit at branch level'
155-
assert_output --partial "Cache is compressed, uncompressing..."
155+
assert_output --partial "Cache is compressed, uncompressing with zip"
156156

157157
unstub cache_dummy
158-
}
158+
}

‎tests/post-command-zip.bats‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,8 +25,9 @@ setup() {
2525
stub cache_dummy \
2626
"save \* \* : echo saving \$3 in \$2"
2727

28+
# if the file is not created, this fails because zip
2829
stub zip \
29-
"echo compressed \${@:2} into \$1"
30+
"touch \$2; echo compressed \${@:3} into \$2"
3031
}
3132

3233
teardown() {
@@ -85,4 +86,4 @@ teardown() {
8586
assert_success
8687
assert_output --partial 'Saving all-level cache'
8788
assert_output --partial 'Compressing tests/data/my_files with zip'
88-
}
89+
}

‎tests/shared.bats‎

Lines changed: 1 addition & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@ teardown() {
5050

5151
assert_success
5252
assert_output 'sha1sum'
53-
53+
5454
unstub hash
5555
}
5656

@@ -155,18 +155,3 @@ teardown() {
155155
assert_success
156156
refute_output "${GENERATED_KEY}"
157157
}
158-
159-
@test 'validate_compression works' {
160-
run validate_compression none
161-
162-
assert_success
163-
164-
run validate_compression tgz
165-
assert_success
166-
167-
run validate_compression zip
168-
assert_success
169-
170-
run validate_compression invalid
171-
assert_failure
172-
}

0 commit comments

Comments
 (0)