Skip to content

Commit 74cc7b2

Browse files
pzeballospzeballos
authored
Merge pull request #101 from buildkite-plugins/zacsweers_zstd_support
zstd support
2 parents f2b4ab0 + e1d535c commit 74cc7b2

File tree

4 files changed

+344
-0
lines changed

4 files changed

+344
-0
lines changed

‎README.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,7 @@ The value `none` is equivalent to not specifying a compression. Any other value
7676
The plugin includes wrappers to provide both examples and backwards-compatibility:
7777
* `tgz`: `tar` with gzip compression
7878
* `zip`: `(un)zip` compression
79+
* `zstd`: `zstd` compression
7980

8081
### `force` (boolean, save only)
8182

‎compression/zstd_wrapper‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
#!/bin/bash
2+
3+
OPERATION=${1?Operation not specified}
4+
SOURCE=${2?Source not specified}
5+
TARGET=${3?Target not specified}
6+
7+
is_absolute_path() {
8+
[ "${1:0:1}" = "/" ]
9+
}
10+
11+
if ! command -v zstd &> /dev/null; then
12+
echo "zstd is not installed"
13+
if [[ "$OSTYPE" == "darwin"* ]]; then
14+
echo "Try 'brew install zstd'"
15+
else
16+
echo "Try 'apt-get install zstd'"
17+
fi
18+
exit 1
19+
fi
20+
21+
if [ "${OPERATION}" = "compress" ]; then
22+
TAR_OPTS=('-c' '-f' '-')
23+
if is_absolute_path "${SOURCE}"; then
24+
TAR_OPTS+=('-P')
25+
fi
26+
tar "${TAR_OPTS[@]}" "${SOURCE}" | zstd -o "${TARGET}"
27+
elif [ "${OPERATION}" = "decompress" ]; then
28+
mkdir -p "${TARGET}"
29+
TAR_OPTS=('-x' '-f' '-')
30+
if is_absolute_path "${TARGET}"; then
31+
TAR_OPTS+=('-P')
32+
zstd -d -c "${SOURCE}" | tar "${TAR_OPTS[@]}"
33+
else
34+
zstd -d -c "${SOURCE}" | tar "${TAR_OPTS[@]}" -C "${TARGET}"
35+
fi
36+
else
37+
echo "Invalid operation"
38+
exit 1
39+
fi

‎tests/post-command-zstd.bats‎

Lines changed: 125 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,125 @@
1+
#!/usr/bin/env bats
2+
3+
# To debug stubs, uncomment these lines:
4+
# export CACHE_DUMMY_STUB_DEBUG=/dev/tty
5+
# export ZSTD_STUB_DEBUG=/dev/tty
6+
7+
setup() {
8+
load "${BATS_PLUGIN_PATH}/load.bash"
9+
10+
export BUILDKITE_COMMAND_EXIT_STATUS=0
11+
12+
mkdir -p tests/data/my_files
13+
echo "all the llamas" > "tests/data/my_files/llamas.txt"
14+
echo "no alpacas" > "tests/data/my_files/alpacas.txt"
15+
16+
export BUILDKITE_PLUGIN_CACHE_BACKEND=dummy
17+
export BUILDKITE_PLUGIN_CACHE_COMPRESSION=zstd
18+
export BUILDKITE_PLUGIN_CACHE_PATH=tests/data/my_files
19+
20+
# to make all test easier
21+
export BUILDKITE_PLUGIN_CACHE_FORCE=true
22+
23+
# necessary for key-calculations
24+
export BUILDKITE_LABEL="step-label"
25+
export BUILDKITE_BRANCH="tests"
26+
export BUILDKITE_ORGANIZATION_SLUG="bk-cache-test"
27+
export BUILDKITE_PIPELINE_SLUG="cache-pipeline"
28+
29+
# stubs are the same for every test
30+
stub zstd \
31+
"\* \* : echo compressed stdin into \$2 with options \$1"
32+
33+
stub cache_dummy \
34+
"save \* \* : echo saving \$3 in \$2"
35+
}
36+
37+
teardown() {
38+
rm -rf tests/data
39+
40+
unstub cache_dummy
41+
unstub zstd
42+
}
43+
44+
@test "File-level saving with compression" {
45+
export BUILDKITE_PLUGIN_CACHE_SAVE=file
46+
export BUILDKITE_PLUGIN_CACHE_MANIFEST=tests/data/my_files/llamas.txt
47+
48+
run "$PWD/hooks/post-command"
49+
50+
assert_success
51+
assert_output --partial 'Saving file-level cache'
52+
assert_output --partial 'Compressing tests/data/my_files with zstd'
53+
assert_output --partial "with options -o"
54+
}
55+
56+
@test "Step-level saving" {
57+
export BUILDKITE_PLUGIN_CACHE_SAVE=step
58+
59+
run "$PWD/hooks/post-command"
60+
61+
assert_success
62+
assert_output --partial 'Saving step-level cache'
63+
assert_output --partial 'Compressing tests/data/my_files with zstd'
64+
}
65+
66+
@test "Branch-level saving" {
67+
export BUILDKITE_PLUGIN_CACHE_SAVE=branch
68+
69+
run "$PWD/hooks/post-command"
70+
71+
assert_success
72+
assert_output --partial 'Saving branch-level cache'
73+
assert_output --partial 'Compressing tests/data/my_files with zstd'
74+
}
75+
76+
@test "Pipeline-level saving" {
77+
export BUILDKITE_PLUGIN_CACHE_SAVE=pipeline
78+
79+
run "$PWD/hooks/post-command"
80+
81+
assert_success
82+
assert_output --partial 'Saving pipeline-level cache'
83+
assert_output --partial 'Compressing tests/data/my_files with zstd'
84+
}
85+
86+
@test "All-level saving" {
87+
export BUILDKITE_PLUGIN_CACHE_SAVE=all
88+
89+
run "$PWD/hooks/post-command"
90+
91+
assert_success
92+
assert_output --partial 'Saving all-level cache'
93+
assert_output --partial 'Compressing tests/data/my_files with zstd'
94+
}
95+
96+
@test "Multiple level saving" {
97+
export BUILDKITE_PLUGIN_CACHE_SAVE_0=all
98+
export BUILDKITE_PLUGIN_CACHE_SAVE_1=pipeline
99+
100+
# add an extra save, but tar should still be called only once
101+
stub cache_dummy \
102+
"save \* \* : echo saving \$3 in \$2"
103+
104+
run "$PWD/hooks/post-command"
105+
106+
assert_success
107+
assert_output --partial 'Saving all-level cache'
108+
assert_output --partial 'Saving pipeline-level cache'
109+
}
110+
111+
112+
@test 'Pipeline-level saving with absolute cache path' {
113+
BUILDKITE_PLUGIN_CACHE_PATH="$(mktemp -d)"
114+
export BUILDKITE_PLUGIN_CACHE_PATH
115+
export BUILDKITE_PLUGIN_CACHE_SAVE=pipeline
116+
117+
run "$PWD/hooks/post-command"
118+
119+
assert_success
120+
assert_output --partial 'Saving pipeline-level cache'
121+
assert_output --partial "Compressing ${BUILDKITE_PLUGIN_CACHE_PATH} with zstd..."
122+
assert_output --partial "with options -o"
123+
124+
rm -rf "${BUILDKITE_PLUGIN_CACHE_PATH}"
125+
}

‎tests/pre-command-zstd.bats‎

Lines changed: 179 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,179 @@
1+
#!/usr/bin/env bats
2+
3+
# To debug stubs, uncomment these lines:
4+
# export CACHE_DUMMY_STUB_DEBUG=/dev/tty
5+
# export TAR_STUB_DEBUG=/dev/tty
6+
# export ZSTD_STUB_DEBUG=/dev/tty
7+
8+
setup() {
9+
load "${BATS_PLUGIN_PATH}/load.bash"
10+
11+
mkdir -p tests/data/my_files
12+
echo "all the llamas" > "tests/data/my_files/llamas.txt"
13+
echo "no alpacas" > "tests/data/my_files/alpacas.txt"
14+
15+
export BUILDKITE_PLUGIN_CACHE_BACKEND=dummy
16+
export BUILDKITE_PLUGIN_CACHE_COMPRESSION=zstd
17+
export BUILDKITE_PLUGIN_CACHE_PATH=tests/data/my_files
18+
export BUILDKITE_PLUGIN_CACHE_MANIFEST=tests/data/my_files/llamas.txt
19+
20+
# necessary for key-calculations
21+
export BUILDKITE_LABEL="step-label"
22+
export BUILDKITE_BRANCH="tests"
23+
export BUILDKITE_ORGANIZATION_SLUG="bk-cache-test"
24+
export BUILDKITE_PIPELINE_SLUG="cache-pipeline"
25+
26+
# stub is the same for all tests
27+
stub zstd \
28+
"-d \* \* : echo uncompressed \$2 into \$3"
29+
30+
stub tar "echo called tar with options \$@ and input; cat"
31+
32+
}
33+
34+
teardown() {
35+
rm -rf tests/data
36+
37+
unstub zstd
38+
unstub tar
39+
}
40+
41+
@test 'Existing file-based restore' {
42+
export BUILDKITE_PLUGIN_CACHE_RESTORE=file
43+
44+
stub cache_dummy \
45+
'exists \* : exit 0' \
46+
"get \* \* : echo restoring \$2 to \$3"
47+
48+
run "$PWD/hooks/pre-command"
49+
50+
assert_success
51+
assert_output --partial 'Cache hit at file level'
52+
assert_output --partial 'Cache is compressed, decompressing with zstd'
53+
54+
unstub cache_dummy
55+
}
56+
57+
@test 'Existing file-based restore even when max-level is higher' {
58+
export BUILDKITE_PLUGIN_CACHE_RESTORE=all
59+
60+
stub cache_dummy \
61+
'exists \* : exit 0' \
62+
"get \* \* : echo restoring \$2 to \$3"
63+
64+
run "$PWD/hooks/pre-command"
65+
66+
assert_success
67+
assert_output --partial 'Cache hit at file level'
68+
assert_output --partial 'Cache is compressed, decompressing with zstd'
69+
70+
unstub cache_dummy
71+
}
72+
73+
@test 'Existing step-based restore' {
74+
export BUILDKITE_PLUGIN_CACHE_RESTORE=step
75+
76+
stub cache_dummy \
77+
'exists \* : exit 1' \
78+
'exists \* : exit 0' \
79+
"get \* \* : echo restoring \$2 to \$3"
80+
81+
run "$PWD/hooks/pre-command"
82+
83+
assert_success
84+
assert_output --partial 'Cache hit at step level'
85+
assert_output --partial 'Cache is compressed, decompressing with zstd'
86+
87+
unstub cache_dummy
88+
}
89+
90+
@test 'Existing branch-based restore' {
91+
export BUILDKITE_PLUGIN_CACHE_RESTORE=branch
92+
93+
stub cache_dummy \
94+
'exists \* : exit 1' \
95+
'exists \* : exit 1' \
96+
'exists \* : exit 0' \
97+
"get \* \* : echo restoring \$2 to \$3"
98+
99+
run "$PWD/hooks/pre-command"
100+
101+
assert_success
102+
assert_output --partial 'Cache hit at branch level'
103+
assert_output --partial 'Cache is compressed, decompressing with zstd'
104+
105+
unstub cache_dummy
106+
}
107+
@test 'Existing pipeline-based restore' {
108+
export BUILDKITE_PLUGIN_CACHE_RESTORE=pipeline
109+
110+
stub cache_dummy \
111+
'exists \* : exit 1' \
112+
'exists \* : exit 1' \
113+
'exists \* : exit 1' \
114+
'exists \* : exit 0' \
115+
"get \* \* : echo restoring \$2 to \$3"
116+
117+
run "$PWD/hooks/pre-command"
118+
119+
assert_success
120+
assert_output --partial 'Cache hit at pipeline level'
121+
assert_output --partial 'Cache is compressed, decompressing with zstd'
122+
123+
unstub cache_dummy
124+
}
125+
126+
@test 'Existing all-based restore' {
127+
export BUILDKITE_PLUGIN_CACHE_RESTORE=all
128+
129+
stub cache_dummy \
130+
'exists \* : exit 1' \
131+
'exists \* : exit 1' \
132+
'exists \* : exit 1' \
133+
'exists \* : exit 1' \
134+
'exists \* : exit 0' \
135+
"get \* \* : echo restoring \$2 to \$3"
136+
137+
run "$PWD/hooks/pre-command"
138+
139+
assert_success
140+
assert_output --partial 'Cache hit at all level'
141+
assert_output --partial 'Cache is compressed, decompressing with zstd'
142+
143+
unstub cache_dummy
144+
}
145+
146+
@test 'Existing lower level restore works' {
147+
export BUILDKITE_PLUGIN_CACHE_RESTORE=all
148+
149+
stub cache_dummy \
150+
'exists \* : exit 1' \
151+
'exists \* : exit 1' \
152+
'exists \* : exit 0' \
153+
"get \* \* : echo restoring \$2 to \$3"
154+
155+
run "$PWD/hooks/pre-command"
156+
157+
assert_success
158+
assert_output --partial 'Cache hit at branch level'
159+
assert_output --partial 'Cache is compressed, decompressing with zstd'
160+
161+
unstub cache_dummy
162+
}
163+
164+
@test 'Existing file-based restore to absolute path' {
165+
export BUILDKITE_PLUGIN_CACHE_RESTORE=file
166+
export BUILDKITE_PLUGIN_CACHE_PATH=/tmp/tests/data/my_files
167+
168+
stub cache_dummy \
169+
'exists \* : exit 0' \
170+
"get \* \* : echo restoring \$2 to \$3"
171+
172+
run "$PWD/hooks/pre-command"
173+
174+
assert_success
175+
assert_output --partial 'Cache hit at file level'
176+
assert_output --partial 'Cache is compressed, decompressing with zstd...'
177+
178+
unstub cache_dummy
179+
}

0 commit comments

Comments
 (0)