Skip to content

Commit 6f0e21e

Browse files
pzeballospzeballos
authored
Merge pull request #28 from buildkite-plugins/toote_s3_backend
S3 as a backend
2 parents 91b9da9 + 7ea9606 commit 6f0e21e

File tree

3 files changed

+190
-3
lines changed

3 files changed

+190
-3
lines changed

‎README.md‎

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -40,9 +40,9 @@ The level to use for saving the cache. See [the available caching levels](#cachi
4040

4141
### `backend` (string)
4242

43-
How cache is stored and restored. Default: `fs`.
44-
45-
Can be any string (see [Customizable Backends](#customizable-backends)), but the plugin natively supports the following.
43+
Defines how the cache is stored and restored. Can be any string (see [Customizable Backends](#customizable-backends)), but the plugin natively supports the following:
44+
* `fs` (default)
45+
* `s3`
4646

4747
#### `fs`
4848

@@ -52,6 +52,13 @@ The `BUILDKITE_PLUGIN_FS_CACHE_FOLDER` environment variable defines where the co
5252

5353
**IMPORTANT**: the `fs` backend just copies files to a different location in the current agent, as it is not a shared or external resource, its caching possibilities are quite limited.
5454

55+
#### `s3`
56+
57+
Store things in an S3 bucket. You need to make sure that the `aws` command is available and appropriately configured.
58+
59+
You also need the agent to have access to the following defined environment variables:
60+
* `BUILDKITE_PLUGIN_S3_CACHE_BUCKET`: the bucket to use (backend will fail if not defined)
61+
* `BUILDKITE_PLUGIN_S3_CACHE_PREFIX`: optional prefix to use for the cache within the bucket
5562

5663
### `manifest` (string, required if using `file` caching level)
5764

‎backends/cache_s3‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
#!/bin/bash
2+
3+
if [ -z "${BUILDKITE_PLUGIN_S3_CACHE_BUCKET}" ]; then
4+
echo '+++ 🚨 Missing S3 bucket configuration'
5+
exit 1
6+
fi
7+
8+
build_key() {
9+
if [ -n "${BUILDKITE_PLUGIN_S3_CACHE_PREFIX}" ]; then
10+
echo "${BUILDKITE_PLUGIN_S3_CACHE_PREFIX}/${1}"
11+
else
12+
echo "$1"
13+
fi
14+
}
15+
16+
restore_cache() {
17+
local from=$1
18+
local to="$2"
19+
aws s3 sync --recursive "s3://${BUILDKITE_PLUGIN_S3_CACHE_BUCKET}/$(build_key "${from}")" "${to}"
20+
}
21+
22+
save_cache() {
23+
local to="$1"
24+
local from="$2"
25+
aws s3 sync --recursive "${from}" "s3://${BUILDKITE_PLUGIN_S3_CACHE_BUCKET}/$(build_key "${to}")"
26+
}
27+
28+
exists_cache() {
29+
if [ -z "$1" ]; then exit 1; fi
30+
aws s3api head-object --bucket "${BUILDKITE_PLUGIN_S3_CACHE_BUCKET}" --key "$(build_key "$1")"
31+
}
32+
33+
OPCODE="$1"
34+
shift
35+
36+
if [ "$OPCODE" = 'exists' ]; then
37+
exists_cache "$@"
38+
elif [ "$OPCODE" = 'get' ]; then
39+
restore_cache "$@"
40+
elif [ "$OPCODE" = 'save' ]; then
41+
save_cache "$@"
42+
else
43+
exit 255
44+
fi

‎tests/cache_s3.bats‎

Lines changed: 136 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,136 @@
1+
#!/usr/bin/env bats
2+
3+
# To debug stubs, uncomment these lines:
4+
# export AWS_STUB_DEBUG=/dev/tty
5+
6+
setup() {
7+
load "${BATS_PLUGIN_PATH}/load.bash"
8+
9+
export BUILDKITE_PLUGIN_S3_CACHE_BUCKET=my-bucket
10+
}
11+
12+
# teardown() {
13+
# rm -rf "${BUILDKITE_PLUGIN_FS_CACHE_FOLDER}"
14+
# }
15+
16+
@test 'Missing bucket configuration makes plugin fail' {
17+
unset BUILDKITE_PLUGIN_S3_CACHE_BUCKET
18+
19+
run "${PWD}/backends/cache_s3"
20+
21+
assert_failure
22+
assert_output --partial 'Missing S3 bucket configuration'
23+
}
24+
25+
@test 'Invalid operation fails silently wtih 255' {
26+
run "${PWD}/backends/cache_s3" invalid
27+
28+
assert_failure 255
29+
assert_output ''
30+
}
31+
32+
@test 'Exists on empty file fails' {
33+
run "${PWD}/backends/cache_s3" exists ""
34+
35+
assert_failure
36+
assert_output ''
37+
}
38+
39+
@test 'Exists on non-existing file fails' {
40+
stub aws 'exit 1'
41+
42+
run "${PWD}/backends/cache_s3" exists PATH/THAT/DOES/NOT/EXIST
43+
44+
assert_failure
45+
assert_output ''
46+
47+
unstub aws
48+
}
49+
50+
@test 'Exists on existing file/folder works' {
51+
stub aws 'exit 0'
52+
53+
run "${PWD}/backends/cache_s3" exists existing
54+
55+
assert_success
56+
assert_output ''
57+
58+
unstub aws
59+
}
60+
61+
@test 'File exists and can be restored after save' {
62+
touch "${BATS_TEST_TMPDIR}/new-file"
63+
mkdir "${BATS_TEST_TMPDIR}/s3-cache"
64+
stub aws \
65+
"test -e $BATS_TEST_TMPDIR/s3-cache/\$(echo s3://\$4/\$6 | md5sum | cut -c-32)" \
66+
"ln -s \$4 $BATS_TEST_TMPDIR/s3-cache/\$(echo \$5 | md5sum | cut -c-32)" \
67+
"test -e $BATS_TEST_TMPDIR/s3-cache/\$(echo s3://\$4/\$6 | md5sum | cut -c-32)" \
68+
"cp -r $BATS_TEST_TMPDIR/s3-cache/\$(echo \$4 | md5sum | cut -c-32) \$5"
69+
70+
run "${PWD}/backends/cache_s3" exists new-file
71+
72+
assert_failure
73+
assert_output ''
74+
75+
run "${PWD}/backends/cache_s3" save new-file "${BATS_TEST_TMPDIR}/new-file"
76+
77+
assert_success
78+
assert_output ''
79+
80+
run "${PWD}/backends/cache_s3" exists new-file
81+
82+
assert_success
83+
assert_output ''
84+
85+
run "${PWD}/backends/cache_s3" get new-file "${BATS_TEST_TMPDIR}/other-file"
86+
87+
assert_success
88+
assert_output ''
89+
90+
diff "${BATS_TEST_TMPDIR}/new-file" "${BATS_TEST_TMPDIR}/other-file"
91+
92+
unstub aws
93+
rm -rf "${BATS_TEST_TMPDIR}/s3-cache"
94+
rm -rf "${BATS_TEST_TMPDIR}/new-file"
95+
}
96+
97+
@test 'Folder exists and can be restored after save' {
98+
mkdir "${BATS_TEST_TMPDIR}/s3-cache"
99+
mkdir "${BATS_TEST_TMPDIR}/new-folder"
100+
echo 'random content' > "${BATS_TEST_TMPDIR}/new-folder/new-file"
101+
102+
stub aws \
103+
"test -e $BATS_TEST_TMPDIR/s3-cache/\$(echo s3://\$4/\$6 | md5sum | cut -c-32)" \
104+
"ln -s \$4 $BATS_TEST_TMPDIR/s3-cache/\$(echo \$5 | md5sum | cut -c-32)" \
105+
"test -e $BATS_TEST_TMPDIR/s3-cache/\$(echo s3://\$4/\$6 | md5sum | cut -c-32)" \
106+
"cp -r $BATS_TEST_TMPDIR/s3-cache/\$(echo \$4 | md5sum | cut -c-32) \$5"
107+
108+
run "${PWD}/backends/cache_s3" exists new-folder
109+
110+
assert_failure
111+
assert_output ''
112+
113+
run "${PWD}/backends/cache_s3" save new-folder "${BATS_TEST_TMPDIR}/new-folder"
114+
115+
assert_success
116+
assert_output ''
117+
118+
run "${PWD}/backends/cache_s3" exists new-folder
119+
120+
assert_success
121+
assert_output ''
122+
123+
run "${PWD}/backends/cache_s3" get new-folder "${BATS_TEST_TMPDIR}/other-folder"
124+
125+
assert_success
126+
assert_output ''
127+
128+
find "${BATS_TEST_TMPDIR}/new-folder"
129+
130+
find "${BATS_TEST_TMPDIR}/other-folder"
131+
diff -r "${BATS_TEST_TMPDIR}/new-folder" "${BATS_TEST_TMPDIR}/other-folder"
132+
133+
rm -rf "${BATS_TEST_TMPDIR}/s3-cache"
134+
rm -rf "${BATS_TEST_TMPDIR}/new-folder"
135+
rm -rf "${BATS_TEST_TMPDIR}/other-folder"
136+
}

0 commit comments

Comments
 (0)