Skip to content

Commit 2704c83

Browse files
fattenederfatteneder
committed
add option to force build using a PR label
1 parent dabb3c9 commit 2704c83

File tree

5 files changed

+280
-17
lines changed

5 files changed

+280
-17
lines changed

‎pipelines/main/launch_unsigned_builders.yml‎

Lines changed: 0 additions & 1 deletion
This file was deleted.

‎pipelines/main/launch_unsigned_builders.yml‎

Lines changed: 220 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,220 @@
1+
# This file launches all the build jobs that _don't_ require secrets access.
2+
# These jobs can pass their output off to jobs that do require secrets access,
3+
# but those privileged steps require signing before they can be run.
4+
#
5+
# Yes, this is creating another layer of indirection; the flow now looks like:
6+
#
7+
# [webui] -> launch_unsigned_builders.yml -> misc/llvmpasses.yml
8+
#
9+
# when we could theoretically just have the `webui` launch `misc/llvmpasses.yml`,
10+
# however this raises the bar for contributors to add new (unsigned) steps to
11+
# our CI configuration, so I'd rather live with an extra layer of indirection
12+
# and only need to touch the webui configuration when we need to alter
13+
# something about the privileged steps.
14+
15+
common:
16+
- diff-filter-build_plugin: &diff-filter-build
17+
https://github.com/fatteneder/diff-filter-buildkite-plugin#main:
18+
name: "build"
19+
ignore:
20+
- "*.md"
21+
- "*.json"
22+
- "CITATION.*"
23+
- ".gitignore"
24+
- ".clangd"
25+
- ".mailmap"
26+
# TODO: Need this option because BUILDKITE_PULL_REQUEST_BRANCH=main
27+
# when running tests in Julia-CI/julia-buildkite,
28+
# but for PRs against Julia/JuliaLang BUILDKITE_PULL_REQUEST_BRANCH=master
29+
target_branch: "master"
30+
- pr-labels_plugin: &pr-labels
31+
sv-oss/github-pr-labels#v0.0.2:
32+
publish-env-var: PULL_REQUEST_LABELS
33+
34+
steps:
35+
- group: "Build"
36+
notify:
37+
- github_commit_status:
38+
context: "Build"
39+
steps:
40+
- label: "Launch build jobs"
41+
plugins:
42+
- JuliaCI/external-buildkite#v1:
43+
version: "./.buildkite-external-version"
44+
repo_url: "https://github.com/JuliaCI/julia-buildkite"
45+
- *diff-filter-build
46+
- *pr-labels
47+
commands: |
48+
# Launch Linux build jobs
49+
FORCE=$(test -n "$(echo $${PULL_REQUEST_LABELS} | grep "\\<ci-force-build\\>")" && echo 1 || echo 0)
50+
BUILD=$(($${BUILDKITE_PLUGIN_DIFF_FILTER_TRIGGERED_BUILD} || ${FORCE}))
51+
GROUP="Build" \
52+
ALLOW_FAIL="false" \
53+
BUILD="${BUILD}" \
54+
bash .buildkite/utilities/arches_pipeline_upload.sh \
55+
.buildkite/pipelines/main/platforms/build_linux.arches \
56+
.buildkite/pipelines/main/platforms/build_linux.yml
57+
# Launch macOS packaging jobs
58+
GROUP="Build" \
59+
ALLOW_FAIL="false" \
60+
BUILD="${BUILD}" \
61+
bash .buildkite/utilities/arches_pipeline_upload.sh \
62+
.buildkite/pipelines/main/platforms/build_macos.arches \
63+
.buildkite/pipelines/main/platforms/build_macos.yml
64+
GROUP="Build" \
65+
ALLOW_FAIL="false" \
66+
BUILD="${BUILD}" \
67+
bash .buildkite/utilities/arches_pipeline_upload.sh \
68+
.buildkite/pipelines/main/platforms/build_freebsd.arches \
69+
.buildkite/pipelines/main/platforms/build_freebsd.yml
70+
GROUP="Build" \
71+
ALLOW_FAIL="false" \
72+
BUILD="${BUILD}" \
73+
bash .buildkite/utilities/arches_pipeline_upload.sh \
74+
.buildkite/pipelines/main/platforms/build_windows.arches \
75+
.buildkite/pipelines/main/platforms/build_windows.yml
76+
agents:
77+
queue: "julia"
78+
os: "linux"
79+
- group: "Check"
80+
notify:
81+
- github_commit_status:
82+
context: "Check"
83+
steps:
84+
- label: "Launch check jobs"
85+
plugins:
86+
- JuliaCI/external-buildkite#v1:
87+
version: "./.buildkite-external-version"
88+
repo_url: "https://github.com/JuliaCI/julia-buildkite"
89+
- *diff-filter-build
90+
- *pr-labels
91+
commands: |
92+
buildkite-agent pipeline upload .buildkite/pipelines/main/misc/doctest.yml
93+
buildkite-agent pipeline upload .buildkite/pipelines/main/misc/pdf_docs/build_pdf_docs.yml
94+
FORCE=$(test -n "$(echo $${PULL_REQUEST_LABELS} | grep "\\<ci-force-build\\>")" && echo 1 || echo 0)
95+
BUILD=$(($${BUILDKITE_PLUGIN_DIFF_FILTER_TRIGGERED_BUILD} || ${FORCE}))
96+
if [[ ${BUILD} == 1 ]]; then
97+
buildkite-agent pipeline upload .buildkite/pipelines/main/misc/analyzegc.yml
98+
buildkite-agent pipeline upload .buildkite/pipelines/main/misc/embedding.yml
99+
buildkite-agent pipeline upload .buildkite/pipelines/main/misc/llvmpasses.yml
100+
# buildkite-agent pipeline upload .buildkite/pipelines/main/misc/whitespace.yml # Currently runs in GitHub Actions instead of Buildkite
101+
102+
buildkite-agent pipeline upload .buildkite/pipelines/main/misc/sanitizers/asan.yml
103+
buildkite-agent pipeline upload .buildkite/pipelines/main/misc/sanitizers/tsan.yml
104+
fi
105+
agents:
106+
queue: "julia"
107+
os: "linux"
108+
- group: "Test"
109+
notify:
110+
- github_commit_status:
111+
context: "Test"
112+
steps:
113+
- label: "Launch test jobs"
114+
plugins:
115+
- JuliaCI/external-buildkite#v1:
116+
version: "./.buildkite-external-version"
117+
repo_url: "https://github.com/JuliaCI/julia-buildkite"
118+
- *diff-filter-build
119+
- *pr-labels
120+
commands: |
121+
export ALLOW_FAIL="false"
122+
FORCE=$(test -n "$(echo $${PULL_REQUEST_LABELS} | grep "\\<ci-force-build\\>")" && echo 1 || echo 0)
123+
BUILD=$(($${BUILDKITE_PLUGIN_DIFF_FILTER_TRIGGERED_BUILD} || ${FORCE}))
124+
if [[ ${BUILD} == 1 ]]; then
125+
buildkite-agent pipeline upload .buildkite/pipelines/main/misc/gcext.yml
126+
buildkite-agent pipeline upload .buildkite/pipelines/main/misc/test_revise.yml
127+
128+
# Launch Linux test jobs
129+
GROUP="Test" \
130+
bash .buildkite/utilities/arches_pipeline_upload.sh \
131+
.buildkite/pipelines/main/platforms/test_linux.arches \
132+
.buildkite/pipelines/main/platforms/test_linux.yml
133+
# Launch i686-linux-gnu test jobs
134+
GROUP="Test" \
135+
bash .buildkite/utilities/arches_pipeline_upload.sh \
136+
.buildkite/pipelines/main/platforms/test_linux.i686.arches \
137+
.buildkite/pipelines/main/platforms/test_linux.i686.yml
138+
# Launch macOS test jobs
139+
GROUP="Test" \
140+
bash .buildkite/utilities/arches_pipeline_upload.sh \
141+
.buildkite/pipelines/main/platforms/test_macos.arches \
142+
.buildkite/pipelines/main/platforms/test_macos.yml
143+
# Launch FreeBSD test jobs
144+
GROUP="Test" \
145+
bash .buildkite/utilities/arches_pipeline_upload.sh \
146+
.buildkite/pipelines/main/platforms/test_freebsd.arches \
147+
.buildkite/pipelines/main/platforms/test_freebsd.yml
148+
# Launch windows test jobs
149+
GROUP="Test" \
150+
bash .buildkite/utilities/arches_pipeline_upload.sh \
151+
.buildkite/pipelines/main/platforms/test_windows.arches \
152+
.buildkite/pipelines/main/platforms/test_windows.yml
153+
fi
154+
agents:
155+
queue: "julia"
156+
os: "linux"
157+
- group: "Allow Fail"
158+
steps:
159+
- label: "Launch allowed-to-fail build jobs"
160+
plugins:
161+
- JuliaCI/external-buildkite#v1:
162+
version: "./.buildkite-external-version"
163+
repo_url: "https://github.com/JuliaCI/julia-buildkite"
164+
- *diff-filter-build
165+
- *pr-labels
166+
commands: |
167+
export ALLOW_FAIL="true"
168+
FORCE=$(test -n "$(echo $${PULL_REQUEST_LABELS} | grep "\\<ci-force-build\\>")" && echo 1 || echo 0)
169+
BUILD=$(($${BUILDKITE_PLUGIN_DIFF_FILTER_TRIGGERED_BUILD} || ${FORCE}))
170+
if [[ ${BUILD} == 1 ]]; then
171+
# Launch Linux allowed-to-fail build jobs
172+
GROUP="Allow Fail" \
173+
bash .buildkite/utilities/arches_pipeline_upload.sh \
174+
.buildkite/pipelines/main/platforms/build_linux.soft_fail.arches \
175+
.buildkite/pipelines/main/platforms/build_linux.yml
176+
# Launch macOS allowed-to-fail build jobs
177+
GROUP="Allow Fail" \
178+
bash .buildkite/utilities/arches_pipeline_upload.sh \
179+
.buildkite/pipelines/main/platforms/build_macos.soft_fail.arches \
180+
.buildkite/pipelines/main/platforms/build_macos.yml
181+
fi
182+
agents:
183+
queue: "julia"
184+
os: "linux"
185+
- label: "Launch allowed-to-fail test jobs"
186+
plugins:
187+
- JuliaCI/external-buildkite#v1:
188+
version: "./.buildkite-external-version"
189+
repo_url: "https://github.com/JuliaCI/julia-buildkite"
190+
- *diff-filter-build
191+
- *pr-labels
192+
commands: |
193+
export ALLOW_FAIL="true"
194+
FORCE=$(test -n "$(echo $${PULL_REQUEST_LABELS} | grep "\\<ci-force-build\\>")" && echo 1 || echo 0)
195+
BUILD=$(($${BUILDKITE_PLUGIN_DIFF_FILTER_TRIGGERED_BUILD} || ${FORCE}))
196+
if [[ ${BUILD} == 1 ]]; then
197+
# Launch Linux allowed-to-fail test jobs
198+
GROUP="Allow Fail" \
199+
bash .buildkite/utilities/arches_pipeline_upload.sh \
200+
.buildkite/pipelines/main/platforms/test_linux.soft_fail.arches \
201+
.buildkite/pipelines/main/platforms/test_linux.yml
202+
# Launch macOS allowed-to-fail test jobs
203+
GROUP="Allow Fail" \
204+
bash .buildkite/utilities/arches_pipeline_upload.sh \
205+
.buildkite/pipelines/main/platforms/test_macos.soft_fail.arches \
206+
.buildkite/pipelines/main/platforms/test_macos.yml
207+
# Launch FreeBSD allowed-to-fail jobs
208+
GROUP="Allow Fail" \
209+
bash .buildkite/utilities/arches_pipeline_upload.sh \
210+
.buildkite/pipelines/main/platforms/test_freebsd.soft_fail.arches \
211+
.buildkite/pipelines/main/platforms/test_freebsd.yml
212+
# Launch windows allowed-to-fail test jobs
213+
GROUP="Allow Fail" \
214+
bash .buildkite/utilities/arches_pipeline_upload.sh \
215+
.buildkite/pipelines/main/platforms/test_windows.soft_fail.arches \
216+
.buildkite/pipelines/main/platforms/test_windows.yml
217+
fi
218+
agents:
219+
queue: "julia"
220+
os: "linux"

‎pipelines/main/launch_unsigned_jobs.yml‎

Lines changed: 26 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,9 @@ common:
2727
# when running tests in Julia-CI/julia-buildkite,
2828
# but for PRs against Julia/JuliaLang BUILDKITE_PULL_REQUEST_BRANCH=master
2929
target_branch: "master"
30+
- pr-labels_plugin: &pr-labels
31+
sv-oss/github-pr-labels#v0.0.2:
32+
publish-env-var: PULL_REQUEST_LABELS
3033

3134
steps:
3235
- group: "Build"
@@ -40,30 +43,33 @@ steps:
4043
version: "./.buildkite-external-version"
4144
repo_url: "https://github.com/JuliaCI/julia-buildkite"
4245
- *diff-filter-build
46+
- *pr-labels
4347
commands: |
4448
# Launch Linux build jobs
49+
FORCE=$(test -n "$(echo $${PULL_REQUEST_LABELS} | grep "\\<ci-force-build\\>")" && echo 1 || echo 0)
50+
BUILD=$(($${BUILDKITE_PLUGIN_DIFF_FILTER_TRIGGERED_BUILD} || ${FORCE}))
4551
GROUP="Build" \
4652
ALLOW_FAIL="false" \
47-
BUILD="$${BUILDKITE_PLUGIN_DIFF_FILTER_TRIGGERED_BUILD}" \
53+
BUILD="${BUILD}" \
4854
bash .buildkite/utilities/arches_pipeline_upload.sh \
4955
.buildkite/pipelines/main/platforms/build_linux.arches \
5056
.buildkite/pipelines/main/platforms/build_linux.yml
5157
# Launch macOS packaging jobs
5258
GROUP="Build" \
5359
ALLOW_FAIL="false" \
54-
BUILD="$${BUILDKITE_PLUGIN_DIFF_FILTER_TRIGGERED_BUILD}" \
60+
BUILD="${BUILD}" \
5561
bash .buildkite/utilities/arches_pipeline_upload.sh \
5662
.buildkite/pipelines/main/platforms/build_macos.arches \
5763
.buildkite/pipelines/main/platforms/build_macos.yml
5864
GROUP="Build" \
5965
ALLOW_FAIL="false" \
60-
BUILD="$${BUILDKITE_PLUGIN_DIFF_FILTER_TRIGGERED_BUILD}" \
66+
BUILD="${BUILD}" \
6167
bash .buildkite/utilities/arches_pipeline_upload.sh \
6268
.buildkite/pipelines/main/platforms/build_freebsd.arches \
6369
.buildkite/pipelines/main/platforms/build_freebsd.yml
6470
GROUP="Build" \
6571
ALLOW_FAIL="false" \
66-
BUILD="$${BUILDKITE_PLUGIN_DIFF_FILTER_TRIGGERED_BUILD}" \
72+
BUILD="${BUILD}" \
6773
bash .buildkite/utilities/arches_pipeline_upload.sh \
6874
.buildkite/pipelines/main/platforms/build_windows.arches \
6975
.buildkite/pipelines/main/platforms/build_windows.yml
@@ -81,10 +87,13 @@ steps:
8187
version: "./.buildkite-external-version"
8288
repo_url: "https://github.com/JuliaCI/julia-buildkite"
8389
- *diff-filter-build
90+
- *pr-labels
8491
commands: |
8592
buildkite-agent pipeline upload .buildkite/pipelines/main/misc/doctest.yml
8693
buildkite-agent pipeline upload .buildkite/pipelines/main/misc/pdf_docs/build_pdf_docs.yml
87-
if [[ $${BUILDKITE_PLUGIN_DIFF_FILTER_TRIGGERED_BUILD} == 1 ]]; then
94+
FORCE=$(test -n "$(echo $${PULL_REQUEST_LABELS} | grep "\\<ci-force-build\\>")" && echo 1 || echo 0)
95+
BUILD=$(($${BUILDKITE_PLUGIN_DIFF_FILTER_TRIGGERED_BUILD} || ${FORCE}))
96+
if [[ ${BUILD} == 1 ]]; then
8897
buildkite-agent pipeline upload .buildkite/pipelines/main/misc/analyzegc.yml
8998
buildkite-agent pipeline upload .buildkite/pipelines/main/misc/embedding.yml
9099
buildkite-agent pipeline upload .buildkite/pipelines/main/misc/llvmpasses.yml
@@ -107,9 +116,12 @@ steps:
107116
version: "./.buildkite-external-version"
108117
repo_url: "https://github.com/JuliaCI/julia-buildkite"
109118
- *diff-filter-build
119+
- *pr-labels
110120
commands: |
111121
export ALLOW_FAIL="false"
112-
if [[ $${BUILDKITE_PLUGIN_DIFF_FILTER_TRIGGERED_BUILD} == 1 ]]; then
122+
FORCE=$(test -n "$(echo $${PULL_REQUEST_LABELS} | grep "\\<ci-force-build\\>")" && echo 1 || echo 0)
123+
BUILD=$(($${BUILDKITE_PLUGIN_DIFF_FILTER_TRIGGERED_BUILD} || ${FORCE}))
124+
if [[ ${BUILD} == 1 ]]; then
113125
buildkite-agent pipeline upload .buildkite/pipelines/main/misc/gcext.yml
114126
buildkite-agent pipeline upload .buildkite/pipelines/main/misc/test_revise.yml
115127
@@ -150,9 +162,12 @@ steps:
150162
version: "./.buildkite-external-version"
151163
repo_url: "https://github.com/JuliaCI/julia-buildkite"
152164
- *diff-filter-build
165+
- *pr-labels
153166
commands: |
154167
export ALLOW_FAIL="true"
155-
if [[ $${BUILDKITE_PLUGIN_DIFF_FILTER_TRIGGERED_BUILD} == 1 ]]; then
168+
FORCE=$(test -n "$(echo $${PULL_REQUEST_LABELS} | grep "\\<ci-force-build\\>")" && echo 1 || echo 0)
169+
BUILD=$(($${BUILDKITE_PLUGIN_DIFF_FILTER_TRIGGERED_BUILD} || ${FORCE}))
170+
if [[ ${BUILD} == 1 ]]; then
156171
# Launch Linux allowed-to-fail build jobs
157172
GROUP="Allow Fail" \
158173
bash .buildkite/utilities/arches_pipeline_upload.sh \
@@ -173,9 +188,12 @@ steps:
173188
version: "./.buildkite-external-version"
174189
repo_url: "https://github.com/JuliaCI/julia-buildkite"
175190
- *diff-filter-build
191+
- *pr-labels
176192
commands: |
177193
export ALLOW_FAIL="true"
178-
if [[ $${BUILDKITE_PLUGIN_DIFF_FILTER_TRIGGERED_BUILD} == 1 ]]; then
194+
FORCE=$(test -n "$(echo $${PULL_REQUEST_LABELS} | grep "\\<ci-force-build\\>")" && echo 1 || echo 0)
195+
BUILD=$(($${BUILDKITE_PLUGIN_DIFF_FILTER_TRIGGERED_BUILD} || ${FORCE}))
196+
if [[ ${BUILD} == 1 ]]; then
179197
# Launch Linux allowed-to-fail test jobs
180198
GROUP="Allow Fail" \
181199
bash .buildkite/utilities/arches_pipeline_upload.sh \

‎pipelines/main/launch_upload_jobs.yml‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,9 @@ common:
1515
# when running tests in Julia-CI/julia-buildkite,
1616
# but for PRs against Julia/JuliaLang BUILDKITE_PULL_REQUEST_BRANCH=master
1717
target_branch: "master"
18+
- pr-labels_plugin: &pr-labels
19+
sv-oss/github-pr-labels#v0.0.2:
20+
publish-env-var: PULL_REQUEST_LABELS
1821

1922
steps:
2023
- group: "Upload"
@@ -25,12 +28,14 @@ steps:
2528
version: "./.buildkite-external-version"
2629
repo_url: "https://github.com/JuliaCI/julia-buildkite"
2730
- *diff-filter-build
31+
- *pr-labels
2832
commands: |
2933
# Explicitly pass along the cryptic token to child pipelines
3034
export BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET
3135
export ALLOW_FAIL="false"
32-
33-
if [[ $${BUILDKITE_PLUGIN_DIFF_FILTER_TRIGGERED_BUILD} == 1 ]]; then
36+
FORCE=$(test -n "$(echo $FLAG | grep "\\<ci-force-build\\>")" && echo 1 || echo 0)
37+
BUILD=$(($${BUILDKITE_PLUGIN_DIFF_FILTER_TRIGGERED_BUILD} || ${FORCE}))
38+
if [[ ${BUILD} == 1 ]]; then
3439
# Launch `upload_*` jobs to store tarballs into S3 once tests are done
3540
GROUP="Upload" \
3641
bash .buildkite/utilities/arches_pipeline_upload.sh \
@@ -77,13 +82,16 @@ steps:
7782
version: "./.buildkite-external-version"
7883
repo_url: "https://github.com/JuliaCI/julia-buildkite"
7984
- *diff-filter-build
85+
- *pr-labels
8086
commands: |
8187
# Explicitly pass along the cryptic token to child pipelines
8288
export BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET
89+
FORCE=$(test -n "$(echo $FLAG | grep "\\<ci-force-build\\>")" && echo 1 || echo 0)
90+
BUILD=$(($${BUILDKITE_PLUGIN_DIFF_FILTER_TRIGGERED_BUILD} || ${FORCE}))
8391
8492
export ALLOW_FAIL="true"
8593
86-
if [[ $${BUILDKITE_PLUGIN_DIFF_FILTER_TRIGGERED_BUILD} == 1 ]]; then
94+
if [[ ${BUILD} == 1 ]]; then
8795
# Launch Linux allowed-to-fail upload jobs
8896
GROUP="Allow Fail (Upload)" \
8997
bash .buildkite/utilities/arches_pipeline_upload.sh \

0 commit comments

Comments
 (0)