Skip to content

Commit 7bc1cd3

Browse files
topolarityDilumAluthge
topolarity
and
DilumAluthge
authored
Use Buildkite to test JuliaSyntax against older Julia versions (#502)
This PR moves the JuliaSyntax GHA matrix (from JuliaLang/julia#59870) to Buildkite. (big thanks to @DilumAluthge for most of the work here) --------- Co-authored-by: Dilum Aluthge <[email protected]>
1 parent 68cf370 commit 7bc1cd3

File tree

3 files changed

+253
-0
lines changed

3 files changed

+253
-0
lines changed

‎pipelines/main/launch_unsigned_jobs.yml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -90,6 +90,13 @@ steps:
9090
9191
buildkite-agent pipeline upload .buildkite/pipelines/main/misc/sanitizers/asan.yml
9292
buildkite-agent pipeline upload .buildkite/pipelines/main/misc/sanitizers/tsan.yml
93+
94+
if [ -e "./JuliaSyntax/Project.toml" ]; then
95+
echo "./JuliaSyntax/Project.toml file exists"
96+
buildkite-agent pipeline upload .buildkite/pipelines/main/misc/juliasyntax.launch.yml
97+
else
98+
echo "./JuliaSyntax/Project.toml file does NOT exist. Will not launch JuliaSyntax test jobs"
99+
fi
93100
agents:
94101
queue: "julia"
95102
os: "linux"

‎pipelines/main/misc/juliasyntax.launch.yml‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
# This is its own YAML file so that we can accomplish the following goal:
2+
# If JuliaSyntax has not yet been moved into the JuliaLang/julia repo,
3+
# then PRs to JuliaLang/julia should not display a `JuliaSyntax` PR commit status.
4+
5+
steps:
6+
- group: "JuliaSyntax"
7+
notify:
8+
- github_commit_status:
9+
context: "JuliaSyntax"
10+
steps:
11+
- label: "Launch JuliaSyntax jobs"
12+
plugins:
13+
- JuliaCI/external-buildkite#v1:
14+
version: "./.buildkite-external-version"
15+
repo_url: "https://github.com/JuliaCI/julia-buildkite"
16+
- JuliaCI/julia#b4d0bfa510bc399417133adae013385af5c71190: # v1.13.5
17+
# A few notes:
18+
# 1. We pin the `JuliaCI/julia` plugin to a full commit hash.
19+
# This is intentional. We want to make sure that this script doesn't suddenly
20+
# break when a new version of the plugin is released.
21+
# 2. We specify the Julia version up to patch, e.g. '1.7.3'.
22+
# This is intentional. We don't do e.g. '1.7', because we want to make sure
23+
# that this script doesn't suddenly break when a new patch release of Julia
24+
# is released.
25+
version: '1.10.10'
26+
commands: |
27+
buildkite-agent pipeline upload .buildkite/pipelines/main/misc/juliasyntax.test.yml
28+
agents:
29+
queue: "julia"
30+
os: "linux"
31+
arch: "x86_64" # prevent from running on PiBots

‎pipelines/main/misc/juliasyntax.test.yml‎

Lines changed: 215 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,215 @@
1+
steps:
2+
- group: "JuliaSyntax"
3+
steps:
4+
# Note: This file only concerns testing JuliaSyntax on older Julia versions
5+
# Testing JuliaSyntax on Julia#master is done separately.
6+
# Specifically, for that case, testing JuliaSyntax is built into the regular `Base.runtests()`,
7+
# so those tests are running in the regular Buildkite platform-specific test jobs.
8+
9+
# This step has most of the JuliaSyntax test jobs
10+
- label: "juliasyntax {{matrix.agent_os}}/{{matrix.julia_arch}}/{{matrix.julia_version}}"
11+
# We comment out the Buildkite step key for this step, because:
12+
# > You can't use matrix values in other attributes, including step keys and ...
13+
# Source: https://buildkite.com/docs/pipelines/configure/workflows/build-matrix
14+
# key: "juliasyntax-step-key"
15+
timeout_in_minutes: 30
16+
agents:
17+
queue: "julia"
18+
os: "{{matrix.agent_os}}"
19+
arch: "{{matrix.agent_arch}}"
20+
matrix:
21+
setup:
22+
agent_os:
23+
- "linux"
24+
- "windows"
25+
# - "macos" # We handle macOS special, see "adjustments" section below
26+
agent_arch: # arch of the Buildkite agent. Distinct from the Julia binary's arch
27+
- "x86_64" # The "x86_64" agent arch is used for both 32-bit Julia and 64-bit Julia
28+
julia_arch: # arch of the Julia binary to install. Distinct from the agent arch
29+
- "x86_64" # 64-bit Julia
30+
# 32-bit Julia on Linux is handled in a separate step later in this YAML file
31+
# 32-bit Julia on Windows is handled in the "adjustments" section below
32+
julia_version:
33+
# Don't put "1" here. Always include at least major and minor version.
34+
- "1.0"
35+
- "1.6" # old LTS
36+
- "1.10" # current LTS
37+
- "1.12" # latest stable
38+
adjustments:
39+
# macOS x86_64 (Intel):
40+
- with:
41+
agent_os: "macos"
42+
agent_arch: "x86_64"
43+
julia_arch: "x86_64"
44+
julia_version: "1.0"
45+
- with:
46+
agent_os: "macos"
47+
agent_arch: "x86_64"
48+
julia_arch: "x86_64"
49+
julia_version: "1.6" # old LTS
50+
- with:
51+
agent_os: "macos"
52+
agent_arch: "x86_64"
53+
julia_arch: "x86_64"
54+
julia_version: "1.10" # current LTS
55+
- with:
56+
agent_os: "macos"
57+
agent_arch: "x86_64"
58+
julia_arch: "x86_64"
59+
julia_version: "1.12" # latest stable
60+
# macOS aarch64 (Apple Silicon):
61+
# Note: Julia 1.0 and 1.6 did not have native aarch64 binaries for macOS, so we skip those
62+
- with:
63+
agent_os: "macos"
64+
agent_arch: "aarch64"
65+
julia_arch: "aarch64"
66+
julia_version: "1.10" # old LTS
67+
- with:
68+
agent_os: "macos"
69+
agent_arch: "aarch64"
70+
julia_arch: "aarch64"
71+
julia_version: "1.12" # latest stable
72+
# For some reason, the Julia 1.0 tarball for Windows x86_64 doesn't seem to exist. Weird. Skip it for now, try to fix this in the future.
73+
- with:
74+
agent_os: "windows"
75+
agent_arch: "x86_64"
76+
julia_arch: "x86_64"
77+
julia_version: "1.0"
78+
skip: true
79+
# 32-bit Julia on Windows
80+
# For some reason, the Julia 1.0 tarball for Windows i686 doesn't seem to exist. Weird. Skip it for now, try to fix this in the future.
81+
# - with:
82+
# agent_os: "windows"
83+
# agent_arch: "x86_64"
84+
# julia_arch: "i686" # 32-bit Julia
85+
# julia_version: "1.0"
86+
- with:
87+
agent_os: "windows"
88+
agent_arch: "x86_64"
89+
julia_arch: "i686" # 32-bit Julia
90+
julia_version: "1.6" # old LTS
91+
- with:
92+
agent_os: "windows"
93+
agent_arch: "x86_64"
94+
julia_arch: "i686" # 32-bit Julia
95+
julia_version: "1.10" # current LTS
96+
- with:
97+
agent_os: "windows"
98+
agent_arch: "x86_64"
99+
julia_arch: "i686" # 32-bit Julia
100+
julia_version: "1.12" # latest stable
101+
plugins:
102+
- JuliaCI/external-buildkite#v1:
103+
version: "./.buildkite-external-version"
104+
repo_url: "https://github.com/JuliaCI/julia-buildkite"
105+
- JuliaCI/julia#v1:
106+
isolated_depot: true
107+
# Drop default "registries" directory, so it is not persisted from execution to execution
108+
persist_depot_dirs: packages,artifacts,compiled
109+
version: '{{matrix.julia_version}}'
110+
arch: '{{matrix.julia_arch}}'
111+
commands: |
112+
echo "--- Julia versioninfo"
113+
julia -e 'import InteractiveUtils; InteractiveUtils.versioninfo()'
114+
115+
echo "--- Pkg.dev-ing (by path) the local copy of JuliaSyntax"
116+
export JULIA_PKG_PRECOMPILE_AUTO=0
117+
# Double dollar signs $$ needed here because of Buildkite stuff
118+
echo "Old JULIA_DEPOT_PATH: $${JULIA_DEPOT_PATH:?}"
119+
# On Windows, the separator is ;
120+
# On other platforms, the separator is :
121+
sep=$$(julia -e 'println(Sys.iswindows() ? ";" : ":")')
122+
# We change the depot path from `depot` to `depot:`
123+
# The trailing colon/semicolon makes sure we can re-use the existing stdlib pkgimages,
124+
# instead of needing to re-precompile them
125+
# https://docs.julialang.org/en/v1/manual/environment-variables/#JULIA_DEPOT_PATH
126+
export JULIA_DEPOT_PATH="$${JULIA_DEPOT_PATH:?}$${sep}"
127+
echo "New JULIA_DEPOT_PATH: $${JULIA_DEPOT_PATH:?}"
128+
julia -e 'import Pkg; pspec = Pkg.PackageSpec(path = "./JuliaSyntax"); Pkg.develop(pspec)'
129+
130+
echo "--- Testing JuliaSyntax on Julia {{matrix.julia_version}}"
131+
unset JULIA_PKG_PRECOMPILE_AUTO
132+
julia -e 'import Pkg; Pkg.test("JuliaSyntax")'
133+
134+
# This step just has the 32-bit Linux jobs for JuliaSyntax.
135+
# For these jobs, we have to run inside a different rootfs image, because the default 64-bit agent rootfs image will complain
136+
# when we try to run a 32-bit executable.
137+
# The logic for the sandbox plugin is annoying, so I've separated it out.
138+
# AFAICT, there is currently no way to conditionally skip a plugin.
139+
# > Plugins run during the job lifecycle, before the conditional is evaluated
140+
# Source: https://buildkite.com/docs/pipelines/configure/conditionals
141+
- label: "juliasyntax {{matrix.agent_os}}/{{matrix.julia_arch}}/{{matrix.julia_version}} in i686 sandbox"
142+
# We comment out the Buildkite step key for this step, because:
143+
# > You can't use matrix values in other attributes, including step keys and ...
144+
# Source: https://buildkite.com/docs/pipelines/configure/workflows/build-matrix
145+
# key: "juliasyntax-step-key"
146+
timeout_in_minutes: 30
147+
agents:
148+
queue: "julia"
149+
os: "{{matrix.agent_os}}"
150+
arch: "{{matrix.agent_arch}}"
151+
matrix:
152+
setup:
153+
agent_os:
154+
- "linux"
155+
agent_arch: # arch of the Buildkite agent. Distinct from the Julia binary's arch
156+
- "x86_64" # The "x86_64" agent arch is used for both 32-bit Julia and 64-bit Julia
157+
julia_arch: # arch of the Julia binary to install. Distinct from the agent arch
158+
- "i686" # 32-bit Julia
159+
julia_version:
160+
# Don't put "1" here. Always include at least major and minor version.
161+
- "1.0"
162+
- "1.6" # old LTS
163+
- "1.10" # current LTS
164+
- "1.12" # latest stable
165+
plugins:
166+
- JuliaCI/external-buildkite#v1:
167+
version: "./.buildkite-external-version"
168+
repo_url: "https://github.com/JuliaCI/julia-buildkite"
169+
- JuliaCI/julia#v1:
170+
# This is the Julia install *outside* the sandbox.
171+
# We only use this Julia install to run Sandbox.jl
172+
# This is not the Julia install that will be used to run Pkg.test()
173+
isolated_depot: true
174+
# Drop default "registries" directory, so it is not persisted from execution to execution
175+
persist_depot_dirs: packages,artifacts,compiled
176+
version: '1.10' # current LTS. It doesn't really matter what version we pick here.
177+
# arch: '{{matrix.julia_arch}}' # no need to specify the julia_arch for this "outside" Julia.
178+
- staticfloat/sandbox#v2:
179+
rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v8.0/agent_linux.i686.tar.gz
180+
rootfs_treehash: "cba648ddb9ac0d292f9c8809652523806674a3d2"
181+
uid: 1000
182+
gid: 1000
183+
workspaces:
184+
# Include `/cache/repos` so that our `git` version introspection works.
185+
- "/cache/repos:/cache/repos"
186+
- JuliaCI/julia#v1:
187+
# This is the Julia install *inside* the sandbox.
188+
# That's the Julia install we actually use for Pkg.test()
189+
isolated_depot: true
190+
# Drop default "registries" directory, so it is not persisted from execution to execution
191+
persist_depot_dirs: packages,artifacts,compiled
192+
version: '{{matrix.julia_version}}'
193+
arch: '{{matrix.julia_arch}}'
194+
commands: |
195+
echo "--- Julia versioninfo"
196+
julia -e 'import InteractiveUtils; InteractiveUtils.versioninfo()'
197+
198+
echo "--- Pkg.dev-ing (by path) the local copy of JuliaSyntax"
199+
export JULIA_PKG_PRECOMPILE_AUTO=0
200+
# Double dollar signs $$ needed here because of Buildkite stuff
201+
echo "Old JULIA_DEPOT_PATH: $${JULIA_DEPOT_PATH:?}"
202+
# On Windows, the separator is ;
203+
# On other platforms, the separator is :
204+
sep=$$(julia -e 'println(Sys.iswindows() ? ";" : ":")')
205+
# We change the depot path from `depot` to `depot:`
206+
# The trailing colon/semicolon makes sure we can re-use the existing stdlib pkgimages,
207+
# instead of needing to re-precompile them
208+
# https://docs.julialang.org/en/v1/manual/environment-variables/#JULIA_DEPOT_PATH
209+
export JULIA_DEPOT_PATH="$${JULIA_DEPOT_PATH:?}$${sep}"
210+
echo "New JULIA_DEPOT_PATH: $${JULIA_DEPOT_PATH:?}"
211+
julia -e 'import Pkg; pspec = Pkg.PackageSpec(path = "./JuliaSyntax"); Pkg.develop(pspec)'
212+
213+
echo "--- Testing JuliaSyntax on Julia {{matrix.julia_version}}"
214+
unset JULIA_PKG_PRECOMPILE_AUTO
215+
julia -e 'import Pkg; Pkg.test("JuliaSyntax")'

0 commit comments

Comments
 (0)