Skip to content

Commit b3ee759

Browse files
ntsemahntsemah
committed
[CI] issue: HPCINFRA-4016 Split compiler step into parallel steps
The compiler step currently takes a long time to run (about 20–40 minutes). Split the compiler step into 6 separate steps (one per compiler) running in parallel to reduce build duration. Signed-off-by: Noam Tsemah <[email protected]>
1 parent c90234e commit b3ee759

File tree

4 files changed

+138
-43
lines changed

4 files changed

+138
-43
lines changed

‎.ci/matrix_job.yaml‎

Lines changed: 89 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -321,16 +321,102 @@ steps:
321321
archiveArtifacts-onfail: |
322322
jenkins/**/arch-*.tar.gz
323323
324-
- name: Compiler
324+
- name: Prepare compiler
325325
enable: ${do_compiler}
326326
containerSelector:
327327
- "{name: 'toolbox-compiler', category: 'tool'}"
328328
agentSelector:
329329
- "{nodeLabel: 'skip-agent'}"
330330
run: |
331-
[ "x${do_compiler}" == "xtrue" ] && action=yes || action=no
332-
env WORKSPACE=$PWD TARGET=${flags} jenkins_test_compiler=${action} ./contrib/test_jenkins.sh
331+
# Prepare dpcp and autogen without compiler tests
332+
env WORKSPACE=$PWD TARGET=${flags} jenkins_test_compiler=no ./contrib/test_jenkins.sh
333333
parallel: false
334+
335+
- name: Compiler (clang-9.0.1)
336+
enable: ${do_compiler}
337+
containerSelector:
338+
- "{name: 'toolbox-compiler', category: 'tool'}"
339+
agentSelector:
340+
- "{nodeLabel: 'skip-agent'}"
341+
run: |
342+
[ "x${do_compiler}" == "xtrue" ] && action=yes || action=no
343+
env WORKSPACE=$PWD TARGET=${flags} COMPILER_SPEC="clang:clang++:dev/clang-9.0.1" jenkins_test_compiler=${action} ./contrib/test_jenkins.sh
344+
parallel: true
345+
onfail: |
346+
./.ci/artifacts.sh
347+
archiveArtifacts-onfail: |
348+
jenkins/**/arch-*.tar.gz
349+
350+
- name: Compiler (icc-18.0.4)
351+
enable: ${do_compiler}
352+
containerSelector:
353+
- "{name: 'toolbox-compiler', category: 'tool'}"
354+
agentSelector:
355+
- "{nodeLabel: 'skip-agent'}"
356+
run: |
357+
[ "x${do_compiler}" == "xtrue" ] && action=yes || action=no
358+
env WORKSPACE=$PWD TARGET=${flags} COMPILER_SPEC="icc:icpc:intel/ics-18.0.4" jenkins_test_compiler=${action} ./contrib/test_jenkins.sh
359+
parallel: true
360+
onfail: |
361+
./.ci/artifacts.sh
362+
archiveArtifacts-onfail: |
363+
jenkins/**/arch-*.tar.gz
364+
365+
- name: Compiler (icc-19.1.1)
366+
enable: ${do_compiler}
367+
containerSelector:
368+
- "{name: 'toolbox-compiler', category: 'tool'}"
369+
agentSelector:
370+
- "{nodeLabel: 'skip-agent'}"
371+
run: |
372+
[ "x${do_compiler}" == "xtrue" ] && action=yes || action=no
373+
env WORKSPACE=$PWD TARGET=${flags} COMPILER_SPEC="icc:icpc:intel/ics-19.1.1" jenkins_test_compiler=${action} ./contrib/test_jenkins.sh
374+
parallel: true
375+
onfail: |
376+
./.ci/artifacts.sh
377+
archiveArtifacts-onfail: |
378+
jenkins/**/arch-*.tar.gz
379+
380+
- name: Compiler (gcc-8.3.0)
381+
enable: ${do_compiler}
382+
containerSelector:
383+
- "{name: 'toolbox-compiler', category: 'tool'}"
384+
agentSelector:
385+
- "{nodeLabel: 'skip-agent'}"
386+
run: |
387+
[ "x${do_compiler}" == "xtrue" ] && action=yes || action=no
388+
env WORKSPACE=$PWD TARGET=${flags} COMPILER_SPEC="gcc:g++:dev/gcc-8.3.0" jenkins_test_compiler=${action} ./contrib/test_jenkins.sh
389+
parallel: true
390+
onfail: |
391+
./.ci/artifacts.sh
392+
archiveArtifacts-onfail: |
393+
jenkins/**/arch-*.tar.gz
394+
395+
- name: Compiler (gcc-9.3.0)
396+
enable: ${do_compiler}
397+
containerSelector:
398+
- "{name: 'toolbox-compiler', category: 'tool'}"
399+
agentSelector:
400+
- "{nodeLabel: 'skip-agent'}"
401+
run: |
402+
[ "x${do_compiler}" == "xtrue" ] && action=yes || action=no
403+
env WORKSPACE=$PWD TARGET=${flags} COMPILER_SPEC="gcc:g++:dev/gcc-9.3.0" jenkins_test_compiler=${action} ./contrib/test_jenkins.sh
404+
parallel: true
405+
onfail: |
406+
./.ci/artifacts.sh
407+
archiveArtifacts-onfail: |
408+
jenkins/**/arch-*.tar.gz
409+
410+
- name: Compiler (gcc-10.1.0)
411+
enable: ${do_compiler}
412+
containerSelector:
413+
- "{name: 'toolbox-compiler', category: 'tool'}"
414+
agentSelector:
415+
- "{nodeLabel: 'skip-agent'}"
416+
run: |
417+
[ "x${do_compiler}" == "xtrue" ] && action=yes || action=no
418+
env WORKSPACE=$PWD TARGET=${flags} COMPILER_SPEC="gcc:g++:dev/gcc-10.1.0" jenkins_test_compiler=${action} ./contrib/test_jenkins.sh
419+
parallel: true
334420
onfail: |
335421
./.ci/artifacts.sh
336422
archiveArtifacts-onfail: |

‎contrib/jenkins_tests/compiler.sh‎

Lines changed: 34 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -1,39 +1,39 @@
11
#!/bin/bash -eExl
22

3-
source $(dirname $0)/globals.sh
3+
[[ -z "${COMPILER_SPEC}" ]] && { echo "ERROR: COMPILER_SPEC must be provided."; exit 1; }
4+
colon_count=$(echo "$COMPILER_SPEC" | tr -cd ':' | wc -c)
5+
[ "${colon_count}" -ne 2 ] && { echo "ERROR: COMPILER_SPEC has invalid format: ${COMPILER_SPEC}"; exit 1; }
6+
7+
source "$(dirname $0)/globals.sh"
48

59
echo "Checking for compiler ..."
610

7-
cd $WORKSPACE
8-
9-
rm -rf $compiler_dir
10-
mkdir -p $compiler_dir
11-
cd $compiler_dir
12-
13-
compiler_list="clang:clang++:dev/clang-9.0.1 icc:icpc:intel/ics-18.0.4 icc:icpc:intel/ics-19.1.1 gcc:g++:dev/gcc-8.3.0 gcc:g++:dev/gcc-9.3.0 gcc:g++:dev/gcc-10.1.0"
14-
15-
compiler_tap=${WORKSPACE}/${prefix}/compiler.tap
16-
echo "1..$(echo $compiler_list | tr " " "\n" | wc -l)" > $compiler_tap
17-
18-
test_id=0
19-
for compiler in $compiler_list; do
20-
IFS=':' read cc cxx module <<< "$compiler"
21-
mkdir -p ${compiler_dir}/${test_id}
22-
cd ${compiler_dir}/${test_id}
23-
[ -z "$module" ] && test_name=$($cc --version | head -n 1) || test_name="$module"
24-
[ ! -z "$module" ] && do_module "$module"
25-
echo "======================================================"
26-
$cc --version
27-
echo
28-
test_exec='${WORKSPACE}/configure --prefix=$compiler_dir-$cc CC=$cc CXX=$cxx --disable-lto $jenkins_test_custom_configure && make $make_opt all'
29-
do_check_result "$test_exec" "$test_id" "$test_name" "$compiler_tap" "${compiler_dir}/compiler-${test_id}"
30-
[ ! -z "$module" ] && module unload "$module"
31-
cd ${compiler_dir}
32-
test_id=$((test_id+1))
33-
pushd ${WORKSPACE}/third_party/json-c
34-
make distclean
35-
popd
36-
done
37-
38-
echo "[${0##*/}]..................exit code = $rc"
39-
exit $rc
11+
# Extract compiler name for unique directory to avoid conflicts between parallel runs
12+
IFS=':' read -r cc cxx module <<< "$COMPILER_SPEC"
13+
compiler_full_dir="${compiler_dir}/${module##*/}"
14+
15+
rm -rf "${compiler_full_dir}"
16+
mkdir -p "${compiler_full_dir}"
17+
18+
# Copy entire source tree to local disk for optimal build performance on NFS
19+
echo "Copying source tree to ${compiler_full_dir}..."
20+
(cd "${WORKSPACE}" && tar --exclude=jenkins --exclude=.git -cf - .) | (cd "${compiler_full_dir}" && tar -xf -)
21+
22+
cd "${compiler_full_dir}"
23+
24+
compiler_tap="${WORKSPACE}/${prefix}/compiler_${module##*/}.tap"
25+
echo "1..1" > "${compiler_tap}"
26+
27+
[ -z "$module" ] && test_name=$($cc --version | head -n 1) || test_name="$module"
28+
[ ! -z "$module" ] && do_module "$module"
29+
echo "======================================================"
30+
$cc --version
31+
echo
32+
33+
test_exec='./configure --prefix=${compiler_full_dir}/install CC=$cc CXX=$cxx --disable-lto $jenkins_test_custom_configure && make $make_opt all'
34+
do_check_result "$test_exec" "0" "$test_name" "$compiler_tap" "${compiler_full_dir}"
35+
36+
[ ! -z "$module" ] && module unload "$module"
37+
38+
echo "[${0##*/}]..................exit code = ${rc}"
39+
exit "${rc}"

‎contrib/jenkins_tests/globals.sh‎

Lines changed: 13 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -303,10 +303,19 @@ do_check_dpcp()
303303
echo "Checking dpcp usage"
304304

305305
ret=0
306-
pushd $(pwd) > /dev/null 2>&1
307-
dpcp_dir=${WORKSPACE}/${prefix}/_dpcp-last
308-
mkdir -p ${dpcp_dir}
309-
cd ${dpcp_dir}
306+
pushd "$(pwd)" > /dev/null 2>&1
307+
dpcp_dir="${WORKSPACE}/${prefix}/_dpcp-last"
308+
309+
# When COMPILER_SPEC is set, reuse dpcp built by "Prepare compiler" step
310+
if [ -n "${COMPILER_SPEC}" ] && [ -d "${dpcp_dir}/install" ]; then
311+
echo "dpcp: reusing shared build"
312+
eval "$1=${dpcp_dir}/install"
313+
popd > /dev/null 2>&1
314+
return 0
315+
fi
316+
317+
mkdir -p "${dpcp_dir}"
318+
cd "${dpcp_dir}"
310319

311320
# libdpcp_path="<repo>|<branch>|<commit>"
312321
# Example:

‎contrib/test_jenkins.sh‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -155,8 +155,8 @@ for target_v in "${target_list[@]}"; do
155155
#
156156
set +e
157157
if [ 1 -lt "$jenkins_opt_exit" -o "$rc" -eq 0 ]; then
158-
if [ "$jenkins_test_compiler" = "yes" ]; then
159-
$WORKSPACE/contrib/jenkins_tests/compiler.sh
158+
if [ "$jenkins_test_compiler" = "yes" ] && [[ ! -z "$COMPILER_SPEC" ]]; then
159+
"${WORKSPACE}/contrib/jenkins_tests/compiler.sh"
160160
ret=$?
161161
if [ $ret -gt 0 ]; then
162162
do_err "case: [compiler: ret=$ret]"

0 commit comments

Comments
 (0)