[CI] issue: HPCINFRA-4016 Split compiler step into parallel steps

The compiler step currently takes a long time to run (about 20–40 minutes).
Split the compiler step into 6 separate steps (one per compiler) running in parallel to reduce build duration.

Signed-off-by: Noam Tsemah <[email protected]>

Author: ntsemah

.ci/matrix_job.yaml

@@ -87,6 +87,10 @@ runs_on_dockers:
      category: 'tool'
     }
   - {name: 'toolbox', url: 'harbor.mellanox.com/hpcx/x86_64/rhel8.6/builder:inbox', category: 'tool', arch: 'x86_64'}
+  - {name: 'toolbox-clang-9.0.1', url: '${registry_host}/hpcx/x86_64/rhel8.6/builder:inbox', category: 'tool', arch: 'x86_64'}
+  - {name: 'toolbox-icc-18.0.4', url: '${registry_host}/hpcx/x86_64/rhel8.6/builder:inbox', category: 'tool', arch: 'x86_64'}
+  - {name: 'toolbox-icc-19.1.1', url: '${registry_host}/hpcx/x86_64/rhel8.6/builder:inbox', category: 'tool', arch: 'x86_64'}
+  - {name: 'toolbox-gcc', url: '${registry_host}/hpcx/x86_64/rhel8.6/builder:inbox', category: 'tool', arch: 'x86_64'}
   - {name: 'header-check', url: 'harbor.mellanox.com/toolbox/header_check:0.0.58', category: 'tool', arch: 'x86_64', tag: '0.0.58'}
 # static tests
   - {
@@ -222,12 +226,12 @@ steps:
       - "{nodeLabel: 'skip-agent'}"
     run: |
       echo "Installing DOCA: ${DOCA_VERSION} ..."
-      .ci/scripts/doca_install.sh 
+      .ci/scripts/doca_install.sh
 
   - name: Install Doca-host on Tools
     run: |
       echo "Installing DOCA: ${DOCA_VERSION} ..."
-      .ci/scripts/doca_install.sh 
+      .ci/scripts/doca_install.sh
     containerSelector:
       - "{name: 'style', category: 'tool', variant: 1}"
     agentSelector:
@@ -320,15 +324,60 @@ steps:
     archiveArtifacts-onfail: |
       jenkins/**/arch-*.tar.gz
 
-  - name: Compiler
+  - name: Compiler (clang-9.0.1)
     enable: ${do_compiler}
     containerSelector:
-      - "{name: 'toolbox', category: 'tool'}"
+      - "{name: 'toolbox-clang-9.0.1', category: 'tool'}"
+    agentSelector:
+      - "{nodeLabel: 'skip-agent'}"
+    run: |
+      [ "x${do_compiler}" == "xtrue" ] && action=yes || action=no
+      env WORKSPACE=$PWD TARGET=${flags} COMPILER_SPEC="clang:clang++:dev/clang-9.0.1" jenkins_test_compiler=${action} ./contrib/test_jenkins.sh
+    parallel: false
+    onfail: |
+      ./.ci/artifacts.sh
+    archiveArtifacts-onfail: |
+      jenkins/**/arch-*.tar.gz
+
+  - name: Compiler (icc-18.0.4)
+    enable: ${do_compiler}
+    containerSelector:
+      - "{name: 'toolbox-icc-18.0.4', category: 'tool'}"
+    agentSelector:
+      - "{nodeLabel: 'skip-agent'}"
+    run: |
+      [ "x${do_compiler}" == "xtrue" ] && action=yes || action=no
+      env WORKSPACE=$PWD TARGET=${flags} COMPILER_SPEC="icc:icpc:intel/ics-18.0.4" jenkins_test_compiler=${action} ./contrib/test_jenkins.sh
+    parallel: false
+    onfail: |
+      ./.ci/artifacts.sh
+    archiveArtifacts-onfail: |
+      jenkins/**/arch-*.tar.gz
+
+  - name: Compiler (icc-19.1.1)
+    enable: ${do_compiler}
+    containerSelector:
+      - "{name: 'toolbox-icc-19.1.1', category: 'tool'}"
+    agentSelector:
+      - "{nodeLabel: 'skip-agent'}"
+    run: |
+      [ "x${do_compiler}" == "xtrue" ] && action=yes || action=no
+      env WORKSPACE=$PWD TARGET=${flags} COMPILER_SPEC="icc:icpc:intel/ics-19.1.1" jenkins_test_compiler=${action} ./contrib/test_jenkins.sh
+    parallel: false
+    onfail: |
+      ./.ci/artifacts.sh
+    archiveArtifacts-onfail: |
+      jenkins/**/arch-*.tar.gz
+
+  - name: Compiler (gcc)
+    enable: ${do_compiler}
+    containerSelector:
+      - "{name: 'toolbox-gcc', category: 'tool'}"
     agentSelector:
       - "{nodeLabel: 'skip-agent'}"
     run: |
       [ "x${do_compiler}" == "xtrue" ] && action=yes || action=no
-      env WORKSPACE=$PWD TARGET=${flags} jenkins_test_compiler=${action} ./contrib/test_jenkins.sh
+      env WORKSPACE=$PWD TARGET=${flags} COMPILER_SPEC="gcc:g++:dev/gcc-8.3.0 gcc:g++:dev/gcc-9.3.0 gcc:g++:dev/gcc-10.1.0" jenkins_test_compiler=${action} ./contrib/test_jenkins.sh
     parallel: false
     onfail: |
       ./.ci/artifacts.sh

contrib/jenkins_tests/compiler.sh

@@ -12,14 +12,18 @@ cd $compiler_dir
 
 compiler_list="clang:clang++:dev/clang-9.0.1 icc:icpc:intel/ics-18.0.4 icc:icpc:intel/ics-19.1.1 gcc:g++:dev/gcc-8.3.0 gcc:g++:dev/gcc-9.3.0 gcc:g++:dev/gcc-10.1.0"
 
+if [[ ! -z "$COMPILER_SPEC" ]]; then
+    compiler_list="${COMPILER_SPEC}"
+fi
+
 compiler_tap=${WORKSPACE}/${prefix}/compiler.tap
-echo "1..$(echo $compiler_list | tr " " "\n" | wc -l)" > $compiler_tap
+echo "1..$(echo "${compiler_list}" | tr " " "\n" | wc -l)" > "${compiler_tap}"
 
 test_id=0
 for compiler in $compiler_list; do
     IFS=':' read cc cxx module <<< "$compiler"
-    mkdir -p ${compiler_dir}/${test_id}
-    cd ${compiler_dir}/${test_id}
+    mkdir -p "${compiler_dir}/${test_id}"
+    cd "${compiler_dir}/${test_id}"
     [ -z "$module" ] && test_name=$($cc --version | head -n 1) || test_name="$module"
     [ ! -z "$module" ] && do_module "$module"
     echo "======================================================"
@@ -28,9 +32,9 @@ for compiler in $compiler_list; do
     test_exec='${WORKSPACE}/configure --prefix=$compiler_dir-$cc CC=$cc CXX=$cxx --disable-lto $jenkins_test_custom_configure && make $make_opt all'
     do_check_result "$test_exec" "$test_id" "$test_name" "$compiler_tap" "${compiler_dir}/compiler-${test_id}"
     [ ! -z "$module" ] && module unload "$module"
-    cd ${compiler_dir}
+    cd "${compiler_dir}"
     test_id=$((test_id+1))
-    pushd ${WORKSPACE}/third_party/json-c
+    pushd "${WORKSPACE}/third_party/json-c"
     make distclean
     popd
 done