[CI] issue: HPCINFRA-4016 Split compiler step into parallel steps

The compiler step currently takes a long time to run (about 20–40 minutes).
Split the compiler step into 6 separate steps (one per compiler) running in parallel to reduce build duration.

Signed-off-by: Noam Tsemah <[email protected]>

Author: ntsemah

.ci/matrix_job.yaml

@@ -3,7 +3,7 @@ job: LIBXLIO
 
 step_allow_single_selector: false
 
-registry_host: harbor.mellanox.com
+registry_host: nbu-harbor.gtm.nvidia.com
 registry_auth: swx-infra_harbor_credentials
 registry_path: /swx-infra/media
 
@@ -17,10 +17,10 @@ kubernetes:
   arch_table:
     x86_64:
       nodeSelector: 'kubernetes.io/arch=amd64'
-      jnlpImage: 'harbor.mellanox.com/toolbox/c3po-jnlp:latest'
+      jnlpImage: '${registry_host}/toolbox/c3po-jnlp:latest'
     aarch64:
       nodeSelector: 'kubernetes.io/arch=arm64'
-      jnlpImage: 'harbor.mellanox.com/toolbox/c3po-jnlp:latest'
+      jnlpImage: '${registry_host}/toolbox/c3po-jnlp:latest'
 
 credentials:
   - {credentialsId: 'media_coverity_credentials', usernameVariable: 'XLIO_COV_USER', passwordVariable: 'XLIO_COV_PASSWORD'}
@@ -86,8 +86,12 @@ runs_on_dockers:
      build_args: '--no-cache --target style',
      category: 'tool'
     }
-  - {name: 'toolbox', url: 'harbor.mellanox.com/hpcx/x86_64/rhel8.6/builder:inbox', category: 'tool', arch: 'x86_64'}
-  - {name: 'header-check', url: 'harbor.mellanox.com/toolbox/header_check:0.0.58', category: 'tool', arch: 'x86_64', tag: '0.0.58'}
+  - {name: 'toolbox', url: '${registry_host}/hpcx/x86_64/rhel8.6/builder:inbox', category: 'tool', arch: 'x86_64'}
+  - {name: 'toolbox-clang-9.0.1', url: '${registry_host}/hpcx/x86_64/rhel8.6/builder:inbox', category: 'tool', arch: 'x86_64'}
+  - {name: 'toolbox-icc-18.0.4', url: '${registry_host}/hpcx/x86_64/rhel8.6/builder:inbox', category: 'tool', arch: 'x86_64'}
+  - {name: 'toolbox-icc-19.1.1', url: '${registry_host}/hpcx/x86_64/rhel8.6/builder:inbox', category: 'tool', arch: 'x86_64'}
+  - {name: 'toolbox-gcc', url: '${registry_host}/hpcx/x86_64/rhel8.6/builder:inbox', category: 'tool', arch: 'x86_64'}
+  - {name: 'header-check', url: '${registry_host}/toolbox/header_check:0.0.58', category: 'tool', arch: 'x86_64', tag: '0.0.58'}
 # static tests
   - {
      file: '.ci/dockerfiles/Dockerfile.rhel8.6',
@@ -106,7 +110,7 @@ runs_on_dockers:
      build_args: '--no-cache --target static',
      category: 'tool'
      }
-  - {name: 'xlio_static.csbuild', url: 'harbor.mellanox.com/swx-infra/media/x86_64/xlio_static.csbuild-clang18:20250515', category: 'tool', arch: 'x86_64' }
+  - {name: 'xlio_static.csbuild', url: '${registry_host}/swx-infra/media/x86_64/xlio_static.csbuild-clang18:20250515', category: 'tool', arch: 'x86_64' }
 # tests
   - {
      file: '.ci/dockerfiles/Dockerfile.ubuntu22.04',
@@ -126,7 +130,7 @@ runs_on_dockers:
   - {
      arch: 'x86_64',
      name: 'vg-worker-threads',
-     url: 'harbor.mellanox.com/swx-infra/media/xlio/x86_64/ubuntu22.04/vg:20250219',
+     url: '${registry_host}/swx-infra/media/xlio/x86_64/ubuntu22.04/vg:20250219',
      category: 'tool',
      annotations: [{ key: 'k8s.v1.cni.cncf.io/networks', value: 'sriov-cx6dx-p2' }],
      limits: '{memory: 10Gi, cpu: 10000m, hugepages-2Mi: 10Gi, nvidia.com/sriov-cx6dx-p2: 1}',
@@ -153,7 +157,7 @@ runs_on_dockers:
   - {
       arch: 'x86_64',
       name: 'sockperf-worker-threads',
-      url: 'harbor.mellanox.com/swx-infra/media/xlio/x86_64/ubuntu22.04/sockperf:20251015',
+      url: '${registry_host}/swx-infra/media/xlio/x86_64/ubuntu22.04/sockperf:20251015',
       category: 'tests',
       annotations: [{ key: 'k8s.v1.cni.cncf.io/networks', value: 'sriov-cx6dx-p1' }],
       limits: '{memory: 10Gi, cpu: 10000m, hugepages-2Mi: 10Gi, nvidia.com/sriov-cx6dx-p1: 1}',
@@ -222,12 +226,12 @@ steps:
       - "{nodeLabel: 'skip-agent'}"
     run: |
       echo "Installing DOCA: ${DOCA_VERSION} ..."
-      .ci/scripts/doca_install.sh 
+      .ci/scripts/doca_install.sh
 
   - name: Install Doca-host on Tools
     run: |
       echo "Installing DOCA: ${DOCA_VERSION} ..."
-      .ci/scripts/doca_install.sh 
+      .ci/scripts/doca_install.sh
     containerSelector:
       - "{name: 'style', category: 'tool', variant: 1}"
     agentSelector:
@@ -320,15 +324,60 @@ steps:
     archiveArtifacts-onfail: |
       jenkins/**/arch-*.tar.gz
 
-  - name: Compiler
+  - name: Compiler (clang-9.0.1)
     enable: ${do_compiler}
     containerSelector:
-      - "{name: 'toolbox', category: 'tool'}"
+      - "{name: 'toolbox-clang-9.0.1', category: 'tool'}"
+    agentSelector:
+      - "{nodeLabel: 'skip-agent'}"
+    run: |
+      [ "x${do_compiler}" == "xtrue" ] && action=yes || action=no
+      env WORKSPACE=$PWD TARGET=${flags} COMPILER_SPEC="clang:clang++:dev/clang-9.0.1" jenkins_test_compiler=${action} ./contrib/test_jenkins.sh
+    parallel: false
+    onfail: |
+      ./.ci/artifacts.sh
+    archiveArtifacts-onfail: |
+      jenkins/**/arch-*.tar.gz
+
+  - name: Compiler (icc-18.0.4)
+    enable: ${do_compiler}
+    containerSelector:
+      - "{name: 'toolbox-icc-18.0.4', category: 'tool'}"
+    agentSelector:
+      - "{nodeLabel: 'skip-agent'}"
+    run: |
+      [ "x${do_compiler}" == "xtrue" ] && action=yes || action=no
+      env WORKSPACE=$PWD TARGET=${flags} COMPILER_SPEC="icc:icpc:intel/ics-18.0.4" jenkins_test_compiler=${action} ./contrib/test_jenkins.sh
+    parallel: false
+    onfail: |
+      ./.ci/artifacts.sh
+    archiveArtifacts-onfail: |
+      jenkins/**/arch-*.tar.gz
+
+  - name: Compiler (icc-19.1.1)
+    enable: ${do_compiler}
+    containerSelector:
+      - "{name: 'toolbox-icc-19.1.1', category: 'tool'}"
+    agentSelector:
+      - "{nodeLabel: 'skip-agent'}"
+    run: |
+      [ "x${do_compiler}" == "xtrue" ] && action=yes || action=no
+      env WORKSPACE=$PWD TARGET=${flags} COMPILER_SPEC="icc:icpc:intel/ics-19.1.1" jenkins_test_compiler=${action} ./contrib/test_jenkins.sh
+    parallel: false
+    onfail: |
+      ./.ci/artifacts.sh
+    archiveArtifacts-onfail: |
+      jenkins/**/arch-*.tar.gz
+
+  - name: Compiler (gcc)
+    enable: ${do_compiler}
+    containerSelector:
+      - "{name: 'toolbox-gcc', category: 'tool'}"
     agentSelector:
       - "{nodeLabel: 'skip-agent'}"
     run: |
       [ "x${do_compiler}" == "xtrue" ] && action=yes || action=no
-      env WORKSPACE=$PWD TARGET=${flags} jenkins_test_compiler=${action} ./contrib/test_jenkins.sh
+      env WORKSPACE=$PWD TARGET=${flags} COMPILER_SPEC="gcc" jenkins_test_compiler=${action} ./contrib/test_jenkins.sh
     parallel: false
     onfail: |
       ./.ci/artifacts.sh

contrib/jenkins_tests/compiler.sh

@@ -4,36 +4,44 @@ source $(dirname $0)/globals.sh
 
 echo "Checking for compiler ..."
 
-cd $WORKSPACE
+cd "${WORKSPACE}"
 
-rm -rf $compiler_dir
-mkdir -p $compiler_dir
-cd $compiler_dir
+rm -rf "${compiler_dir}"
+mkdir -p "${compiler_dir}"
+cd "${compiler_dir}"
 
-compiler_list="clang:clang++:dev/clang-9.0.1 icc:icpc:intel/ics-18.0.4 icc:icpc:intel/ics-19.1.1 gcc:g++:dev/gcc-8.3.0 gcc:g++:dev/gcc-9.3.0 gcc:g++:dev/gcc-10.1.0"
+default_compilers="clang:clang++:dev/clang-9.0.1 icc:icpc:intel/ics-18.0.4 icc:icpc:intel/ics-19.1.1 gcc:g++:dev/gcc-8.3.0 gcc:g++:dev/gcc-9.3.0 gcc:g++:dev/gcc-10.1.0"
 
-compiler_tap=${WORKSPACE}/${prefix}/compiler.tap
-echo "1..$(echo $compiler_list | tr " " "\n" | wc -l)" > $compiler_tap
+if [[ -z "$COMPILER_SPEC" ]]; then
+    compiler_list="${default_compilers}"
+elif [[ "$COMPILER_SPEC" == "gcc" ]] || [[ "$COMPILER_SPEC" == "clang" ]] || [[ "$COMPILER_SPEC" == "icc" ]]; then
+    compiler_list=$(echo "${default_compilers}" | tr " " "\n" | grep "^${COMPILER_SPEC}:" | tr "\n" " ")
+else
+    compiler_list="${COMPILER_SPEC}"
+fi
+
+compiler_tap="${WORKSPACE}/${prefix}/compiler.tap"
+echo "1..$(echo "${compiler_list}" | tr " " "\n" | wc -l)" > "${compiler_tap}"
 
 test_id=0
 for compiler in $compiler_list; do
-    IFS=':' read cc cxx module <<< "$compiler"
-    mkdir -p ${compiler_dir}/${test_id}
-    cd ${compiler_dir}/${test_id}
+    IFS=':' read -r cc cxx module <<< "$compiler"
+    mkdir -p "${compiler_dir}/${test_id}"
+    cd "${compiler_dir}/${test_id}"
     [ -z "$module" ] && test_name=$($cc --version | head -n 1) || test_name="$module"
     [ ! -z "$module" ] && do_module "$module"
     echo "======================================================"
     $cc --version
     echo
     test_exec='${WORKSPACE}/configure --prefix=$compiler_dir-$cc CC=$cc CXX=$cxx --disable-lto $jenkins_test_custom_configure && make $make_opt all'
     do_check_result "$test_exec" "$test_id" "$test_name" "$compiler_tap" "${compiler_dir}/compiler-${test_id}"
-    [ ! -z "$module" ] && module unload "$module"
-    cd ${compiler_dir}
+    [ ! -z "${module}" ] && module unload "$module"
+    cd "${compiler_dir}"
     test_id=$((test_id+1))
-    pushd ${WORKSPACE}/third_party/json-c
+    pushd "${WORKSPACE}/third_party/json-c"
     make distclean
     popd
 done
 
-echo "[${0##*/}]..................exit code = $rc"
-exit $rc
+echo "[${0##*/}]..................exit code = ${rc}"
+exit "${rc}"

contrib/test_jenkins.sh

@@ -156,7 +156,7 @@ for target_v in "${target_list[@]}"; do
     set +e
     if [ 1 -lt "$jenkins_opt_exit" -o "$rc" -eq 0 ]; then
 	    if [ "$jenkins_test_compiler" = "yes" ]; then
-	        $WORKSPACE/contrib/jenkins_tests/compiler.sh
+	        "${WORKSPACE}/contrib/jenkins_tests/compiler.sh"
 	        ret=$?
 	        if [ $ret -gt 0 ]; then
 	           do_err "case: [compiler: ret=$ret]"