[CI] migrate CI test steps to containers

Today we use benni09 static agent to run test/gtest/valgrind steps which
is unscaleable since it can only run one pipeline at a time, causing
delays in builds that can be stuck waiting for hours

The idea is to move these steps to containers, allowing running them in
parallel as well as running multiple pipelines at the same time
(depending on the capacity of the k8s cluster)

Issue: HPCINFRA-3249
Signed-off-by: NirWolfer <[email protected]>

Author: NirWolfer

.ci/dockerfiles/Dockerfile.ubuntu22.04

@@ -1,8 +1,35 @@
-FROM harbor.mellanox.com/hpcx/x86_64/ubuntu22.04/base AS style
+ARG ARCH=x86_64
+FROM harbor.mellanox.com/hpcx/x86_64/ubuntu22.04/base AS build
+FROM build AS tests
+RUN apt-get update && \
+    apt-get install -y \
+    net-tools unzip iproute2 wget \
+    && apt-get clean && rm -rf /var/lib/apt/lists/*
+FROM tests AS vg
+RUN apt-get update && \
+    apt-get install -y \
+    valgrind \
+    && apt-get clean && rm -rf /var/lib/apt/lists/*
+FROM tests AS test
+RUN apt-get update && \
+    apt-get install -y \
+    openssh-server psmisc \
+    && apt-get clean && rm -rf /var/lib/apt/lists/*
+# setup ssh server and passwordless login for root for tests flows (verifyer.pl)
+RUN mkdir -p /var/run/sshd ~/.ssh && \
+    rm -rf ~/.ssh/id_rsa* && ssh-keygen -t rsa -N '' -f ~/.ssh/id_rsa && \
+    cat ~/.ssh/id_rsa.pub > ~/.ssh/authorized_keys && \
+    sed -i 's|#PermitRootLogin.*|PermitRootLogin without-password|g' /etc/ssh/sshd_config && \
+    sed -i 's|#PasswordAuthentication.*|PasswordAuthentication no|g' /etc/ssh/sshd_config && \
+    echo "Host *" >> ~/.ssh/config && \
+    echo "   StrictHostKeyChecking no" >> ~/.ssh/config && \
+    echo "   UserKnownHostsFile /dev/null" >> ~/.ssh/config && \
+    echo "   LogLevel ERROR" >> ~/.ssh/config
+FROM tests AS gtest
+FROM build AS style
 RUN apt-get update \
  && apt-get install -y clang-15 clang-format-15 \
  && update-alternatives --install /usr/bin/clang-format clang-format /usr/bin/clang-format-15 100 \
  && update-alternatives --install /usr/bin/clang clang /usr/bin/clang-15 100 \
                     --slave /usr/bin/clang++ clang++ /usr/bin/clang++-15 \
  && apt-get clean && rm -rf /var/lib/apt/lists/*
- 

.ci/matrix_job.yaml

@@ -12,8 +12,8 @@ kubernetes:
   cloud: il-ipp-blossom-prod
   nodeSelector: 'beta.kubernetes.io/os=linux'
   namespace: swx-media
-  limits: '{memory: 8Gi, cpu: 7000m}'
-  requests: '{memory: 8Gi, cpu: 7000m}'
+  limits: '{memory: 10Gi, cpu: 10000m}'
+  requests: '{memory: 10Gi, cpu: 10000m}'
 
 credentials:
   - {credentialsId: 'media_coverity_credentials', usernameVariable: 'XLIO_COV_USER', passwordVariable: 'XLIO_COV_PASSWORD'}
@@ -67,14 +67,14 @@ runs_on_dockers:
       tag: '20241001',
       build_args: '--build-arg ARCH=aarch64 --no-cache',
       category: 'base'
-    }
+   }
 # tool
   - {
      file: '.ci/dockerfiles/Dockerfile.ubuntu22.04',
      arch: 'x86_64',
      name: 'style',
      uri: 'xlio/$arch/ubuntu22.04/$name',
-     tag: '20240812',
+     tag: '20250219',
      build_args: '--no-cache --target style',
      category: 'tool'
     }
@@ -98,9 +98,58 @@ runs_on_dockers:
      build_args: '--no-cache',
      category: 'tool'
      }
-
-runs_on_agents:
-  - {nodeLabel: 'beni09', category: 'base'}
+# tests
+  - {
+      file: '.ci/dockerfiles/Dockerfile.ubuntu22.04',
+      arch: 'x86_64',
+      name: 'test',
+      uri: 'xlio/$arch/ubuntu22.04/$name',
+      tag: '20250219',
+      build_args: '--no-cache --target test',
+      category: 'tests',
+      annotations: [{ key: 'k8s.v1.cni.cncf.io/networks', value: 'sriov-cx6dx-p1' }],
+      limits: '{memory: 10Gi, cpu: 10000m, hugepages-2Mi: 10Gi, mellanox.com/sriov_cx6dx_p1: 1}',
+      requests: '{memory: 10Gi, cpu: 10000m, hugepages-2Mi: 10Gi, mellanox.com/sriov_cx6dx_p1: 1}',
+      caps_add: '[ IPC_LOCK, SYS_RESOURCE ]',
+      runAsUser: '0',
+      runAsGroup: '0',
+      cloud: swx-k8s-spray,
+      namespace: xlio-ci
+    }
+  - {
+     file: '.ci/dockerfiles/Dockerfile.ubuntu22.04',
+     arch: 'x86_64',
+     name: 'vg',
+     uri: 'xlio/$arch/ubuntu22.04/$name',
+     tag: '20250219',
+     build_args: '--no-cache --target vg',
+     category: 'tool',
+     annotations: [{ key: 'k8s.v1.cni.cncf.io/networks', value: 'sriov-cx6dx-p2' }],
+     limits: '{memory: 10Gi, cpu: 10000m, hugepages-2Mi: 10Gi, mellanox.com/sriov_cx6dx_p2: 1}',
+     requests: '{memory: 10Gi, cpu: 10000m, hugepages-2Mi: 10Gi, mellanox.com/sriov_cx6dx_p2: 1}',
+     caps_add: '[ IPC_LOCK, SYS_RESOURCE ]',
+     runAsUser: '0',
+     runAsGroup: '0',
+     cloud: swx-k8s-spray,
+     namespace: xlio-ci
+    }
+  - {
+     file: '.ci/dockerfiles/Dockerfile.ubuntu22.04',
+     arch: 'x86_64',
+     name: 'gtest',
+     uri: 'xlio/$arch/ubuntu22.04/$name',
+     tag: '20250219',
+     build_args: '--no-cache --target gtest',
+     category: 'tests',
+     annotations: [{ key: 'k8s.v1.cni.cncf.io/networks', value: 'sriov-cx6dx-p1@net1,sriov-cx6dx-p2@net2' }],
+     limits: '{memory: 10Gi, cpu: 10000m, hugepages-2Mi: 8Gi, mellanox.com/sriov_cx6dx_p1: 1, mellanox.com/sriov_cx6dx_p2: 1}',
+     requests: '{memory: 10Gi, cpu: 10000m, hugepages-2Mi: 8Gi, mellanox.com/sriov_cx6dx_p1: 1, mellanox.com/sriov_cx6dx_p2: 1}',
+     caps_add: '[ IPC_LOCK, SYS_RESOURCE ]',
+     runAsUser: '0',
+     runAsGroup: '0',
+     cloud: swx-k8s-spray,
+     namespace: xlio-ci
+    }
 
 matrix:
   axes:
@@ -133,8 +182,9 @@ steps:
   - name: Install Doca-host
     containerSelector:
       - "{category: 'base'}"
-    agentSelector:
-      - "{nodeLabel: 'skip-agent'}"
+      - "{name: 'test'}"
+      - "{name: 'gtest'}"
+      - "{name: 'vg'}"
     run: |
       echo "Installing DOCA: ${DOCA_VERSION} ..."
       .ci/scripts/doca_install.sh 
@@ -145,17 +195,13 @@ steps:
       .ci/scripts/doca_install.sh 
     containerSelector:
       - "{name: 'style', category: 'tool', variant: 1}"
-    agentSelector:
-      - "{nodeLabel: 'skip-agent'}"
 
   - name: Copyrights
     enable: ${do_copyrights}
     credentialsId: 'mellanox_github_credentials'
     run: env WORKSPACE=$PWD GITHUB_TOKEN=$MELLANOX_GH_TOKEN ./contrib/jenkins_tests/copyrights.sh
     containerSelector:
       - "{name: 'header-check', category: 'tool', variant: 1}"
-    agentSelector:
-      - "{nodeLabel: 'skip-agent'}"
     archiveArtifacts: '*.log,*.tar.gz'
     parallel: false
 
@@ -167,9 +213,6 @@ steps:
   - name: Build
     enable: ${do_build}
     containerSelector:
-      - "{category: 'base'}"
-    agentSelector:
-      - "{category: 'base'}"
     run: |
       [ "x${do_build}" == "xtrue" ] && action=yes || action=no
       env WORKSPACE=$PWD TARGET=${flags} jenkins_test_build=${action} ./contrib/test_jenkins.sh
@@ -183,8 +226,6 @@ steps:
     enable: ${do_service}
     containerSelector:
       - "{category: 'base', variant:1}"
-    agentSelector:
-      - "{category: 'base', variant:1}"
     run: |
       [ "x${do_service}" == "xtrue" ] && action=yes || action=no
       env WORKSPACE=$PWD TARGET=${flags} jenkins_test_tool=${action} ./contrib/test_jenkins.sh
@@ -198,8 +239,6 @@ steps:
     enable: ${do_package}
     containerSelector:
       - "{category: 'base'}"
-    agentSelector:
-      - "{nodeLabel: 'skip-agent'}"
     run: |
       [ "x${do_package}" == "xtrue" ] && action=yes || action=no
       env WORKSPACE=$PWD TARGET=${flags} jenkins_test_rpm=${action} ./contrib/test_jenkins.sh
@@ -213,8 +252,6 @@ steps:
     enable: ${do_antivirus}
     containerSelector:
       - "{name: 'rhel8.3-mofed-x86_64', category: 'base', variant: 1}"
-    agentSelector:
-      - "{nodeLabel: 'skip-agent'}"
     run: |
       env WORKSPACE=$PWD .ci/antivirus.sh ${release_folder}
     archiveArtifacts: 'logs/'
@@ -223,8 +260,6 @@ steps:
     enable: ${do_style}
     containerSelector:
       - "{name: 'style', category: 'tool'}"
-    agentSelector:
-      - "{nodeLabel: 'skip-agent'}"
     run: |
       [ "x${do_style}" == "xtrue" ] && action=yes || action=no
       env WORKSPACE=$PWD TARGET=${flags} jenkins_test_style=${action} ./contrib/test_jenkins.sh
@@ -238,8 +273,6 @@ steps:
     enable: ${do_compiler}
     containerSelector:
       - "{name: 'toolbox', category: 'tool'}"
-    agentSelector:
-      - "{nodeLabel: 'skip-agent'}"
     run: |
       [ "x${do_compiler}" == "xtrue" ] && action=yes || action=no
       env WORKSPACE=$PWD TARGET=${flags} jenkins_test_compiler=${action} ./contrib/test_jenkins.sh
@@ -254,8 +287,6 @@ steps:
     credentialsId: 'media_coverity_credentials'
     containerSelector:
       - "{name: 'toolbox', category: 'tool'}"
-    agentSelector:
-      - "{nodeLabel: 'skip-agent'}"
     run: |
       [ "x${do_coverity}" == "xtrue" ] && action=yes || action=no
       env WORKSPACE=$PWD TARGET=${flags} jenkins_test_cov=${action} ./contrib/test_jenkins.sh
@@ -270,8 +301,6 @@ steps:
     enable: ${do_cppcheck}
     containerSelector:
       - "{name: 'xlio_static.cppcheck', category: 'tool', variant: 1}"
-    agentSelector:
-      - "{nodeLabel: 'skip-agent'}"
     run: |
       [ "x${do_cppcheck}" == "xtrue" ] && action=yes || action=no
       env WORKSPACE=$PWD TARGET=${flags} jenkins_test_cppcheck=${action} ./contrib/test_jenkins.sh
@@ -285,8 +314,6 @@ steps:
     enable: ${do_csbuild}
     containerSelector:
       - "{name: 'xlio_static.csbuild', category: 'tool', variant: 1}"
-    agentSelector:
-      - "{nodeLabel: 'skip-agent'}"
     run: |
       [ "x${do_csbuild}" == "xtrue" ] && action=yes || action=no
       env WORKSPACE=$PWD TARGET=${flags} jenkins_test_csbuild=${action} ./contrib/test_jenkins.sh
@@ -299,9 +326,7 @@ steps:
   - name: Test
     enable: ${do_test}
     containerSelector:
-      - "{name: 'skip-container'}"
-    agentSelector:
-      - "{nodeLabel: 'beni09'}"
+      - "{name: 'test'}"
     run: |
       [ "x${do_test}" == "xtrue" ] && action=yes || action=no
       env WORKSPACE=$PWD TARGET=${flags} jenkins_test_run=${action} ./contrib/test_jenkins.sh
@@ -314,9 +339,7 @@ steps:
   - name: Gtest
     enable: ${do_gtest}
     containerSelector:
-      - "{name: 'skip-container'}"
-    agentSelector:
-      - "{nodeLabel: 'beni09'}"
+      - "{name: 'gtest'}"
     run: |
       [ "x${do_gtest}" == "xtrue" ] && action=yes || action=no
       env WORKSPACE=$PWD TARGET=${flags} jenkins_test_gtest=${action} ./contrib/test_jenkins.sh
@@ -331,9 +354,7 @@ steps:
   - name: Valgrind
     enable: ${do_valgrind}
     containerSelector:
-      - "{name: 'skip-container'}"
-    agentSelector:
-      - "{nodeLabel: 'beni09'}"
+      - "{name: 'vg'}"
     run: |
       [ "x${do_valgrind}" == "xtrue" ] && action=yes || action=no
       env WORKSPACE=$PWD TARGET=${flags} jenkins_test_vg=${action} ./contrib/test_jenkins.sh
@@ -348,8 +369,6 @@ steps:
     enable: ${do_commit}
     containerSelector:
       - "{name: 'toolbox', category: 'tool', variant:1}"
-    agentSelector:
-      - "{nodeLabel: 'skip-agent'}"
     run: |
       [ "x${do_commit}" == "xtrue" ] && action=yes || action=no
       env WORKSPACE=$PWD TARGET=${flags} jenkins_test_commit=${action} ./contrib/test_jenkins.sh
@@ -373,8 +392,6 @@ steps:
     enable: ${do_blackduck}
     containerSelector:
       - "{name: 'blackduck', category:'tool', variant:1}"
-    agentSelector:
-      - "{nodeLabel: 'skip-agent'}"
     shell: action
     module: ngci
     run: NGCIBlackDuckScan
@@ -401,4 +418,4 @@ pipeline_stop:
 
 failFast: false
 
-taskName: '${flags}/${name}/${axis_index}'
+taskName: '${flags}/${arch}/${name}/${axis_index}'

contrib/jenkins_tests/globals.sh

@@ -82,6 +82,17 @@ function do_archive()
 # Otherwise, return error code.
 # $1 - module name
 #
+
+function do_hugepages()
+{
+    if [[ -f /.dockerenv && ! $(grep -q hugetlbfs /proc/mounts) ]]; then
+        mkdir -p /mnt/huge
+        mount -t hugetlbfs nodev /mnt/huge
+        grep hugetlbfs /proc/mounts
+        echo $?
+    fi
+}
+
 function do_module()
 {
     [ -z "$1" ] && return

contrib/jenkins_tests/gtest.sh

@@ -2,6 +2,9 @@
 
 source $(dirname $0)/globals.sh
 
+# Fix hugepages for docker environments
+do_hugepages
+
 echo "Checking for gtest ..."
 
 if [[ -z "${MANUAL_RUN}" ]]; then
@@ -11,11 +14,6 @@ if [[ -z "${MANUAL_RUN}" ]]; then
 		exit 1
 	fi
 
-	if [ $(command -v ibdev2netdev >/dev/null 2>&1 || echo $?) ]; then
-		echo "[SKIP] ibdev2netdev tool does not exist"
-		exit 1
-	fi
-
 	cd $WORKSPACE
 
 	rm -rf $gtest_dir
@@ -36,9 +34,6 @@ else
 	opt2=${MANUAL_RUN_ADAPTER:-'ConnectX-7'}
 fi
 
-# Retrieve server/client addresses for the test.
-# $1 - [ib|eth|inet6] to select link type or empty to select the first found
-#
 function do_get_addrs()
 {
 	gtest_ip_list="$(do_get_ip $1 $2)"
@@ -58,8 +53,13 @@ function do_get_addrs()
 	echo $gtest_ip_list
 }
 
-gtest_opt="--addr=$(do_get_addrs 'eth' ${opt2})"
-gtest_opt_ipv6="--addr=$(do_get_addrs 'inet6' ${opt2}) -r fdff:ffff:ffff:ffff:ffff:ffff:ffff:ffff" # Remote - Dummy Address
+if [[ -f /.dockerenv ]] || [[ -f /run/.containerenv ]] || [[ -n "${KUBERNETES_SERVICE_HOST}" ]]; then
+	gtest_opt="--addr=$(ip -f inet addr show net1 | awk '/inet / {print $2}' | cut -d/ -f1),$(ip -f inet addr show net2 | awk '/inet / {print $2}' | cut -d/ -f1)"
+	gtest_opt_ipv6="--addr=$(ip -f inet6 addr show net1 | grep global | awk '/inet6 / {print $2}' | cut -d/ -f1),$(ip -f inet6 addr show net2 | grep global | awk '/inet6 / {print $2}' | cut -d/ -f1) -r fdff:ffff:ffff:ffff:ffff:ffff:ffff:ffff" # Remote - Dummy Address
+else
+	gtest_opt="--addr=$(do_get_addrs 'eth' ${opt2})"
+	gtest_opt_ipv6="--addr=$(do_get_addrs 'inet6' ${opt2}) -r fdff:ffff:ffff:ffff:ffff:ffff:ffff:ffff" # Remote - Dummy Address
+fi
 
 set +eE