migrate CI test steps to containers

Today we use benni09 static agent to run test/gtest/valgrind steps which
is unscaleable since it can only run one pipeline at a time, causing
delays in builds that can be stuck waiting for hours

The idea is to move these steps to containers, allowing running them in
parallel as well as running multiple pipelines at the same time
(depending on the capacity of the k8s cluster)

Issue: HPCINFRA-3250
Signed-off-by: NirWolfer <[email protected]>

Author: NirWolfer

.ci/dockerfiles/Dockerfile.ubuntu22.04

@@ -0,0 +1,35 @@
+ARG ARCH=x86_64
+FROM harbor.mellanox.com/hpcx/x86_64/ubuntu22.04/base AS build
+FROM build AS tests
+RUN apt-get update && \
+    apt-get install -y \
+    net-tools unzip iproute2 wget \
+    && apt-get clean && rm -rf /var/lib/apt/lists/*
+FROM tests AS vg
+RUN apt-get update && \
+    apt-get install -y \
+    valgrind \
+    && apt-get clean && rm -rf /var/lib/apt/lists/*
+FROM tests AS test
+RUN apt-get update && \
+    apt-get install -y \
+    openssh-server psmisc \
+    && apt-get clean && rm -rf /var/lib/apt/lists/*
+# setup ssh server and passwordless login for root for tests flows (verifyer.pl)
+RUN mkdir -p /var/run/sshd ~/.ssh && \
+    rm -rf ~/.ssh/id_rsa* && ssh-keygen -t rsa -N '' -f ~/.ssh/id_rsa && \
+    cat ~/.ssh/id_rsa.pub > ~/.ssh/authorized_keys && \
+    sed -i 's|#PermitRootLogin.*|PermitRootLogin without-password|g' /etc/ssh/sshd_config && \
+    sed -i 's|#PasswordAuthentication.*|PasswordAuthentication no|g' /etc/ssh/sshd_config && \
+    echo "Host *" >> ~/.ssh/config && \
+    echo "   StrictHostKeyChecking no" >> ~/.ssh/config && \
+    echo "   UserKnownHostsFile /dev/null" >> ~/.ssh/config && \
+    echo "   LogLevel ERROR" >> ~/.ssh/config
+FROM tests AS gtest
+FROM build AS style
+RUN apt-get update \
+ && apt-get install -y clang-15 clang-format-15 \
+ && update-alternatives --install /usr/bin/clang-format clang-format /usr/bin/clang-format-15 100 \
+ && update-alternatives --install /usr/bin/clang clang /usr/bin/clang-15 100 \
+                    --slave /usr/bin/clang++ clang++ /usr/bin/clang++-15 \
+ && apt-get clean && rm -rf /var/lib/apt/lists/*

.ci/matrix_job.yaml

@@ -4,7 +4,8 @@ job: LIBVMA
 step_allow_single_selector: false
 
 registry_host: harbor.mellanox.com
-registry_auth: swx-storage
+registry_auth: swx-infra_harbor_credentials
+registry_path: /swx-infra/media
 
 kubernetes:
   privileged: true
@@ -43,9 +44,58 @@ runs_on_dockers:
   - {name: 'toolbox', url: 'harbor.mellanox.com/hpcx/x86_64/rhel8.6/builder:inbox', category: 'tool', arch: 'x86_64'}
   - {name: 'blackduck', url: 'harbor.mellanox.com/toolbox/ngci-centos:7.9.2009.2', category: 'tool', arch: 'x86_64'}
   - {name: 'header-check', url: 'harbor.mellanox.com/toolbox/header_check:0.0.51', category: 'tool', arch: 'x86_64', tag: '0.0.51'}
-
-runs_on_agents:
-  - {nodeLabel: 'beni09', category: 'base'}
+# tests
+  - {
+      file: '.ci/dockerfiles/Dockerfile.ubuntu22.04',
+      arch: 'x86_64',
+      name: 'test',
+      uri: 'vma/$arch/ubuntu22.04/$name',
+      tag: '20250219',
+      build_args: '--no-cache --target test',
+      category: 'tests',
+      annotations: [{ key: 'k8s.v1.cni.cncf.io/networks', value: 'sriov-cx6dx-p1' }],
+      limits: '{memory: 10Gi, cpu: 10000m, hugepages-2Mi: 10Gi, mellanox.com/sriov_cx6dx_p1: 1}',
+      requests: '{memory: 10Gi, cpu: 10000m, hugepages-2Mi: 10Gi, mellanox.com/sriov_cx6dx_p1: 1}',
+      caps_add: '[ IPC_LOCK, SYS_RESOURCE ]',
+      runAsUser: '0',
+      runAsGroup: '0',
+      cloud: swx-k8s-spray,
+      namespace: default
+    }
+  - {
+     file: '.ci/dockerfiles/Dockerfile.ubuntu22.04',
+     arch: 'x86_64',
+     name: 'vg',
+     uri: 'vma/$arch/ubuntu22.04/$name',
+     tag: '20250219',
+     build_args: '--no-cache --target vg',
+     category: 'tool',
+     annotations: [{ key: 'k8s.v1.cni.cncf.io/networks', value: 'sriov-cx6dx-p2' }],
+     limits: '{memory: 10Gi, cpu: 10000m, hugepages-2Mi: 10Gi, mellanox.com/sriov_cx6dx_p2: 1}',
+     requests: '{memory: 10Gi, cpu: 10000m, hugepages-2Mi: 10Gi, mellanox.com/sriov_cx6dx_p2: 1}',
+     caps_add: '[ IPC_LOCK, SYS_RESOURCE ]',
+     runAsUser: '0',
+     runAsGroup: '0',
+     cloud: swx-k8s-spray,
+     namespace: default
+    }
+  - {
+     file: '.ci/dockerfiles/Dockerfile.ubuntu22.04',
+     arch: 'x86_64',
+     name: 'gtest',
+     uri: 'vma/$arch/ubuntu22.04/$name',
+     tag: '20250219',
+     build_args: '--no-cache --target gtest',
+     category: 'tests',
+     annotations: [{ key: 'k8s.v1.cni.cncf.io/networks', value: 'sriov-cx6dx-p1@net1,sriov-cx6dx-p2@net2' }],
+     limits: '{memory: 10Gi, cpu: 10000m, hugepages-2Mi: 8Gi, mellanox.com/sriov_cx6dx_p1: 1, mellanox.com/sriov_cx6dx_p2: 1}',
+     requests: '{memory: 10Gi, cpu: 10000m, hugepages-2Mi: 8Gi, mellanox.com/sriov_cx6dx_p1: 1, mellanox.com/sriov_cx6dx_p2: 1}',
+     caps_add: '[ IPC_LOCK, SYS_RESOURCE ]',
+     runAsUser: '0',
+     runAsGroup: '0',
+     cloud: swx-k8s-spray,
+     namespace: default
+    }
 
 matrix:
   axes:
@@ -62,6 +112,12 @@ env:
 
 steps:
   - name: Setup
+    containerSelector:
+      - "{category: 'base'}"
+      - "{name: 'test', variant:1}"
+      - "{name: 'gtest', variant:1}"
+      - "{name: 'vg', variant:1}"
+      - "{name: 'style'}"
     run: |
       set +x
       echo
@@ -90,27 +146,34 @@ steps:
   - name: Install Doca-host
     containerSelector:
       - "{category: 'base'}"
+      - "{name: 'test', variant:1}"
+      - "{name: 'gtest', variant:1}"
+      - "{name: 'vg', variant:1}"
+      - "{name: 'style'}"
     agentSelector:
       - "{nodeLabel: 'skip-agent'}"
     run: |
       echo "Installing DOCA: ${DOCA_VERSION} ..."
       .ci/scripts/doca_install.sh 
 
-  - name: Install Doca-host on Tools
-    run: |
-      echo "Installing DOCA: ${DOCA_VERSION} ..."
-      .ci/scripts/doca_install.sh 
-    containerSelector:
-      - "{name: 'style', category: 'tool'}"
-    agentSelector:
-      - "{nodeLabel: 'skip-agent'}"
-
   - name: Autogen
+    containerSelector:
+      - "{category: 'base'}"
+      - "{name: 'test', variant:1}"
+      - "{name: 'gtest', variant:1}"
+      - "{name: 'vg', variant:1}"
+      - "{name: 'style'}"
     run: |
       ./autogen.sh -s
     parallel: false
 
   - name: Build
+    containerSelector:
+      - "{category: 'base'}"
+      - "{name: 'test', variant:1}"
+      - "{name: 'gtest', variant:1}"
+      - "{name: 'vg', variant:1}"
+      - "{name: 'style'}"
     enable: ${do_build}
     run: |
       [ "x${do_build}" == "xtrue" ] && action=yes || action=no
@@ -227,12 +290,13 @@ steps:
   - name: Test
     enable: ${do_test}
     containerSelector:
-      - "{name: 'skip-container'}"
+      - "{name: 'test', variant:1}"
     agentSelector:
-      - "{nodeLabel: 'beni09', variant:1}"
+      - "{nodeLabel: 'skip-agent'}"
     run: |
       [ "x${do_test}" == "xtrue" ] && action=yes || action=no
       env WORKSPACE=$PWD TARGET=${flags} jenkins_test_run=${action} ./contrib/test_jenkins.sh
+      sleep 80000
     parallel: false
     onfail: |
       ./.ci/artifacts.sh
@@ -242,9 +306,9 @@ steps:
   - name: Gtest
     enable: ${do_gtest}
     containerSelector:
-      - "{name: 'skip-container'}"
+      - "{name: 'gtest', variant:1}"
     agentSelector:
-      - "{nodeLabel: 'beni09', variant:1}"
+      - "{nodeLabel: 'skip-agent'}"
     run: |
       [ "x${do_gtest}" == "xtrue" ] && action=yes || action=no
       env WORKSPACE=$PWD TARGET=${flags} jenkins_test_gtest=${action} ./contrib/test_jenkins.sh
@@ -259,9 +323,9 @@ steps:
   - name: Valgrind
     enable: ${do_valgrind}
     containerSelector:
-      - "{name: 'skip-container'}"
+      - "{name: 'vg', variant:1}"
     agentSelector:
-      - "{nodeLabel: 'beni09', variant:1}"
+      - "{nodeLabel: 'skip-agent'}"
     run: |
       [ "x${do_valgrind}" == "xtrue" ] && action=yes || action=no
       env WORKSPACE=$PWD TARGET=${flags} jenkins_test_vg=${action} ./contrib/test_jenkins.sh

contrib/jenkins_tests/globals.sh

@@ -75,6 +75,16 @@ function do_archive()
     set -e
 }
 
+function do_hugepages()
+{
+	if [[ -f /.dockerenv && ! $(grep -q hugetlbfs /proc/mounts) ]]; then
+        mkdir -p /mnt/huge
+        mount -t hugetlbfs nodev /mnt/huge
+        grep hugetlbfs /proc/mounts
+        echo $?
+    fi
+}
+
 # Test if an environment module exists and load it if yes.
 # Otherwise, return error code.
 # $1 - module name

contrib/jenkins_tests/gtest.sh

@@ -2,6 +2,9 @@
 
 source $(dirname $0)/globals.sh
 
+# Fix hugepages for docker environments
+do_hugepages
+
 echo "Checking for gtest ..."
 
 # Check dependencies
@@ -10,11 +13,6 @@ if [ $(test -d ${install_dir} >/dev/null 2>&1 || echo $?) ]; then
 	exit 1
 fi
 
-if [ $(command -v ibdev2netdev >/dev/null 2>&1 || echo $?) ]; then
-	echo "[SKIP] ibdev2netdev tool does not exist"
-	exit 1
-fi
-
 cd $WORKSPACE
 
 rm -rf $gtest_dir
@@ -25,16 +23,21 @@ gtest_app="$PWD/tests/gtest/gtest"
 gtest_lib=$install_dir/lib/${prj_lib}
 
 gtest_ip_list=""
-if [ ! -z $(do_get_ip 'eth') ]; then
-	gtest_ip_list="$(do_get_ip 'eth')"
-fi
-if [ ! -z $(do_get_ip 'eth' '' $gtest_ip_list) ]; then
-	gtest_ip_list="${gtest_ip_list}:$(do_get_ip 'eth' '' $gtest_ip_list)"
+
+if [[ -f /.dockerenv ]] || [[ -f /run/.containerenv ]] || [[ -n "${KUBERNETES_SERVICE_HOST}" ]]; then
+	gtest_opt="--addr=$(ip -f inet addr show net1 | awk '/inet / {print $2}' | cut -d/ -f1):$(ip -f inet addr show net2 | awk '/inet / {print $2}' | cut -d/ -f1)"
 else
-	echo "[SKIP] two eth interfaces are required. found: ${gtest_ip_list}"
-	exit 0
+	if [ ! -z $(do_get_ip 'eth') ]; then
+		gtest_ip_list="$(do_get_ip 'eth')"
+	fi
+	if [ ! -z $(do_get_ip 'eth' '' $gtest_ip_list) ]; then
+		gtest_ip_list="${gtest_ip_list}:$(do_get_ip 'eth' '' $gtest_ip_list)"
+	else
+		echo "[SKIP] two eth interfaces are required. found: ${gtest_ip_list}"
+		exit 0
+	fi
+	gtest_opt="--addr=${gtest_ip_list}"
 fi
-gtest_opt="--addr=${gtest_ip_list}"
 
 set +eE