tests: add test_cd_failover.bats and support

Signed-off-by: Dr. Jan-Philip Gehrcke <[email protected]>

Author: jgehrcke

tests/bats/Dockerfile

@@ -2,8 +2,9 @@ FROM debian:trixie
 
 # GNU parallel: bats may want to use that
 # gettext-base: provides envsubst, used by nickelpie
+# bc: by bash wrappers, for calculation
 RUN apt-get update && apt-get install -y -q --no-install-recommends \
-    parallel git ca-certificates curl make gettext-base jq && \
+    parallel git ca-certificates curl make gettext-base jq bc yq && \
     rm -rf /var/lib/apt/lists/*
 
 # Set by BuiltKit, of the form amd64/arm64.
@@ -15,6 +16,7 @@ RUN git clone https://github.com/bats-core/bats-core.git && cd bats-core && \
     git checkout 658f442f0fcdd6f9e2ea01625999217e8f7bfe7d && ./install.sh /usr/local
 
 RUN mkdir -p /bats-libraries
+# These are conservatively patched -- so far; maybe pin them in the future
 RUN git clone https://github.com/bats-core/bats-support /bats-libraries/bats-support
 RUN git clone https://github.com/bats-core/bats-assert /bats-libraries/bats-assert
 RUN git clone https://github.com/bats-core/bats-file /bats-libraries/bats-file

tests/bats/Makefile

@@ -110,5 +110,6 @@ tests: image
 			tests/bats/test_cd_imex_chan_inject.bats \
 			tests/bats/test_cd_mnnvl_workload.bats \
 			tests/bats/test_cd_logging.bats \
+			tests/bats/test_cd_failover.bats \
 			tests/bats/test_cd_updowngrade.bats \
 		"

tests/bats/cleanup-from-previous-run.sh

@@ -56,6 +56,8 @@ timeout -v 5 kubectl delete -f demo/specs/imex/channel-injection-all.yaml 2> /de
 timeout -v 5 kubectl delete jobs nickelpie-test 2> /dev/null
 timeout -v 5 kubectl delete computedomain nickelpie-test-compute-domain 2> /dev/null
 timeout -v 5 kubectl delete -f demo/specs/imex/nvbandwidth-test-job-1.yaml 2> /dev/null
+timeout -v 5 kubectl delete -f demo/specs/imex/nvbandwidth-test-job-2.yaml 2> /dev/null
+timeout -v 5 kubectl delete -f tests/bats/specs/nvb2.yaml 2> /dev/null
 timeout -v 5 kubectl delete pods -l env=batssuite 2> /dev/null
 timeout -v 2 kubectl delete resourceclaim batssuite-rc-bad-opaque-config --force 2> /dev/null
 

tests/bats/specs/nvb2.yaml

@@ -0,0 +1,88 @@
+---
+apiVersion: resource.nvidia.com/v1beta1
+kind: ComputeDomain
+metadata:
+  name: test-failover-cd
+spec:
+  numNodes: 2
+  channel:
+    resourceClaimTemplate:
+      name: test-failover-rct-channel
+---
+apiVersion: kubeflow.org/v2beta1
+kind: MPIJob
+metadata:
+  name: test-failover-job
+spec:
+  slotsPerWorker: 2
+  launcherCreationPolicy: WaitForWorkersReady
+  runPolicy:
+    cleanPodPolicy: Running
+  sshAuthMountPath: /home/mpiuser/.ssh
+  mpiReplicaSpecs:
+    Launcher:
+      replicas: 1
+      template:
+        metadata:
+          labels:
+            mpi-memcpy-dra-test-replica: mpi-launcher
+        spec:
+          containers:
+          - image: ghcr.io/nvidia/k8s-samples:nvbandwidth-6dc12f17
+            name: mpi-launcher
+            securityContext:
+              runAsUser: 1000
+            command:
+            - mpirun
+            args:
+            - --show-progress
+            - --bind-to
+            - core
+            - --map-by
+            - ppr:2:node
+            - -np
+            - "4"
+            - --report-bindings
+            - -q
+            - nvbandwidth
+            - --verbose
+            - --testSamples
+            - "20"
+            - --bufferSize
+            - "2048"
+            - -t
+            - multinode_device_to_device_memcpy_read_ce
+          affinity:
+            nodeAffinity:
+              requiredDuringSchedulingIgnoredDuringExecution:
+                nodeSelectorTerms:
+                - matchExpressions:
+                  - key: node-role.kubernetes.io/control-plane
+                    operator: Exists
+    Worker:
+      replicas: 2
+      template:
+        metadata:
+          labels:
+            mpi-memcpy-dra-test-replica: mpi-worker
+        spec:
+          containers:
+          - image: ghcr.io/nvidia/k8s-samples:nvbandwidth-6dc12f17
+            name: mpi-worker
+            securityContext:
+              runAsUser: 1000
+            env:
+            command:
+            - /usr/sbin/sshd
+            args:
+            - -De
+            - -f
+            - /home/mpiuser/.sshd_config
+            resources:
+              limits:
+                nvidia.com/gpu: 2
+              claims:
+              - name: test-failover-rc-channel
+          resourceClaims:
+          - name: test-failover-rc-channel
+            resourceClaimTemplateName: test-failover-rct-channel

tests/bats/test_cd_failover.bats

@@ -0,0 +1,25 @@
+# shellcheck disable=SC2148
+# shellcheck disable=SC2329
+
+setup_file() {
+  load 'helpers.sh'
+  _common_setup
+
+  local _iargs=("--set" "logVerbosity=6")
+  iupgrade_wait "${TEST_CHART_REPO}" "${TEST_CHART_VERSION}" _iargs
+}
+
+
+@test "CD failover nvb2 0" {
+  bash tests/bats/lib/test_cd_nvb_failover.sh tests/bats/specs/nvb2.yaml 0
+}
+
+
+@test "CD failover nvb2 1" {
+  bash tests/bats/lib/test_cd_nvb_failover.sh tests/bats/specs/nvb2.yaml 1
+}
+
+
+@test "CD failover nvb2 2" {
+  bash tests/bats/lib/test_cd_nvb_failover.sh tests/bats/specs/nvb2.yaml 2
+}