Merge pull request #677 from jgehrcke/jp/test-abort-on-failure

tests: abort suite on first failure, misc

Author: jgehrcke

tests/bats/Dockerfile

@@ -12,7 +12,7 @@ ARG BUILDARCH
 # Install bats for running cmdline tests.
 # This is the image used when invoking `make bats-test`.
 RUN git clone https://github.com/bats-core/bats-core.git && cd bats-core && \
-    git checkout 855844b8344e67d60dc0f43fa39817ed7787f141 && ./install.sh /usr/local
+    git checkout 658f442f0fcdd6f9e2ea01625999217e8f7bfe7d && ./install.sh /usr/local
 
 RUN mkdir -p /bats-libraries
 RUN git clone https://github.com/bats-core/bats-support /bats-libraries/bats-support

tests/bats/Makefile

@@ -77,8 +77,9 @@ image:
 # cmdline args).
 .PHONY: tests
 tests: image
-	mkdir -p tests-out && \
-	export _RUNDIR=$$(mktemp -p tests-out -d -t bats-tests-$$(date +%s)-XXXXX) && \
+	export _RUNDIR_PARENT=/tmp/k8s-dra-driver-gpu-tests-out && \
+	mkdir -p "$${_RUNDIR_PARENT}" && \
+	export _RUNDIR=$$(mktemp -p $${_RUNDIR_PARENT} -d -t bats-tests-$$(date +%s)-XXXXX) && \
 	docker run \
 		--rm \
 		$(DOCKER_RUN_FLAGS) \
@@ -99,9 +100,10 @@ tests: image
 		-c "set -ex; cd /cwd; \
 			echo 'Running k8s cluster cleanup (invasive)... '; \
 			bash tests/bats/cleanup-from-previous-run.sh 2>&1 | tee -a $${_RUNDIR}/cleanup.outerr; \
-			TMPDIR=/cwd/$${_RUNDIR} bats \
+			TMPDIR=$${_RUNDIR} bats \
 			--print-output-on-failure \
 			--no-tempdir-cleanup \
 			--timing \
+			--abort \
 			tests/bats/tests.bats \
 		"

tests/bats/README.md

@@ -67,7 +67,8 @@ Make wise usage of
 
 Misc notes:
 
+* The test suite stops on first failure (using the [new](https://github.com/bats-core/bats-core/issues/209) `--abort` flag for bats).
+  The tests are not perfectly independent yet, and hence that's sane default behavior.
 * Don't skip the section about when [not to use `run`](https://bats-core.readthedocs.io/en/stable/writing-tests.html#when-not-to-use-run).
 * Take inspiration from [cri-o tests](https://github.com/cri-o/cri-o/tree/81e69a58c7e6ec8699b3bdd8696b1d0e25e32bfb/test).
-* Stop test suite on first failure? No first-class support in bats. See [this](https://github.com/bats-core/bats-core/issues/209) discussion.
 * We can and should radically iterate on the test suite's config interface to satisfy our needs.

tests/bats/helpers.sh

@@ -112,15 +112,27 @@ get_all_cd_daemon_logs_for_cd_name() {
   CD_NAME="$1"
   CD_UID=$(kubectl describe computedomains.resource.nvidia.com "${CD_NAME}" | grep UID | awk '{print $2}')
   CD_LABEL_KV="resource.nvidia.com/computeDomain=${CD_UID}"
+  echo "CD daemon logs for CD: $CD_UID"
+  kubectl logs \
+    -n nvidia-dra-driver-gpu \
+    -l "${CD_LABEL_KV}" \
+    --tail=-1 \
+    --prefix \
+    --all-containers
+}
+
+show_kubelet_plugin_error_logs() {
+  echo -e "\nKUBELET PLUGIN ERROR LOGS START"
+  (
     kubectl logs \
-      -n nvidia-dra-driver-gpu \
-      -l "${CD_LABEL_KV}" \
-      --tail=-1 \
-      --prefix \
-      --all-containers \
-      --timestamps
+    -l nvidia-dra-driver-gpu-component=kubelet-plugin \
+    -n nvidia-dra-driver-gpu \
+    --prefix --tail=-1 | grep -E "^(E|W)[0-9]{4}"
+  ) || true
+  echo -e "KUBELET PLUGIN ERROR LOGS END\n\n"
 }
 
+
 # Intended use case: one pod in Running or ContainerCreating state; then this
 # function returns the specific name of that pod. Specifically, ignore pods that
 # were just deleted or are terminating (this is important during the small time

tests/bats/tests.bats

@@ -42,16 +42,23 @@ setup_file() {
   helm repo add nvidia https://helm.ngc.nvidia.com/nvidia && helm repo update
 }
 
+bats::on_failure() {
+  echo -e "\n\nFAILURE HOOK START"
+  log_objects
+  show_kubelet_plugin_error_logs
+  get_all_cd_daemon_logs_for_cd_name "imex-channel-injection" || true
+  echo -e "FAILURE HOOK END\n\n"
+}
+
 apply_check_delete_workload_imex_chan_inject() {
   kubectl apply -f demo/specs/imex/channel-injection.yaml
   kubectl wait --for=condition=READY pods imex-channel-injection --timeout=100s
   run kubectl logs imex-channel-injection
-  kubectl delete -f demo/specs/imex/channel-injection.yaml
-  # Check output after attempted deletion.
   assert_output --partial "channel0"
 
   # Wait for deletion to complete; this is critical before moving on to the next
   # test (as long as we don't wipe state entirely between tests).
+  kubectl delete -f demo/specs/imex/channel-injection.yaml
   kubectl wait --for=delete pods imex-channel-injection --timeout=10s
 }
 
@@ -101,12 +108,9 @@ log_objects() {
   kubectl get crd computedomains.resource.nvidia.com
 }
 
-@test "wait for kubelet plugin pods READY" {
+@test "wait for plugin & controller pods READY" {
   kubectl wait --for=condition=READY pods -A \
     -l nvidia-dra-driver-gpu-component=kubelet-plugin --timeout=10s
-}
-
-@test "wait for controller pod READY" {
   kubectl wait --for=condition=READY pods -A \
     -l nvidia-dra-driver-gpu-component=controller --timeout=10s
 }