Merge pull request #600 from jgehrcke/jp/bats-for-local-dev

bats-tests: add TEST_CHART_LOCAL, add tests, adjust readme

Author: jgehrcke

.dockerignore

@@ -3,4 +3,5 @@ deployments/container
 *.tar
 *.tgz
 demo
-.git
+.git
+tests-out

tests/bats/Dockerfile

@@ -3,7 +3,7 @@ FROM debian:trixie
 # GNU parallel: bats may want to use that
 # gettext-base: provides envsubst, used by nickelpie
 RUN apt-get update && apt-get install -y -q --no-install-recommends \
-    parallel git ca-certificates curl make gettext-base && \
+    parallel git ca-certificates curl make gettext-base jq && \
     rm -rf /var/lib/apt/lists/*
 
 # Set by BuiltKit, of the form amd64/arm64.

tests/bats/Makefile

@@ -16,18 +16,45 @@
 include $(CURDIR)/versions.mk
 include $(CURDIR)/common.mk
 
-BATS_IMAGE = batstests:$(GIT_COMMIT_SHORT)
 
-KUBECONFIG ?= $(HOME)/.kube/config
+# The to-be-tested Helm chart. Ignored when setting TEST_CHART_LOCAL.
 TEST_CHART_REPO ?= "oci://ghcr.io/nvidia/k8s-dra-driver-gpu"
-TEST_CHART_VERSION ?= $(VERSION_GHCR_CHART)
+TEST_CHART_VERSION ?= "$(VERSION_GHCR_CHART)"
+
+# The baseline Helm chart to test upgrades from and downgrades to.
 TEST_CHART_LASTSTABLE_REPO ?= "oci://ghcr.io/nvidia/k8s-dra-driver-gpu"
 TEST_CHART_LASTSTABLE_VERSION ?= "25.3.2-2c250af3-chart"
+
+# If not "false": the to-be-tested Helm chart is installed from the local
+# filesystem (from `deployments/helm/nvidia-dra-driver-gpu`). Make sure
+# (out-of-band) that the container images that the chart refers to are available
+# (placed directly on the involved nodes or pullable). This is a convenince
+# parameter for setting
+#
+#        TEST_CHART_REPO="deployments/helm/nvidia-dra-driver-gpu/"
+#        TEST_CHART_VERSION="$(make print-VERSION)" (but w/o v prefix)
+#
+TEST_CHART_LOCAL ?= "false"
+
+# Consumed in upgrade test via kubectl apply -f <URL>
+# (can be a branch, tag, or commit). TODO: parse default
+# from `TEST_CHART_VERSION`.
+TEST_CRD_UPGRADE_TARGET_GIT_REF ?= "main"
+
 TEST_NVIDIA_DRIVER_ROOT ?= "/run/nvidia/driver"
 
-# Currently consumed in upgrade test via
-# kubectl apply -f <URL> (can be a branch, tag, or commit)
-TEST_CRD_UPGRADE_TARGET_GIT_REF ?= $(GIT_COMMIT_SHORT)
+# TODO: calculate precise, expected container image spec,
+# and test for that in test suite, for example:
+# `nvcr.io/nvidia/k8s-dra-driver-gpu:v25.8.0-dev`
+TEST_EXPECTED_IMAGE_SPEC_SUBSTRING ?= $(VERSION)
+
+ifneq ($(TEST_CHART_LOCAL),"false")
+TEST_CHART_REPO = "deployments/helm/nvidia-dra-driver-gpu/"
+TEST_CHART_VERSION = $(VERSION:v%=%)
+endif
+
+BATS_IMAGE = batstests:$(GIT_COMMIT_SHORT)
+KUBECONFIG ?= $(HOME)/.kube/config
 
 default: tests
 
@@ -57,6 +84,7 @@ tests: image
 		--env TEST_CHART_LASTSTABLE_VERSION=$(TEST_CHART_LASTSTABLE_VERSION) \
 		--env TEST_CRD_UPGRADE_TARGET_GIT_REF=$(TEST_CRD_UPGRADE_TARGET_GIT_REF) \
 		--env TEST_NVIDIA_DRIVER_ROOT=$(TEST_NVIDIA_DRIVER_ROOT) \
+		--env TEST_EXPECTED_IMAGE_SPEC_SUBSTRING=$(TEST_EXPECTED_IMAGE_SPEC_SUBSTRING) \
 		-u $(shell id -u ${USER}):$(shell id -g ${USER}) \
 		--entrypoint "/bin/bash"\
 		$(BATS_IMAGE) \

tests/bats/README.md

@@ -2,70 +2,72 @@
 
 ## Usage
 
-Invoke `make bats` in the root of this repository.
+Review the `TEST_*` variables [at around the top of the Makefile](https://github.com/NVIDIA/k8s-dra-driver-gpu/blob/main/tests/bats/Makefile#L22). Most of them can be overridden via environment.
+Use this configuration interface to customize your test run.
 
+Then invoke `make bats` in the root of the repository.
 
-### Test local dev state (artifacts not pushed)
+Some examples are shown below.
 
-Not yet supported.
-Let's change this ASAP.
+### Test a specific GHCR chart version
 
-This test suite for now assumes public availability of a Helm chart on GHCR or NGC, pointing to a container image publicly available on GHCR or NGC.
+Example:
 
-### Test Helm chart from registery
+```console
+$ export TEST_CHART_VERSION="25.8.0-dev-b823882b-chart"
+$ make bats
+...
+12 tests, 0 failures in 166 seconds
+```
 
-#### Default versions
+Note: by default, the test suite assumes availability of a Helm chart on `oci://ghcr.io/nvidia/k8s-dra-driver-gpu`, pointing to a container image also publicly available in that registry.
 
-Say, this is the current local git revision:
 
-```console
-$ git rev-parse --short=8 HEAD
-e6e1dde4
-```
+### Test local dev state (artifacts not pushed)
+
+To test the Helm chart currently specified in `deployments/helm/nvidia-dra-driver-gpu` in the local checkout, run
 
-Then the test suite runs with the default configuration, for example:
 ```console
-$ make bats
-...
-        --env TEST_CHART_REPO="oci://ghcr.io/nvidia/k8s-dra-driver-gpu" \
-        --env TEST_CHART_VERSION=25.8.0-dev-e6e1dde4-chart \
-        --env TEST_CHART_LASTSTABLE_REPO="oci://ghcr.io/nvidia/k8s-dra-driver-gpu" \
-        --env TEST_CHART_LASTSTABLE_VERSION="25.3.2-7020737a-chart" \
-        --env TEST_CRD_UPGRADE_TARGET_GIT_REF=e6e1dde4 \
-...
-12 tests, 0 failures in 166 seconds
+TEST_CHART_LOCAL=1 make bats
 ```
 
-As you can see, this currently requires a Helm chart corresponding to the local revision to be available on GHCR.
+This overrides `TEST_CHART_REPO` and `TEST_CHART_VERSION`.
 
-#### Test specific versions
+Make sure (out-of-band) that the container images that the local chart refers to are available to all nodes in the Kubernetes cluster -- placed directly (TODO: how-to) or pullable.
 
-Set the correponding `TEST_*` environment variables before invoking the Makefile target.
+### Defaults
 
-For example:
+By default, `make bats` tries to install a Helm chart from  `oci://ghcr.io/nvidia/k8s-dra-driver-gpu` corresponding to the git revision of the local checkout:
 
 ```console
-$ export TEST_CHART_VERSION="25.8.0-dev-b823882b-chart"
-$ export TEST_CRD_UPGRADE_TARGET_GIT_REF="main"
+$ git rev-parse --short=8 HEAD
+e6e1dde4
 $ make bats
 ...
-12 tests, 0 failures in 166 seconds
+        --env TEST_CHART_REPO="oci://ghcr.io/nvidia/k8s-dra-driver-gpu" \
+        --env TEST_CHART_VERSION=25.8.0-dev-e6e1dde4-chart \
+        --env TEST_CRD_UPGRADE_TARGET_GIT_REF=e6e1dde4 \
+...
 ```
 
+That's CI-oriented.
+We may want to change that.
+
 
 ## Development
 
 Bats is a workable solution.
 Developing new tests might however probe your patience.
 Make wise usage of
 
+* bats' [`run`](https://bats-core.readthedocs.io/en/stable/writing-tests.html#run-test-other-commands) command.
 * [skipping tests](https://bats-core.readthedocs.io/en/stable/writing-tests.html#skip-easily-skip-tests)
 * [tagging tests with `bats:focus`](https://bats-core.readthedocs.io/en/stable/writing-tests.html#special-tags)
 * [CLI args](https://bats-core.readthedocs.io/en/stable/usage.html) such as `--verbose-run`, `--show-output-of-passing-tests`.
 
+Misc notes:
 
-Also, familiarize yourself with bat's [`run`](https://bats-core.readthedocs.io/en/stable/writing-tests.html#run-test-other-commands) command.
-
-Don't skip the section about when [not to use `run`](https://bats-core.readthedocs.io/en/stable/writing-tests.html#when-not-to-use-run).
-
-Take inspiration from [cri-o tests](https://github.com/cri-o/cri-o/tree/81e69a58c7e6ec8699b3bdd8696b1d0e25e32bfb/test).
+* Don't skip the section about when [not to use `run`](https://bats-core.readthedocs.io/en/stable/writing-tests.html#when-not-to-use-run).
+* Take inspiration from [cri-o tests](https://github.com/cri-o/cri-o/tree/81e69a58c7e6ec8699b3bdd8696b1d0e25e32bfb/test).
+* Stop test suite on first failure? No first-class support in bats. See [this](https://github.com/bats-core/bats-core/issues/209) discussion.
+* We can and should radically iterate on the test suite's config interface to satisfy our needs.

tests/bats/tests.bats

@@ -48,7 +48,7 @@ setup_file() {
 # Install or upgrade, and wait for pods to be READY.
 # 1st arg: helm chart repo
 # 2nd arg: helm chart version
-# 3rd arg: array with additional args (provide `NOARGS`` if none)
+# 3rd arg: array with additional args (provide `NOARGS` if none)
 iupgrade_wait() {
   # E.g. `nvidia/nvidia-dra-driver-gpu` or
   # `oci://ghcr.io/nvidia/k8s-dra-driver-gpu`
@@ -83,7 +83,9 @@ apply_check_delete_workload_imex_chan_inject() {
 
 # A test that covers local dev tooling, we don't want to
 # unintentionally change/break these targets.
-@test "test VERSION_W_COMMIT, VERSION_GHCR_CHART" {
+@test "test VERSION_W_COMMIT, VERSION_GHCR_CHART, VERSION" {
+  run make print-VERSION
+  assert_output --regexp '^v[0-9]+\.[0-9]+\.[0-9]+-dev$'
   run make print-VERSION_W_COMMIT
   assert_output --regexp '^v[0-9]+\.[0-9]+\.[0-9]+-dev-[0-9a-f]{8}$'
   run make print-VERSION_GHCR_CHART
@@ -101,6 +103,17 @@ apply_check_delete_workload_imex_chan_inject() {
   iupgrade_wait "${TEST_CHART_REPO}" "${TEST_CHART_VERSION}" _iargs
 }
 
+@test "helm list: validate output" {
+  # Sanity check: one chart installed.
+  helm list -n nvidia-dra-driver-gpu -o json | jq 'length == 1'
+
+  # Confirm consistency between the various version-related parameters. Note
+  # that the --version arg provided to `helm install/upgrade` does not directly
+  # set app_version; it is just a version constraint. `app_version` tested here
+  # is AFAIU defined solely by the chart's appVersion YAML spec.
+  helm list -n nvidia-dra-driver-gpu -o json | jq '.[].app_version' | grep "${TEST_CHART_VERSION}"
+}
+
 @test "get crd computedomains.resource.nvidia.com" {
   kubectl get crd computedomains.resource.nvidia.com
 }
@@ -115,6 +128,22 @@ apply_check_delete_workload_imex_chan_inject() {
     -l nvidia-dra-driver-gpu-component=controller --timeout=10s
 }
 
+@test "validate CD controller container image spec" {
+  local ACTUAL_IMAGE_SPEC
+  ACTUAL_IMAGE_SPEC=$(kubectl get pod \
+    -n nvidia-dra-driver-gpu \
+    -l nvidia-dra-driver-gpu-component=controller \
+    -o json | \
+      jq -r '.items[].spec.containers[] | select(.name=="compute-domain") | .image')
+
+  # Emit once, unfiltered, for debuggability
+  echo "$ACTUAL_IMAGE_SPEC"
+
+  # Confirm substring; TODO: make tighter with precise
+  # TEST_EXPECTED_IMAGE_SPEC_SUBSTRING
+  echo "$ACTUAL_IMAGE_SPEC" | grep "${TEST_EXPECTED_IMAGE_SPEC_SUBSTRING}"
+}
+
 @test "IMEX channel injection (single)" {
   apply_check_delete_workload_imex_chan_inject
 }