Add E2E tests

Signed-off-by: Carlos Eduardo Arango Gutierrez <[email protected]>

Author: ArangoGutierrez

tests/e2e/cleanup_test.go

@@ -0,0 +1,270 @@
+package e2e
+
+import (
+	"fmt"
+	"time"
+
+	. "github.com/onsi/gomega"
+
+	"k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+)
+
+// waitForDeletion polls the provided checkFunc until a NotFound error is returned,
+// confirming that the resource is deleted.
+func waitForDeletion(resourceName string, checkFunc func() error) error {
+	timeout := 2 * time.Minute
+	interval := 5 * time.Second
+	start := time.Now()
+	for {
+		err := checkFunc()
+		if err != nil && errors.IsNotFound(err) {
+			return nil
+		}
+		if time.Since(start) > timeout {
+			return fmt.Errorf("timed out waiting for deletion of %s", resourceName)
+		}
+		time.Sleep(interval)
+	}
+}
+
+// cleanup aggressively uninstalls deployed Helm releases and deletes resource claim templates.
+func cleanup() {
+	deployed, err := helmClient.ListDeployedReleases()
+	Expect(err).NotTo(HaveOccurred())
+
+	for _, release := range deployed {
+		switch release.Name {
+		case helmReleaseName:
+			err := helmClient.UninstallReleaseByName(helmReleaseName)
+			Expect(err).NotTo(HaveOccurred())
+		case "gfd":
+			// Uninstall GFD if it was enabled.
+			if EnableGFD {
+				err := helmClient.UninstallReleaseByName("gfd")
+				Expect(err).NotTo(HaveOccurred())
+			}
+		default:
+			// Skip other releases.
+			continue
+		}
+	}
+
+	// Delete all ResourceClaimTemplates in the test namespace.
+	resourceClaimTemplates, err := clientSet.ResourceV1beta1().ResourceClaimTemplates(testNamespace.Name).List(ctx, metav1.ListOptions{})
+	Expect(err).NotTo(HaveOccurred())
+
+	for _, tmpl := range resourceClaimTemplates.Items {
+		// Remove finalizers before deletion.
+		patchBytes := []byte(`[{"op": "remove", "path": "/metadata/finalizers"}]`)
+		_, _ = clientSet.ResourceV1beta1().ResourceClaimTemplates(testNamespace.Name).Patch(ctx, tmpl.Name, types.JSONPatchType, patchBytes, metav1.PatchOptions{})
+		err = clientSet.ResourceV1beta1().ResourceClaimTemplates(testNamespace.Name).Delete(ctx, tmpl.Name, metav1.DeleteOptions{})
+		Expect(err).NotTo(HaveOccurred())
+	}
+}
+
+// cleanupCRDs deletes specific CRDs used during testing.
+func cleanupCRDs() {
+	crds := []string{
+		"computedomains.resource.nvidia.com",
+	}
+	if EnableGFD {
+		crds = append(crds, "nodefeatures.nfd.k8s-sigs.io", "nodefeaturerules.nfd.k8s-sigs.io")
+	}
+
+	for _, crd := range crds {
+		err := extClient.ApiextensionsV1().CustomResourceDefinitions().Delete(ctx, crd, metav1.DeleteOptions{})
+		Expect(err).NotTo(HaveOccurred())
+		// Optionally, wait for deletion confirmation:
+		_ = waitForDeletion(crd, func() error {
+			_, err := extClient.ApiextensionsV1().CustomResourceDefinitions().Get(ctx, crd, metav1.GetOptions{})
+			return err
+		})
+	}
+}
+
+// cleanupNamespaceResources removes all resources in the specified namespace.
+func cleanupNamespaceResources(namespace string) {
+	cleanupFuncs := []func(string) error{
+		cleanupHelmDeployments,
+		cleanupResourceClaims,
+		cleanupResourceClaimTemplates,
+		cleanupDaemonSets,
+		cleanupPods,
+	}
+	for _, fn := range cleanupFuncs {
+		Expect(fn(namespace)).NotTo(HaveOccurred())
+	}
+}
+
+// cleanupResourceClaims forcibly removes finalizers and deletes all ResourceClaims.
+func cleanupResourceClaims(namespace string) error {
+	claims, err := clientSet.ResourceV1beta1().ResourceClaims(namespace).List(ctx, metav1.ListOptions{})
+	if err != nil {
+		return err
+	}
+	patchBytes := []byte(`[{"op": "remove", "path": "/metadata/finalizers"}]`)
+	zero := int64(0)
+	policy := metav1.DeletePropagationForeground
+	deleteOptions := metav1.DeleteOptions{GracePeriodSeconds: &zero, PropagationPolicy: &policy}
+	for _, rc := range claims.Items {
+		_, _ = clientSet.ResourceV1beta1().ResourceClaims(namespace).Patch(ctx, rc.Name, types.JSONPatchType, patchBytes, metav1.PatchOptions{})
+		if err = clientSet.ResourceV1beta1().ResourceClaims(namespace).Delete(ctx, rc.Name, deleteOptions); err != nil {
+			return err
+		}
+		if err = waitForDeletion(rc.Name, func() error {
+			_, err := clientSet.ResourceV1beta1().ResourceClaims(namespace).Get(ctx, rc.Name, metav1.GetOptions{})
+			return err
+		}); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// cleanupResourceClaimTemplates forcibly removes finalizers and deletes all ResourceClaimTemplates.
+func cleanupResourceClaimTemplates(namespace string) error {
+	templates, err := clientSet.ResourceV1beta1().ResourceClaimTemplates(namespace).List(ctx, metav1.ListOptions{})
+	if err != nil {
+		return err
+	}
+	patchBytes := []byte(`[{"op": "remove", "path": "/metadata/finalizers"}]`)
+	zero := int64(0)
+	policy := metav1.DeletePropagationForeground
+	deleteOptions := metav1.DeleteOptions{GracePeriodSeconds: &zero, PropagationPolicy: &policy}
+	for _, tmpl := range templates.Items {
+		_, _ = clientSet.ResourceV1beta1().ResourceClaimTemplates(namespace).Patch(ctx, tmpl.Name, types.JSONPatchType, patchBytes, metav1.PatchOptions{})
+		if err = clientSet.ResourceV1beta1().ResourceClaimTemplates(namespace).Delete(ctx, tmpl.Name, deleteOptions); err != nil {
+			return err
+		}
+		if err = waitForDeletion(tmpl.Name, func() error {
+			_, err := clientSet.ResourceV1beta1().ResourceClaimTemplates(namespace).Get(ctx, tmpl.Name, metav1.GetOptions{})
+			return err
+		}); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// cleanupDaemonSets forcibly removes finalizers and deletes all DaemonSets in the namespace.
+func cleanupDaemonSets(namespace string) error {
+	dsList, err := clientSet.AppsV1().DaemonSets(namespace).List(ctx, metav1.ListOptions{})
+	if err != nil {
+		return err
+	}
+	patchBytes := []byte(`[{"op": "remove", "path": "/metadata/finalizers"}]`)
+	zero := int64(0)
+	policy := metav1.DeletePropagationForeground
+	deleteOptions := metav1.DeleteOptions{GracePeriodSeconds: &zero, PropagationPolicy: &policy}
+	for _, ds := range dsList.Items {
+		_, _ = clientSet.AppsV1().DaemonSets(namespace).Patch(ctx, ds.Name, types.JSONPatchType, patchBytes, metav1.PatchOptions{})
+		if err = clientSet.AppsV1().DaemonSets(namespace).Delete(ctx, ds.Name, deleteOptions); err != nil {
+			return err
+		}
+		if err = waitForDeletion(ds.Name, func() error {
+			_, err := clientSet.AppsV1().DaemonSets(namespace).Get(ctx, ds.Name, metav1.GetOptions{})
+			return err
+		}); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// cleanupPods forcibly removes finalizers and force-deletes all Pods in the namespace.
+func cleanupPods(namespace string) error {
+	podList, err := clientSet.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{})
+	if err != nil {
+		return err
+	}
+	patchBytes := []byte(`[{"op": "remove", "path": "/metadata/finalizers"}]`)
+	zero := int64(0)
+	deleteOptions := metav1.DeleteOptions{GracePeriodSeconds: &zero}
+	for _, pod := range podList.Items {
+		_, _ = clientSet.CoreV1().Pods(namespace).Patch(ctx, pod.Name, types.JSONPatchType, patchBytes, metav1.PatchOptions{})
+		if err = clientSet.CoreV1().Pods(namespace).Delete(ctx, pod.Name, deleteOptions); err != nil {
+			return err
+		}
+		if err = waitForDeletion(pod.Name, func() error {
+			_, err := clientSet.CoreV1().Pods(namespace).Get(ctx, pod.Name, metav1.GetOptions{})
+			return err
+		}); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// cleanupNodeLabels removes the "resource.nvidia.com/computeDomain" label from all nodes.
+func cleanupNodeLabels() error {
+	nodes, err := clientSet.CoreV1().Nodes().List(ctx, metav1.ListOptions{})
+	if err != nil {
+		return err
+	}
+	for _, node := range nodes.Items {
+		if _, exists := node.Labels["resource.nvidia.com/computeDomain"]; exists {
+			delete(node.Labels, "resource.nvidia.com/computeDomain")
+			_, err := clientSet.CoreV1().Nodes().Update(ctx, &node, metav1.UpdateOptions{})
+			if err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+// cleanupTestResources deletes the pod, computeDomain, and resource claim template,
+// and then removes the "resource.nvidia.com/computeDomain" label from all nodes.
+// It returns an error if any step fails.
+func cleanupTestResources(namespace, podName, cdName, rctName string) error {
+	// Delete the Pod with default deletion options.
+	if err := clientSet.CoreV1().Pods(namespace).Delete(ctx, podName, metav1.DeleteOptions{}); err != nil {
+		return err
+	}
+
+	// Prepare foreground deletion options with zero grace period.
+	zero := int64(0)
+	policy := metav1.DeletePropagationForeground
+	deleteOptions := metav1.DeleteOptions{
+		GracePeriodSeconds: &zero,
+		PropagationPolicy:  &policy,
+	}
+
+	// Delete the computeDomain.
+	if err := resourceClient.ResourceV1beta1().ComputeDomains(namespace).Delete(ctx, cdName, deleteOptions); err != nil {
+		return err
+	}
+
+	// Delete the resource claim template.
+	if err := clientSet.ResourceV1beta1().ResourceClaimTemplates(namespace).Delete(ctx, rctName, deleteOptions); err != nil {
+		return err
+	}
+
+	// Remove the "resource.nvidia.com/computeDomain" label from all nodes.
+	if err := cleanupNodeLabels(); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// cleanupHelmDeployments uninstalls all deployed Helm releases in the specified namespace.
+func cleanupHelmDeployments(namespace string) error {
+	releases, err := helmClient.ListDeployedReleases()
+	if err != nil {
+		return fmt.Errorf("failed to list deployed releases: %w", err)
+	}
+
+	for _, release := range releases {
+		// Check if the release is deployed in the target namespace.
+		// Depending on your helmClient configuration the release might carry the namespace information.
+		if release.Namespace == namespace {
+			if err := helmClient.UninstallReleaseByName(release.Name); err != nil {
+				return fmt.Errorf("failed to uninstall release %q: %w", release.Name, err)
+			}
+		}
+	}
+	return nil
+}

tests/e2e/data/test-pod1.yaml

@@ -0,0 +1,58 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: gpu-test1
+
+---
+apiVersion: resource.k8s.io/v1beta1
+kind: ResourceClaimTemplate
+metadata:
+  namespace: gpu-test1
+  name: single-gpu
+spec:
+  spec:
+    devices:
+      requests:
+      - name: gpu
+        deviceClassName: gpu.nvidia.com
+
+---
+apiVersion: resource.nvidia.com/v1beta1
+kind: ComputeDomain
+metadata:
+  namespace: gpu-test1
+  name: imex-channel-injection
+spec:
+  numNodes: 1
+  channel:
+    resourceClaimTemplate:
+      name: imex-channel-0
+
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  namespace: gpu-test1
+  name: pod1
+  labels:
+    app: pod
+spec:
+  containers:
+  - name: ctr
+    image: ubuntu:22.04
+    command: ["bash", "-c"]
+    args: ["nvidia-smi -L; trap 'exit 0' TERM; sleep 9999 & wait"]
+    resources:
+      claims:
+      - name: gpu
+      - name: imex-channel-0
+  resourceClaims:
+  - name: gpu
+    resourceClaimTemplateName: single-gpu
+  - name: imex-channel-0
+    resourceClaimTemplateName: imex-channel-0
+  tolerations:
+  - key: "nvidia.com/gpu"
+    operator: "Exists"
+    effect: "NoSchedule"