Replace kubectl usage with utility program to apply CRDs

kubectl is heavyweight and often has CVEs, so we are forced to bump its
version even if the final gpu-operator image does not contain kubectl
bits. This change lets us remove that dependency and use client-go
functions to manage CRDs.

Signed-off-by: Rajath Agasthya <[email protected]>

Author: rajathagasthya

cmd/apply-crds/main.go

@@ -0,0 +1,129 @@
+/*
+Copyright (c), NVIDIA CORPORATION.  All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"context"
+	"fmt"
+	"os"
+
+	"github.com/NVIDIA/k8s-operator-libs/pkg/crdutil"
+	log "github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+
+	"github.com/NVIDIA/gpu-operator/internal/info"
+)
+
+var logger = log.New()
+
+type config struct {
+	Debug     bool
+	crdsPaths *cli.StringSlice
+}
+
+func main() {
+	config := config{
+		crdsPaths: cli.NewStringSlice(),
+	}
+
+	// Create the top-level CLI
+	c := cli.NewApp()
+	c.Name = "apply-crds"
+	c.Usage = "Tools for managing Custom Resource Definitions (CRDs) for NVIDIA GPU Operator"
+	c.Version = info.GetVersionString()
+
+	// Setup the flags for this command
+	c.Flags = []cli.Flag{
+		&cli.BoolFlag{
+			Name:        "debug",
+			Aliases:     []string{"d"},
+			Usage:       "Enable debug-level logging",
+			Destination: &config.Debug,
+			EnvVars:     []string{"DEBUG"},
+		},
+	}
+
+	// Set log-level for all subcommands
+	c.Before = func(c *cli.Context) error {
+		logLevel := log.InfoLevel
+		if config.Debug {
+			logLevel = log.DebugLevel
+		}
+		logger.SetLevel(logLevel)
+		return nil
+	}
+
+	// Common flags for both apply and delete subcommands
+	commonFlags := []cli.Flag{
+		&cli.StringSliceFlag{
+			Name:        "crds-path",
+			Usage:       "Path to CRD manifest file or directory (can be specified multiple times, directories are searched recursively)",
+			Required:    true,
+			Destination: config.crdsPaths,
+		},
+	}
+
+	// Define the subcommands
+	c.Commands = []*cli.Command{
+		{
+			Name:  "apply",
+			Usage: "Apply CRDs from the specified path",
+			Flags: commonFlags,
+			Action: func(c *cli.Context) error {
+				return runApply(c.Context, config)
+			},
+		},
+		{
+			Name:  "delete",
+			Usage: "Delete CRDs from the specified path",
+			Flags: commonFlags,
+			Action: func(c *cli.Context) error {
+				return runDelete(c.Context, config)
+			},
+		},
+	}
+
+	err := c.Run(os.Args)
+	if err != nil {
+		log.Errorf("%v", err)
+		log.Exit(1)
+	}
+}
+
+func runApply(ctx context.Context, cfg config) error {
+	paths := cfg.crdsPaths.Value()
+	logger.Infof("Applying CRDs from %d path(s): %v", len(paths), paths)
+
+	if err := crdutil.ProcessCRDs(ctx, crdutil.CRDOperationApply, paths...); err != nil {
+		return fmt.Errorf("failed to apply CRDs: %w", err)
+	}
+
+	logger.Info("Successfully applied CRDs")
+	return nil
+}
+
+func runDelete(ctx context.Context, cfg config) error {
+	paths := cfg.crdsPaths.Value()
+	logger.Infof("Deleting CRDs from %d path(s): %v", len(paths), paths)
+
+	if err := crdutil.ProcessCRDs(ctx, crdutil.CRDOperationDelete, paths...); err != nil {
+		return fmt.Errorf("failed to delete CRDs: %w", err)
+	}
+
+	logger.Info("Successfully deleted CRDs")
+	return nil
+}

deployments/gpu-operator/templates/cleanup_crd.yaml

@@ -35,16 +35,12 @@ spec:
           image: {{ include "gpu-operator.fullimage" . }}
           imagePullPolicy: {{ .Values.operator.imagePullPolicy }}
           command:
-          - sh
-          - -c
-          - >
-              kubectl delete clusterpolicy cluster-policy;
-              kubectl delete crd clusterpolicies.nvidia.com;
-              kubectl delete crd nvidiadrivers.nvidia.com --ignore-not-found=true;
-            {{- if .Values.nfd.enabled -}}
-              kubectl delete crd nodefeatures.nfd.k8s-sigs.io --ignore-not-found=true;
-              kubectl delete crd nodefeaturegroups.nfd.k8s-sigs.io --ignore-not-found=true;
-              kubectl delete crd nodefeaturerules.nfd.k8s-sigs.io --ignore-not-found=true;
-            {{- end }}
+            - /usr/bin/apply-crds
+          args:
+            - --crds-path=/opt/gpu-operator/nvidia.com_clusterpolicies.yaml
+            - --crds-path=/opt/gpu-operator/nvidia.com_nvidiadrivers.yaml;
+        {{- if .Values.nfd.enabled }}
+            - --crds-path=/opt/gpu-operator/nfd-api-crds.yaml;
+        {{- end }}
       restartPolicy: OnFailure
 {{- end }}

deployments/gpu-operator/templates/upgrade_crd.yaml

@@ -83,13 +83,12 @@ spec:
           image: {{ include "gpu-operator.fullimage" . }}
           imagePullPolicy: {{ .Values.operator.imagePullPolicy }}
           command:
-          - sh
-          - -c
-          - >
-            kubectl apply -f /opt/gpu-operator/nvidia.com_clusterpolicies.yaml;
-            kubectl apply -f /opt/gpu-operator/nvidia.com_nvidiadrivers.yaml;
+            - /usr/bin/apply-crds
+          args:
+            - --crds-path=/opt/gpu-operator/nvidia.com_clusterpolicies.yaml
+            - --crds-path=/opt/gpu-operator/nvidia.com_nvidiadrivers.yaml;
         {{- if .Values.nfd.enabled }}
-            kubectl apply -f /opt/gpu-operator/nfd-api-crds.yaml;
+            - --crds-path=/opt/gpu-operator/nfd-api-crds.yaml;
         {{- end }}
       restartPolicy: OnFailure
 {{- end }}

docker/Dockerfile

@@ -47,12 +47,6 @@ RUN dnf install -y --allowerasing \
 
 WORKDIR /workspace
 
-# Install must-gather dependency: `kubectl`
-ARG TARGETARCH
-RUN OS_ARCH=${TARGETARCH/x86_64/amd64} && OS_ARCH=${OS_ARCH/aarch64/arm64} && \
-    curl -LO https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/${OS_ARCH}/kubectl && \
-    chmod +x ./kubectl
-
 FROM nvcr.io/nvidia/cuda:12.9.1-base-ubi9 AS sample-builder
 
 RUN dnf install -y --allowerasing \
@@ -96,7 +90,7 @@ LABEL vsc-ref=${GIT_COMMIT}
 
 WORKDIR /
 COPY --from=builder /workspace/gpu-operator /usr/bin/
-COPY --from=cuda-base /workspace/kubectl /usr/bin/
+COPY --from=builder /workspace/apply-crds /usr/bin/
 COPY --from=builder /workspace/nvidia-validator /usr/bin/
 COPY --from=sample-builder /build/vectorAdd /usr/bin/vectorAdd
 # TODO: Copy the compat libs from the 'sample-builder' image instead.

go.mod

@@ -6,11 +6,11 @@ require (
 	github.com/Masterminds/sprig/v3 v3.3.0
 	github.com/NVIDIA/go-nvlib v0.8.1
 	github.com/NVIDIA/k8s-kata-manager v0.2.3
-	github.com/NVIDIA/k8s-operator-libs v0.0.0-20250709180754-c80af13d73e3
+	github.com/NVIDIA/k8s-operator-libs v0.0.0-20251027171627-45ccd0c3dd32
 	github.com/NVIDIA/nvidia-container-toolkit v1.18.0
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc
 	github.com/go-logr/logr v1.4.3
-	github.com/onsi/ginkgo/v2 v2.26.0
+	github.com/onsi/ginkgo/v2 v2.27.1
 	github.com/onsi/gomega v1.38.2
 	github.com/openshift/api v0.0.0-20251021124544-a2cb0c5d994d
 	github.com/openshift/client-go v0.0.0-20251015124057-db0dee36e235

go.sum

@@ -16,8 +16,8 @@ github.com/NVIDIA/go-nvlib v0.8.1 h1:OPEHVvn3zcV5OXB68A7WRpeCnYMRSPl7LdeJH/d3gZI
 github.com/NVIDIA/go-nvlib v0.8.1/go.mod h1:7mzx9FSdO9fXWP9NKuZmWkCwhkEcSWQFe2tmFwtLb9c=
 github.com/NVIDIA/k8s-kata-manager v0.2.3 h1:d5+gRFqU5el/fKMXhHUaPY7haj+dbHL4nDsO/q05LBo=
 github.com/NVIDIA/k8s-kata-manager v0.2.3/go.mod h1:xx5OUiMsHyKbyX0JjKHqAftvqS8vx00LFn/5EaMdtB4=
-github.com/NVIDIA/k8s-operator-libs v0.0.0-20250709180754-c80af13d73e3 h1:vGT+oyUY7kOGLd71Cz0NfRVEep23jdd4fi+PYsZEj88=
-github.com/NVIDIA/k8s-operator-libs v0.0.0-20250709180754-c80af13d73e3/go.mod h1:0GPZJRwr6nY1IVhGUyzG9YfKhNFQq8UlhYe4u7jVF0U=
+github.com/NVIDIA/k8s-operator-libs v0.0.0-20251027171627-45ccd0c3dd32 h1:TWudaaTt7QwN/cQwPOm1wgesGLOc8hoik9GubKgnph0=
+github.com/NVIDIA/k8s-operator-libs v0.0.0-20251027171627-45ccd0c3dd32/go.mod h1:WbVhWGKqRcwjRKj8MYsYJas73G1YdU3oLW5ggDvTWXs=
 github.com/NVIDIA/nvidia-container-toolkit v1.18.0 h1:bXoKq9C1WHU5fF6VqXvX3RkMzpp4ihTUgBPrh66vTf0=
 github.com/NVIDIA/nvidia-container-toolkit v1.18.0/go.mod h1:ZxWSG7fnFo2Z7xSGtMyZVF7WnTbj1lgx4dMrBLUq90g=
 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
@@ -59,8 +59,8 @@ github.com/gkampitakis/ciinfo v0.3.2 h1:JcuOPk8ZU7nZQjdUhctuhQofk7BGHuIy0c9Ez8BN
 github.com/gkampitakis/ciinfo v0.3.2/go.mod h1:1NIwaOcFChN4fa/B0hEBdAb6npDlFL8Bwx4dfRLRqAo=
 github.com/gkampitakis/go-diff v1.3.2 h1:Qyn0J9XJSDTgnsgHRdz9Zp24RaJeKMUHg2+PDZZdC4M=
 github.com/gkampitakis/go-diff v1.3.2/go.mod h1:LLgOrpqleQe26cte8s36HTWcTmMEur6OPYerdAAS9tk=
-github.com/gkampitakis/go-snaps v0.5.14 h1:3fAqdB6BCPKHDMHAKRwtPUwYexKtGrNuw8HX/T/4neo=
-github.com/gkampitakis/go-snaps v0.5.14/go.mod h1:HNpx/9GoKisdhw9AFOBT1N7DBs9DiHo/hGheFGBZ+mc=
+github.com/gkampitakis/go-snaps v0.5.15 h1:amyJrvM1D33cPHwVrjo9jQxX8g/7E2wYdZ+01KS3zGE=
+github.com/gkampitakis/go-snaps v0.5.15/go.mod h1:HNpx/9GoKisdhw9AFOBT1N7DBs9DiHo/hGheFGBZ+mc=
 github.com/go-errors/errors v1.5.1 h1:ZwEMSLRCapFLflTpT7NKaAc7ukJ8ZPEjzlxt8rPN8bk=
 github.com/go-errors/errors v1.5.1/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og=
 github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
@@ -150,8 +150,8 @@ github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J
 github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
 github.com/olareg/olareg v0.1.2 h1:75G8X6E9FUlzL/CSjgFcYfMgNzlc7CxULpUUNsZBIvI=
 github.com/olareg/olareg v0.1.2/go.mod h1:TWs+N6pO1S4bdB6eerzUm/ITRQ6kw91mVf9ZYeGtw+Y=
-github.com/onsi/ginkgo/v2 v2.26.0 h1:1J4Wut1IlYZNEAWIV3ALrT9NfiaGW2cDCJQSFQMs/gE=
-github.com/onsi/ginkgo/v2 v2.26.0/go.mod h1:qhEywmzWTBUY88kfO0BRvX4py7scov9yR+Az2oavUzw=
+github.com/onsi/ginkgo/v2 v2.27.1 h1:0LJC8MpUSQnfnp4n/3W3GdlmJP3ENGF0ZPzjQGLPP7s=
+github.com/onsi/ginkgo/v2 v2.27.1/go.mod h1:wmy3vCqiBjirARfVhAqFpYt8uvX0yaFe+GudAqqcCqA=
 github.com/onsi/gomega v1.38.2 h1:eZCjf2xjZAqe+LeWvKb5weQ+NcPwX84kqJ0cZNxok2A=
 github.com/onsi/gomega v1.38.2/go.mod h1:W2MJcYxRGV63b418Ai34Ud0hEdTVXq9NW9+Sx6uXf3k=
 github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=