Mock NVIDIA userspace + CDI integration

  - Add mock driver tree generation (pkg/gpu/mockdriver)
  - Integrate nvidia-container-toolkit nvcdi for CDI spec generation
  - Extend go-nvml dgxa100 mock with MIG stubs for nvcdi compatibility
  - Add DaemonSet to deploy CDI mock on all nodes
  - Add comprehensive smoke tests for CDI verification
  - Update gpu-mockctl CLI with fs/cdi/all modes
  - Use __NVCT_TESTING_DEVICES_ARE_FILES from nvidia-container-toolkit tests
  - Generate empty files with versioned naming

Signed-off-by: Carlos Eduardo Arango Gutierrez <[email protected]>

Author: ArangoGutierrez

cmd/gpu-mockctl/main.go

@@ -1,4 +1,4 @@
-// Copyright (c) 2024, NVIDIA CORPORATION.  All rights reserved.
+// Copyright (c) 2025, NVIDIA CORPORATION.  All rights reserved.
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
@@ -19,9 +19,12 @@ import (
 	"log"
 	"os"
 	"path/filepath"
+	"strings"
 
 	"github.com/urfave/cli/v3"
 
+	"github.com/NVIDIA/k8s-test-infra/pkg/gpu/cdi"
+	"github.com/NVIDIA/k8s-test-infra/pkg/gpu/mockdriver"
 	mockfs "github.com/NVIDIA/k8s-test-infra/pkg/gpu/mockfs"
 	mocktopo "github.com/NVIDIA/k8s-test-infra/pkg/gpu/mocktopo"
 )
@@ -31,10 +34,25 @@ func main() {
 		Name:  "gpu-mockctl",
 		Usage: "Generate mock NVIDIA driver filesystem for testing",
 		Flags: []cli.Flag{
+			&cli.StringFlag{
+				Name:  "mode",
+				Value: "all",
+				Usage: "operation mode: fs, cdi, or all",
+			},
 			&cli.StringFlag{
 				Name:  "base",
 				Value: "/run/nvidia/driver",
-				Usage: "mock driver root directory",
+				Usage: "mock driver root directory (for fs mode)",
+			},
+			&cli.StringFlag{
+				Name:  "driver-root",
+				Value: "/var/lib/nvidia-mock/driver",
+				Usage: "host mock driver tree root (for cdi mode)",
+			},
+			&cli.StringFlag{
+				Name:  "cdi-output",
+				Value: cdi.DefaultSpecPath,
+				Usage: "CDI spec output path",
 			},
 			&cli.StringFlag{
 				Name: "machine",
@@ -46,12 +64,31 @@ func main() {
 				}(),
 				Usage: "machine type (only dgxa100 supported)",
 			},
+			&cli.BoolFlag{
+				Name:  "with-dri",
+				Usage: "include DRI render node",
+			},
+			&cli.BoolFlag{
+				Name:  "with-hook",
+				Usage: "include CDI hook references",
+			},
+			&cli.StringFlag{
+				Name:  "toolkit-root",
+				Value: "/usr/local/nvidia-container-toolkit",
+				Usage: "toolkit root for hook paths",
+			},
 		},
 		Action: func(ctx context.Context, cmd *cli.Command) error {
-			return run(
-				cmd.String("base"),
-				cmd.String("machine"),
-			)
+			return run(&config{
+				mode:        cmd.String("mode"),
+				base:        cmd.String("base"),
+				driverRoot:  cmd.String("driver-root"),
+				cdiOutput:   cmd.String("cdi-output"),
+				machine:     cmd.String("machine"),
+				withDRI:     cmd.Bool("with-dri"),
+				withHook:    cmd.Bool("with-hook"),
+				toolkitRoot: cmd.String("toolkit-root"),
+			})
 		},
 	}
 
@@ -60,17 +97,51 @@ func main() {
 	}
 }
 
-func run(base, machine string) error {
-	topo, err := mocktopo.New(machine)
+type config struct {
+	mode        string
+	base        string
+	driverRoot  string
+	cdiOutput   string
+	machine     string
+	withDRI     bool
+	withHook    bool
+	toolkitRoot string
+}
+
+func run(cfg *config) error {
+	// Get topology (A100-only for now)
+	topo, err := mocktopo.New(cfg.machine)
 	if err != nil {
 		if os.Getenv("ALLOW_UNSUPPORTED") == "true" {
-			log.Printf("unsupported machine %q, using fallback", machine)
+			log.Printf("unsupported machine %q, using fallback",
+				cfg.machine)
 			topo = mocktopo.NewFallback(8, "NVIDIA A100-SXM4-40GB")
 		} else {
 			return fmt.Errorf("failed to create topology: %w", err)
 		}
 	}
 
+	gpuCount := len(topo.GPUs)
+	modes := strings.Split(cfg.mode, ",")
+
+	// Mode: fs (Step 1 behavior - proc/dev mock under -base)
+	if contains(modes, "fs") || contains(modes, "all") {
+		if err := runFS(cfg.base, topo); err != nil {
+			return err
+		}
+	}
+
+	// Mode: cdi (Step 2 - mock driver tree + CDI spec)
+	if contains(modes, "cdi") || contains(modes, "all") {
+		if err := runCDI(cfg, topo, gpuCount); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func runFS(base string, topo *mocktopo.Topology) error {
 	layout := mockfs.Layout{Base: filepath.Clean(base)}
 	for _, g := range topo.GPUs {
 		layout.GPUs = append(layout.GPUs, mockfs.GPU{
@@ -84,10 +155,83 @@ func run(base, machine string) error {
 		return fmt.Errorf("failed to write mock filesystem: %w", err)
 	}
 
-	log.Printf(
-		"mock filesystem written under %s (%d GPUs)\n",
-		layout.Base,
-		len(layout.GPUs),
-	)
+	log.Printf("mock filesystem (fs mode) written under %s (%d GPUs)\n",
+		layout.Base, len(layout.GPUs))
+	return nil
+}
+
+func runCDI(cfg *config, topo *mocktopo.Topology, gpuCount int) error {
+	// Create mock driver tree
+	files := mockdriver.DefaultFiles(cfg.driverRoot)
+	if err := mockdriver.WriteAll(files); err != nil {
+		return fmt.Errorf("failed to write driver files: %w", err)
+	}
+	log.Printf("mock driver tree written to %s\n", cfg.driverRoot)
+
+	// Create device nodes (both host /dev and under driverRoot/dev)
+	// Host /dev nodes for CDI runtime compatibility
+	hostDevNodes := mockdriver.DeviceNodes("/dev", gpuCount, cfg.withDRI)
+	if err := mockdriver.WriteAll(hostDevNodes); err != nil {
+		log.Printf("warning: failed to create host /dev nodes: %v", err)
+	}
+
+	// Also create under driverRoot/dev for completeness
+	driverDevNodes := mockdriver.DeviceNodes(cfg.driverRoot, gpuCount,
+		cfg.withDRI)
+	if err := mockdriver.WriteAll(driverDevNodes); err != nil {
+		log.Printf("warning: failed to create %s/dev nodes: %v",
+			cfg.driverRoot, err)
+	}
+
+	// Get the mock NVML library for CDI generation
+	// This uses the same mock topology we're already using
+	mockNVML, err := mocktopo.New(cfg.machine)
+	if err != nil {
+		if os.Getenv("ALLOW_UNSUPPORTED") == "true" {
+			log.Printf("warning: using fallback mock for CDI generation")
+			mockNVML = mocktopo.NewFallback(8, "NVIDIA A100-SXM4-40GB")
+		} else {
+			return fmt.Errorf("failed to create mock NVML: %w", err)
+		}
+	}
+
+	// Generate CDI spec using nvidia-container-toolkit nvcdi library
+	cdiOpts := cdi.Options{
+		NVMLLib:           mockNVML.NVMLInterface(),
+		DriverRoot:        cfg.driverRoot,
+		DevRoot:           "/host/dev", // DevRoot is already prefixed by the DaemonSet mount
+		NVIDIACDIHookPath: cfg.toolkitRoot + "/bin/nvidia-cdi-hook",
+	}
+
+	specYAML, err := cdi.Generate(cdiOpts)
+	if err != nil {
+		return fmt.Errorf("failed to generate CDI spec: %w", err)
+	}
+
+	// Validate before writing
+	if err := cdi.Validate(specYAML); err != nil {
+		return fmt.Errorf("CDI spec validation failed: %w", err)
+	}
+
+	// Write CDI spec
+	if err := os.MkdirAll(filepath.Dir(cfg.cdiOutput), 0o755); err != nil {
+		return fmt.Errorf("failed to create CDI directory: %w", err)
+	}
+
+	if err := os.WriteFile(cfg.cdiOutput, specYAML, 0o644); err != nil {
+		return fmt.Errorf("failed to write CDI spec: %w", err)
+	}
+
+	log.Printf("CDI spec written to %s (generated via nvidia-container-toolkit)\n",
+		cfg.cdiOutput)
 	return nil
 }
+
+func contains(slice []string, item string) bool {
+	for _, s := range slice {
+		if s == item {
+			return true
+		}
+	}
+	return false
+}

deployments/devel/gpu-mock/00-namespace.yaml

@@ -1,4 +1,4 @@
-# Copyright (c) 2024, NVIDIA CORPORATION.  All rights reserved.
+# Copyright (c) 2025, NVIDIA CORPORATION.  All rights reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at

deployments/devel/gpu-mock/10-configmap-verify-script.yaml

@@ -1,4 +1,4 @@
-# Copyright (c) 2024, NVIDIA CORPORATION.  All rights reserved.
+# Copyright (c) 2025, NVIDIA CORPORATION.  All rights reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at

deployments/devel/gpu-mock/20-job-gpu-mock-verify.yaml

@@ -1,4 +1,4 @@
-# Copyright (c) 2024, NVIDIA CORPORATION.  All rights reserved.
+# Copyright (c) 2025, NVIDIA CORPORATION.  All rights reserved.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
@@ -43,7 +43,7 @@ spec:
           volumeMounts:
             - name: mock-driver
               mountPath: /run/nvidia/driver
-          command: ["/usr/local/bin/gpu-mockctl","-base","/run/nvidia/driver"]
+          command: ["/usr/local/bin/gpu-mockctl","-mode","fs","-base","/run/nvidia/driver"]
       containers:
         - name: verify
           image: alpine:3.20

deployments/devel/gpu-mock/30-daemonset-cdi-mock.yaml

@@ -0,0 +1,72 @@
+# Copyright (c) 2025, NVIDIA CORPORATION.  All rights reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: nvidia-cdi-mock
+  namespace: gpu-mock
+spec:
+  selector:
+    matchLabels:
+      app: nvidia-cdi-mock
+  template:
+    metadata:
+      labels:
+        app: nvidia-cdi-mock
+    spec:
+      hostPID: true
+      tolerations:
+        - operator: Exists
+      restartPolicy: Always
+      volumes:
+        - name: host-cdi
+          hostPath:
+            path: /etc/cdi
+            type: DirectoryOrCreate
+        - name: host-driver
+          hostPath:
+            path: /var/lib/nvidia-mock/driver
+            type: DirectoryOrCreate
+        - name: host-dev
+          hostPath:
+            path: /dev
+      initContainers:
+        - name: setup
+          image: local/gpu-mockctl:dev
+          env:
+            - name: __NVCT_TESTING_DEVICES_ARE_FILES
+              value: "true"
+          securityContext:
+            privileged: true
+            runAsUser: 0
+          volumeMounts:
+            - name: host-cdi
+              mountPath: /host/etc/cdi
+            - name: host-driver
+              mountPath: /host/var/lib/nvidia-mock/driver
+            - name: host-dev
+              mountPath: /host/dev
+          command: ["/usr/local/bin/gpu-mockctl"]
+          args:
+            - "-mode"
+            - "cdi"
+            - "-driver-root"
+            - "/host/var/lib/nvidia-mock/driver"
+            - "-cdi-output"
+            - "/host/etc/cdi/nvidia.yaml"
+      containers:
+        - name: hold
+          image: debian:bookworm-slim
+          command: ["/bin/sh", "-c", "sleep infinity"]
+