Conditionally mount the host's /dev over the container's /dev

Currently, in the compute domain driver, we unconditionally mount the
host's /dev over the container's /dev regardless of what the devRoot of
the GPU driver is. This change was introduced in
NVIDIA#307.

Unfortunately, this causes some problems on certain systems as described
in NVIDIA#477.

This patch makes this mount contingent on devRoot == "/" (which is the
only case where we actually ever wanted / needed to have this mount in
the first place.

Signed-off-by: Kevin Klues <[email protected]>

Author: klueska

cmd/compute-domain-kubelet-plugin/nvlib.go

@@ -43,6 +43,7 @@ const (
 	nvidiaCapsDeviceName             = "nvidia-caps"
 	nvidiaCapsImexChannelsDeviceName = "nvidia-caps-imex-channels"
 	nvidiaCapFabricImexMgmtPath      = "/proc/driver/nvidia/capabilities/fabric-imex-mgmt"
+	hostDevPath                      = "/host/dev"
 )
 
 type deviceLib struct {
@@ -89,6 +90,10 @@ func newDeviceLib(driverRoot root) (*deviceLib, error) {
 		return nil, fmt.Errorf("error recursively unmounting %s: %w", procDriverNvidiaPath, err)
 	}
 
+	if err := d.conditionallyBindMountHostDev(); err != nil {
+		return nil, fmt.Errorf("error conditionally bind mounting host dev: %w", err)
+	}
+
 	return &d, nil
 }
 
@@ -418,3 +423,25 @@ func (l deviceLib) unmountRecursively(root string) error {
 
 	return helper(root)
 }
+
+// conditionallyBindMountHostDev bind mounts hostDevPath over /dev when devRoot is "/".
+func (l deviceLib) conditionallyBindMountHostDev() error {
+	// If devRoot != "/" then we don't need to do the mount
+	if l.devRoot != "/" {
+		return nil
+	}
+
+	// Get a reference to the mount executable.
+	mountExecutable, err := exec.LookPath("mount")
+	if err != nil {
+		return fmt.Errorf("error looking up mount executable: %w", err)
+	}
+	mounter := mount.New(mountExecutable)
+
+	// Bind mount hostDevPath over /dev
+	if err := mounter.Mount(hostDevPath, "/dev", "", []string{"bind"}); err != nil {
+		return fmt.Errorf("failed to bind mount %s over /dev: %w", hostDevPath, err)
+	}
+
+	return nil
+}

deployments/helm/nvidia-dra-driver-gpu/templates/kubeletplugin.yaml

@@ -141,7 +141,7 @@ spec:
         # TODO: make this more surgical, see discussion in
         # https://github.com/NVIDIA/k8s-dra-driver-gpu/pull/307.
         - name: host-dev
-          mountPath: /dev
+          mountPath: /host/dev
       {{- end }}
       {{- if .Values.resources.gpus.enabled }}
       - name: gpus