Merge pull request #479 from klueska/conditionally-mount-host-dev

Conditionally mount the host's /dev over the container's /dev

Author: klueska

cmd/compute-domain-kubelet-plugin/nvlib.go

@@ -43,6 +43,7 @@ const (
 	nvidiaCapsDeviceName             = "nvidia-caps"
 	nvidiaCapsImexChannelsDeviceName = "nvidia-caps-imex-channels"
 	nvidiaCapFabricImexMgmtPath      = "/proc/driver/nvidia/capabilities/fabric-imex-mgmt"
+	hostDevContainerPath             = "/host/dev"
 )
 
 type deviceLib struct {
@@ -89,6 +90,10 @@ func newDeviceLib(driverRoot root) (*deviceLib, error) {
 		return nil, fmt.Errorf("error recursively unmounting %s: %w", procDriverNvidiaPath, err)
 	}
 
+	if err := d.conditionallyBindMountHostDev(); err != nil {
+		return nil, fmt.Errorf("error conditionally bind mounting host dev: %w", err)
+	}
+
 	return &d, nil
 }
 
@@ -418,3 +423,27 @@ func (l deviceLib) unmountRecursively(root string) error {
 
 	return helper(root)
 }
+
+// conditionallyBindMountHostDev bind mounts hostDevContainerPath over /dev when devRoot is "/".
+// Introduced to address the issues described in https://github.com/NVIDIA/k8s-dra-driver-gpu/issues/477.
+// TODO: Revisit with a more comprehensive solution as proposed in https://github.com/NVIDIA/k8s-dra-driver-gpu/pull/307.
+func (l deviceLib) conditionallyBindMountHostDev() error {
+	// If devRoot != "/" then we don't need to do the mount
+	if l.devRoot != "/" {
+		return nil
+	}
+
+	// Get a reference to the mount executable.
+	mountExecutable, err := exec.LookPath("mount")
+	if err != nil {
+		return fmt.Errorf("error looking up mount executable: %w", err)
+	}
+	mounter := mount.New(mountExecutable)
+
+	// Bind mount hostDevContainerPath over /dev
+	if err := mounter.Mount(hostDevContainerPath, "/dev", "", []string{"bind"}); err != nil {
+		return fmt.Errorf("failed to bind mount %s over /dev: %w", hostDevContainerPath, err)
+	}
+
+	return nil
+}

deployments/helm/nvidia-dra-driver-gpu/templates/kubeletplugin.yaml

@@ -149,7 +149,7 @@ spec:
         # TODO: make this more surgical, see discussion in
         # https://github.com/NVIDIA/k8s-dra-driver-gpu/pull/307.
         - name: host-dev
-          mountPath: /dev
+          mountPath: /host/dev
       {{- end }}
       {{- if .Values.resources.gpus.enabled }}
       - name: gpus