Ensure that the fabric-imex-mgmt nvcap is created and injected always

Signed-off-by: Kevin Klues <[email protected]>

Author: klueska

cmd/compute-domain-kubelet-plugin/computedomain.go

@@ -152,7 +152,8 @@ func (s *ComputeDomainDaemonSettings) GetDomain() string {
 	return s.domain
 }
 
-func (s *ComputeDomainDaemonSettings) GetCDIContainerEdits() *cdiapi.ContainerEdits {
+func (s *ComputeDomainDaemonSettings) GetCDIContainerEdits(devRoot string, info *nvcapDeviceInfo) *cdiapi.ContainerEdits {
+
 	return &cdiapi.ContainerEdits{
 		ContainerEdits: &cdispec.ContainerEdits{
 			Mounts: []*cdispec.Mount{
@@ -162,6 +163,12 @@ func (s *ComputeDomainDaemonSettings) GetCDIContainerEdits() *cdiapi.ContainerEd
 					Options:       []string{"rw", "nosuid", "nodev", "bind"},
 				},
 			},
+			DeviceNodes: []*cdispec.DeviceNode{
+				{
+					Path:     info.path,
+					HostPath: filepath.Join(devRoot, info.path),
+				},
+			},
 		},
 	}
 }

cmd/compute-domain-kubelet-plugin/device_state.go

@@ -371,6 +371,17 @@ func (s *DeviceState) applyComputeDomainDaemonConfig(ctx context.Context, config
 		return nil, fmt.Errorf("only expected 1 device for requests '%v' in claim '%v'", requests, claim.UID)
 	}
 
+	// Parse the device node info for the fabic-imex-mgmt nvcap.
+	nvcapDeviceInfo, err := s.nvdevlib.parseNVCapDeviceInfo(nvidiaCapFabricImexMgmtPath)
+	if err != nil {
+		return nil, fmt.Errorf("error parsing nvcap device info for fabic-imex-mgmt: %w", err)
+	}
+
+	// Create the device node for the fabic-imex-mgmt nvcap.
+	if err := s.nvdevlib.createNvCapDevice(nvidiaCapFabricImexMgmtPath); err != nil {
+		return nil, fmt.Errorf("error creating nvcap device for fabic-imex-mgmt: %w", err)
+	}
+
 	// Declare a device group state object to populate.
 	var configState DeviceConfigState
 
@@ -385,7 +396,7 @@ func (s *DeviceState) applyComputeDomainDaemonConfig(ctx context.Context, config
 	// Store information about the ComputeDomain daemon in the configState.
 	configState.Type = ComputeDomainDaemonType
 	configState.ComputeDomain = config.DomainID
-	configState.containerEdits = computeDomainDaemonSettings.GetCDIContainerEdits()
+	configState.containerEdits = computeDomainDaemonSettings.GetCDIContainerEdits(s.cdi.devRoot, nvcapDeviceInfo)
 
 	return &configState, nil
 }

cmd/compute-domain-kubelet-plugin/nvlib.go

@@ -32,7 +32,9 @@ import (
 
 const (
 	procDevicesPath                  = "/proc/devices"
+	nvidiaCapsDeviceName             = "nvidia-caps"
 	nvidiaCapsImexChannelsDeviceName = "nvidia-caps-imex-channels"
+	nvidiaCapFabricImexMgmtPath      = "/proc/driver/nvidia/capabilities/fabric-imex-mgmt"
 )
 
 type deviceLib struct {
@@ -43,6 +45,14 @@ type deviceLib struct {
 	nvidiaSMIPath     string
 }
 
+type nvcapDeviceInfo struct {
+	major  int
+	minor  int
+	mode   int
+	modify int
+	path   string
+}
+
 func newDeviceLib(driverRoot root) (*deviceLib, error) {
 	driverLibraryPath, err := driverRoot.getDriverLibraryPath()
 	if err != nil {
@@ -128,7 +138,7 @@ func (l deviceLib) getImexChannelCount() (int, error) {
 	return 2048, nil
 }
 
-func (l deviceLib) getImexChannelMajor() (int, error) {
+func (l deviceLib) getDeviceMajor(name string) (int, error) {
 	file, err := os.Open(procDevicesPath)
 	if err != nil {
 		return -1, err
@@ -163,7 +173,7 @@ func (l deviceLib) getImexChannelMajor() (int, error) {
 		// If we've passed the character devices section, check for nvidiaCapsImexChannelsDeviceName
 		if foundCharDevices {
 			parts := strings.Fields(line)
-			if len(parts) == 2 && parts[1] == nvidiaCapsImexChannelsDeviceName {
+			if len(parts) == 2 && parts[1] == name {
 				return strconv.Atoi(parts[0])
 			}
 		}
@@ -172,14 +182,57 @@ func (l deviceLib) getImexChannelMajor() (int, error) {
 	return -1, scanner.Err()
 }
 
+func (l deviceLib) parseNVCapDeviceInfo(nvcapsFilePath string) (*nvcapDeviceInfo, error) {
+	file, err := os.Open(nvcapsFilePath)
+	if err != nil {
+		return nil, err
+	}
+	defer file.Close()
+
+	info := &nvcapDeviceInfo{}
+
+	major, err := l.getDeviceMajor(nvidiaCapsDeviceName)
+	if err != nil {
+		return nil, fmt.Errorf("error getting device major: %w", err)
+	}
+	info.major = major
+
+	scanner := bufio.NewScanner(file)
+	for scanner.Scan() {
+		line := scanner.Text()
+		parts := strings.SplitN(line, ":", 2)
+		if len(parts) != 2 {
+			continue
+		}
+		key := strings.TrimSpace(parts[0])
+		value := strings.TrimSpace(parts[1])
+
+		switch key {
+		case "DeviceFileMinor":
+			_, _ = fmt.Sscanf(value, "%d", &info.minor)
+		case "DeviceFileMode":
+			_, _ = fmt.Sscanf(value, "%d", &info.mode)
+		case "DeviceFileModify":
+			_, _ = fmt.Sscanf(value, "%d", &info.modify)
+		}
+	}
+	info.path = fmt.Sprintf("/dev/nvidia-caps/nvidia-cap%d", info.minor)
+
+	if err := scanner.Err(); err != nil {
+		return nil, err
+	}
+
+	return info, nil
+}
+
 func (l deviceLib) createComputeDomainChannelDevice(channel int) error {
 	// Construct the properties of the device node to create.
 	path := fmt.Sprintf("/dev/nvidia-caps-imex-channels/channel%d", channel)
 	path = filepath.Join(l.devRoot, path)
 	mode := uint32(unix.S_IFCHR | 0666)
 
 	// Get the IMEX channel major and build a /dev device from it
-	major, err := l.getImexChannelMajor()
+	major, err := l.getDeviceMajor(nvidiaCapsImexChannelsDeviceName)
 	if err != nil {
 		return fmt.Errorf("error getting IMEX channel major: %w", err)
 	}
@@ -202,3 +255,33 @@ func (l deviceLib) createComputeDomainChannelDevice(channel int) error {
 
 	return nil
 }
+
+func (l deviceLib) createNvCapDevice(nvcapFilePath string) error {
+	// Get the nvcapDeviceInfo for the nvcap file.
+	deviceInfo, err := l.parseNVCapDeviceInfo(nvcapFilePath)
+	if err != nil {
+		return fmt.Errorf("error parsing nvcap file for fabric-imex-mgmt: %w", err)
+	}
+
+	// Construct the necessary information to create the device node
+	path := filepath.Join(l.devRoot, deviceInfo.path)
+	mode := unix.S_IFCHR | uint32(deviceInfo.mode)
+	dev := unix.Mkdev(uint32(deviceInfo.major), uint32(deviceInfo.minor))
+
+	// Recursively create any parent directories of the device.
+	if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {
+		return fmt.Errorf("error creating directory for nvcaps device nodes: %w", err)
+	}
+
+	// Remove the device if it already exists.
+	if err := os.Remove(path); err != nil && !os.IsNotExist(err) {
+		return fmt.Errorf("error removing existing nvcap device node: %w", err)
+	}
+
+	// Create the device node using syscall.Mknod
+	if err := unix.Mknod(path, mode, int(dev)); err != nil {
+		return fmt.Errorf("mknod of nvcap device failed: %w", err)
+	}
+
+	return nil
+}