Merge branch 'fix-load-kernel-modules' into 'main'

Split internal system package

See merge request nvidia/container-toolkit/container-toolkit!420

Author: Evan Lezar

CHANGELOG.md

@@ -6,6 +6,7 @@
 * Fix bug causing incorrect nvidia-smi symlink to be created on WSL2 systems with multiple driver roots.
 * Fix bug when using driver versions that do not include a patch component in their version number.
 * Skip additional modifications in CDI mode.
+* Fix loading of kernel modules and creation of device nodes in containerized use cases.
 
 * [toolkit-container] Allow same envars for all runtime configs
 

cmd/nvidia-ctk/system/create-dev-char-symlinks/all.go

@@ -28,14 +28,14 @@ import (
 
 type allPossible struct {
 	logger       *logrus.Logger
-	driverRoot   string
+	devRoot      string
 	deviceMajors devices.Devices
 	migCaps      nvcaps.MigCaps
 }
 
 // newAllPossible returns a new allPossible device node lister.
 // This lister lists all possible device nodes for NVIDIA GPUs, control devices, and capability devices.
-func newAllPossible(logger *logrus.Logger, driverRoot string) (nodeLister, error) {
+func newAllPossible(logger *logrus.Logger, devRoot string) (nodeLister, error) {
 	deviceMajors, err := devices.GetNVIDIADevices()
 	if err != nil {
 		return nil, fmt.Errorf("failed reading device majors: %v", err)
@@ -61,7 +61,7 @@ func newAllPossible(logger *logrus.Logger, driverRoot string) (nodeLister, error
 
 	l := allPossible{
 		logger:       logger,
-		driverRoot:   driverRoot,
+		devRoot:      devRoot,
 		deviceMajors: deviceMajors,
 		migCaps:      migCaps,
 	}
@@ -72,15 +72,15 @@ func newAllPossible(logger *logrus.Logger, driverRoot string) (nodeLister, error
 // DeviceNodes returns a list of all possible device nodes for NVIDIA GPUs, control devices, and capability devices.
 func (m allPossible) DeviceNodes() ([]deviceNode, error) {
 	gpus, err := nvpci.NewFrom(
-		filepath.Join(m.driverRoot, nvpci.PCIDevicesRoot),
+		filepath.Join(m.devRoot, nvpci.PCIDevicesRoot),
 	).GetGPUs()
 	if err != nil {
 		return nil, fmt.Errorf("failed to get GPU information: %v", err)
 	}
 
 	count := len(gpus)
 	if count == 0 {
-		m.logger.Infof("No NVIDIA devices found in %s", m.driverRoot)
+		m.logger.Infof("No NVIDIA devices found in %s", m.devRoot)
 		return nil, nil
 	}
 
@@ -179,7 +179,7 @@ func (m allPossible) newDeviceNode(deviceName devices.Name, path string, minor i
 	major, _ := m.deviceMajors.Get(deviceName)
 
 	return deviceNode{
-		path:  filepath.Join(m.driverRoot, path),
+		path:  filepath.Join(m.devRoot, path),
 		major: uint32(major),
 		minor: uint32(minor),
 	}

cmd/nvidia-ctk/system/create-dev-char-symlinks/create-dev-char-symlinks.go

@@ -24,7 +24,8 @@ import (
 	"strings"
 	"syscall"
 
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/system"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/system/nvdevices"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/system/nvmodules"
 	"github.com/fsnotify/fsnotify"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli/v2"
@@ -216,6 +217,7 @@ type linkCreator struct {
 	logger            *logrus.Logger
 	lister            nodeLister
 	driverRoot        string
+	devRoot           string
 	devCharPath       string
 	dryRun            bool
 	createAll         bool
@@ -243,6 +245,9 @@ func NewSymlinkCreator(opts ...Option) (Creator, error) {
 	if c.driverRoot == "" {
 		c.driverRoot = "/"
 	}
+	if c.devRoot == "" {
+		c.devRoot = "/"
+	}
 	if c.devCharPath == "" {
 		c.devCharPath = defaultDevCharPath
 	}
@@ -252,13 +257,13 @@ func NewSymlinkCreator(opts ...Option) (Creator, error) {
 	}
 
 	if c.createAll {
-		lister, err := newAllPossible(c.logger, c.driverRoot)
+		lister, err := newAllPossible(c.logger, c.devRoot)
 		if err != nil {
 			return nil, fmt.Errorf("failed to create all possible device lister: %v", err)
 		}
 		c.lister = lister
 	} else {
-		c.lister = existing{c.logger, c.driverRoot}
+		c.lister = existing{c.logger, c.devRoot}
 	}
 	return c, nil
 }
@@ -268,36 +273,48 @@ func (m linkCreator) setup() error {
 		return nil
 	}
 
-	s, err := system.New(
-		system.WithLogger(m.logger),
-		system.WithDryRun(m.dryRun),
-	)
-	if err != nil {
-		return err
-	}
-
 	if m.loadKernelModules {
-		if err := s.LoadNVIDIAKernelModules(); err != nil {
+		modules := nvmodules.New(
+			nvmodules.WithLogger(m.logger),
+			nvmodules.WithDryRun(m.dryRun),
+			nvmodules.WithRoot(m.driverRoot),
+		)
+		if err := modules.LoadAll(); err != nil {
 			return fmt.Errorf("failed to load NVIDIA kernel modules: %v", err)
 		}
 	}
 
 	if m.createDeviceNodes {
-		if err := s.CreateNVIDIAControlDeviceNodesAt(m.driverRoot); err != nil {
+		devices, err := nvdevices.New(
+			nvdevices.WithLogger(m.logger),
+			nvdevices.WithDryRun(m.dryRun),
+			nvdevices.WithDevRoot(m.devRoot),
+		)
+		if err != nil {
+			return err
+		}
+		if err := devices.CreateNVIDIAControlDevices(); err != nil {
 			return fmt.Errorf("failed to create NVIDIA device nodes: %v", err)
 		}
 	}
-
 	return nil
 }
 
 // WithDriverRoot sets the driver root path.
+// This is the path in which kernel modules must be loaded.
 func WithDriverRoot(root string) Option {
 	return func(c *linkCreator) {
 		c.driverRoot = root
 	}
 }
 
+// WithDevRoot sets the root path for the /dev directory.
+func WithDevRoot(root string) Option {
+	return func(c *linkCreator) {
+		c.devRoot = root
+	}
+}
+
 // WithDevCharPath sets the path at which the symlinks will be created.
 func WithDevCharPath(path string) Option {
 	return func(c *linkCreator) {

cmd/nvidia-ctk/system/create-dev-char-symlinks/existing.go

@@ -30,16 +30,16 @@ type nodeLister interface {
 }
 
 type existing struct {
-	logger     *logrus.Logger
-	driverRoot string
+	logger  *logrus.Logger
+	devRoot string
 }
 
 // DeviceNodes returns a list of NVIDIA device nodes in the specified root.
 // The nvidia-nvswitch* and nvidia-nvlink devices are excluded.
 func (m existing) DeviceNodes() ([]deviceNode, error) {
 	locator := lookup.NewCharDeviceLocator(
 		lookup.WithLogger(m.logger),
-		lookup.WithRoot(m.driverRoot),
+		lookup.WithRoot(m.devRoot),
 		lookup.WithOptional(true),
 	)
 
@@ -54,7 +54,7 @@ func (m existing) DeviceNodes() ([]deviceNode, error) {
 	}
 
 	if len(devices) == 0 && len(capDevices) == 0 {
-		m.logger.Infof("No NVIDIA devices found in %s", m.driverRoot)
+		m.logger.Infof("No NVIDIA devices found in %s", m.devRoot)
 		return nil, nil
 	}
 

cmd/nvidia-ctk/system/create-device-nodes/create-device-nodes.go

@@ -19,7 +19,8 @@ package createdevicenodes
 import (
 	"fmt"
 
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/system"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/system/nvdevices"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/system/nvmodules"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli/v2"
 )
@@ -96,19 +97,29 @@ func (m command) validateFlags(r *cli.Context, opts *options) error {
 }
 
 func (m command) run(c *cli.Context, opts *options) error {
-	s, err := system.New(
-		system.WithLogger(m.logger),
-		system.WithDryRun(opts.dryRun),
-		system.WithLoadKernelModules(opts.loadKernelModules),
-	)
-	if err != nil {
-		return fmt.Errorf("failed to create library: %v", err)
+	if opts.loadKernelModules {
+		modules := nvmodules.New(
+			nvmodules.WithLogger(m.logger),
+			nvmodules.WithDryRun(opts.dryRun),
+			nvmodules.WithRoot(opts.driverRoot),
+		)
+		if err := modules.LoadAll(); err != nil {
+			return fmt.Errorf("failed to load NVIDIA kernel modules: %v", err)
+		}
 	}
 
 	if opts.control {
+		devices, err := nvdevices.New(
+			nvdevices.WithLogger(m.logger),
+			nvdevices.WithDryRun(opts.dryRun),
+			nvdevices.WithDevRoot(opts.driverRoot),
+		)
+		if err != nil {
+			return err
+		}
 		m.logger.Infof("Creating control device nodes at %s", opts.driverRoot)
-		if err := s.CreateNVIDIAControlDeviceNodesAt(opts.driverRoot); err != nil {
-			return fmt.Errorf("failed to create control device nodes: %v", err)
+		if err := devices.CreateNVIDIAControlDevices(); err != nil {
+			return fmt.Errorf("failed to create NVIDIA control device nodes: %v", err)
 		}
 	}
 	return nil

internal/system/nvdevices/devices.go

@@ -0,0 +1,154 @@
+/**
+# Copyright (c) NVIDIA CORPORATIOm.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package nvdevices
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/info/proc/devices"
+	"github.com/sirupsen/logrus"
+)
+
+var errInvalidDeviceNode = errors.New("invalid device node")
+
+// Interface provides a set of utilities for interacting with NVIDIA devices on the system.
+type Interface struct {
+	devices.Devices
+
+	logger *logrus.Logger
+
+	dryRun bool
+	// devRoot is the root directory where device nodes are expected to exist.
+	devRoot string
+
+	mknoder
+}
+
+// New constructs a new Interface struct with the specified options.
+func New(opts ...Option) (*Interface, error) {
+	i := &Interface{}
+	for _, opt := range opts {
+		opt(i)
+	}
+
+	if i.logger == nil {
+		i.logger = logrus.StandardLogger()
+	}
+	if i.devRoot == "" {
+		i.devRoot = "/"
+	}
+	if i.Devices == nil {
+		devices, err := devices.GetNVIDIADevices()
+		if err != nil {
+			return nil, fmt.Errorf("failed to create devices info: %v", err)
+		}
+		i.Devices = devices
+	}
+
+	if i.dryRun {
+		i.mknoder = &mknodLogger{i.logger}
+	} else {
+		i.mknoder = &mknodUnix{}
+	}
+	return i, nil
+}
+
+// CreateNVIDIAControlDevices creates the NVIDIA control device nodes at the configured devRoot.
+func (m *Interface) CreateNVIDIAControlDevices() error {
+	controlNodes := []string{"nvidiactl", "nvidia-modeset", "nvidia-uvm", "nvidia-uvm-tools"}
+	for _, node := range controlNodes {
+		err := m.CreateNVIDIADevice(node)
+		if err != nil {
+			return fmt.Errorf("failed to create device node %s: %w", node, err)
+		}
+	}
+	return nil
+}
+
+// CreateNVIDIADevice creates the specified NVIDIA device node at the configured devRoot.
+func (m *Interface) CreateNVIDIADevice(node string) error {
+	node = filepath.Base(node)
+	if !strings.HasPrefix(node, "nvidia") {
+		return fmt.Errorf("invalid device node %q: %w", node, errInvalidDeviceNode)
+	}
+
+	major, err := m.Major(node)
+	if err != nil {
+		return fmt.Errorf("failed to determine major: %w", err)
+	}
+
+	minor, err := m.Minor(node)
+	if err != nil {
+		return fmt.Errorf("failed to determine minor: %w", err)
+	}
+
+	return m.createDeviceNode(filepath.Join("dev", node), int(major), int(minor))
+}
+
+// createDeviceNode creates the specified device node with the require major and minor numbers.
+// If a devRoot is configured, this is prepended to the path.
+func (m *Interface) createDeviceNode(path string, major int, minor int) error {
+	path = filepath.Join(m.devRoot, path)
+	if _, err := os.Stat(path); err == nil {
+		m.logger.Infof("Skipping: %s already exists", path)
+		return nil
+	} else if !os.IsNotExist(err) {
+		return fmt.Errorf("failed to stat %s: %v", path, err)
+	}
+
+	return m.Mknode(path, major, minor)
+}
+
+// Major returns the major number for the specified NVIDIA device node.
+// If the device node is not supported, an error is returned.
+func (m *Interface) Major(node string) (int64, error) {
+	var valid bool
+	var major devices.Major
+	switch node {
+	case "nvidia-uvm", "nvidia-uvm-tools":
+		major, valid = m.Get(devices.NVIDIAUVM)
+	case "nvidia-modeset", "nvidiactl":
+		major, valid = m.Get(devices.NVIDIAGPU)
+	}
+
+	if valid {
+		return int64(major), nil
+	}
+
+	return 0, errInvalidDeviceNode
+}
+
+// Minor returns the minor number for the specified NVIDIA device node.
+// If the device node is not supported, an error is returned.
+func (m *Interface) Minor(node string) (int64, error) {
+	switch node {
+	case "nvidia-modeset":
+		return devices.NVIDIAModesetMinor, nil
+	case "nvidia-uvm-tools":
+		return devices.NVIDIAUVMToolsMinor, nil
+	case "nvidia-uvm":
+		return devices.NVIDIAUVMMinor, nil
+	case "nvidiactl":
+		return devices.NVIDIACTLMinor, nil
+	}
+
+	return 0, errInvalidDeviceNode
+}