Unify GetDevices logic at internal/config/image

Signed-off-by: Carlos Eduardo Arango Gutierrez <[email protected]>
Signed-off-by: Evan Lezar <[email protected]>

Author: ArangoGutierrez

cmd/nvidia-container-runtime-hook/container_config.go

@@ -13,10 +13,6 @@ import (
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/config/image"
 )
 
-const (
-	capSysAdmin = "CAP_SYS_ADMIN"
-)
-
 type nvidiaConfig struct {
 	Devices            []string
 	MigConfigDevices   string
@@ -103,9 +99,9 @@ func loadSpec(path string) (spec *Spec) {
 	return
 }
 
-func isPrivileged(s *Spec) bool {
-	if s.Process.Capabilities == nil {
-		return false
+func (s *Spec) GetCapabilities() []string {
+	if s == nil || s.Process == nil || s.Process.Capabilities == nil {
+		return nil
 	}
 
 	var caps []string
@@ -118,67 +114,22 @@ func isPrivileged(s *Spec) bool {
 		if err != nil {
 			log.Panicln("could not decode Process.Capabilities in OCI spec:", err)
 		}
-		for _, c := range caps {
-			if c == capSysAdmin {
-				return true
-			}
-		}
-		return false
+		return caps
 	}
 
 	// Otherwise, parse s.Process.Capabilities as:
 	// github.com/opencontainers/runtime-spec/blob/v1.0.0/specs-go/config.go#L30-L54
-	process := specs.Process{
-		Env: s.Process.Env,
-	}
-
-	err := json.Unmarshal(*s.Process.Capabilities, &process.Capabilities)
+	capabilities := specs.LinuxCapabilities{}
+	err := json.Unmarshal(*s.Process.Capabilities, &capabilities)
 	if err != nil {
 		log.Panicln("could not decode Process.Capabilities in OCI spec:", err)
 	}
 
-	fullSpec := specs.Spec{
-		Version: *s.Version,
-		Process: &process,
-	}
-
-	return image.IsPrivileged(&fullSpec)
-}
-
-func getDevicesFromEnvvar(containerImage image.CUDA, swarmResourceEnvvars []string) []string {
-	// We check if the image has at least one of the Swarm resource envvars defined and use this
-	// if specified.
-	for _, envvar := range swarmResourceEnvvars {
-		if containerImage.HasEnvvar(envvar) {
-			return containerImage.DevicesFromEnvvars(swarmResourceEnvvars...).List()
-		}
-	}
-
-	return containerImage.VisibleDevicesFromEnvVar()
+	return image.OCISpecCapabilities(capabilities).GetCapabilities()
 }
 
-func (hookConfig *hookConfig) getDevices(image image.CUDA, privileged bool) []string {
-	// If enabled, try and get the device list from volume mounts first
-	if hookConfig.AcceptDeviceListAsVolumeMounts {
-		devices := image.VisibleDevicesFromMounts()
-		if len(devices) > 0 {
-			return devices
-		}
-	}
-
-	// Fallback to reading from the environment variable if privileges are correct
-	devices := getDevicesFromEnvvar(image, hookConfig.getSwarmResourceEnvvars())
-	if len(devices) == 0 {
-		return nil
-	}
-	if privileged || hookConfig.AcceptEnvvarUnprivileged {
-		return devices
-	}
-
-	configName := hookConfig.getConfigOption("AcceptEnvvarUnprivileged")
-	log.Printf("Ignoring devices specified in NVIDIA_VISIBLE_DEVICES (privileged=%v, %v=%v) ", privileged, configName, hookConfig.AcceptEnvvarUnprivileged)
-
-	return nil
+func isPrivileged(s *Spec) bool {
+	return image.IsPrivileged(s)
 }
 
 func getMigConfigDevices(i image.CUDA) *string {
@@ -225,7 +176,6 @@ func (hookConfig *hookConfig) getDriverCapabilities(cudaImage image.CUDA, legacy
 	// We use the default driver capabilities by default. This is filtered to only include the
 	// supported capabilities
 	supportedDriverCapabilities := image.NewDriverCapabilities(hookConfig.SupportedDriverCapabilities)
-
 	capabilities := supportedDriverCapabilities.Intersection(image.DefaultDriverCapabilities)
 
 	capsEnvSpecified := cudaImage.HasEnvvar(image.EnvVarNvidiaDriverCapabilities)
@@ -251,7 +201,7 @@ func (hookConfig *hookConfig) getDriverCapabilities(cudaImage image.CUDA, legacy
 func (hookConfig *hookConfig) getNvidiaConfig(image image.CUDA, privileged bool) *nvidiaConfig {
 	legacyImage := image.IsLegacy()
 
-	devices := hookConfig.getDevices(image, privileged)
+	devices := image.VisibleDevices()
 	if len(devices) == 0 {
 		// empty devices means this is not a GPU container.
 		return nil
@@ -306,20 +256,30 @@ func (hookConfig *hookConfig) getContainerConfig() (config containerConfig) {
 
 	s := loadSpec(path.Join(b, "config.json"))
 
-	image, err := image.New(
+	privileged := isPrivileged(s)
+
+	opts := []image.Option{
 		image.WithEnv(s.Process.Env),
 		image.WithMounts(s.Mounts),
+		image.WithPrivileged(privileged),
 		image.WithDisableRequire(hookConfig.DisableRequire),
-	)
+		image.WithAcceptDeviceListAsVolumeMounts(hookConfig.AcceptDeviceListAsVolumeMounts),
+		image.WithAcceptEnvvarUnprivileged(hookConfig.AcceptEnvvarUnprivileged),
+	}
+
+	if len(hookConfig.getSwarmResourceEnvvars()) > 0 {
+		opts = append(opts, image.WithVisibleDevicesEnvVars(hookConfig.getSwarmResourceEnvvars()...))
+	}
+
+	i, err := image.New(opts...)
 	if err != nil {
 		log.Panicln(err)
 	}
 
-	privileged := isPrivileged(s)
 	return containerConfig{
 		Pid:    h.Pid,
 		Rootfs: s.Root.Path,
-		Image:  image,
-		Nvidia: hookConfig.getNvidiaConfig(image, privileged),
+		Image:  i,
+		Nvidia: hookConfig.getNvidiaConfig(i, privileged),
 	}
 }

cmd/nvidia-container-runtime-hook/container_config_test.go

@@ -477,9 +477,19 @@ func TestGetNvidiaConfig(t *testing.T) {
 	}
 	for _, tc := range tests {
 		t.Run(tc.description, func(t *testing.T) {
-			image, _ := image.New(
+			opts := []image.Option{
 				image.WithEnvMap(tc.env),
-			)
+				image.WithPrivileged(tc.privileged),
+				image.WithAcceptEnvvarUnprivileged(true),
+			}
+
+			if tc.hookConfig != nil {
+				if tc.hookConfig.SwarmResource != "" {
+					opts = append(opts, image.WithVisibleDevicesEnvVars(tc.hookConfig.SwarmResource))
+				}
+			}
+			image, _ := image.New(opts...)
+
 			// Wrap the call to getNvidiaConfig() in a closure.
 			var cfg *nvidiaConfig
 			getConfig := func() {
@@ -622,12 +632,15 @@ func TestDeviceListSourcePriority(t *testing.T) {
 						},
 					),
 					image.WithMounts(tc.mountDevices),
+					image.WithPrivileged(tc.privileged),
+					image.WithAcceptDeviceListAsVolumeMounts(tc.acceptMounts),
+					image.WithAcceptEnvvarUnprivileged(tc.acceptUnprivileged),
 				)
 				defaultConfig, _ := config.GetDefault()
 				cfg := &hookConfig{defaultConfig}
 				cfg.AcceptEnvvarUnprivileged = tc.acceptUnprivileged
 				cfg.AcceptDeviceListAsVolumeMounts = tc.acceptMounts
-				devices = cfg.getDevices(image, tc.privileged)
+				devices = image.VisibleDevices()
 			}
 
 			// For all other tests, just grab the devices and check the results
@@ -843,10 +856,20 @@ func TestGetDevicesFromEnvvar(t *testing.T) {
 
 	for _, tc := range tests {
 		t.Run(tc.description, func(t *testing.T) {
-			image, _ := image.New(
+			opts := []image.Option{
 				image.WithEnvMap(tc.env),
-			)
-			devices := getDevicesFromEnvvar(image, tc.swarmResourceEnvvars)
+				image.WithPrivileged(true),
+				image.WithAcceptDeviceListAsVolumeMounts(false),
+				image.WithAcceptEnvvarUnprivileged(false),
+			}
+
+			if len(tc.swarmResourceEnvvars) > 0 {
+				opts = append(opts, image.WithVisibleDevicesEnvVars(tc.swarmResourceEnvvars...))
+			}
+
+			image, _ := image.New(opts...)
+
+			devices := image.VisibleDevices()
 			require.EqualValues(t, tc.expectedDevices, devices)
 		})
 	}

cmd/nvidia-container-runtime-hook/hook_test.go

@@ -24,11 +24,14 @@ import (
 )
 
 type builder struct {
-	env            map[string]string
-	mounts         []specs.Mount
+	CUDA
+
 	disableRequire bool
 }
 
+// Option is a functional option for creating a CUDA image.
+type Option func(*builder) error
+
 // New creates a new CUDA image from the input options.
 func New(opt ...Option) (CUDA, error) {
 	b := &builder{}
@@ -50,15 +53,36 @@ func (b builder) build() (CUDA, error) {
 		b.env[EnvVarNvidiaDisableRequire] = "true"
 	}
 
-	c := CUDA{
-		env:    b.env,
-		mounts: b.mounts,
+	return b.CUDA, nil
+}
+
+func WithAnnotationPrefixes(annotationPrefixes []string) Option {
+	return func(b *builder) error {
+		b.annotationPrefixes = annotationPrefixes
+		return nil
 	}
-	return c, nil
 }
 
-// Option is a functional option for creating a CUDA image.
-type Option func(*builder) error
+func WithAnnotations(annotations map[string]string) Option {
+	return func(b *builder) error {
+		b.annotations = annotations
+		return nil
+	}
+}
+
+func WithAcceptDeviceListAsVolumeMounts(acceptDeviceListAsVolumeMounts bool) Option {
+	return func(b *builder) error {
+		b.acceptDeviceListAsVolumeMounts = acceptDeviceListAsVolumeMounts
+		return nil
+	}
+}
+
+func WithAcceptEnvvarUnprivileged(acceptEnvvarUnprivileged bool) Option {
+	return func(b *builder) error {
+		b.acceptEnvvarUnprivileged = acceptEnvvarUnprivileged
+		return nil
+	}
+}
 
 // WithDisableRequire sets the disable require option.
 func WithDisableRequire(disableRequire bool) Option {
@@ -100,3 +124,35 @@ func WithMounts(mounts []specs.Mount) Option {
 		return nil
 	}
 }
+
+// WithPrivileged sets whether an image is privileged or not.
+func WithPrivileged(isPrivileged bool) Option {
+	return func(b *builder) error {
+		b.isPrivileged = isPrivileged
+		return nil
+	}
+}
+
+// WithVisibleDevicesEnvVars sets the swarm resource for the CUDA image.
+func WithVisibleDevicesEnvVars(visibleDevicesEnvVars ...string) Option {
+	return func(b *builder) error {
+		if len(visibleDevicesEnvVars) == 0 {
+			return fmt.Errorf("visible devices env vars cannot be empty")
+		}
+		b.visibleDevicesEnvVars = []string{}
+		// if resource is a single string, split it by comma
+		if len(visibleDevicesEnvVars) == 1 && strings.Contains(visibleDevicesEnvVars[0], ",") {
+			candidates := strings.Split(visibleDevicesEnvVars[0], ",")
+			for _, c := range candidates {
+				trimmed := strings.TrimSpace(c)
+				if len(trimmed) > 0 {
+					b.visibleDevicesEnvVars = append(b.visibleDevicesEnvVars, trimmed)
+				}
+			}
+			return nil
+		}
+
+		b.visibleDevicesEnvVars = append(b.visibleDevicesEnvVars, visibleDevicesEnvVars...)
+		return nil
+	}
+}