Add compat-libs hook to ensure that compat libs are used

Signed-off-by: Evan Lezar <[email protected]>

Author: elezar

cmd/nvidia-cdi-hook/compat-libs/compat-libs.go

@@ -0,0 +1,187 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package compatlibs
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/urfave/cli/v2"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/oci"
+)
+
+type command struct {
+	logger logger.Interface
+}
+
+type options struct {
+	driverVersion string
+	containerSpec string
+}
+
+// NewCommand constructs an compat-libs command with the specified logger
+func NewCommand(logger logger.Interface) *cli.Command {
+	c := command{
+		logger: logger,
+	}
+	return c.build()
+}
+
+// build the compat-libs command
+func (m command) build() *cli.Command {
+	cfg := options{}
+
+	// Create the 'compat-libs' command
+	c := cli.Command{
+		Name: "compat-libs",
+		Before: func(c *cli.Context) error {
+			return m.validateFlags(c, &cfg)
+		},
+		Action: func(c *cli.Context) error {
+			return m.run(c, &cfg)
+		},
+	}
+
+	c.Flags = []cli.Flag{
+		&cli.StringFlag{
+			Name:        "driver-version",
+			Usage:       "Specify the host driver version",
+			Destination: &cfg.driverVersion,
+		},
+		&cli.StringFlag{
+			Name:        "container-spec",
+			Usage:       "Specify the path to the OCI container spec. If empty or '-' the spec will be read from STDIN",
+			Destination: &cfg.containerSpec,
+		},
+	}
+
+	return &c
+}
+
+func (m command) validateFlags(c *cli.Context, cfg *options) error {
+	return nil
+}
+
+func (m command) run(c *cli.Context, cfg *options) error {
+	s, err := oci.LoadContainerState(cfg.containerSpec)
+	if err != nil {
+		return fmt.Errorf("failed to load container state: %v", err)
+	}
+
+	containerRoot, err := s.GetContainerRoot()
+	if err != nil {
+		return fmt.Errorf("failed to determined container root: %v", err)
+	}
+
+	if !root(containerRoot).hasPath("/usr/local/cuda/compat") {
+		return nil
+	}
+
+	if !root(containerRoot).hasPath("/etc/ld.so.cache") {
+		// If there is no ldcache in the container, the hook is a no-op
+		return nil
+	}
+	if !root(containerRoot).hasPath("/etc/ld.so.conf.d") {
+		return nil
+	}
+
+	libs, err := root(containerRoot).glob("/usr/local/cuda/compat/libcuda.so.*.*")
+	if err != nil {
+		m.logger.Warningf("Failed to find CUDA compat library: %v", err)
+		return nil
+	}
+
+	if len(libs) == 0 {
+		return nil
+	}
+
+	if len(libs) != 1 {
+		m.logger.Warningf("Unexpected number of CUDA compat libraries: %v", libs)
+		return nil
+	}
+
+	compatVersion := strings.TrimPrefix(filepath.Base(libs[0]), "libcuda.so.")
+	compatMajor := strings.SplitN(compatVersion, ".", 2)[0]
+	driverMajor := strings.SplitN(cfg.driverVersion, ".", 2)[0]
+
+	if driverMajor > compatMajor {
+		return nil
+	}
+
+	return m.createConfig(containerRoot, []string{"/usr/local/cuda/compat"})
+}
+
+type root string
+
+func (r root) hasPath(path string) bool {
+	_, err := os.Stat(filepath.Join(string(r), path))
+	if err != nil && os.IsNotExist(err) {
+		return false
+	}
+	return true
+}
+
+func (r root) glob(pattern string) ([]string, error) {
+	return filepath.Glob(filepath.Join(string(r), pattern))
+}
+
+// createConfig creates (or updates) /etc/ld.so.conf.d/00-compat-<RANDOM_STRING>.conf in the container
+// to include the required paths.
+// Note that the 00-nvcr prefix is chosen to ensure that these libraries have
+// a higher precedence than other libraries on the system but are applied AFTER
+// 00-cuda-compat.conf.
+func (m command) createConfig(root string, folders []string) error {
+	if len(folders) == 0 {
+		m.logger.Debugf("No folders to add to /etc/ld.so.conf")
+		return nil
+	}
+
+	if err := os.MkdirAll(filepath.Join(root, "/etc/ld.so.conf.d"), 0755); err != nil {
+		return fmt.Errorf("failed to create ld.so.conf.d: %v", err)
+	}
+
+	configFile, err := os.CreateTemp(filepath.Join(root, "/etc/ld.so.conf.d"), "00-compat-*.conf")
+	if err != nil {
+		return fmt.Errorf("failed to create config file: %v", err)
+	}
+	defer configFile.Close()
+
+	m.logger.Debugf("Adding folders %v to %v", folders, configFile.Name())
+
+	configured := make(map[string]bool)
+	for _, folder := range folders {
+		if configured[folder] {
+			continue
+		}
+		_, err = configFile.WriteString(fmt.Sprintf("%s\n", folder))
+		if err != nil {
+			return fmt.Errorf("failed to update ld.so.conf.d: %v", err)
+		}
+		configured[folder] = true
+	}
+
+	// The created file needs to be world readable for the cases where the container is run as a non-root user.
+	if err := os.Chmod(configFile.Name(), 0644); err != nil {
+		return fmt.Errorf("failed to chmod config file: %v", err)
+	}
+
+	return nil
+}