Create .so and SONAME symlinks for driver libraries

This change ensures that .so and SONAME symlinks are created for
driver libraries in the container.

Signed-off-by: Evan Lezar <[email protected]>

Author: elezar

CHANGELOG.md

@@ -2,6 +2,7 @@
 
 ## v1.18.0-rc.2
 
+- Ensure that .so symlinks are created for driver libraries in the container
 - Load settings from config.toml file during CDI generation
 - Use securejoin to resolve /proc
 - Refactor nvml CDI spec generation for consistency

internal/discover/symlinks.go

@@ -17,10 +17,21 @@
 package discover
 
 import (
+	"debug/elf"
 	"fmt"
 	"path/filepath"
+	"strings"
 )
 
+type Symlink struct {
+	target string
+	link   string
+}
+
+func (s *Symlink) String() string {
+	return fmt.Sprintf("%s::%s", s.target, s.link)
+}
+
 type additionalSymlinks struct {
 	Discover
 	version     string
@@ -60,7 +71,14 @@ func (d *additionalSymlinks) Hooks() ([]Hook, error) {
 		}
 		processedPaths[mount.Path] = true
 
-		for _, link := range d.getLinksForMount(mount.Path) {
+		linksForMount := d.getLinksForMount(mount.Path)
+		soSymlinks, err := d.getDotSoSymlinks(mount.HostPath)
+		if err != nil {
+			soSymlinks = nil
+		}
+		linksForMount = append(linksForMount, soSymlinks...)
+
+		for _, link := range linksForMount {
 			if processedLinks[link] {
 				continue
 			}
@@ -110,3 +128,73 @@ func (d additionalSymlinks) isDriverLibrary(libraryName string, filename string)
 	match, _ := filepath.Match(pattern, filename)
 	return match
 }
+
+func (d *additionalSymlinks) getDotSoSymlinks(libraryPath string) ([]string, error) {
+	libraryDir, libraryName := filepath.Split(libraryPath)
+	if !d.isDriverLibrary("*", libraryName) {
+		return nil, nil
+	}
+
+	soname, err := getSoname(libraryPath)
+	if err != nil {
+		return nil, err
+	}
+	if soname == "" || soname == libraryName {
+		return nil, nil
+	}
+
+	var soSymlinks []string
+	// Create the SONAME -> libraryName symlink.
+	sonameLinkPath := filepath.Join(libraryDir, soname)
+	s := Symlink{
+		target: libraryName,
+		link:   sonameLinkPath,
+	}
+	soSymlinks = append(soSymlinks, s.String())
+
+	// Create the .so -> SONAME symlink
+	soLink := getSoLink(soname)
+	if soLink != "" && soLink != soname && soLink != libraryName {
+		s := Symlink{
+			target: filepath.Base(soname),
+			link:   filepath.Join(libraryDir, soLink),
+		}
+		soSymlinks = append(soSymlinks, s.String())
+	}
+	return soSymlinks, nil
+}
+
+// getSoname returns the soname for the specified library path.
+// We use a function variable here to allow this to be overridden for testing.
+var getSoname = func(libraryPath string) (string, error) {
+	lib, err := elf.Open(libraryPath)
+	if err != nil {
+		return "", err
+	}
+	defer lib.Close()
+
+	sonames, err := lib.DynString(elf.DT_SONAME)
+	if err != nil {
+		return "", err
+	}
+	if len(sonames) == 0 {
+		return "", nil
+	}
+	if len(sonames) != 1 {
+		return "", fmt.Errorf("multiple SONAMEs detected for %v: %v", libraryPath, sonames)
+	}
+	return sonames[0], nil
+}
+
+// getSoLink returns the filename for the .so symlink that should point to the
+// soname symlink for the specified library.
+func getSoLink(soname string) string {
+	ext := filepath.Ext(soname)
+	if ext == "" {
+		return ""
+	}
+	if ext == ".so" {
+		return soname
+	}
+	return getSoLink(strings.TrimSuffix(soname, ext))
+}

internal/discover/symlinks_test.go

@@ -334,3 +334,118 @@ func TestWithWithDriverDotSoSymlinks(t *testing.T) {
 		})
 	}
 }
+
+func TestGetDotSoSymlinks(t *testing.T) {
+	testCases := []struct {
+		description      string
+		libraryPath      string
+		getSonameFunc    func(string) (string, error)
+		expectedError    error
+		expectedSymlinks []string
+	}{
+		{
+			description: "libcuda.soname links",
+			libraryPath: "/usr/lib/libcuda.so.999.88.77",
+			getSonameFunc: func(s string) (string, error) {
+				return "libcuda.so.1", nil
+			},
+			expectedError: nil,
+			expectedSymlinks: []string{
+				"libcuda.so.999.88.77::/usr/lib/libcuda.so.1",
+				"libcuda.so.1::/usr/lib/libcuda.so",
+			},
+		},
+		{
+			description: "empty soname is ignored",
+			libraryPath: "/usr/lib/libcuda.so.999.88.77",
+			getSonameFunc: func(s string) (string, error) {
+				return "", nil
+			},
+			expectedError:    nil,
+			expectedSymlinks: nil,
+		},
+		{
+			description: "equal soname is ignored",
+			libraryPath: "/usr/lib/libcuda.so.999.88.77",
+			getSonameFunc: func(s string) (string, error) {
+				return "libcuda.so.999.88.77", nil
+			},
+			expectedError:    nil,
+			expectedSymlinks: nil,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.description, func(t *testing.T) {
+			defer setGetSoname(tc.getSonameFunc)()
+
+			sut := &additionalSymlinks{version: "*.*"}
+			symlinks, err := sut.getDotSoSymlinks(tc.libraryPath)
+
+			if tc.expectedError == nil {
+				require.NoError(t, err)
+			} else {
+				require.EqualError(t, err, tc.expectedError.Error())
+			}
+
+			require.EqualValues(t, tc.expectedSymlinks, symlinks)
+		})
+	}
+}
+
+func TestGetSoLink(t *testing.T) {
+	testCases := []struct {
+		description    string
+		input          string
+		expectedSoLink string
+	}{
+		{
+			description:    "empty string",
+			input:          "",
+			expectedSoLink: "",
+		},
+		{
+			description:    "cuda driver library",
+			input:          "libcuda.so.999.88.77",
+			expectedSoLink: "libcuda.so",
+		},
+		{
+			description:    "beta cuda driver library",
+			input:          "libcuda.so.999.88",
+			expectedSoLink: "libcuda.so",
+		},
+		{
+			description:    "no .so in libname",
+			input:          "foo.bar.baz",
+			expectedSoLink: "",
+		},
+		{
+			description:    "multiple .so in libname",
+			input:          "foo.so.so.566",
+			expectedSoLink: "foo.so.so",
+		},
+		{
+			description:    "no suffix after so",
+			input:          "foo.so",
+			expectedSoLink: "foo.so",
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.description, func(t *testing.T) {
+
+			soLink := getSoLink(tc.input)
+
+			require.Equal(t, tc.expectedSoLink, soLink)
+		})
+	}
+}
+
+func setGetSoname(override func(string) (string, error)) func() {
+	original := getSoname
+	getSoname = override
+
+	return func() {
+		getSoname = original
+	}
+}

tests/e2e/nvidia-container-toolkit_test.go

@@ -29,6 +29,8 @@ import (
 // Integration tests for Docker runtime
 var _ = Describe("docker", Ordered, ContinueOnFailure, func() {
 	var runner Runner
+	var hostDriverVersion string
+	var hostDriverMajor string
 
 	// Install the NVIDIA Container Toolkit
 	BeforeAll(func(ctx context.Context) {
@@ -50,6 +52,15 @@ var _ = Describe("docker", Ordered, ContinueOnFailure, func() {
 			err = installer.Install()
 			Expect(err).ToNot(HaveOccurred())
 		}
+
+		driverOutput, _, err := runner.Run("nvidia-smi -q | grep \"Driver Version\"")
+		Expect(err).ToNot(HaveOccurred())
+		parts := strings.SplitN(driverOutput, ":", 2)
+		Expect(parts).To(HaveLen(2))
+
+		hostDriverVersion = strings.TrimSpace(parts[1])
+		Expect(hostDriverVersion).ToNot(BeEmpty())
+		hostDriverMajor = strings.SplitN(hostDriverVersion, ".", 2)[0]
 	})
 
 	// GPUs are accessible in a container: Running nvidia-smi -L inside the
@@ -184,16 +195,7 @@ var _ = Describe("docker", Ordered, ContinueOnFailure, func() {
 			compatDriverVersion := strings.TrimPrefix(filepath.Base(compatOutput), "libcuda.so.")
 			compatMajor := strings.SplitN(compatDriverVersion, ".", 2)[0]
 
-			driverOutput, _, err := runner.Run("nvidia-smi -q | grep \"Driver Version\"")
-			Expect(err).ToNot(HaveOccurred())
-			parts := strings.SplitN(driverOutput, ":", 2)
-			Expect(parts).To(HaveLen(2))
-
-			hostDriverVersion := strings.TrimSpace(parts[1])
-			Expect(hostDriverVersion).ToNot(BeEmpty())
-			driverMajor := strings.SplitN(hostDriverVersion, ".", 2)[0]
-
-			if driverMajor >= compatMajor {
+			if hostDriverMajor >= compatMajor {
 				GinkgoLogr.Info("CUDA Forward Compatibility tests require an older driver version", "hostDriverVersion", hostDriverVersion, "compatDriverVersion", compatDriverVersion)
 				Skip("CUDA Forward Compatibility tests require an older driver version")
 			}
@@ -241,6 +243,8 @@ var _ = Describe("docker", Ordered, ContinueOnFailure, func() {
 		BeforeAll(func(ctx context.Context) {
 			_, _, err := runner.Run("docker pull ubuntu")
 			Expect(err).ToNot(HaveOccurred())
+			_, _, err = runner.Run("docker pull busybox")
+			Expect(err).ToNot(HaveOccurred())
 		})
 
 		It("should include libcuda.so in the ldcache", func(ctx context.Context) {
@@ -257,6 +261,91 @@ var _ = Describe("docker", Ordered, ContinueOnFailure, func() {
 
 			Expect(libs).To(ContainElements([]string{"libcuda.so", "libcuda.so.1"}))
 		})
+
+		It("should include .so and SONAME symlinks", func(ctx context.Context) {
+			symlinkOutput, _, err := runner.Run("docker run --rm -i --runtime=nvidia -e NVIDIA_VISIBLE_DEVICES=runtime.nvidia.com/gpu=all busybox ls -l /usr/lib/x86_64-linux-gnu/ | awk '{print $1, $9, $11}'")
+			Expect(err).ToNot(HaveOccurred())
+			Expect(symlinkOutput).ToNot(BeEmpty())
+
+			// This produces output similar to:
+			// We check this to ensure that we have valid driver library symlink
+			// chains.
+			// lrwxrwxrwx libcuda.so libcuda.so.1
+			// lrwxrwxrwx libcuda.so.1 libcuda.so.570.133.20
+			// -rw-r--r-- libcuda.so.570.133.20
+			// lrwxrwxrwx libcudadebugger.so libcudadebugger.so.1
+			// lrwxrwxrwx libcudadebugger.so.1 libcudadebugger.so.570.133.20
+			// -rw-r--r-- libcudadebugger.so.570.133.20
+			// lrwxrwxrwx libnvidia-ml.so libnvidia-ml.so.1
+			// lrwxrwxrwx libnvidia-ml.so.1 libnvidia-ml.so.570.133.20
+			// -rw-r--r-- libnvidia-ml.so.570.133.20
+			// lrwxrwxrwx libnvidia-nvvm.so libnvidia-nvvm.so.4
+			// lrwxrwxrwx libnvidia-nvvm.so.4 libnvidia-nvvm.so.570.133.20
+			// -rw-r--r-- libnvidia-nvvm.so.570.133.20
+			// lrwxrwxrwx libnvidia-opencl.so libnvidia-opencl.so.1
+			// lrwxrwxrwx libnvidia-opencl.so.1 libnvidia-opencl.so.570.133.20
+			// -rw-r--r-- libnvidia-opencl.so.570.133.20
+			// -rw-r--r-- libnvidia-pkcs11-openssl3.so.570.133.20
+			// -rw-r--r-- libnvidia-pkcs11.so.570.133.20
+			// lrwxrwxrwx libnvidia-ptxjitcompiler.so libnvidia-ptxjitcompiler.so.1
+			// lrwxrwxrwx libnvidia-ptxjitcompiler.so.1 libnvidia-ptxjitcompiler.so.570.133.20
+			// -rw-r--r-- libnvidia-ptxjitcompiler.so.570.133.20
+
+			symlinkOutputLines := strings.Split(symlinkOutput, "\n")
+			symlinks := make(map[string][]string)
+			var soSymlink string
+			for _, line := range symlinkOutputLines {
+				trimmed := strings.TrimSpace(line)
+				if trimmed == "" {
+					soSymlink = ""
+					continue
+				}
+				// We only consider links and regular files.
+				if trimmed[0] != 'l' && trimmed[0] != '-' {
+					soSymlink = ""
+					continue
+				}
+				fmt.Printf("trimmed = %q\n", trimmed)
+				parts := strings.SplitN(trimmed, " ", 3)
+				permissionString := parts[0]
+
+				// If the line represents a symlink we may have to handle a new symlink chain.
+				if strings.HasPrefix(permissionString, "l") && soSymlink == "" {
+					if strings.HasSuffix(parts[1], ".so") {
+						soSymlink = parts[1]
+					}
+				}
+
+				// Keep track of the symlink chain.
+				symlinks[soSymlink] = append(symlinks[soSymlink], parts[1:]...)
+
+				// We assume a regular file is the end of a symlink chain.
+				if strings.HasPrefix(permissionString, "-") {
+					soSymlink = ""
+				}
+
+			}
+
+			// The symlink chains have the pattern:
+			// [A.so, A.so.1, A.so.1, A.so.driverVersion, A.so.driverVersion]
+			// A has the suffix .so.
+			Expect(symlinks).ToNot(BeEmpty())
+			for soSymlink, chain := range symlinks {
+				if soSymlink == "" {
+					continue
+				}
+				Expect(chain).To(HaveLen(5))
+				for _, c := range chain {
+					Expect(c).To(HavePrefix(soSymlink))
+				}
+				Expect(chain[0]).To(HaveSuffix(".so"))
+				Expect(chain[1]).To(Equal(chain[2]))
+				Expect(chain[3]).To(Equal(chain[4]))
+				Expect(chain[3]).To(HaveSuffix(hostDriverVersion))
+				Expect(chain[4]).To(HaveSuffix(hostDriverVersion))
+			}
+			Expect(symlinks).To(And(HaveKey("libcuda.so"), HaveKey("libnvidia-ml.so")))
+		})
 	})
 
 	When("Running containers with shared mount propagation", Ordered, func() {