Write system paths to lexicographically last ld.so.conf.d drop-in

In most cases, the hook will be executing a host ldconfig that may be
configured widely differently from what the container image expects. The
common case is Debian vs non-Debian. But there are also hosts that
configure ldconfig to search in a glibc prefix (e.g. /usr/lib/glibc). To
avoid all these cases, write the container's expected system search
paths to a drop-in conf file that is likely to be last in lexicographic
order. Entries in the top-level ld.so.conf file may be processed after
this drop-in, but this hook does not modify the top-level file if it
exists.

Signed-off-by: Jean-Francois Roy <[email protected]>

Author: jfroy

internal/ldconfig/ldconfig.go

@@ -37,6 +37,9 @@ const (
 	// higher precedence than other libraries on the system, but lower than
 	// the 00-cuda-compat that is included in some containers.
 	ldsoconfdFilenamePattern = "00-nvcr-*.conf"
+	// ldsoconfdSystemDirsFilenamePattern specifies the filename pattern for the drop-in conf file
+	// that includes the expected system directories for the container.
+	ldsoconfdSystemDirsFilenamePattern = "99-nvcr-*.conf"
 )
 
 type Ldconfig struct {
@@ -123,7 +126,7 @@ func (l *Ldconfig) UpdateLDCache() error {
 	// Explicitly specify using /etc/ld.so.conf since the host's ldconfig may
 	// be configured to use a different config file by default.
 	const topLevelLdsoconfFilePath = "/etc/ld.so.conf"
-	filteredDirectories, ldconfigDirs, err := l.filterDirectories(topLevelLdsoconfFilePath, l.directories...)
+	filteredDirectories, err := l.filterDirectories(topLevelLdsoconfFilePath, l.directories...)
 	if err != nil {
 		return err
 	}
@@ -133,22 +136,20 @@ func (l *Ldconfig) UpdateLDCache() error {
 		"-f", topLevelLdsoconfFilePath,
 		"-C", "/etc/ld.so.cache",
 	}
-	// If we are running in a non-debian container on a debian host we also
-	// need to add the system directories for non-debian hosts to the list of
-	// folders processed by ldconfig.
-	// We only do this if they are not already tracked, since the folders on
-	// on the command line have a higher priority than folders in ld.so.conf.
-	if l.isDebianLikeHost && !l.isDebianLikeContainer {
-		for _, systemSearchPath := range l.getSystemSearchPaths() {
-			if _, ok := ldconfigDirs[systemSearchPath]; ok {
-				continue
-			}
-			args = append(args, "/lib64", "/usr/lib64")
-		}
-	}
 
 	if err := createLdsoconfdFile(ldsoconfdFilenamePattern, filteredDirectories...); err != nil {
-		return fmt.Errorf("failed to update ld.so.conf.d: %w", err)
+		return fmt.Errorf("failed to write %s drop-in: %w", ldsoconfdFilenamePattern, err)
+	}
+
+	// In most cases, the hook will be executing a host ldconfig that may be configured widely
+	// differently from what the container image expects. The common case is Debian vs non-Debian.
+	// But there are also hosts that configure ldconfig to search in a glibc prefix
+	// (e.g. /usr/lib/glibc). To avoid all these cases, write the container's expected system search
+	// paths to a drop-in conf file that is likely to be last in lexicographic order. Entries in the
+	// top-level ld.so.conf file may be processed after this drop-in, but this hook does not modify
+	// the top-level file if it exists.
+	if err := createLdsoconfdFile(ldsoconfdSystemDirsFilenamePattern, l.getSystemSearchPaths()...); err != nil {
+		return fmt.Errorf("failed to write %s drop-in: %w", ldsoconfdSystemDirsFilenamePattern, err)
 	}
 
 	return SafeExec(ldconfigPath, args, nil)
@@ -183,10 +184,10 @@ func (l *Ldconfig) prepareRoot() (string, error) {
 	return ldconfigPath, nil
 }
 
-func (l *Ldconfig) filterDirectories(configFilePath string, directories ...string) ([]string, map[string]struct{}, error) {
+func (l *Ldconfig) filterDirectories(configFilePath string, directories ...string) ([]string, error) {
 	ldconfigDirs, err := l.getLdsoconfDirectories(configFilePath)
 	if err != nil {
-		return nil, nil, err
+		return nil, err
 	}
 
 	var filtered []string
@@ -197,7 +198,7 @@ func (l *Ldconfig) filterDirectories(configFilePath string, directories ...strin
 		filtered = append(filtered, d)
 		ldconfigDirs[d] = struct{}{}
 	}
-	return filtered, ldconfigDirs, nil
+	return filtered, nil
 }
 
 // createLdsoconfdFile creates a file at /etc/ld.so.conf.d/.
@@ -241,6 +242,24 @@ func createLdsoconfdFile(pattern string, dirs ...string) error {
 	return nil
 }
 
+func appendSystemSearchPathsToLdsoconf(configFilePath string, dirs ...string) error {
+	if len(dirs) == 0 {
+		return nil
+	}
+	configFile, err := os.OpenFile(configFilePath, os.O_APPEND|os.O_WRONLY, 0644)
+	if err != nil {
+		return fmt.Errorf("failed to open config file: %w", err)
+	}
+	defer configFile.Close()
+	for _, dir := range dirs {
+		_, err = fmt.Fprintf(configFile, "%s\n", dir)
+		if err != nil {
+			return fmt.Errorf("failed to update config file: %w", err)
+		}
+	}
+	return nil
+}
+
 // getLdsoconfDirectories returns a map of ldsoconf directories to the conf
 // files that refer to the directory.
 func (l *Ldconfig) getLdsoconfDirectories(configFilePath string) (map[string]struct{}, error) {
@@ -324,22 +343,61 @@ func isDebian() bool {
 	return !info.IsDir()
 }
 
-// nonDebianSystemSearchPaths returns the system search paths for non-Debian
-// systems.
+// nonDebianSystemSearchPaths returns the system search paths for non-Debian systems.
+//
+// glibc ldconfig's calls `add_system_dir` with `SLIBDIR` and `LIBDIR` (if they are not equal). On
+// aarch64 and x86_64, `add_system_dir` is a macro that scans the provided path. If the path ends
+// with "/lib64" (or "/libx32", x86_64 only), it strips those suffixes. Then it registers the
+// resulting path. Then if the path ends with "/lib", it registers "path"+"64" (and "path"+"x32",
+// x86_64 only).
+//
+// By default, "LIBDIR" is "/usr/lib" and "SLIBDIR" is "/lib". Note that on modern distributions,
+// "/lib" is usually a symlink to "/usr/lib" and "/lib64" to "/usr/lib64". ldconfig resolves
+// symlinks and skips duplicate directory entries.
+//
+// To get the list of system paths, you can invoke the dynamic linker with `--list-diagnostics` and
+// look for "path.system_dirs". For example
+// `docker run --rm -ti fedora:latest /lib64/ld-linux-x86-64.so.2 --list-diagnostics | grep path.system_dirs`.
 //
-// This list was taken from the output of:
+// On most distributions, including Fedora and derivatives, this yields the following
+// ldconfig system search paths.
 //
-//	docker run --rm -ti redhat/ubi9 /usr/lib/ld-linux-aarch64.so.1 --help | grep -A6 "Shared library search path"
+// TODO: Add other architectures that have custom `add_system_dir` macros (e.g. riscv)
+// TODO: Replace with executing the container's dynamlic linker with `--list-diagnostics`?
 func nonDebianSystemSearchPaths() []string {
-	return []string{"/lib64", "/usr/lib64"}
+	var paths []string
+	paths = append(paths, "/lib", "/usr/lib")
+	switch runtime.GOARCH {
+	case "amd64":
+		paths = append(paths,
+			"/lib/lib64",
+			"/usr/lib64",
+			"/libx32",
+			"/usr/libx32",
+		)
+	case "arm64":
+		paths = append(paths,
+			"/lib/lib64",
+			"/usr/lib64",
+		)
+	}
+	return paths
 }
 
-// debianSystemSearchPaths returns the system search paths for Debian-like
-// systems.
+// debianSystemSearchPaths returns the system search paths for Debian-like systems.
+//
+// Debian (and derivatives) apply their multi-arch patch to glibc, which modifies ldconfig to
+// use the same set of system paths as the dynamic linker. These paths are going to include the
+// multi-arch directory _and_ by default "/lib" and "/usr/lib" for compatibility.
 //
-// This list was taken from the output of:
+// To get the list of system paths, you can invoke the dynamic linker with `--list-diagnostics` and
+// look for "path.system_dirs". For example
+// `docker run --rm -ti ubuntu:latest /lib64/ld-linux-x86-64.so.2 --list-diagnostics | grep path.system_dirs`.
 //
-//	docker run --rm -ti ubuntu /usr/lib/aarch64-linux-gnu/ld-linux-aarch64.so.1 --help | grep -A6 "Shared library search path"
+// This yields the following ldconfig system search paths.
+//
+// TODO: Add other architectures that have custom `add_system_dir` macros (e.g. riscv)
+// TODO: Replace with executing the container's dynamlic linker with `--list-diagnostics`?
 func debianSystemSearchPaths() []string {
 	var paths []string
 	switch runtime.GOARCH {
@@ -355,6 +413,5 @@ func debianSystemSearchPaths() []string {
 		)
 	}
 	paths = append(paths, "/lib", "/usr/lib")
-
 	return paths
 }

internal/ldconfig/ldconfig_test.go

@@ -19,6 +19,7 @@ package ldconfig
 
 import (
 	"os"
+	"path/filepath"
 	"strings"
 	"testing"
 
@@ -117,10 +118,79 @@ include INCLUDED_PATTERN*
 			l := &Ldconfig{
 				isDebianLikeContainer: true,
 			}
-			filtered, _, err := l.filterDirectories(topLevelConfPath, tc.input...)
+			filtered, err := l.filterDirectories(topLevelConfPath, tc.input...)
 
 			require.NoError(t, err)
 			require.Equal(t, tc.expected, filtered)
 		})
 	}
 }
+
+func TestAppendSystemSearchPathsToLdsoconf(t *testing.T) {
+	testCases := []struct {
+		description    string
+		initialContent string
+		noFile         bool
+		dirs           []string
+		expectedFinal  string
+		expectError    bool
+	}{
+		{
+			description:    "append to empty file",
+			initialContent: "",
+			dirs:           []string{"/lib", "/usr/lib"},
+			expectedFinal:  "/lib\n/usr/lib\n",
+		},
+		{
+			description:    "append to existing content",
+			initialContent: "# existing config\n/existing/path\n",
+			dirs:           []string{"/lib", "/usr/lib"},
+			expectedFinal:  "# existing config\n/existing/path\n/lib\n/usr/lib\n",
+		},
+		{
+			description:    "append empty does nothing",
+			initialContent: "# existing config\n",
+			dirs:           []string{},
+			expectedFinal:  "# existing config\n",
+		},
+		{
+			description: "append empty does not create file",
+			noFile:      true,
+			dirs:        []string{},
+		},
+		{
+			description: "append to non-existent file fails",
+			noFile:      true,
+			dirs:        []string{"/lib"},
+			expectError: true,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.description, func(t *testing.T) {
+			tmpDir := t.TempDir()
+			configPath := filepath.Join(tmpDir, "ld.so.conf")
+
+			if !tc.noFile {
+				err := os.WriteFile(configPath, []byte(tc.initialContent), 0644)
+				require.NoError(t, err)
+			}
+
+			err := appendSystemSearchPathsToLdsoconf(configPath, tc.dirs...)
+			if tc.expectError {
+				require.Error(t, err)
+				return
+			}
+			require.NoError(t, err)
+
+			if !tc.noFile {
+				content, err := os.ReadFile(configPath)
+				require.NoError(t, err)
+				require.Equal(t, tc.expectedFinal, string(content))
+			} else {
+				_, err := os.Stat(configPath)
+				require.True(t, os.IsNotExist(err))
+			}
+		})
+	}
+}