Write system paths to lexicographically last ld.so.conf.d drop-in

In most cases, the hook will be executing a host ldconfig that may be
configured widely differently from what the container image expects. The
common case is Debian vs non-Debian. But there are also hosts that
configure ldconfig to search in a glibc prefix (e.g. /usr/lib/glibc). To
avoid all these cases, write the container's expected system search
paths to a drop-in conf file that is likely to be last in lexicographic
order. Entries in the top-level ld.so.conf file may be processed after
this drop-in, but this hook does not modify the top-level file if it
exists.

Signed-off-by: Jean-Francois Roy <[email protected]>

Author: jfroy

internal/ldconfig/ldconfig.go

@@ -46,6 +46,9 @@ const (
 	// ld.so.conf file, but some may not. And some container images may not have a top-level
 	// ld.so.conf file at all.
 	defaultLdsoconfdDir = "/etc/ld.so.conf.d"
+	// ldsoconfdSystemDirsFilenamePattern specifies the filename pattern for the drop-in conf file
+	// that includes the expected system directories for the container.
+	ldsoconfdSystemDirsFilenamePattern = "zz-nvcr-*.conf"
 )
 
 type Ldconfig struct {
@@ -131,7 +134,7 @@ func (l *Ldconfig) UpdateLDCache() error {
 
 	// Explicitly specify using /etc/ld.so.conf since the host's ldconfig may
 	// be configured to use a different config file by default.
-	filteredDirectories, ldconfigDirs, err := l.filterDirectories(defaultTopLevelLdsoconfFilePath, l.directories...)
+	filteredDirectories, err := l.filterDirectories(defaultTopLevelLdsoconfFilePath, l.directories...)
 	if err != nil {
 		return err
 	}
@@ -141,25 +144,23 @@ func (l *Ldconfig) UpdateLDCache() error {
 		"-f", defaultTopLevelLdsoconfFilePath,
 		"-C", "/etc/ld.so.cache",
 	}
-	// If we are running in a non-debian container on a debian host we also
-	// need to add the system directories for non-debian hosts to the list of
-	// folders processed by ldconfig.
-	// We only do this if they are not already tracked, since the folders on
-	// on the command line have a higher priority than folders in ld.so.conf.
-	if l.isDebianLikeHost && !l.isDebianLikeContainer {
-		for _, systemSearchPath := range l.getSystemSearchPaths() {
-			if _, ok := ldconfigDirs[systemSearchPath]; ok {
-				continue
-			}
-			args = append(args, "/lib64", "/usr/lib64")
-		}
-	}
 
 	if err := ensureLdsoconfFile(defaultTopLevelLdsoconfFilePath, defaultLdsoconfdDir); err != nil {
 		return fmt.Errorf("failed to ensure ld.so.conf file: %w", err)
 	}
 	if err := createLdsoconfdFile(defaultLdsoconfdDir, ldsoconfdFilenamePattern, filteredDirectories...); err != nil {
-		return fmt.Errorf("failed to create ld.so.conf.d drop-in file: %w", err)
+		return fmt.Errorf("failed to write %s drop-in: %w", ldsoconfdFilenamePattern, err)
+	}
+
+	// In most cases, the hook will be executing a host ldconfig that may be configured widely
+	// differently from what the container image expects. The common case is Debian vs non-Debian.
+	// But there are also hosts that configure ldconfig to search in a glibc prefix
+	// (e.g. /usr/lib/glibc). To avoid all these cases, write the container's expected system search
+	// paths to a drop-in conf file that is likely to be last in lexicographic order. Entries in the
+	// top-level ld.so.conf file may be processed after this drop-in, but this hook does not modify
+	// the top-level file if it exists.
+	if err := createLdsoconfdFile(defaultLdsoconfdDir, ldsoconfdSystemDirsFilenamePattern, l.getSystemSearchPaths()...); err != nil {
+		return fmt.Errorf("failed to write %s drop-in: %w", ldsoconfdSystemDirsFilenamePattern, err)
 	}
 
 	return SafeExec(ldconfigPath, args, nil)
@@ -194,10 +195,10 @@ func (l *Ldconfig) prepareRoot() (string, error) {
 	return ldconfigPath, nil
 }
 
-func (l *Ldconfig) filterDirectories(configFilePath string, directories ...string) ([]string, map[string]struct{}, error) {
+func (l *Ldconfig) filterDirectories(configFilePath string, directories ...string) ([]string, error) {
 	ldconfigDirs, err := l.getLdsoconfDirectories(configFilePath)
 	if err != nil {
-		return nil, nil, err
+		return nil, err
 	}
 
 	var filtered []string
@@ -208,7 +209,7 @@ func (l *Ldconfig) filterDirectories(configFilePath string, directories ...strin
 		filtered = append(filtered, d)
 		ldconfigDirs[d] = struct{}{}
 	}
-	return filtered, ldconfigDirs, nil
+	return filtered, nil
 }
 
 // createLdsoconfdFile creates a ld.so.conf.d drop-in file with the specified directories on each
@@ -350,22 +351,61 @@ func isDebian() bool {
 	return !info.IsDir()
 }
 
-// nonDebianSystemSearchPaths returns the system search paths for non-Debian
-// systems.
+// nonDebianSystemSearchPaths returns the system search paths for non-Debian systems.
+//
+// glibc ldconfig's calls `add_system_dir` with `SLIBDIR` and `LIBDIR` (if they are not equal). On
+// aarch64 and x86_64, `add_system_dir` is a macro that scans the provided path. If the path ends
+// with "/lib64" (or "/libx32", x86_64 only), it strips those suffixes. Then it registers the
+// resulting path. Then if the path ends with "/lib", it registers "path"+"64" (and "path"+"x32",
+// x86_64 only).
+//
+// By default, "LIBDIR" is "/usr/lib" and "SLIBDIR" is "/lib". Note that on modern distributions,
+// "/lib" is usually a symlink to "/usr/lib" and "/lib64" to "/usr/lib64". ldconfig resolves
+// symlinks and skips duplicate directory entries.
 //
-// This list was taken from the output of:
+// To get the list of system paths, you can invoke the dynamic linker with `--list-diagnostics` and
+// look for "path.system_dirs". For example
+// `docker run --rm -ti fedora:latest /lib64/ld-linux-x86-64.so.2 --list-diagnostics | grep path.system_dirs`.
 //
-//	docker run --rm -ti redhat/ubi9 /usr/lib/ld-linux-aarch64.so.1 --help | grep -A6 "Shared library search path"
+// On most distributions, including Fedora and derivatives, this yields the following
+// ldconfig system search paths.
+//
+// TODO: Add other architectures that have custom `add_system_dir` macros (e.g. riscv)
+// TODO: Replace with executing the container's dynamlic linker with `--list-diagnostics`?
 func nonDebianSystemSearchPaths() []string {
-	return []string{"/lib64", "/usr/lib64"}
+	var paths []string
+	paths = append(paths, "/lib", "/usr/lib")
+	switch runtime.GOARCH {
+	case "amd64":
+		paths = append(paths,
+			"/lib/lib64",
+			"/usr/lib64",
+			"/libx32",
+			"/usr/libx32",
+		)
+	case "arm64":
+		paths = append(paths,
+			"/lib/lib64",
+			"/usr/lib64",
+		)
+	}
+	return paths
 }
 
-// debianSystemSearchPaths returns the system search paths for Debian-like
-// systems.
+// debianSystemSearchPaths returns the system search paths for Debian-like systems.
 //
-// This list was taken from the output of:
+// Debian (and derivatives) apply their multi-arch patch to glibc, which modifies ldconfig to
+// use the same set of system paths as the dynamic linker. These paths are going to include the
+// multi-arch directory _and_ by default "/lib" and "/usr/lib" for compatibility.
 //
-//	docker run --rm -ti ubuntu /usr/lib/aarch64-linux-gnu/ld-linux-aarch64.so.1 --help | grep -A6 "Shared library search path"
+// To get the list of system paths, you can invoke the dynamic linker with `--list-diagnostics` and
+// look for "path.system_dirs". For example
+// `docker run --rm -ti ubuntu:latest /lib64/ld-linux-x86-64.so.2 --list-diagnostics | grep path.system_dirs`.
+//
+// This yields the following ldconfig system search paths.
+//
+// TODO: Add other architectures that have custom `add_system_dir` macros (e.g. riscv)
+// TODO: Replace with executing the container's dynamlic linker with `--list-diagnostics`?
 func debianSystemSearchPaths() []string {
 	var paths []string
 	switch runtime.GOARCH {
@@ -381,6 +421,5 @@ func debianSystemSearchPaths() []string {
 		)
 	}
 	paths = append(paths, "/lib", "/usr/lib")
-
 	return paths
 }

internal/ldconfig/ldconfig_test.go

@@ -118,7 +118,7 @@ include INCLUDED_PATTERN*
 			l := &Ldconfig{
 				isDebianLikeContainer: true,
 			}
-			filtered, _, err := l.filterDirectories(topLevelConfPath, tc.input...)
+			filtered, err := l.filterDirectories(topLevelConfPath, tc.input...)
 
 			require.NoError(t, err)
 			require.Equal(t, tc.expected, filtered)