Skip to content

Write system paths to lexicographically last ld.so.conf.d drop-in #1462

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 2 commits into from
+90 −14
Closed

Write system paths to lexicographically last ld.so.conf.d drop-in #1462

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
5790adc
Fix update of ldcache on non-debian containers
elezar Nov 11, 2025
b709f1d
Write system paths to lexicographically last ld.so.conf.d drop-in
jfroy Nov 17, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
85 changes: 71 additions & 14 deletions internal/ldconfig/ldconfig.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,9 @@ const (
// ld.so.conf file, but some may not. And some container images may not have a top-level
// ld.so.conf file at all.
defaultLdsoconfdDir = "/etc/ld.so.conf.d"
// ldsoconfdSystemDirsFilenamePattern specifies the filename pattern for the drop-in conf file
// that includes the expected system directories for the container.
ldsoconfdSystemDirsFilenamePattern = "zz-nvcr-*.conf"
)

type Ldconfig struct {
Expand Down Expand Up @@ -76,7 +79,8 @@ func NewRunner(id string, ldconfigPath string, containerRoot string, additionala
// This struct is used to perform operations on the ldcache and libraries in a
// particular root (e.g. a container).
//
// args[0] is the reexec initializer function name
// args[0] is the reexec initializer function name and is required.
//
// The following flags are required:
//
// --ldconfig-path=LDCONFIG_PATH the path to ldconfig on the host
Expand All @@ -85,16 +89,20 @@ func NewRunner(id string, ldconfigPath string, containerRoot string, additionala
// The following flags are optional:
//
// --is-debian-like-host Indicates that the host system is debian-based.
// See https://github.com/NVIDIA/nvidia-container-toolkit/pull/1444
//
// The remaining args are folders where soname symlinks need to be created.
func NewFromArgs(args ...string) (*Ldconfig, error) {
if len(args) < 1 {
return nil, fmt.Errorf("incorrect arguments: %v", args)
}
fs := flag.NewFlagSet(args[1], flag.ExitOnError)
fs := flag.NewFlagSet("ldconfig-options", flag.ExitOnError)
ldconfigPath := fs.String("ldconfig-path", "", "the path to ldconfig on the host")
containerRoot := fs.String("container-root", "", "the path in which ldconfig must be run")
isDebianLikeHost := fs.Bool("is-debian-like-host", false, "the hook is running from a Debian-like host")
isDebianLikeHost := fs.Bool("is-debian-like-host", false, `indicates that the host system is debian-based.
This allows us to handle the case where there are differences in behavior
between the ldconfig from the host (as executed from an update-ldcache hook) and
ldconfig in the container. Such differences include system search paths.`)
if err := fs.Parse(args[1:]); err != nil {
return nil, err
}
Expand Down Expand Up @@ -141,7 +149,18 @@ func (l *Ldconfig) UpdateLDCache() error {
return fmt.Errorf("failed to ensure ld.so.conf file: %w", err)
}
if err := createLdsoconfdFile(defaultLdsoconfdDir, ldsoconfdFilenamePattern, filteredDirectories...); err != nil {
return fmt.Errorf("failed to create ld.so.conf.d drop-in file: %w", err)
return fmt.Errorf("failed to write %s drop-in: %w", ldsoconfdFilenamePattern, err)
}

// In most cases, the hook will be executing a host ldconfig that may be configured widely
// differently from what the container image expects. The common case is Debian vs non-Debian.
// But there are also hosts that configure ldconfig to search in a glibc prefix
// (e.g. /usr/lib/glibc). To avoid all these cases, write the container's expected system search
// paths to a drop-in conf file that is likely to be last in lexicographic order. Entries in the
// top-level ld.so.conf file may be processed after this drop-in, but this hook does not modify
// the top-level file if it exists.
if err := createLdsoconfdFile(defaultLdsoconfdDir, ldsoconfdSystemDirsFilenamePattern, l.getSystemSearchPaths()...); err != nil {
return fmt.Errorf("failed to write %s drop-in: %w", ldsoconfdSystemDirsFilenamePattern, err)
}

return SafeExec(ldconfigPath, args, nil)
Expand Down Expand Up @@ -331,22 +350,61 @@ func isDebian() bool {
return !info.IsDir()
}

// nonDebianSystemSearchPaths returns the system search paths for non-Debian
// systems.
// nonDebianSystemSearchPaths returns the system search paths for non-Debian systems.
//
// glibc ldconfig's calls `add_system_dir` with `SLIBDIR` and `LIBDIR` (if they are not equal). On
// aarch64 and x86_64, `add_system_dir` is a macro that scans the provided path. If the path ends
// with "/lib64" (or "/libx32", x86_64 only), it strips those suffixes. Then it registers the
// resulting path. Then if the path ends with "/lib", it registers "path"+"64" (and "path"+"x32",
// x86_64 only).
//
// By default, "LIBDIR" is "/usr/lib" and "SLIBDIR" is "/lib". Note that on modern distributions,
// "/lib" is usually a symlink to "/usr/lib" and "/lib64" to "/usr/lib64". ldconfig resolves
// symlinks and skips duplicate directory entries.
//
// This list was taken from the output of:
// To get the list of system paths, you can invoke the dynamic linker with `--list-diagnostics` and
// look for "path.system_dirs". For example
// `docker run --rm -ti fedora:latest /lib64/ld-linux-x86-64.so.2 --list-diagnostics | grep path.system_dirs`.
//
// docker run --rm -ti redhat/ubi9 /usr/lib/ld-linux-aarch64.so.1 --help | grep -A6 "Shared library search path"
// On most distributions, including Fedora and derivatives, this yields the following
// ldconfig system search paths.
//
// TODO: Add other architectures that have custom `add_system_dir` macros (e.g. riscv)
// TODO: Replace with executing the container's dynamlic linker with `--list-diagnostics`?
Copy link
Member

@elezar elezar Nov 18, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Just a note on this. Since containers are user-supplied, we need to be careful about executing something from the container. This (and the fact that not all containers include ldconfig) is the reason that we don't run ldconfig form the container.

Copy link
Collaborator Author

@jfroy jfroy Nov 18, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah this is why I wrote it as a question. Two follow up:

  • The suggestion/question is about the dynamic linker, not ldconfig. I imagine almost all container images that expect to run our software are going to have it.
  • I probably lack the historical or technical background why this hook is not a startContainer hook that could run the container image's ldconfig (if any) or ld.so. Naively, the runtime is about to execute the main container process, which is just as untrusted. Anyways I don't mean to start a big technical discussion with this comment; I am just curious about the way things are and the security posture.

Copy link
Member

@elezar elezar Nov 18, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The reason this is a createContainer hook and not a startContainer hook is that the hooks we run (i.e. nvidia-cdi-hook) are not available in the container. This is not to say that it is impossible to ensure that these are injected and available to the container, but it is not something that we have worked on.

Note that logic like "if any" is not really expressible in the current OCI Runtime hook spec which is why we rely on more complex logic being backed into an executable. What we could consider doing is:

  1. Run createContainer hooks to:
    2. create /etc/ld.so.conf.d/ drop in files for injected libraries and CUDA compat libraries.
    3. create a hook at a well known path in the container that covers optionally running ldconfig / ldconfig.real in the container.
  2. Run a startContainer hook referencing the created hook.

Note that since we would then be running ldconfig in the container as a startContainer hook, we would be able to leverage the isolation that is already provided by low-level runtimes such as runc and it would also simplify the logic around running ldconfig since we would not have to handle differences between the host and the container distributions.

One caveat here is that we would NOT be able to handle containers that do not have ldconfig in the container -- although in this case we may be able to fall back to a host executable mounted into the container.

Copy link
Collaborator Author

@jfroy jfroy Nov 18, 2025 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It's appealing to consider a startContainer approach, but because of containers without an ldconfig binary and the need to emit one or more drop-ins, it wouldn't save any code or reduce complexity.

func nonDebianSystemSearchPaths() []string {
return []string{"/lib64", "/usr/lib64"}
var paths []string
paths = append(paths, "/lib", "/usr/lib")
switch runtime.GOARCH {
case "amd64":
paths = append(paths,
"/lib64",
"/usr/lib64",
"/libx32",
"/usr/libx32",
)
case "arm64":
paths = append(paths,
"/lib64",
"/usr/lib64",
)
}
return paths
}

// debianSystemSearchPaths returns the system search paths for Debian-like
// systems.
// debianSystemSearchPaths returns the system search paths for Debian-like systems.
//
// Debian (and derivatives) apply their multi-arch patch to glibc, which modifies ldconfig to
// use the same set of system paths as the dynamic linker. These paths are going to include the
// multi-arch directory _and_ by default "/lib" and "/usr/lib" for compatibility.
//
// This list was taken from the output of:
// To get the list of system paths, you can invoke the dynamic linker with `--list-diagnostics` and
// look for "path.system_dirs". For example
// `docker run --rm -ti ubuntu:latest /lib64/ld-linux-x86-64.so.2 --list-diagnostics | grep path.system_dirs`.
//
// docker run --rm -ti ubuntu /usr/lib/aarch64-linux-gnu/ld-linux-aarch64.so.1 --help | grep -A6 "Shared library search path"
// This yields the following ldconfig system search paths.
//
// TODO: Add other architectures that have custom `add_system_dir` macros (e.g. riscv)
// TODO: Replace with executing the container's dynamlic linker with `--list-diagnostics`?
func debianSystemSearchPaths() []string {
var paths []string
switch runtime.GOARCH {
Expand All @@ -362,6 +420,5 @@ func debianSystemSearchPaths() []string {
)
}
paths = append(paths, "/lib", "/usr/lib")

return paths
}
19 changes: 19 additions & 0 deletions tests/e2e/nvidia-container-toolkit_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -570,4 +570,23 @@ EOF`)
Expect(output).To(Equal(expectedOutput))
})
})

When("running a ubi9 container", Ordered, func() {
var (
expectedOutput string
)
BeforeAll(func(ctx context.Context) {
_, _, err := runner.Run(`docker pull redhat/ubi9`)
Expect(err).ToNot(HaveOccurred())

expectedOutput, _, err = runner.Run(`docker run --rm --runtime=runc redhat/ubi9 bash -c "ldconfig -p | grep libc.so."`)
Expect(err).ToNot(HaveOccurred())
})

It("should include the system libraries when using the nvidia-container-runtime", func(ctx context.Context) {
output, _, err := runner.Run(`docker run --rm --runtime=nvidia -e NVIDIA_VISIBLE_DEVICES=all redhat/ubi9 bash -c "ldconfig -p | grep libc.so."`)
Expect(err).ToNot(HaveOccurred())
Expect(output).To(Equal(expectedOutput))
})
})
})