Allow update-ldcache to work when pivot-root is not supported

This change updates the update-ldcache logic to use an alternative to
pivot-root when this is not supported. This includes cases where the
root filesystem is in a ramfs (e.g. when running from the kata-agent).

Signed-off-by: Evan Lezar <[email protected]>

Author: elezar

internal/ldconfig/ldconfig.go

@@ -27,6 +27,8 @@ import (
 	"runtime"
 	"strings"
 
+	"github.com/prometheus/procfs"
+
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/config"
 )
 
@@ -44,6 +46,7 @@ type Ldconfig struct {
 	inRoot                string
 	isDebianLikeHost      bool
 	isDebianLikeContainer bool
+	noPivotRoot           bool
 	directories           []string
 }
 
@@ -57,6 +60,11 @@ func NewRunner(id string, ldconfigPath string, containerRoot string, additionala
 	if isDebian() {
 		args = append(args, "--is-debian-like-host")
 	}
+
+	if noPivotRoot() {
+		args = append(args, "--no-pivot")
+	}
+
 	args = append(args, additionalargs...)
 
 	return createReexecCommand(args)
@@ -76,6 +84,7 @@ func NewRunner(id string, ldconfigPath string, containerRoot string, additionala
 // The following flags are optional:
 //
 //	--is-debian-like-host	Indicates that the host system is debian-based.
+//	--no-pivot           	pivot_root should not be used to provide process isolation.
 //
 // The remaining args are folders where soname symlinks need to be created.
 func NewFromArgs(args ...string) (*Ldconfig, error) {
@@ -86,6 +95,7 @@ func NewFromArgs(args ...string) (*Ldconfig, error) {
 	ldconfigPath := fs.String("ldconfig-path", "", "the path to ldconfig on the host")
 	containerRoot := fs.String("container-root", "", "the path in which ldconfig must be run")
 	isDebianLikeHost := fs.Bool("is-debian-like-host", false, "the hook is running from a Debian-like host")
+	noPivot := fs.Bool("no-pivot", false, "don't use pivot_root to perform isolation")
 	if err := fs.Parse(args[1:]); err != nil {
 		return nil, err
 	}
@@ -101,6 +111,7 @@ func NewFromArgs(args ...string) (*Ldconfig, error) {
 		ldconfigPath:     *ldconfigPath,
 		inRoot:           *containerRoot,
 		isDebianLikeHost: *isDebianLikeHost,
+		noPivotRoot:      *noPivot,
 		directories:      fs.Args(),
 	}
 	return l, nil
@@ -158,7 +169,7 @@ func (l *Ldconfig) prepareRoot() (string, error) {
 
 	// We pivot to the container root for the new process, this further limits
 	// access to the host.
-	if err := pivotRoot(root.Name()); err != nil {
+	if err := l.pivotRoot(root); err != nil {
 		return "", fmt.Errorf("error running pivot_root: %w", err)
 	}
 
@@ -339,3 +350,50 @@ func debianSystemSearchPaths() []string {
 
 	return paths
 }
+
+func (l *Ldconfig) pivotRoot(root *os.Root) error {
+	rootDir := root.Name()
+	// We select the function to pivot the root based on whether pivot_root is
+	// supported.
+	// See https://github.com/opencontainers/runc/blob/c3d127f6e8d9f6c06d78b8329cafa8dd39f6236e/libcontainer/rootfs_linux.go#L207-L216
+	if l.noPivotRoot {
+		return msMoveRoot(rootDir)
+	}
+	return pivotRoot(rootDir)
+}
+
+// noPivotRoot checks whether the current root filesystem supports a pivot_root.
+// See https://github.com/opencontainers/runc/blob/main/libcontainer/SPEC.md#filesystem
+// for a discussion on when this is not the case.
+// If we fail to detect whether pivot-root is supported, we assume that it is supported.
+// The logic to check for support is adapted from kata-containers:
+//
+//	https://github.com/kata-containers/kata-containers/blob/e7b9eddcede4bbe2edeb9c3af7b2358dc65da76f/src/agent/src/sandbox.rs#L150
+//
+// and checks whether "/" is mounted as a rootfs.
+func noPivotRoot() bool {
+	rootFsType, err := getRootfsType("/")
+	if err != nil {
+		return false
+	}
+	return rootFsType == "rootfs"
+}
+
+func getRootfsType(path string) (string, error) {
+	procSelf, err := procfs.Self()
+	if err != nil {
+		return "", err
+	}
+
+	mountStats, err := procSelf.MountStats()
+	if err != nil {
+		return "", err
+	}
+
+	for _, mountStat := range mountStats {
+		if mountStat.Mount == path {
+			return mountStat.Type, nil
+		}
+	}
+	return "", fmt.Errorf("mount stats for %q not found", path)
+}

internal/ldconfig/ldconfig_linux.go

@@ -20,14 +20,17 @@
 package ldconfig
 
 import (
+	"errors"
 	"fmt"
 	"os"
 	"os/exec"
 	"path/filepath"
 	"strconv"
+	"strings"
 	"syscall"
 
 	"github.com/google/uuid"
+	"github.com/moby/sys/mountinfo"
 	"github.com/moby/sys/reexec"
 
 	"github.com/opencontainers/runc/libcontainer/utils"
@@ -98,6 +101,86 @@ func pivotRoot(rootfs string) error {
 	return nil
 }
 
+// msMoveRoot is used in cases where pivot root is not supported.
+// This includes initramfs filesystems where the root is read-only.
+// This is adapted from the implementation here:
+//
+//	https://github.com/opencontainers/runc/blob/e89a29929c775025419ab0d218a43588b4c12b9a/libcontainer/rootfs_linux.go#L1115
+//
+// With the `mount` and `unmount` calls changed to direct unix.Mount and unix.Unmount calls.
+func msMoveRoot(rootfs string) error {
+	// Before we move the root and chroot we have to mask all "full" sysfs and
+	// procfs mounts which exist on the host. This is because while the kernel
+	// has protections against mounting procfs if it has masks, when using
+	// chroot(2) the *host* procfs mount is still reachable in the mount
+	// namespace and the kernel permits procfs mounts inside --no-pivot
+	// containers.
+	//
+	// Users shouldn't be using --no-pivot except in exceptional circumstances,
+	// but to avoid such a trivial security flaw we apply a best-effort
+	// protection here. The kernel only allows a mount of a pseudo-filesystem
+	// like procfs or sysfs if there is a *full* mount (the root of the
+	// filesystem is mounted) without any other locked mount points covering a
+	// subtree of the mount.
+	//
+	// So we try to unmount (or mount tmpfs on top of) any mountpoint which is
+	// a full mount of either sysfs or procfs (since those are the most
+	// concerning filesystems to us).
+	mountinfos, err := mountinfo.GetMounts(func(info *mountinfo.Info) (skip, stop bool) {
+		// Collect every sysfs and procfs filesystem, except for those which
+		// are non-full mounts or are inside the rootfs of the container.
+		if info.Root != "/" ||
+			(info.FSType != "proc" && info.FSType != "sysfs") ||
+			strings.HasPrefix(info.Mountpoint, rootfs) {
+			skip = true
+		}
+		return
+	})
+	if err != nil {
+		return err
+	}
+	for _, info := range mountinfos {
+		p := info.Mountpoint
+		// Be sure umount events are not propagated to the host.
+		if err := unix.Mount("", p, "", unix.MS_SLAVE|unix.MS_REC, ""); err != nil {
+			if errors.Is(err, unix.ENOENT) {
+				// If the mountpoint doesn't exist that means that we've
+				// already blasted away some parent directory of the mountpoint
+				// and so we don't care about this error.
+				continue
+			}
+			return err
+		}
+		if err := unix.Unmount(p, unix.MNT_DETACH); err != nil {
+			if !errors.Is(err, unix.EINVAL) && !errors.Is(err, unix.EPERM) {
+				return err
+			} else {
+				// If we have not privileges for umounting (e.g. rootless), then
+				// cover the path.
+				if err := unix.Mount("tmpfs", p, "tmpfs", 0, ""); err != nil {
+					return err
+				}
+			}
+		}
+	}
+
+	// Move the rootfs on top of "/" in our mount namespace.
+	if err := unix.Mount(rootfs, "/", "", unix.MS_MOVE, ""); err != nil {
+		return err
+	}
+	return chroot()
+}
+
+func chroot() error {
+	if err := unix.Chroot("."); err != nil {
+		return &os.PathError{Op: "chroot", Path: ".", Err: err}
+	}
+	if err := unix.Chdir("/"); err != nil {
+		return &os.PathError{Op: "chdir", Path: "/", Err: err}
+	}
+	return nil
+}
+
 // mountLdConfig mounts the host ldconfig to the mount namespace of the hook.
 // We use WithProcfd to perform the mount operations to ensure that the changes
 // are persisted across the pivot root.

internal/ldconfig/ldconfig_other.go

@@ -29,6 +29,10 @@ func pivotRoot(newroot string) error {
 	return fmt.Errorf("not supported")
 }
 
+func msMoveRoot(rootfs string) error {
+	return fmt.Errorf("not supported")
+}
+
 func mountLdConfig(hostLdconfigPath string, containerRoot *os.Root) (string, error) {
 	return "", fmt.Errorf("not supported")
 }