[DO NOT MERGE] Add memory policy support

Implement support for Linux memory policy in OCI spec PR:
opencontainers/runtime-spec#1282

TODO:
- remove the replace from go.mod when OCI spec is merged

Signed-off-by: Antti Kervinen <[email protected]>

Author: askervin

go.mod

@@ -39,3 +39,5 @@ require (
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/vishvananda/netns v0.0.4 // indirect
 )
+
+replace github.com/opencontainers/runtime-spec => github.com/askervin/runtime-spec v1.0.3-0.20250328150043-68936b63f0db

go.sum

@@ -1,4 +1,6 @@
 github.com/BurntSushi/toml v1.4.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho=
+github.com/askervin/runtime-spec v1.0.3-0.20250328150043-68936b63f0db h1:vfXC+spYsKfEjaQxvkj2vGbSMuaTy475Z3ctMoe2Uf8=
+github.com/askervin/runtime-spec v1.0.3-0.20250328150043-68936b63f0db/go.mod h1:0ccwhiCQXxLwvWvVsdVdxTe+IFfXyJTjr/wNue5fNJY=
 github.com/checkpoint-restore/go-criu/v7 v7.2.0 h1:qGiWA4App1gGlEfIJ68WR9jbezV9J7yZdjzglezcqKo=
 github.com/checkpoint-restore/go-criu/v7 v7.2.0/go.mod h1:u0LCWLg0w4yqqu14aXhiB4YD3a1qd8EcCEg7vda5dwo=
 github.com/cilium/ebpf v0.17.3 h1:FnP4r16PWYSE4ux6zN+//jMcW4nMVRvuTLVTvCjyyjg=
@@ -47,8 +49,6 @@ github.com/mrunalp/fileutils v0.5.1 h1:F+S7ZlNKnrwHfSwdlgNSkKo67ReVf8o9fel6C3dkm
 github.com/mrunalp/fileutils v0.5.1/go.mod h1:M1WthSahJixYnrXQl/DFQuteStB1weuxD2QJNHXfbSQ=
 github.com/opencontainers/cgroups v0.0.1 h1:MXjMkkFpKv6kpuirUa4USFBas573sSAY082B4CiHEVA=
 github.com/opencontainers/cgroups v0.0.1/go.mod h1:s8lktyhlGUqM7OSRL5P7eAW6Wb+kWPNvt4qvVfzA5vs=
-github.com/opencontainers/runtime-spec v1.2.1 h1:S4k4ryNgEpxW1dzyqffOmhI1BHYcjzU8lpJfSlR0xww=
-github.com/opencontainers/runtime-spec v1.2.1/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
 github.com/opencontainers/selinux v1.12.0 h1:6n5JV4Cf+4y0KNXW48TLj5DwfXpvWlxXplUkdTrmPb8=
 github.com/opencontainers/selinux v1.12.0/go.mod h1:BTPX+bjVbWGXw7ZZWUbdENt8w0htPSrlgOOysQaU62U=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=

libcontainer/configs/config.go

@@ -208,6 +208,9 @@ type Config struct {
 	// to limit the resources (e.g., L3 cache, memory bandwidth) the container has available
 	IntelRdt *IntelRdt `json:"intel_rdt,omitempty"`
 
+	// MemoryPolicy specifies NUMA memory policy for the container.
+	MemoryPolicy *LinuxMemoryPolicy `json:"memoryPolicy,omitempty"`
+
 	// RootlessEUID is set when the runc was launched with non-zero EUID.
 	// Note that RootlessEUID is set to false when launched with EUID=0 in userns.
 	// When RootlessEUID is set, runc creates a new userns for the container.

libcontainer/configs/memorypolicy.go

@@ -0,0 +1,6 @@
+package configs
+
+type LinuxMemoryPolicy struct {
+	Mode  uint
+	Nodes []int
+}

libcontainer/init_linux.go

@@ -659,6 +659,10 @@ func setupIOPriority(config *initConfig) error {
 	return nil
 }
 
+func setupMemoryPolicy(config *configs.Config) error {
+	return system.SetMempolicy(config.MemoryPolicy.Mode, config.MemoryPolicy.Nodes)
+}
+
 func setupPersonality(config *configs.Config) error {
 	return system.SetLinuxPersonality(config.Personality.Domain)
 }

libcontainer/setns_init_linux.go

@@ -110,6 +110,11 @@ func (l *linuxSetnsInit) Init() error {
 	if err := apparmor.ApplyProfile(l.config.AppArmorProfile); err != nil {
 		return err
 	}
+	if l.config.Config.MemoryPolicy != nil {
+		if err := setupMemoryPolicy(l.config.Config); err != nil {
+			return err
+		}
+	}
 	if l.config.Config.Personality != nil {
 		if err := setupPersonality(l.config.Config); err != nil {
 			return err

libcontainer/specconv/spec_linux.go

@@ -9,6 +9,7 @@ import (
 	"os"
 	"path/filepath"
 	"sort"
+	"strconv"
 	"strings"
 	"sync"
 	"time"
@@ -41,6 +42,8 @@ var (
 		flag  int
 	}
 	complexFlags map[string]func(*configs.Mount)
+	mpolModeMap  map[specs.MemoryPolicyModeType]uint
+	mpolModeFMap map[specs.MemoryPolicyFlagType]uint
 )
 
 func initMaps() {
@@ -148,6 +151,21 @@ func initMaps() {
 				m.IDMapping.Recursive = true
 			},
 		}
+
+		mpolModeMap = map[specs.MemoryPolicyModeType]uint{
+			specs.MpolDefault:            0,
+			specs.MpolPreferred:          1,
+			specs.MpolBind:               2,
+			specs.MpolInterleave:         3,
+			specs.MpolLocal:              4,
+			specs.MpolWeightedInterleave: 6,
+		}
+
+		mpolModeFMap = map[specs.MemoryPolicyFlagType]uint{
+			specs.MpolFStaticNodes:   1 << 15,
+			specs.MpolFRelativeNodes: 1 << 14,
+			specs.MpolFNumaBalancing: 1 << 13,
+		}
 	})
 }
 
@@ -467,6 +485,32 @@ func CreateLibcontainerConfig(opts *CreateOpts) (*configs.Config, error) {
 				MemBwSchema:   spec.Linux.IntelRdt.MemBwSchema,
 			}
 		}
+		if spec.Linux.MemoryPolicy != nil &&
+			(spec.Linux.MemoryPolicy.Mode != "" ||
+				spec.Linux.MemoryPolicy.Nodes != "" ||
+				len(spec.Linux.MemoryPolicy.Flags) > 0) {
+			var ok bool
+			specMp := spec.Linux.MemoryPolicy
+			confMp := &configs.LinuxMemoryPolicy{}
+			if confMp.Mode, ok = mpolModeMap[specMp.Mode]; !ok {
+				return nil, fmt.Errorf("invalid memory policy mode %q", specMp.Mode)
+			}
+			// MAX_NODE is a sensibility check to user-provided
+			// nodes, not reflecting currently onlined nodes.
+			// set_mempolicy() accepts non-existent nodes.
+			MAX_NODE := 1023
+			if confMp.Nodes, err = parseListSet(specMp.Nodes, 0, MAX_NODE); err != nil {
+				return nil, fmt.Errorf("invalid memory policy nodes %q: %v", specMp.Nodes, err)
+			}
+			for _, specFlag := range specMp.Flags {
+				confModeFlag, ok := mpolModeFMap[specFlag]
+				if !ok {
+					return nil, fmt.Errorf("invalid memory policy flag %q", specFlag)
+				}
+				confMp.Mode |= confModeFlag
+			}
+			config.MemoryPolicy = confMp
+		}
 		if spec.Linux.Personality != nil {
 			if len(spec.Linux.Personality.Flags) > 0 {
 				logrus.Warnf("ignoring unsupported personality flags: %+v because personality flag has not supported at this time", spec.Linux.Personality.Flags)
@@ -1127,6 +1171,53 @@ func parseMountOptions(options []string) *configs.Mount {
 	return &m
 }
 
+// parseListSet parses "list set" syntax ("0,61-63,2") into a list ([0, 61, 62, 63, 2])
+func parseListSet(listSet string, minValue, maxValue int) ([]int, error) {
+	var result []int
+	if listSet == "" {
+		return result, nil
+	}
+	parts := strings.Split(listSet, ",")
+	for _, part := range parts {
+		switch {
+		case part == "":
+			continue
+		case strings.Contains(part, "-"):
+			rangeParts := strings.Split(part, "-")
+			if len(rangeParts) != 2 {
+				return nil, fmt.Errorf("invalid range: %s", part)
+			}
+			start, err := strconv.Atoi(rangeParts[0])
+			if err != nil {
+				return nil, err
+			}
+			end, err := strconv.Atoi(rangeParts[1])
+			if err != nil {
+				return nil, err
+			}
+			if start > end {
+				return nil, fmt.Errorf("invalid range %s: start > end", part)
+			}
+			if start < minValue || end > maxValue {
+				return nil, fmt.Errorf("invalid range %s: not in %d-%d", part, minValue, maxValue)
+			}
+			for i := start; i <= end; i++ {
+				result = append(result, i)
+			}
+		default:
+			num, err := strconv.Atoi(part)
+			if err != nil {
+				return nil, err
+			}
+			if num < minValue || num > maxValue {
+				return nil, fmt.Errorf("invalid value %d: not in %d-%d", num, minValue, maxValue)
+			}
+			result = append(result, num)
+		}
+	}
+	return result, nil
+}
+
 func SetupSeccomp(config *specs.LinuxSeccomp) (*configs.Seccomp, error) {
 	if config == nil {
 		return nil, nil

libcontainer/standard_init_linux.go

@@ -238,6 +238,13 @@ func (l *linuxStandardInit) Init() error {
 		}
 	}
 
+	// Set memory policy if specified.
+	if l.config.Config.MemoryPolicy != nil {
+		if err := setupMemoryPolicy(l.config.Config); err != nil {
+			return err
+		}
+	}
+
 	// Set personality if specified.
 	if l.config.Config.Personality != nil {
 		if err := setupPersonality(l.config.Config); err != nil {

libcontainer/system/linux.go

@@ -151,6 +151,31 @@ fallback:
 	return io.Copy(dst, src)
 }
 
+func bitmaskFromInts(bits []int) []uint64 {
+	maxBit := 0
+	for _, bit := range bits {
+		if bit > maxBit {
+			maxBit = bit
+		}
+	}
+	mask := make([]uint64, (maxBit/64)+1)
+	for _, bit := range bits {
+		mask[bit/64] |= (1 << (bit % 64))
+	}
+	return mask
+}
+
+// SetMempolicy sets the NUMA memory policy. For more information see the set_mempolicy syscall documentation.
+func SetMempolicy(mode uint, nodes []int) error {
+	nodemask := bitmaskFromInts(nodes)
+	nodemaskPtr := unsafe.Pointer(&nodemask[0])
+	_, _, errno := unix.Syscall(unix.SYS_SET_MEMPOLICY, uintptr(mode), uintptr(nodemaskPtr), uintptr(len(nodemask)*64))
+	if errno != 0 {
+		return &os.SyscallError{Syscall: "set_mempolicy", Err: errno}
+	}
+	return nil
+}
+
 // SetLinuxPersonality sets the Linux execution personality. For more information see the personality syscall documentation.
 // checkout getLinuxPersonalityFromStr() from libcontainer/specconv/spec_linux.go for type conversion.
 func SetLinuxPersonality(personality int) error {

tests/integration/memorypolicy.bats

@@ -0,0 +1,134 @@
+#!/usr/bin/env bats
+
+load helpers
+
+function setup() {
+	setup_busybox
+}
+
+function teardown() {
+	teardown_bundle
+}
+
+@test "runc run memory policy interleave without flags" {
+	update_config '
+        .process.args = ["/bin/sh", "-c", "head -n 1 /proc/self/numa_maps | cut -d \" \" -f 2"]
+        | .linux.memoryPolicy = {
+		"mode": "MPOL_INTERLEAVE",
+		"nodes": "0"
+	}'
+	runc run test_busybox
+	[ "$status" -eq 0 ]
+	[[ "${lines[0]}" == "interleave:0" ]]
+}
+
+@test "runc run memory policy bind static" {
+	update_config '
+        .process.args = ["/bin/sh", "-c", "head -n 1 /proc/self/numa_maps | cut -d \" \" -f 2"]
+        | .linux.memoryPolicy = {
+		"mode": "MPOL_BIND",
+		"nodes": "0",
+                "flags": ["MPOL_F_STATIC_NODES"]
+	}'
+	runc run test_busybox
+	[ "$status" -eq 0 ]
+	[[ "${lines[0]}" == "bind"*"static"*"0" ]]
+}
+
+@test "runc run and exec memory policy prefer relative" {
+	update_config '
+ 	.linux.memoryPolicy = {
+                "mode": "MPOL_PREFERRED",
+                "nodes": "0",
+                "flags": ["MPOL_F_RELATIVE_NODES"]
+	}'
+	runc run -d --console-socket "$CONSOLE_SOCKET" test_busybox
+	[ "$status" -eq 0 ]
+
+	runc exec test_busybox /bin/sh -c "head -n 1 /proc/self/numa_maps | cut -d \" \" -f 2"
+	[ "$status" -eq 0 ]
+        [[ "${lines[0]}" == "prefer"*"relative"*"0" ]]
+}
+
+@test "runc run empty memory policy" {
+        update_config '
+        .process.args = ["/bin/sh", "-c", "head -n 1 /proc/self/numa_maps | cut -d \" \" -f 2"]
+        | .linux.memoryPolicy = {
+        }'
+	runc run test_busybox
+	[ "$status" -eq 0 ]
+	[[ "${lines[0]}" == "default" ]]
+}
+
+@test "runc run memory policy with non-existing mode" {
+	update_config '
+        .process.args = ["/bin/sh", "-c", "head -n 1 /proc/self/numa_maps | cut -d \" \" -f 2"]
+	| .linux.memoryPolicy = {
+                "mode": "INTERLEAVE",
+                "nodes": "0"
+	}'
+	runc run test_busybox
+	[ "$status" -eq 1 ]
+        [[ "${lines[0]}" == *"invalid memory policy"* ]]
+}
+
+@test "runc run memory policy with invalid flag" {
+	update_config '
+        .process.args = ["/bin/sh", "-c", "head -n 1 /proc/self/numa_maps | cut -d \" \" -f 2"]
+	| .linux.memoryPolicy = {
+                "mode": "MPOL_PREFERRED",
+                "nodes": "0",
+                "flags": ["MPOL_F_RELATIVE_NODES", "badflag"]
+        }'
+	runc run test_busybox
+	[ "$status" -eq 1 ]
+        [[ "${lines[0]}" == *"invalid memory policy flag"* ]]
+}
+
+@test "runc run memory policy default with missing nodes" {
+	update_config '
+        .process.args = ["/bin/sh", "-c", "head -n 1 /proc/self/numa_maps | cut -d \" \" -f 2"]
+	| .linux.memoryPolicy = {
+                "mode": "MPOL_DEFAULT"
+        }'
+	runc run test_busybox
+	[ "$status" -eq 0 ]
+        [[ "${lines[0]}" == *"default"* ]]
+}
+
+@test "runc run memory policy with missing mode" {
+	update_config '
+        .process.args = ["/bin/sh", "-c", "head -n 1 /proc/self/numa_maps | cut -d \" \" -f 2"]
+	| .linux.memoryPolicy = {
+                "nodes": "0-7"
+        }'
+	runc run test_busybox
+	[ "$status" -eq 1 ]
+        [[ "${lines[0]}" == *"invalid memory policy mode"* ]]
+}
+
+@test "runc run memory policy calls syscall with invalid arguments" {
+	update_config '
+        .process.args = ["/bin/sh", "-c", "head -n 1 /proc/self/numa_maps | cut -d \" \" -f 2"]
+	| .linux.memoryPolicy = {
+                "mode": "MPOL_DEFAULT",
+                "nodes": "0-7",
+                "flags": ["MPOL_F_NUMA_BALANCING", "MPOL_F_STATIC_NODES", "MPOL_F_RELATIVE_NODES"]
+        }'
+	runc run test_busybox
+	[ "$status" -eq 1 ]
+        [[ "${lines[0]}" == *"set_mempolicy"*"invalid argument"* ]]
+}
+
+@test "runc run memory policy bind way too large a node number" {
+	update_config '
+        .process.args = ["/bin/sh", "-c", "head -n 1 /proc/self/numa_maps | cut -d \" \" -f 2"]
+        | .linux.memoryPolicy = {
+		"mode": "MPOL_BIND",
+		"nodes": "0-9876543210",
+                "flags": []
+	}'
+	runc run test_busybox
+	[ "$status" -eq 1 ]
+	[[ "${lines[0]}" == *"invalid memory policy node"* ]]
+}