fix: allow containers to start using a large numbers of ports

Suppose we have a compose.yaml that allocates a large numbers of ports as
follows.

```
> cat compose.yaml
services:
  svc0:
    image: alpine
    command: "sleep infinity"
    ports:
      - '32000-32060:32000-32060'
```

When we run `nerdctl compose up -d` using this compose.yaml, we will get
the following error.

```
FATA[0000] create container failed validation: containers.Labels: label key and value length (4711 bytes) greater than maximum size (4096 bytes), key: nerdctl/ports: invalid argument
FATA[0000] error while creating container haytok-svc0-1: error while creating container haytok-svc0-1: exit status 1
```

This issue is reported in the following issue.

- #4027

This issue is considered to be the same as the one with errors when
trying to perform many port mappings, such as `nerdctl run -p 80:80 -p 81:81 ~ -p 1000:1000 ...`

The current implementation is processing to create a container with the
information specified in -p to the label.
And as can be seen from the error message, as the number of ports to be
port mapped increases, the creation of the container fails because it
violates the limit of the maximum number of bytes on the containerd side
that can be allocated for a label.

Therefore, this PR modifies the container creation process so that
containers can be launched without having to assign the information
specified in the -p option to the labels.

Specifically, port mapping information is stored in the following path,
and when port mapping information is required, it is retrieved from this
file.

```
<DATAROOT>/<ADDRHASH>/containers/<NAMESPACE>/<CID>/port-mappings.json
```

Signed-off-by: Hayato Kiwata <[email protected]>

Author: haytok

cmd/nerdctl/compose/compose_port.go

@@ -88,11 +88,18 @@ func portAction(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
+	dataStore, err := clientutil.DataStore(globalOptions.DataRoot, globalOptions.Address)
+	if err != nil {
+		return err
+	}
+
 	po := composer.PortOptions{
 		ServiceName: args[0],
 		Index:       index,
 		Port:        port,
 		Protocol:    protocol,
+		DataStore:   dataStore,
+		Namespace:   globalOptions.Namespace,
 	}
 
 	return c.Port(ctx, cmd.OutOrStdout(), po)

cmd/nerdctl/compose/compose_port_linux_test.go

@@ -18,9 +18,16 @@ package compose
 
 import (
 	"fmt"
+	"strings"
 	"testing"
 
+	"gotest.tools/v3/assert"
+
+	"github.com/containerd/nerdctl/mod/tigron/expect"
+	"github.com/containerd/nerdctl/mod/tigron/test"
+
 	"github.com/containerd/nerdctl/v2/pkg/testutil"
+	"github.com/containerd/nerdctl/v2/pkg/testutil/nerdtest"
 )
 
 func TestComposePort(t *testing.T) {
@@ -75,3 +82,44 @@ services:
 	base.ComposeCmd("-f", comp.YAMLFullPath(), "port", "--protocol", "udp", "svc0", "10000").AssertFail()
 	base.ComposeCmd("-f", comp.YAMLFullPath(), "port", "--protocol", "tcp", "svc0", "10001").AssertFail()
 }
+
+// TestComposeMultiplePorts tests whether it is possible to allocate a large
+// number of ports. (https://github.com/containerd/nerdctl/issues/4027)
+func TestComposeMultiplePorts(t *testing.T) {
+	var dockerComposeYAML = fmt.Sprintf(`
+services:
+  svc0:
+    image: %s
+    command: "sleep infinity"
+    ports:
+    - '32000-32060:32000-32060'
+`, testutil.AlpineImage)
+
+	testCase := nerdtest.Setup()
+
+	testCase.Setup = func(data test.Data, helpers test.Helpers) {
+		compYamlPath := data.Temp().Save(dockerComposeYAML, "compose.yaml")
+		data.Labels().Set("composeYaml", compYamlPath)
+
+		helpers.Ensure("compose", "-f", compYamlPath, "up", "-d")
+	}
+
+	testCase.Cleanup = func(data test.Data, helpers test.Helpers) {
+		helpers.Anyhow("compose", "-f", data.Temp().Path("compose.yaml"), "down", "-v")
+	}
+
+	testCase.SubTests = []*test.Case{
+		{
+			Description: "Issue #4027 - Allocate a large number of ports.",
+			NoParallel:  true,
+			Command: func(data test.Data, helpers test.Helpers) test.TestableCommand {
+				return helpers.Command("compose", "-f", data.Labels().Get("composeYaml"), "port", "svc0", "32000")
+			},
+			Expected: test.Expects(expect.ExitCodeSuccess, nil, func(stdout, info string, t *testing.T) {
+				assert.Assert(t, strings.Contains(stdout, "0.0.0.0:32000"))
+			}),
+		},
+	}
+
+	testCase.Run(t)
+}

cmd/nerdctl/compose/compose_ps.go

@@ -32,6 +32,7 @@ import (
 	"github.com/containerd/log"
 
 	"github.com/containerd/nerdctl/v2/cmd/nerdctl/helpers"
+	"github.com/containerd/nerdctl/v2/pkg/api/types"
 	"github.com/containerd/nerdctl/v2/pkg/clientutil"
 	"github.com/containerd/nerdctl/v2/pkg/cmd/compose"
 	"github.com/containerd/nerdctl/v2/pkg/containerutil"
@@ -183,9 +184,9 @@ func psAction(cmd *cobra.Command, args []string) error {
 			var p composeContainerPrintable
 			var err error
 			if format == "json" {
-				p, err = composeContainerPrintableJSON(ctx, container)
+				p, err = composeContainerPrintableJSON(ctx, container, globalOptions)
 			} else {
-				p, err = composeContainerPrintableTab(ctx, container)
+				p, err = composeContainerPrintableTab(ctx, container, globalOptions)
 			}
 			if err != nil {
 				return err
@@ -234,7 +235,7 @@ func psAction(cmd *cobra.Command, args []string) error {
 
 // composeContainerPrintableTab constructs composeContainerPrintable with fields
 // only for console output.
-func composeContainerPrintableTab(ctx context.Context, container containerd.Container) (composeContainerPrintable, error) {
+func composeContainerPrintableTab(ctx context.Context, container containerd.Container, gOptions types.GlobalCommandOptions) (composeContainerPrintable, error) {
 	info, err := container.Info(ctx, containerd.WithoutRefreshedMetadata)
 	if err != nil {
 		return composeContainerPrintable{}, err
@@ -251,20 +252,28 @@ func composeContainerPrintableTab(ctx context.Context, container containerd.Cont
 	if err != nil {
 		return composeContainerPrintable{}, err
 	}
+	dataStore, err := clientutil.DataStore(gOptions.DataRoot, gOptions.Address)
+	if err != nil {
+		return composeContainerPrintable{}, err
+	}
+	ports, err := portutil.LoadPortMappings(dataStore, gOptions.Namespace, info.ID)
+	if err != nil {
+		return composeContainerPrintable{}, err
+	}
 
 	return composeContainerPrintable{
 		Name:    info.Labels[labels.Name],
 		Image:   image.Metadata().Name,
 		Command: formatter.InspectContainerCommandTrunc(spec),
 		Service: info.Labels[labels.ComposeService],
 		State:   status,
-		Ports:   formatter.FormatPorts(info.Labels),
+		Ports:   formatter.FormatPorts(ports),
 	}, nil
 }
 
 // composeContainerPrintableJSON constructs composeContainerPrintable with fields
 // only for json output and compatible docker output.
-func composeContainerPrintableJSON(ctx context.Context, container containerd.Container) (composeContainerPrintable, error) {
+func composeContainerPrintableJSON(ctx context.Context, container containerd.Container, gOptions types.GlobalCommandOptions) (composeContainerPrintable, error) {
 	info, err := container.Info(ctx, containerd.WithoutRefreshedMetadata)
 	if err != nil {
 		return composeContainerPrintable{}, err
@@ -294,6 +303,14 @@ func composeContainerPrintableJSON(ctx context.Context, container containerd.Con
 	if err != nil {
 		return composeContainerPrintable{}, err
 	}
+	dataStore, err := clientutil.DataStore(gOptions.DataRoot, gOptions.Address)
+	if err != nil {
+		return composeContainerPrintable{}, err
+	}
+	portsJSON, err := portutil.LoadPortMappingsData(dataStore, gOptions.Namespace, info.ID)
+	if err != nil {
+		return composeContainerPrintable{}, err
+	}
 
 	return composeContainerPrintable{
 		ID:         container.ID(),
@@ -305,7 +322,7 @@ func composeContainerPrintableJSON(ctx context.Context, container containerd.Con
 		State:      state,
 		Health:     "",
 		ExitCode:   exitCode,
-		Publishers: formatPublishers(info.Labels),
+		Publishers: formatPublishers(portsJSON),
 	}, nil
 }
 
@@ -321,7 +338,7 @@ type PortPublisher struct {
 
 // formatPublishers parses and returns docker-compatible []PortPublisher from
 // label map. If an error happens, an empty slice is returned.
-func formatPublishers(labelMap map[string]string) []PortPublisher {
+func formatPublishers(portsJSON string) []PortPublisher {
 	mapper := func(pm cni.PortMapping) PortPublisher {
 		return PortPublisher{
 			URL:           pm.HostIP,
@@ -332,7 +349,7 @@ func formatPublishers(labelMap map[string]string) []PortPublisher {
 	}
 
 	var dockerPorts []PortPublisher
-	if portMappings, err := portutil.ParsePortsLabel(labelMap); err == nil {
+	if portMappings, err := portutil.ParsePortsLabel(portsJSON); err == nil {
 		for _, p := range portMappings {
 			dockerPorts = append(dockerPorts, mapper(p))
 		}

cmd/nerdctl/container/container_port.go

@@ -29,6 +29,7 @@ import (
 	"github.com/containerd/nerdctl/v2/pkg/clientutil"
 	"github.com/containerd/nerdctl/v2/pkg/containerutil"
 	"github.com/containerd/nerdctl/v2/pkg/idutil/containerwalker"
+	"github.com/containerd/nerdctl/v2/pkg/portutil"
 )
 
 func PortCommand() *cobra.Command {
@@ -81,13 +82,22 @@ func portAction(cmd *cobra.Command, args []string) error {
 	}
 	defer cancel()
 
+	dataStore, err := clientutil.DataStore(globalOptions.DataRoot, globalOptions.Address)
+	if err != nil {
+		return err
+	}
+
 	walker := &containerwalker.ContainerWalker{
 		Client: client,
 		OnFound: func(ctx context.Context, found containerwalker.Found) error {
 			if found.MatchCount > 1 {
 				return fmt.Errorf("multiple IDs found with provided prefix: %s", found.Req)
 			}
-			return containerutil.PrintHostPort(ctx, cmd.OutOrStdout(), found.Container, argPort, argProto)
+			ports, err := portutil.LoadPortMappings(dataStore, globalOptions.Namespace, found.Container.ID())
+			if err != nil {
+				return err
+			}
+			return containerutil.PrintHostPort(ctx, cmd.OutOrStdout(), found.Container, argPort, argProto, ports)
 		},
 	}
 	req := args[0]

cmd/nerdctl/container/container_run_network_linux_test.go

@@ -36,7 +36,6 @@ import (
 
 	"github.com/containerd/containerd/v2/defaults"
 	"github.com/containerd/containerd/v2/pkg/netns"
-	"github.com/containerd/errdefs"
 	"github.com/containerd/nerdctl/mod/tigron/expect"
 	"github.com/containerd/nerdctl/mod/tigron/require"
 	"github.com/containerd/nerdctl/mod/tigron/test"
@@ -366,8 +365,8 @@ func TestRunWithInvalidPortThenCleanUp(t *testing.T) {
 			},
 			Expected: func(data test.Data, helpers test.Helpers) *test.Expected {
 				return &test.Expected{
-					ExitCode: 1,
-					Errors:   []error{errdefs.ErrInvalidArgument},
+					ExitCode: 0,
+					Errors:   []error{},
 					Output: func(stdout string, info string, t *testing.T) {
 						getAddrHash := func(addr string) string {
 							const addrHashLen = 8

pkg/cmd/container/create.go

@@ -36,7 +36,6 @@ import (
 	"github.com/containerd/containerd/v2/core/containers"
 	"github.com/containerd/containerd/v2/pkg/cio"
 	"github.com/containerd/containerd/v2/pkg/oci"
-	"github.com/containerd/go-cni"
 	"github.com/containerd/log"
 
 	"github.com/containerd/nerdctl/v2/pkg/annotations"
@@ -58,6 +57,7 @@ import (
 	"github.com/containerd/nerdctl/v2/pkg/mountutil"
 	"github.com/containerd/nerdctl/v2/pkg/namestore"
 	"github.com/containerd/nerdctl/v2/pkg/platformutil"
+	"github.com/containerd/nerdctl/v2/pkg/portutil"
 	"github.com/containerd/nerdctl/v2/pkg/referenceutil"
 	"github.com/containerd/nerdctl/v2/pkg/rootlessutil"
 	"github.com/containerd/nerdctl/v2/pkg/store"
@@ -379,6 +379,11 @@ func Create(ctx context.Context, client *containerd.Client, args []string, netMa
 	}
 	cOpts = append(cOpts, ilOpt)
 
+	err = portutil.GeneratePortMappingsConfig(dataStore, options.GOptions.Namespace, id, netLabelOpts.PortMappings)
+	if err != nil {
+		return nil, generateRemoveOrphanedDirsFunc(ctx, id, dataStore, internalLabels), fmt.Errorf("Error writing to port-mappings.json: %v", err)
+	}
+
 	opts = append(opts, propagateInternalContainerdLabelsToOCIAnnotations(),
 		oci.WithAnnotations(strutil.ConvertKVStringsToMap(options.Annotations)))
 
@@ -678,7 +683,6 @@ type internalLabels struct {
 	networks             []string
 	ipAddress            string
 	ip6Address           string
-	ports                []cni.PortMapping
 	macAddress           string
 	dnsServers           []string
 	dnsSearchDomains     []string
@@ -730,13 +734,6 @@ func withInternalLabels(internalLabels internalLabels) (containerd.NewContainerO
 		return nil, err
 	}
 	m[labels.Networks] = string(networksJSON)
-	if len(internalLabels.ports) > 0 {
-		portsJSON, err := json.Marshal(internalLabels.ports)
-		if err != nil {
-			return nil, err
-		}
-		m[labels.Ports] = string(portsJSON)
-	}
 	if internalLabels.logURI != "" {
 		m[labels.LogURI] = internalLabels.logURI
 		logConfigJSON, err := json.Marshal(internalLabels.logConfig)
@@ -838,7 +835,6 @@ func withInternalLabels(internalLabels internalLabels) (containerd.NewContainerO
 func (il *internalLabels) loadNetOpts(opts types.NetworkOptions) {
 	il.hostname = opts.Hostname
 	il.domainname = opts.Domainname
-	il.ports = opts.PortMappings
 	il.ipAddress = opts.IPAddress
 	il.ip6Address = opts.IP6Address
 	il.networks = opts.NetworkSlice

pkg/cmd/container/inspect.go

@@ -25,27 +25,36 @@ import (
 	"github.com/containerd/containerd/v2/core/snapshots"
 
 	"github.com/containerd/nerdctl/v2/pkg/api/types"
+	"github.com/containerd/nerdctl/v2/pkg/clientutil"
 	"github.com/containerd/nerdctl/v2/pkg/containerdutil"
 	"github.com/containerd/nerdctl/v2/pkg/containerinspector"
 	"github.com/containerd/nerdctl/v2/pkg/idutil/containerwalker"
 	"github.com/containerd/nerdctl/v2/pkg/imgutil"
 	"github.com/containerd/nerdctl/v2/pkg/inspecttypes/dockercompat"
+	"github.com/containerd/nerdctl/v2/pkg/portutil"
 )
 
 // Inspect prints detailed information for each container in `containers`.
 func Inspect(ctx context.Context, client *containerd.Client, containers []string, options types.ContainerInspectOptions) ([]any, error) {
+	dataStore, err := clientutil.DataStore(options.GOptions.DataRoot, options.GOptions.Address)
+	if err != nil {
+		return []any{}, err
+	}
+
 	f := &containerInspector{
 		mode:        options.Mode,
 		size:        options.Size,
 		snapshotter: containerdutil.SnapshotService(client, options.GOptions.Snapshotter),
+		dataStore:   dataStore,
+		namespace:   options.GOptions.Namespace,
 	}
 
 	walker := &containerwalker.ContainerWalker{
 		Client:  client,
 		OnFound: f.Handler,
 	}
 
-	err := walker.WalkAll(ctx, containers, true)
+	err = walker.WalkAll(ctx, containers, true)
 	if err != nil {
 		return []any{}, err
 	}
@@ -58,6 +67,8 @@ type containerInspector struct {
 	size        bool
 	snapshotter snapshots.Snapshotter
 	entries     []interface{}
+	dataStore   string
+	namespace   string
 }
 
 func (x *containerInspector) Handler(ctx context.Context, found containerwalker.Found) error {
@@ -68,6 +79,15 @@ func (x *containerInspector) Handler(ctx context.Context, found containerwalker.
 	if err != nil {
 		return err
 	}
+
+	ports, err := portutil.LoadPortMappings(x.dataStore, x.namespace, n.ID)
+	if err != nil {
+		return err
+	}
+	if n.Process.NetNS != nil && len(ports) > 0 {
+		n.Process.NetNS.PortMappings = ports
+	}
+
 	switch x.mode {
 	case "native":
 		x.entries = append(x.entries, n)

pkg/cmd/container/kill.go

@@ -33,6 +33,7 @@ import (
 	"github.com/containerd/log"
 
 	"github.com/containerd/nerdctl/v2/pkg/api/types"
+	"github.com/containerd/nerdctl/v2/pkg/clientutil"
 	"github.com/containerd/nerdctl/v2/pkg/containerutil"
 	"github.com/containerd/nerdctl/v2/pkg/idutil/containerwalker"
 	"github.com/containerd/nerdctl/v2/pkg/labels"
@@ -122,14 +123,14 @@ func killContainer(ctx context.Context, container containerd.Container, signal s
 // cleanupNetwork removes cni network setup, specifically the forwards
 func cleanupNetwork(ctx context.Context, container containerd.Container, globalOpts types.GlobalCommandOptions) error {
 	return rootlessutil.WithDetachedNetNSIfAny(func() error {
-		// retrieve info to get current active port mappings
-		info, err := container.Info(ctx, containerd.WithoutRefreshedMetadata)
+		// retrieve current active port mappings
+		dataStore, err := clientutil.DataStore(globalOpts.DataRoot, globalOpts.Address)
 		if err != nil {
 			return err
 		}
-		ports, portErr := portutil.ParsePortsLabel(info.Labels)
-		if portErr != nil {
-			return fmt.Errorf("no oci spec: %q", portErr)
+		ports, err := portutil.LoadPortMappings(dataStore, globalOpts.Namespace, container.ID())
+		if err != nil {
+			return fmt.Errorf("no oci spec: %q", err)
 		}
 		portMappings := []cni.NamespaceOpts{
 			cni.WithCapabilityPortMap(ports),