Skip to content

Commit 0c8723a

Browse files
elezarelezar
committed
Add a hook to disable device node creation in a container
If required, this hook creates a modified params file (with ModifyDeviceFiles: 0) in a tmpfs and mounts this over /proc/driver/nvidia/params. This prevents device node creation when running tools such as nvidia-smi. In general the creation of these devices is cosmetic as a container does not have the required cgroup access for the devices. Signed-off-by: Evan Lezar <[email protected]>
1 parent fdcd250 commit 0c8723a

File tree

5 files changed

+327
-0
lines changed

5 files changed

+327
-0
lines changed

‎cmd/nvidia-cdi-hook/commands/commands.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ import (
2222
"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-cdi-hook/chmod"
2323
symlinks "github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-cdi-hook/create-symlinks"
2424
"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-cdi-hook/cudacompat"
25+
disabledevicenodemodification "github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-cdi-hook/disable-device-node-modification"
2526
ldcache "github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-cdi-hook/update-ldcache"
2627
"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
2728
)
@@ -34,6 +35,7 @@ func New(logger logger.Interface) []*cli.Command {
3435
symlinks.NewCommand(logger),
3536
chmod.NewCommand(logger),
3637
cudacompat.NewCommand(logger),
38+
disabledevicenodemodification.NewCommand(logger),
3739
}
3840
}
3941

‎cmd/nvidia-cdi-hook/disable-device-node-modification/disable-device-node-modification.go‎

Lines changed: 144 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,144 @@
1+
/**
2+
# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3+
# SPDX-License-Identifier: Apache-2.0
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
**/
17+
18+
package disabledevicenodemodification
19+
20+
import (
21+
"bufio"
22+
"bytes"
23+
"errors"
24+
"fmt"
25+
"io"
26+
"os"
27+
"strings"
28+
29+
"github.com/urfave/cli/v2"
30+
31+
"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
32+
"github.com/NVIDIA/nvidia-container-toolkit/internal/oci"
33+
)
34+
35+
const (
36+
nvidiaDriverParamsPath = "/proc/driver/nvidia/params"
37+
)
38+
39+
type options struct {
40+
containerSpec string
41+
}
42+
43+
// NewCommand constructs an disable-device-node-modification subcommand with the specified logger
44+
func NewCommand(logger logger.Interface) *cli.Command {
45+
cfg := options{}
46+
47+
c := cli.Command{
48+
Name: "disable-device-node-modification",
49+
Usage: "Ensure that the /proc/driver/nvidia/params file present in the container does not allow device node modifications.",
50+
Before: func(c *cli.Context) error {
51+
return validateFlags(c, &cfg)
52+
},
53+
Action: func(c *cli.Context) error {
54+
return run(c, &cfg)
55+
},
56+
}
57+
58+
c.Flags = []cli.Flag{
59+
&cli.StringFlag{
60+
Name: "container-spec",
61+
Hidden: true,
62+
Usage: "Specify the path to the OCI container spec. If empty or '-' the spec will be read from STDIN",
63+
Destination: &cfg.containerSpec,
64+
},
65+
}
66+
67+
return &c
68+
}
69+
70+
func validateFlags(c *cli.Context, cfg *options) error {
71+
return nil
72+
}
73+
74+
func run(_ *cli.Context, cfg *options) error {
75+
modifiedParamsFileContents, err := getModifiedNVIDIAParamsContents()
76+
if err != nil {
77+
return fmt.Errorf("failed to get modified params file contents: %w", err)
78+
}
79+
if len(modifiedParamsFileContents) == 0 {
80+
return nil
81+
}
82+
83+
s, err := oci.LoadContainerState(cfg.containerSpec)
84+
if err != nil {
85+
return fmt.Errorf("failed to load container state: %w", err)
86+
}
87+
88+
containerRootDirPath, err := s.GetContainerRoot()
89+
if err != nil {
90+
return fmt.Errorf("failed to determined container root: %w", err)
91+
}
92+
93+
return createParamsFileInContainer(containerRootDirPath, modifiedParamsFileContents)
94+
}
95+
96+
func getModifiedNVIDIAParamsContents() ([]byte, error) {
97+
hostNvidiaParamsFile, err := os.Open(nvidiaDriverParamsPath)
98+
if errors.Is(err, os.ErrNotExist) {
99+
return nil, nil
100+
}
101+
if err != nil {
102+
return nil, fmt.Errorf("failed to load params file: %w", err)
103+
}
104+
defer hostNvidiaParamsFile.Close()
105+
106+
modifiedContents, err := getModifiedParamsFileContentsFromReader(hostNvidiaParamsFile)
107+
if err != nil {
108+
return nil, fmt.Errorf("failed to get modfied params file contents: %w", err)
109+
}
110+
111+
return modifiedContents, nil
112+
}
113+
114+
// getModifiedParamsFileContentsFromReader returns the contents of a modified params file from the specified reader.
115+
func getModifiedParamsFileContentsFromReader(r io.Reader) ([]byte, error) {
116+
var modified bytes.Buffer
117+
scanner := bufio.NewScanner(r)
118+
119+
var requiresModification bool
120+
for scanner.Scan() {
121+
line := scanner.Text()
122+
if strings.HasPrefix(line, "ModifyDeviceFiles: ") {
123+
if line == "ModifyDeviceFiles: 0" {
124+
return nil, nil
125+
}
126+
if line == "ModifyDeviceFiles: 1" {
127+
line = "ModifyDeviceFiles: 0"
128+
requiresModification = true
129+
}
130+
}
131+
if _, err := modified.WriteString(line + "\n"); err != nil {
132+
return nil, fmt.Errorf("failed to create output buffer: %w", err)
133+
}
134+
}
135+
if err := scanner.Err(); err != nil {
136+
return nil, fmt.Errorf("failed to read params file: %w", err)
137+
}
138+
139+
if !requiresModification {
140+
return nil, nil
141+
}
142+
143+
return modified.Bytes(), nil
144+
}

‎cmd/nvidia-cdi-hook/disable-device-node-modification/disable-device-node-modification_test.go‎

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
/**
2+
# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3+
# SPDX-License-Identifier: Apache-2.0
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
**/
17+
18+
package disabledevicenodemodification
19+
20+
import (
21+
"bytes"
22+
"testing"
23+
24+
"github.com/stretchr/testify/require"
25+
)
26+
27+
func TestGetModifiedParamsFileContentsFromReader(t *testing.T) {
28+
testCases := map[string]struct {
29+
contents []byte
30+
expectedError error
31+
expectedContents []byte
32+
}{
33+
"no contents": {
34+
contents: nil,
35+
expectedError: nil,
36+
expectedContents: nil,
37+
},
38+
"other contents are ignored": {
39+
contents: []byte(`# Some other content
40+
that we don't care about
41+
`),
42+
expectedError: nil,
43+
expectedContents: nil,
44+
},
45+
"already zero requires no modification": {
46+
contents: []byte("ModifyDeviceFiles: 0"),
47+
expectedError: nil,
48+
expectedContents: nil,
49+
},
50+
"leading spaces require no modification": {
51+
contents: []byte(" ModifyDeviceFiles: 1"),
52+
},
53+
"Trailing spaces require no modification": {
54+
contents: []byte("ModifyDeviceFiles: 1 "),
55+
},
56+
"Not 1 require no modification": {
57+
contents: []byte("ModifyDeviceFiles: 11"),
58+
},
59+
"single line requires modification": {
60+
contents: []byte("ModifyDeviceFiles: 1"),
61+
expectedError: nil,
62+
expectedContents: []byte("ModifyDeviceFiles: 0\n"),
63+
},
64+
"single line with trailing newline requires modification": {
65+
contents: []byte("ModifyDeviceFiles: 1\n"),
66+
expectedError: nil,
67+
expectedContents: []byte("ModifyDeviceFiles: 0\n"),
68+
},
69+
"other content is maintained": {
70+
contents: []byte(`ModifyDeviceFiles: 1
71+
other content
72+
that
73+
is maintained`),
74+
expectedError: nil,
75+
expectedContents: []byte(`ModifyDeviceFiles: 0
76+
other content
77+
that
78+
is maintained
79+
`),
80+
},
81+
}
82+
83+
for description, tc := range testCases {
84+
t.Run(description, func(t *testing.T) {
85+
contents, err := getModifiedParamsFileContentsFromReader(bytes.NewReader(tc.contents))
86+
require.EqualValues(t, tc.expectedError, err)
87+
require.EqualValues(t, string(tc.expectedContents), string(contents))
88+
})
89+
}
90+
91+
}

‎cmd/nvidia-cdi-hook/disable-device-node-modification/params_linux.go‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
//go:build linux
2+
3+
/**
4+
# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
5+
# SPDX-License-Identifier: Apache-2.0
6+
#
7+
# Licensed under the Apache License, Version 2.0 (the "License");
8+
# you may not use this file except in compliance with the License.
9+
# You may obtain a copy of the License at
10+
#
11+
# http://www.apache.org/licenses/LICENSE-2.0
12+
#
13+
# Unless required by applicable law or agreed to in writing, software
14+
# distributed under the License is distributed on an "AS IS" BASIS,
15+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16+
# See the License for the specific language governing permissions and
17+
# limitations under the License.
18+
**/
19+
20+
package disabledevicenodemodification
21+
22+
import (
23+
"fmt"
24+
"os"
25+
"path/filepath"
26+
27+
"github.com/opencontainers/runc/libcontainer/utils"
28+
"golang.org/x/sys/unix"
29+
)
30+
31+
func createParamsFileInContainer(containerRootDirPath string, contents []byte) error {
32+
tmpRoot, err := os.MkdirTemp("", "nvct-empty-dir*")
33+
if err != nil {
34+
return fmt.Errorf("failed to create temp root: %w", err)
35+
}
36+
37+
if err := createTmpFs(tmpRoot, len(contents)); err != nil {
38+
return fmt.Errorf("failed to create tmpfs mount for params file: %w", err)
39+
}
40+
41+
modifiedParamsFile, err := os.OpenFile(filepath.Join(tmpRoot, "nvct-params"), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0444)
42+
if err != nil {
43+
return fmt.Errorf("failed to open modified params file: %w", err)
44+
}
45+
defer modifiedParamsFile.Close()
46+
47+
if _, err := modifiedParamsFile.Write(contents); err != nil {
48+
return fmt.Errorf("failed to write temporary params file: %w", err)
49+
}
50+
51+
err = utils.WithProcfd(containerRootDirPath, nvidiaDriverParamsPath, func(nvidiaDriverParamsFdPath string) error {
52+
return unix.Mount(modifiedParamsFile.Name(), nvidiaDriverParamsFdPath, "", unix.MS_BIND|unix.MS_RDONLY|unix.MS_NODEV|unix.MS_PRIVATE|unix.MS_NOSYMFOLLOW, "")
53+
})
54+
if err != nil {
55+
return fmt.Errorf("failed to mount modified params file: %w", err)
56+
}
57+
58+
return nil
59+
}
60+
61+
func createTmpFs(target string, size int) error {
62+
return unix.Mount("tmpfs", target, "tmpfs", 0, fmt.Sprintf("size=%d", size))
63+
}

‎cmd/nvidia-cdi-hook/disable-device-node-modification/params_other.go‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
//go:build !linux
2+
// +build !linux
3+
4+
/**
5+
# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
6+
# SPDX-License-Identifier: Apache-2.0
7+
#
8+
# Licensed under the Apache License, Version 2.0 (the "License");
9+
# you may not use this file except in compliance with the License.
10+
# You may obtain a copy of the License at
11+
#
12+
# http://www.apache.org/licenses/LICENSE-2.0
13+
#
14+
# Unless required by applicable law or agreed to in writing, software
15+
# distributed under the License is distributed on an "AS IS" BASIS,
16+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17+
# See the License for the specific language governing permissions and
18+
# limitations under the License.
19+
**/
20+
21+
package disabledevicenodemodification
22+
23+
import "fmt"
24+
25+
func createParamsFileInContainer(containerRootDirPath string, contents []byte) error {
26+
return fmt.Errorf("not supported")
27+
}

0 commit comments

Comments
 (0)