Skip to content

Commit becddb7

Browse files
elezarelezar
authored
Merge pull request #1082 from elezar/backport-add-cuda-compat-mode
Add cuda-compat-mode config option
2 parents f8b0b43 + 8069346 commit becddb7

File tree

8 files changed

+119
-22
lines changed

8 files changed

+119
-22
lines changed

‎cmd/nvidia-container-runtime-hook/hook_config.go‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -104,3 +104,26 @@ func (c *hookConfig) getSwarmResourceEnvvars() []string {
104104

105105
return envvars
106106
}
107+
108+
// nvidiaContainerCliCUDACompatModeFlags returns required --cuda-compat-mode
109+
// flag(s) depending on the hook and runtime configurations.
110+
func (c *hookConfig) nvidiaContainerCliCUDACompatModeFlags() []string {
111+
var flag string
112+
switch c.NVIDIAContainerRuntimeConfig.Modes.Legacy.CUDACompatMode {
113+
case config.CUDACompatModeLdconfig:
114+
flag = "--cuda-compat-mode=ldconfig"
115+
case config.CUDACompatModeMount:
116+
flag = "--cuda-compat-mode=mount"
117+
case config.CUDACompatModeDisabled, config.CUDACompatModeHook:
118+
flag = "--cuda-compat-mode=disabled"
119+
default:
120+
if !c.Features.AllowCUDACompatLibsFromContainer.IsEnabled() {
121+
flag = "--cuda-compat-mode=disabled"
122+
}
123+
}
124+
125+
if flag == "" {
126+
return nil
127+
}
128+
return []string{flag}
129+
}

‎cmd/nvidia-container-runtime-hook/main.go‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -114,9 +114,8 @@ func doPrestart() {
114114
}
115115
args = append(args, "configure")
116116

117-
if !hook.Features.AllowCUDACompatLibsFromContainer.IsEnabled() {
118-
args = append(args, "--no-cntlibs")
119-
}
117+
args = append(args, hook.nvidiaContainerCliCUDACompatModeFlags()...)
118+
120119
if ldconfigPath := cli.NormalizeLDConfigPath(); ldconfigPath != "" {
121120
args = append(args, fmt.Sprintf("--ldconfig=%s", ldconfigPath))
122121
}

‎internal/config/config.go‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -121,6 +121,9 @@ func GetDefault() (*Config, error) {
121121
AnnotationPrefixes: []string{cdi.AnnotationPrefix},
122122
SpecDirs: cdi.DefaultSpecDirs,
123123
},
124+
Legacy: legacyModeConfig{
125+
CUDACompatMode: defaultCUDACompatMode,
126+
},
124127
},
125128
},
126129
NVIDIAContainerRuntimeHookConfig: RuntimeHookConfig{

‎internal/config/config_test.go‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,9 @@ func TestGetConfig(t *testing.T) {
7474
AnnotationPrefixes: []string{"cdi.k8s.io/"},
7575
SpecDirs: []string{"/etc/cdi", "/var/run/cdi"},
7676
},
77+
Legacy: legacyModeConfig{
78+
CUDACompatMode: "ldconfig",
79+
},
7780
},
7881
},
7982
NVIDIAContainerRuntimeHookConfig: RuntimeHookConfig{
@@ -93,6 +96,7 @@ func TestGetConfig(t *testing.T) {
9396
"nvidia-container-cli.load-kmods = false",
9497
"nvidia-container-cli.ldconfig = \"@/foo/bar/ldconfig\"",
9598
"nvidia-container-cli.user = \"foo:bar\"",
99+
"nvidia-container-cli.cuda-compat-mode = \"mount\"",
96100
"nvidia-container-runtime.debug = \"/foo/bar\"",
97101
"nvidia-container-runtime.discover-mode = \"not-legacy\"",
98102
"nvidia-container-runtime.log-level = \"debug\"",
@@ -102,6 +106,7 @@ func TestGetConfig(t *testing.T) {
102106
"nvidia-container-runtime.modes.cdi.annotation-prefixes = [\"cdi.k8s.io/\", \"example.vendor.com/\",]",
103107
"nvidia-container-runtime.modes.cdi.spec-dirs = [\"/except/etc/cdi\", \"/not/var/run/cdi\",]",
104108
"nvidia-container-runtime.modes.csv.mount-spec-path = \"/not/etc/nvidia-container-runtime/host-files-for-container.d\"",
109+
"nvidia-container-runtime.modes.legacy.cuda-compat-mode = \"mount\"",
105110
"nvidia-container-runtime-hook.path = \"/foo/bar/nvidia-container-runtime-hook\"",
106111
"nvidia-ctk.path = \"/foo/bar/nvidia-ctk\"",
107112
},
@@ -134,6 +139,9 @@ func TestGetConfig(t *testing.T) {
134139
"/not/var/run/cdi",
135140
},
136141
},
142+
Legacy: legacyModeConfig{
143+
CUDACompatMode: "mount",
144+
},
137145
},
138146
},
139147
NVIDIAContainerRuntimeHookConfig: RuntimeHookConfig{
@@ -178,6 +186,9 @@ func TestGetConfig(t *testing.T) {
178186
"/var/run/cdi",
179187
},
180188
},
189+
Legacy: legacyModeConfig{
190+
CUDACompatMode: "ldconfig",
191+
},
181192
},
182193
},
183194
NVIDIAContainerRuntimeHookConfig: RuntimeHookConfig{
@@ -200,6 +211,7 @@ func TestGetConfig(t *testing.T) {
200211
"root = \"/bar/baz\"",
201212
"load-kmods = false",
202213
"ldconfig = \"@/foo/bar/ldconfig\"",
214+
"cuda-compat-mode = \"mount\"",
203215
"user = \"foo:bar\"",
204216
"[nvidia-container-runtime]",
205217
"debug = \"/foo/bar\"",
@@ -213,6 +225,8 @@ func TestGetConfig(t *testing.T) {
213225
"spec-dirs = [\"/except/etc/cdi\", \"/not/var/run/cdi\",]",
214226
"[nvidia-container-runtime.modes.csv]",
215227
"mount-spec-path = \"/not/etc/nvidia-container-runtime/host-files-for-container.d\"",
228+
"[nvidia-container-runtime.modes.legacy]",
229+
"cuda-compat-mode = \"mount\"",
216230
"[nvidia-container-runtime-hook]",
217231
"path = \"/foo/bar/nvidia-container-runtime-hook\"",
218232
"[nvidia-ctk]",
@@ -247,6 +261,9 @@ func TestGetConfig(t *testing.T) {
247261
"/not/var/run/cdi",
248262
},
249263
},
264+
Legacy: legacyModeConfig{
265+
CUDACompatMode: "mount",
266+
},
250267
},
251268
},
252269
NVIDIAContainerRuntimeHookConfig: RuntimeHookConfig{
@@ -283,6 +300,9 @@ func TestGetConfig(t *testing.T) {
283300
AnnotationPrefixes: []string{"cdi.k8s.io/"},
284301
SpecDirs: []string{"/etc/cdi", "/var/run/cdi"},
285302
},
303+
Legacy: legacyModeConfig{
304+
CUDACompatMode: "ldconfig",
305+
},
286306
},
287307
},
288308
NVIDIAContainerRuntimeHookConfig: RuntimeHookConfig{
@@ -322,6 +342,9 @@ func TestGetConfig(t *testing.T) {
322342
AnnotationPrefixes: []string{"cdi.k8s.io/"},
323343
SpecDirs: []string{"/etc/cdi", "/var/run/cdi"},
324344
},
345+
Legacy: legacyModeConfig{
346+
CUDACompatMode: "ldconfig",
347+
},
325348
},
326349
},
327350
NVIDIAContainerRuntimeHookConfig: RuntimeHookConfig{

‎internal/config/runtime.go‎

Lines changed: 31 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,8 +29,9 @@ type RuntimeConfig struct {
2929

3030
// modesConfig defines (optional) per-mode configs
3131
type modesConfig struct {
32-
CSV csvModeConfig `toml:"csv"`
33-
CDI cdiModeConfig `toml:"cdi"`
32+
CSV csvModeConfig `toml:"csv"`
33+
CDI cdiModeConfig `toml:"cdi"`
34+
Legacy legacyModeConfig `toml:"legacy"`
3435
}
3536

3637
type cdiModeConfig struct {
@@ -45,3 +46,31 @@ type cdiModeConfig struct {
4546
type csvModeConfig struct {
4647
MountSpecPath string `toml:"mount-spec-path"`
4748
}
49+
50+
type legacyModeConfig struct {
51+
// CUDACompatMode sets the mode to be used to make CUDA Forward Compat
52+
// libraries discoverable in the container.
53+
CUDACompatMode cudaCompatMode `toml:"cuda-compat-mode,omitempty"`
54+
}
55+
56+
type cudaCompatMode string
57+
58+
const (
59+
defaultCUDACompatMode = CUDACompatModeLdconfig
60+
// CUDACompatModeDisabled explicitly disables the handling of CUDA Forward
61+
// Compatibility in the NVIDIA Container Runtime and NVIDIA Container
62+
// Runtime Hook.
63+
CUDACompatModeDisabled = cudaCompatMode("disabled")
64+
// CUDACompatModeHook uses a container lifecycle hook to implement CUDA
65+
// Forward Compatibility support. This requires the use of the NVIDIA
66+
// Container Runtime and is not compatible with use cases where only the
67+
// NVIDIA Container Runtime Hook is used (e.g. the Docker --gpus flag).
68+
CUDACompatModeHook = cudaCompatMode("hook")
69+
// CUDACompatModeLdconfig adds the folders containing CUDA Forward Compat
70+
// libraries to the ldconfig command invoked from the NVIDIA Container
71+
// Runtime Hook.
72+
CUDACompatModeLdconfig = cudaCompatMode("ldconfig")
73+
// CUDACompatModeMount mounts CUDA Forward Compat folders from the container
74+
// to the container when using the NVIDIA Container Runtime Hook.
75+
CUDACompatModeMount = cudaCompatMode("mount")
76+
)

‎internal/config/toml_test.go‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,9 @@ spec-dirs = ["/etc/cdi", "/var/run/cdi"]
7474
[nvidia-container-runtime.modes.csv]
7575
mount-spec-path = "/etc/nvidia-container-runtime/host-files-for-container.d"
7676
77+
[nvidia-container-runtime.modes.legacy]
78+
cuda-compat-mode = "ldconfig"
79+
7780
[nvidia-container-runtime-hook]
7881
path = "nvidia-container-runtime-hook"
7982
skip-mode-detection = false

‎internal/modifier/gated.go‎

Lines changed: 33 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -79,24 +79,41 @@ func NewFeatureGatedModifier(logger logger.Interface, cfg *config.Config, image
7979
discoverers = append(discoverers, d)
8080
}
8181

82-
if !cfg.Features.AllowCUDACompatLibsFromContainer.IsEnabled() && !cfg.Features.DisableCUDACompatLibHook.IsEnabled() {
83-
compatLibHookDiscoverer := discover.NewCUDACompatHookDiscoverer(logger, cfg.NVIDIACTKConfig.Path, driver)
84-
discoverers = append(discoverers, compatLibHookDiscoverer)
85-
// For legacy mode, we also need to inject a hook to update the LDCache
86-
// after we have modifed the configuration.
87-
if cfg.NVIDIAContainerRuntimeConfig.Mode == "legacy" {
88-
ldcacheUpdateHookDiscoverer, err := discover.NewLDCacheUpdateHook(
89-
logger,
90-
discover.None{},
91-
cfg.NVIDIACTKConfig.Path,
92-
"",
93-
)
94-
if err != nil {
95-
return nil, fmt.Errorf("failed to construct ldcache update discoverer: %w", err)
96-
}
97-
discoverers = append(discoverers, ldcacheUpdateHookDiscoverer)
82+
// If the feature flag has explicitly been toggled, we don't make any modification.
83+
if !cfg.Features.DisableCUDACompatLibHook.IsEnabled() {
84+
cudaCompatDiscoverer, err := getCudaCompatModeDiscoverer(logger, cfg, driver)
85+
if err != nil {
86+
return nil, fmt.Errorf("failed to construct CUDA Compat discoverer: %w", err)
9887
}
88+
discoverers = append(discoverers, cudaCompatDiscoverer)
9989
}
10090

10191
return NewModifierFromDiscoverer(logger, discover.Merge(discoverers...))
10292
}
93+
94+
func getCudaCompatModeDiscoverer(logger logger.Interface, cfg *config.Config, driver *root.Driver) (discover.Discover, error) {
95+
// For legacy mode, we only include the enable-cuda-compat hook if cuda-compat-mode is set to hook.
96+
if cfg.NVIDIAContainerRuntimeConfig.Mode == "legacy" && cfg.NVIDIAContainerRuntimeConfig.Modes.Legacy.CUDACompatMode != config.CUDACompatModeHook {
97+
return nil, nil
98+
}
99+
100+
compatLibHookDiscoverer := discover.NewCUDACompatHookDiscoverer(logger, cfg.NVIDIACTKConfig.Path, driver)
101+
// For non-legacy modes we return the hook as is. These modes *should* already include the update-ldcache hook.
102+
if cfg.NVIDIAContainerRuntimeConfig.Mode != "legacy" {
103+
return compatLibHookDiscoverer, nil
104+
}
105+
106+
// For legacy mode, we also need to inject a hook to update the LDCache
107+
// after we have modifed the configuration.
108+
ldcacheUpdateHookDiscoverer, err := discover.NewLDCacheUpdateHook(
109+
logger,
110+
discover.None{},
111+
cfg.NVIDIACTKConfig.Path,
112+
"",
113+
)
114+
if err != nil {
115+
return nil, fmt.Errorf("failed to construct ldcache update discoverer: %w", err)
116+
}
117+
118+
return discover.Merge(compatLibHookDiscoverer, ldcacheUpdateHookDiscoverer), nil
119+
}

‎third_party/libnvidia-container‎

0 commit comments

Comments
 (0)