Skip to content

Commit 8b410a6

Browse files
elezarelezar
committed
Skip creation of libcuda.so symlink on tegra
This change skips the creation of an additional libcuda.so symlink on Tegra-based systems where this is already handled by the entries in the CSV files. Note that on such systems one typically has a target library (libcuda.so.1.1) which is pointed to by both libcuda.so and libcuda.so.1. Signed-off-by: Evan Lezar <[email protected]>
1 parent 5bc0315 commit 8b410a6

File tree

11 files changed

+442
-2
lines changed

11 files changed

+442
-2
lines changed

‎internal/discover/symlinks.go‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,18 +25,23 @@ type additionalSymlinks struct {
2525
Discover
2626
version string
2727
nvidiaCDIHookPath string
28+
// skipLibCudaSo removes the creation of the libcuda.so symlink.
29+
// This can be set on Tegra-based systems where the creation of the libcuda.so symlink is
30+
// determined by the drivers.csv file entries.
31+
skipLibCudaSo bool
2832
}
2933

3034
// WithDriverDotSoSymlinks decorates the provided discoverer.
3135
// A hook is added that checks for specific driver symlinks that need to be created.
32-
func WithDriverDotSoSymlinks(mounts Discover, version string, nvidiaCDIHookPath string) Discover {
36+
func WithDriverDotSoSymlinks(mounts Discover, version string, nvidiaCDIHookPath string, skipLibCudaSo bool) Discover {
3337
if version == "" {
3438
version = "*.*"
3539
}
3640
return &additionalSymlinks{
3741
Discover: mounts,
3842
nvidiaCDIHookPath: nvidiaCDIHookPath,
3943
version: version,
44+
skipLibCudaSo: skipLibCudaSo,
4045
}
4146
}
4247

@@ -81,7 +86,7 @@ func (d *additionalSymlinks) Hooks() ([]Hook, error) {
8186
func (d additionalSymlinks) getLinksForMount(path string) []string {
8287
dir, filename := filepath.Split(path)
8388
switch {
84-
case d.isDriverLibrary("libcuda.so", filename):
89+
case !d.skipLibCudaSo && d.isDriverLibrary("libcuda.so", filename):
8590
// XXX Many applications wrongly assume that libcuda.so exists (e.g. with dlopen).
8691
// create libcuda.so -> libcuda.so.1 symlink
8792
link := fmt.Sprintf("%s::%s", "libcuda.so.1", filepath.Join(dir, "libcuda.so"))

‎internal/discover/symlinks_test.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -312,6 +312,7 @@ func TestWithWithDriverDotSoSymlinks(t *testing.T) {
312312
tc.discover,
313313
tc.version,
314314
"/path/to/nvidia-cdi-hook",
315+
false,
315316
)
316317

317318
devices, err := d.Devices()

‎internal/platform-support/tegra/csv.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,7 @@ func (o tegraOptions) newDiscovererFromCSVFiles() (discover.Discover, error) {
6060
),
6161
"",
6262
o.nvidiaCDIHookPath,
63+
true,
6364
)
6465

6566
// We process the explicitly requested symlinks.

‎internal/platform-support/tegra/tegra_test.go‎

Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,104 @@
1+
/**
2+
# Copyright 2024 NVIDIA CORPORATION
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
**/
16+
17+
package tegra
18+
19+
import (
20+
"path/filepath"
21+
"testing"
22+
23+
testlog "github.com/sirupsen/logrus/hooks/test"
24+
"github.com/stretchr/testify/require"
25+
26+
"github.com/NVIDIA/nvidia-container-toolkit/internal/discover"
27+
"github.com/NVIDIA/nvidia-container-toolkit/internal/platform-support/tegra/csv"
28+
)
29+
30+
func TestTegraDiscoverer(t *testing.T) {
31+
logger, _ := testlog.NewNullLogger()
32+
33+
moduleRoot := "/Users/elezar/dev/container-toolkit"
34+
35+
root := filepath.Join(moduleRoot, "testdata", "lookup", "rootfs-tegra")
36+
// TODO: We could already deduplicate these mounts at discovery.
37+
expectedMounts := []discover.Mount{
38+
{
39+
HostPath: "/Users/elezar/dev/container-toolkit/testdata/lookup/rootfs-tegra/usr/lib64/nvidia/libcuda.so.1.1",
40+
Path: "/usr/lib64/nvidia/libcuda.so.1.1",
41+
Options: []string{
42+
"ro",
43+
"nosuid",
44+
"nodev",
45+
"bind",
46+
},
47+
},
48+
{
49+
HostPath: "/Users/elezar/dev/container-toolkit/testdata/lookup/rootfs-tegra/usr/lib64/nvidia/libcuda.so.1.1",
50+
Path: "/usr/lib64/nvidia/libcuda.so.1.1",
51+
Options: []string{
52+
"ro",
53+
"nosuid",
54+
"nodev",
55+
"bind",
56+
},
57+
},
58+
}
59+
60+
expectedHooks := []discover.Hook{
61+
{
62+
Lifecycle: "createContainer",
63+
Path: "/usr/bin/nvidia-cdi-hook",
64+
Args: []string{
65+
// TODO: The create-symlinks hooks should NOT reference the container root.
66+
"nvidia-cdi-hook", "create-symlinks",
67+
"--link", "nvidia/libcuda.so.1.1::/Users/elezar/dev/container-toolkit/testdata/lookup/rootfs-tegra/usr/lib64/libcuda.so",
68+
"--link", "libcuda.so.1.1::/Users/elezar/dev/container-toolkit/testdata/lookup/rootfs-tegra/usr/lib64/nvidia/libcuda.so",
69+
"--link", "libcuda.so.1.1::/Users/elezar/dev/container-toolkit/testdata/lookup/rootfs-tegra/usr/lib64/nvidia/libcuda.so.1",
70+
},
71+
},
72+
{
73+
Lifecycle: "createContainer",
74+
Path: "/usr/bin/nvidia-cdi-hook",
75+
Args: []string{
76+
"nvidia-cdi-hook", "update-ldcache",
77+
"--folder", "/usr/lib64/nvidia",
78+
},
79+
},
80+
}
81+
82+
var csvFiles []string
83+
for _, csvFile := range csv.DefaultFileList() {
84+
csvFiles = append(csvFiles, filepath.Join(root, csvFile))
85+
}
86+
87+
d, err := New(
88+
WithLogger(logger),
89+
WithDriverRoot(root),
90+
WithCSVFiles(csvFiles),
91+
WithNVIDIACDIHookPath("/usr/bin/nvidia-cdi-hook"),
92+
)
93+
94+
require.NoError(t, err)
95+
96+
mounts, err := d.Mounts()
97+
require.NoError(t, err)
98+
99+
require.EqualValues(t, expectedMounts, mounts)
100+
101+
hooks, err := d.Hooks()
102+
require.NoError(t, err)
103+
require.EqualValues(t, expectedHooks, hooks)
104+
}

‎pkg/nvcdi/driver-nvml.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -104,6 +104,7 @@ func NewDriverLibraryDiscoverer(logger logger.Interface, driver *root.Driver, nv
104104
libraries,
105105
version,
106106
nvidiaCDIHookPath,
107+
false,
107108
),
108109
updateLDCache,
109110
)

‎testdata/lookup/rootfs-tegra/etc/nvidia-container-runtime/host-files-for-container.d/devices.csv‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
dev, /dev/dri/card*
2+
dev, /dev/dri/renderD*
3+
dir, /dev/dri/by-path
4+
dev, /dev/fb0
5+
dev, /dev/fb1
6+
dev, /dev/host1x-fence
7+
dev, /dev/nvhost-as-gpu
8+
dev, /dev/nvhost-ctrl-gpu
9+
dev, /dev/nvhost-ctrl-nvdla0
10+
dev, /dev/nvhost-ctrl-nvdla1
11+
dev, /dev/nvhost-ctrl-pva0
12+
dev, /dev/nvhost-ctxsw-gpu
13+
dev, /dev/nvhost-dbg-gpu
14+
dev, /dev/nvhost-gpu
15+
dev, /dev/nvhost-nvsched-gpu
16+
dev, /dev/nvhost-power-gpu
17+
dev, /dev/nvhost-prof-ctx-gpu
18+
dev, /dev/nvhost-prof-dev-gpu
19+
dev, /dev/nvhost-prof-gpu
20+
dev, /dev/nvhost-sched-gpu
21+
dev, /dev/nvhost-tsg-gpu
22+
dev, /dev/nvgpu/igpu0/as
23+
dev, /dev/nvgpu/igpu0/channel
24+
dev, /dev/nvgpu/igpu0/ctrl
25+
dev, /dev/nvgpu/igpu0/ctxsw
26+
dev, /dev/nvgpu/igpu0/dbg
27+
dev, /dev/nvgpu/igpu0/nvsched
28+
dev, /dev/nvgpu/igpu0/power
29+
dev, /dev/nvgpu/igpu0/prof
30+
dev, /dev/nvgpu/igpu0/prof-ctx
31+
dev, /dev/nvgpu/igpu0/prof-dev
32+
dev, /dev/nvgpu/igpu0/sched
33+
dev, /dev/nvgpu/igpu0/tsg
34+
dev, /dev/nvidia-modeset
35+
dev, /dev/nvidia0
36+
dev, /dev/nvidiactl
37+
dev, /dev/nvmap
38+
dev, /dev/nvsciipc
39+
dev, /dev/v4l2-nvdec
40+
dev, /dev/v4l2-nvenc
41+
dev, /dev/video*

0 commit comments

Comments
 (0)