Mount NVIDIA firmware directory if present

This change mounts the NVIDIA firmware directory into the container
if present on the host.

Starting with A100 devices the firmware from
/lib/firmware/nvidia/<driver_version> is loaded when a device
is initialised. This means that this folder is required in
the container when persistence mode is not enabled on the host.

Signed-off-by: Evan Lezar <[email protected]>

Author: elezar

src/nvc.h

@@ -59,6 +59,8 @@ struct nvc_driver_info {
         size_t nipcs;
         struct nvc_device_node *devs;
         size_t ndevs;
+        char **dirs;
+        size_t ndirs;
 };
 
 struct nvc_mig_device {

src/nvc_info.c

@@ -47,8 +47,10 @@ static int find_library_paths(struct error *, struct dxcore_context *, struct nv
 static int find_binary_paths(struct error *, struct dxcore_context*, struct nvc_driver_info *, const char *, const char * const [], size_t);
 static int find_device_node(struct error *, const char *, const char *, struct nvc_device_node *);
 static int find_ipc_path(struct error *, const char *, const char *, char **);
+static int lookup_paths(struct error *, struct dxcore_context *, struct nvc_driver_info *, const char *, int32_t, const char *);
 static int lookup_libraries(struct error *, struct dxcore_context *, struct nvc_driver_info *, const char *, int32_t, const char *);
 static int lookup_binaries(struct error *, struct dxcore_context *, struct nvc_driver_info *, const char *, int32_t);
+static int lookup_directories(struct error *, struct dxcore_context *, struct nvc_driver_info *, const char *, int32_t);
 static int lookup_devices(struct error *, const struct nvc_context *, struct nvc_driver_info *, const char *, int32_t);
 static int lookup_ipcs(struct error *, struct nvc_driver_info *, const char *, int32_t);
 static int fill_mig_device_info(struct nvc_context *, bool mig_enabled, struct driver_device *, struct nvc_device *);
@@ -350,6 +352,27 @@ find_ipc_path(struct error *err, const char *root, const char *ipc, char **buf)
         return (0);
 }
 
+static int
+lookup_paths(struct error *err, struct dxcore_context *dxcore, struct nvc_driver_info *info, const char *root, int32_t flags, const char *ldcache)
+{
+        if (lookup_libraries(err, dxcore, info, root, flags, ldcache) < 0) {
+                log_err("error looking up libraries");
+                return (-1);
+        }
+
+        if (lookup_binaries(err, dxcore, info, root, flags) < 0) {
+                log_err("error looking up binaries");
+                return (-1);
+        }
+
+        if (lookup_directories(err, dxcore, info, root, flags) < 0) {
+                log_err("error looking up additional paths");
+                return (-1);
+        }
+
+        return (0);
+}
+
 static int
 lookup_libraries(struct error *err, struct dxcore_context *dxcore, struct nvc_driver_info *info, const char *root, int32_t flags, const char *ldcache)
 {
@@ -406,6 +429,38 @@ lookup_binaries(struct error *err, struct dxcore_context* dxcore, struct nvc_dri
         return (0);
 }
 
+static int
+lookup_directories(struct error *err, struct dxcore_context *dxcore, struct nvc_driver_info *info, const char *root, int32_t flags) {
+        int fd;
+        char *firmware_path = NULL;
+
+        if (dxcore->initialized) {
+                log_info("skipping path lookup for dxcore");
+                return 0;
+        }
+
+        // If the NVIDIA driver firmware path exists, include this in the mounted folders.
+        if (xasprintf(err, &firmware_path, NV_FIRMWARE_DRIVER_PATH, info->nvrm_version) < 0) {
+                log_errf("error constructing firmware path for %s", info->nvrm_version);
+                return (-1);
+        }
+        if ((fd = xopen(err, firmware_path, O_PATH|O_DIRECTORY)) < 0) {
+                log_infof("missing firmware path %s", firmware_path);
+                return (0);
+        }
+        close(fd);
+
+        info->dirs = array_new(err, 1);
+        if (info->dirs == NULL) {
+                log_err("error creating path array");
+                return (-1);
+        }
+        info->dirs[0] = firmware_path;
+        info->ndirs = 1;
+
+        return (0);
+}
+
 static int
 lookup_devices(struct error *err, const struct nvc_context *ctx, struct nvc_driver_info *info, const char *root, int32_t flags)
 {
@@ -762,9 +817,7 @@ nvc_driver_info_new(struct nvc_context *ctx, const char *opts)
                 goto fail;
         if (driver_get_cuda_version(&ctx->drv, &info->cuda_version) < 0)
                 goto fail;
-        if (lookup_libraries(&ctx->err, &ctx->dxcore, info, ctx->cfg.root, flags, ctx->cfg.ldcache) < 0)
-                goto fail;
-        if (lookup_binaries(&ctx->err, &ctx->dxcore, info, ctx->cfg.root, flags) < 0)
+        if (lookup_paths(&ctx->err, &ctx->dxcore, info, ctx->cfg.root, flags, ctx->cfg.ldcache) < 0)
                 goto fail;
         if (lookup_devices(&ctx->err, ctx, info, ctx->cfg.root, flags) < 0)
                 goto fail;

src/nvc_internal.h

@@ -64,6 +64,9 @@
 
 #define CUDA_RUNTIME_DIR         "/usr/local/cuda"
 
+#define NV_FIRMWARE_PATH        "/lib/firmware/nvidia"
+#define NV_FIRMWARE_DRIVER_PATH NV_FIRMWARE_PATH "/%s"
+
 #define MSFT_DXG_DEVICE_PATH     _PATH_DEV "dxg"
 
 struct nvc_context {

src/nvc_mount.c

@@ -720,7 +720,7 @@ nvc_driver_mount(struct nvc_context *ctx, const struct nvc_container *cnt, const
         if (ns_enter(&ctx->err, cnt->mnt_ns, CLONE_NEWNS) < 0)
                 return (-1);
 
-        nmnt = 2 + info->nbins + info->nlibs + cnt->nlibs + info->nlibs32 + info->nipcs + info->ndevs;
+        nmnt = 2 + info->nbins + info->nlibs + cnt->nlibs + info->nlibs32 + info->nipcs + info->ndevs + info->ndirs;
         mnt = ptr = (const char **)array_new(&ctx->err, nmnt);
         if (mnt == NULL)
                 goto fail;
@@ -778,6 +778,18 @@ nvc_driver_mount(struct nvc_context *ctx, const struct nvc_container *cnt, const
                 free(libs);
         }
 
+        /* Directory mounts */
+        for (size_t i = 0; i < info->ndirs; ++i) {
+                if (str_has_prefix(NV_FIRMWARE_PATH, info->dirs[i])) {
+                        if (!(cnt->flags & OPT_UTILITY_LIBS))
+                                continue;
+                }
+                if ((*ptr++ = mount_directory(&ctx->err, ctx->cfg.root, cnt, info->dirs[i])) == NULL) {
+                        log_errf("error mounting directory %s", info->dirs[i]);
+                        goto fail;
+                }
+        }
+
         /* IPC mounts */
         for (size_t i = 0; i < info->nipcs; ++i) {
                 /* XXX Only utility libraries require persistenced or fabricmanager IPC, everything else is compute only. */