From d7fc52eb82e83e747876d4f2bf03a050f09f53eb Mon Sep 17 00:00:00 2001 From: Sergio Oller Moreno Date: Mon, 24 Feb 2025 15:22:01 +0100 Subject: [PATCH 1/4] Lockdir prefix for install packages Some distributed file systems do not support opening files in append mode. These file systems are often used in data analysis cloud platforms. R package installation relies on appending to files, for instance collating R code or when installing help pages. Therefore, packages can't be installed in those filesystems. Instead, users are forced to install packages into a local directory and copy them afterwards. However, the current package installation procedure already uses a 00LOCK directory to install packages there, before copying them to the final library directory. By globally modifying the location of the 00LOCK directory, it is possible to use a local filesystem to install packages, where append is allowed. The installation process takes care of copying the resulting package into the final out directory. This change introduces the environment variable PKG_LOCKDIR_PREFIX that, when set to a directory like "/tmp/r-lockdir", uses that root path to create all 00LOCK folders. This allows to install packages on file systems that do not support opening files in append mode. --- src/library/tools/R/install.R | 38 ++++++++++++++++++++++++++--------- 1 file changed, 28 insertions(+), 10 deletions(-) diff --git a/src/library/tools/R/install.R b/src/library/tools/R/install.R index eb026dc267d..8cfd53cd93c 100644 --- a/src/library/tools/R/install.R +++ b/src/library/tools/R/install.R @@ -54,7 +54,23 @@ if(FALSE) { ## global variables curPkg <- character() # list of packages in current pkg + + ## Path to where lockdirs will be created. If empty, use lib directory + lockdir_prefix <- Sys.getenv("PKG_LOCKDIR_PREFIX", "") + if (lockdir_prefix == "") + lockdir_prefix <- NULL + else + lockdir_prefix <- path.expand(lockdir_prefix) + ## The lockdir, relative to lockdir_prefix lockdir <- "" + ## The lockdir, including the prefix + lockdir_with_prefix <- "" + get_lockdir_with_prefix <- function(lockdir) { + if (!is.null(lockdir_prefix)) + file.path(lockdir_prefix, lockdir) + else + lockdir + } is_first_package <- TRUE stars <- "*" user.tmpdir <- Sys.getenv("PKG_BUILD_DIR") @@ -105,8 +121,8 @@ if(FALSE) { } if (nzchar(lockdir) && - dir.exists(lp <- file.path(lockdir, curPkg)) && - is_subdir(lp, lockdir)) { + dir.exists(lp <- file.path(lockdir_with_prefix, curPkg)) && + is_subdir(lp, lockdir_with_prefix)) { starsmsg(stars, "restoring previous ", sQuote(pkgdir)) if (WINDOWS) { file.copy(lp, dirname(pkgdir), recursive = TRUE, @@ -134,7 +150,7 @@ if(FALSE) { if (lib == .Library && "html" %in% build_help_types) utils::make.packages.html(.Library, docdir = R.home("doc")) } - if (nzchar(lockdir)) unlink(lockdir, recursive = TRUE) + if (nzchar(lockdir_with_prefix)) unlink(lockdir_with_prefix, recursive = TRUE) } do_cleanup_tmpdir <- function() @@ -478,7 +494,7 @@ if(FALSE) { if (file.exists(file.path(instdir, "DESCRIPTION"))) { if (nzchar(lockdir)) system(paste("mv -f", shQuote(instdir), - shQuote(file.path(lockdir, pkg)))) + shQuote(file.path(lockdir_with_prefix, pkg)))) dir.create(instdir, recursive = TRUE, showWarnings = FALSE) } TAR <- Sys.getenv("TAR", 'tar') @@ -1041,9 +1057,9 @@ if(FALSE) { if (more_than_libs) unlink(instdir, recursive = TRUE) } else if (more_than_libs) system(paste("mv -f ", shQuote(instdir), - shQuote(file.path(lockdir, pkg_name)))) + shQuote(file.path(lockdir_with_prefix, pkg_name)))) else - file.copy(instdir, lockdir, recursive = TRUE, + file.copy(instdir, lockdir_with_prefix, recursive = TRUE, copy.date = TRUE) } else if (more_than_libs) unlink(instdir, recursive = TRUE) if (more_than_libs && dir.exists(instdir)) @@ -1073,10 +1089,10 @@ if(FALSE) { final_rlibs <- Sys.getenv("R_LIBS") final_libpaths <- .libPaths() - instdir <- file.path(lockdir, "00new", pkg_name) + instdir <- file.path(lockdir_with_prefix, "00new", pkg_name) Sys.setenv(R_PACKAGE_DIR = instdir) dir.create(instdir, recursive = TRUE, showWarnings = FALSE) - lib <- file.path(lockdir, "00new") + lib <- file.path(lockdir_with_prefix, "00new") rlibs <- if (nzchar(final_rlibs)) paste(lib, final_rlibs, sep = .Platform$path.sep) @@ -2373,7 +2389,8 @@ if(FALSE) { } if (lock && !pkglock) { lockdir <- file.path(lib, "00LOCK") - mk_lockdir(lockdir) + lockdir_with_prefix <- get_lockdir_with_prefix(lockdir) + mk_lockdir(lockdir_with_prefix) } if (is.na(staged_install)) { # environment variable intended as temporary @@ -2418,7 +2435,8 @@ if(FALSE) { for(pkg in allpkgs) { if (pkglock) { lockdir <- file.path(lib, paste0("00LOCK-", basename(pkg))) - mk_lockdir(lockdir) + lockdir_with_prefix <- get_lockdir_with_prefix(lockdir) + mk_lockdir(lockdir_with_prefix) } do_install(pkg) } From 7b7291c5fd06fcad8ee97420f993bab96b9f9f78 Mon Sep 17 00:00:00 2001 From: Sergio Oller Moreno Date: Thu, 27 Mar 2025 10:56:33 +0100 Subject: [PATCH 2/4] Rename the env variable so it uses the R_ prefix Suggested by Duncan Murdoch --- src/library/tools/R/install.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/library/tools/R/install.R b/src/library/tools/R/install.R index 8cfd53cd93c..0ad1b2c7ce6 100644 --- a/src/library/tools/R/install.R +++ b/src/library/tools/R/install.R @@ -56,7 +56,7 @@ if(FALSE) { curPkg <- character() # list of packages in current pkg ## Path to where lockdirs will be created. If empty, use lib directory - lockdir_prefix <- Sys.getenv("PKG_LOCKDIR_PREFIX", "") + lockdir_prefix <- Sys.getenv("R_LOCKDIR_PREFIX", "") if (lockdir_prefix == "") lockdir_prefix <- NULL else From 386ac9c6467499c383b1426a82ee593f653c593a Mon Sep 17 00:00:00 2001 From: Sergio Oller Moreno Date: Thu, 27 Mar 2025 10:57:34 +0100 Subject: [PATCH 3/4] NEWS and R-Admin documentation --- doc/NEWS.Rd | 9 +++++++++ doc/manual/R-admin.texi | 14 ++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/doc/NEWS.Rd b/doc/NEWS.Rd index 76778d9d912..5d2666f52ec 100644 --- a/doc/NEWS.Rd +++ b/doc/NEWS.Rd @@ -423,6 +423,15 @@ \item There is preliminary support for C++26 with GCC \eqn{>=} 14, Apple \command{clang++} \eqn{>=} 16 and \I{LLVM} \command{clang++} \eqn{>=} 17. + + \item \code{install.packages()} supports setting a different path for the + intermediate \code{00LOCK} directories through the environment variable + \code{R_LOCKDIR_PREFIX}. This helps source package installation in some + distributed filesystems available in cloud environments like databricks. + Check the + \dQuote{R Installation and Administration} manual for further details on + this option. + } } diff --git a/doc/manual/R-admin.texi b/doc/manual/R-admin.texi index 17b71c783de..4f0086b4d98 100644 --- a/doc/manual/R-admin.texi +++ b/doc/manual/R-admin.texi @@ -2061,6 +2061,20 @@ comparable age to your version of @R{}). Naive users sometimes forget that as well as installing a package, they have to use @code{library} to make its functionality available. +The package installation process has some standard requirements on the +file system where packages are installed. Specifically, when a \emph{source +package} needs to be installed, R needs to be able +to open files in append mode \code{open(filename, "a")}. R also expects to +be able to perform atomic directory moves during package installation +to ensure correct error recovery from package installation errors (see +@uref{https://developer.r-project.org/Blog/public/2019/02/14/staged-install/index.html}). +Both requirements are fullfilled by POSIX and Windows filesystems, but not by +some distributed file systems. In scenarios where opening append mode is not available, the +@env{R_LOCKDIR_PREFIX} environment variable can be set to specify the absolute path to +a different directory (in a compliant file system) where the package can be preinstalled +before being moved to its final destination. While this approach makes package installation +in those file systems possible, the error recovery capabilities will not be as robust as +on supported filesystems. @node Windows packages @subsection Windows From 0b259c7ab22ebcf00f4c3733f5187d408aabf8ff Mon Sep 17 00:00:00 2001 From: Sergio Oller Moreno Date: Thu, 27 Mar 2025 13:24:06 +0100 Subject: [PATCH 4/4] Move from r-4.5.0 to r-devel --- doc/NEWS.Rd | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/doc/NEWS.Rd b/doc/NEWS.Rd index 5d2666f52ec..87401a41bbb 100644 --- a/doc/NEWS.Rd +++ b/doc/NEWS.Rd @@ -26,7 +26,15 @@ \subsection{PACKAGE INSTALLATION}{ \itemize{ - \item . + + \item \code{install.packages()} supports setting a different path for the + intermediate \code{00LOCK} directories through the environment variable + \code{R_LOCKDIR_PREFIX}. This helps source package installation in some + distributed filesystems available in cloud environments like databricks. + Check the + \dQuote{R Installation and Administration} manual for further details on + this option. + } } @@ -424,14 +432,6 @@ \item There is preliminary support for C++26 with GCC \eqn{>=} 14, Apple \command{clang++} \eqn{>=} 16 and \I{LLVM} \command{clang++} \eqn{>=} 17. - \item \code{install.packages()} supports setting a different path for the - intermediate \code{00LOCK} directories through the environment variable - \code{R_LOCKDIR_PREFIX}. This helps source package installation in some - distributed filesystems available in cloud environments like databricks. - Check the - \dQuote{R Installation and Administration} manual for further details on - this option. - } }