copy_symlinks: allow extract to simulate symlinks with copies (#63)

The logic to figure out which symlinks can be created as copies and
in what order is fairly bonkers, but this seems to do the trick.

Author: StefanKarpinski

Project.toml

@@ -5,6 +5,7 @@ version = "1.5.0"
 
 [deps]
 ArgTools = "0dad84c5-d112-42e6-8d28-ef12dabb789f"
+Logging = "56ddb016-857b-54e1-b83d-db4d58db5568"
 SHA = "ea8e919c-243c-51af-8825-aaa63cd721ce"
 
 [compat]
@@ -13,8 +14,9 @@ julia = "1.3"
 
 [extras]
 Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
+Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
 Tar_jll = "9b64493d-8859-5bf3-93d7-7c32dd38186f"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 
 [targets]
-test = ["Test", "Pkg", "Tar_jll"]
+test = ["Pkg", "Random", "Tar_jll", "Test"]

README.md

@@ -147,12 +147,14 @@ recreated. The `skeleton` and `predicate` arguments cannot be used together.
 
 ### Tar.extract
 
-    extract([ predicate, ] tarball, [ dir ]; [ skeleton ]) -> dir
+    extract([ predicate, ] tarball, [ dir ];
+            [ skeleton, ] [ copy_symlinks ]) -> dir
 
-* `predicate :: Header --> Bool`
-* `tarball   :: Union{AbstractString, IO}`
-* `dir       :: AbstractString`
-* `skeleton  :: Union{AbstractString, IO}`
+* `predicate     :: Header --> Bool`
+* `tarball       :: Union{AbstractString, AbstractCmd, IO}`
+* `dir           :: AbstractString`
+* `skeleton      :: Union{AbstractString, AbstractCmd, IO}`
+* `copy_symlinks :: Bool`
 
 Extract a tar archive ("tarball") located at the path `tarball` into the
 directory `dir`. If `tarball` is an IO object instead of a path, then the
@@ -172,6 +174,14 @@ is written to the file or IO handle given. This skeleton file can be used to
 recreate an identical tarball by passing the `skeleton` keyword to the `create`
 function. The `skeleton` and `predicate` arguments cannot be used together.
 
+If `copy_symlinks` is `true` then instead of extracting symbolic links as such,
+they will be extracted as copies of what they link to if they are internal to
+the tarball and if it is possible to do so. Non-internal symlinks, such as a
+link to `/etc/passwd` will not be copied. Symlinks which are in any way cyclic
+will also not be copied and will instead be skipped. By default, `extract` will
+detect whether symlinks can be created in `dir` or not and will automatically
+copy symlinks if they cannot be created.
+
 ### Tar.list
 
     list(tarball; [ strict = true ]) -> Vector{Header}

src/Tar.jl

@@ -1,6 +1,8 @@
 module Tar
 
 using ArgTools
+using Logging
+
 const true_predicate = _ -> true
 
 # 2 MiB to take advantage of THP if enabled
@@ -17,6 +19,23 @@ function Base.skip(io::Union{Base.Process, Base.ProcessChain}, n::Integer)
 end
 const skip_buffer = UInt8[]
 
+function can_symlink(dir::AbstractString)
+    # guaranteed to be an empty directory
+    link_path = joinpath(dir, "link")
+    log_level = Logging.min_enabled_level(Logging.current_logger())
+    return try
+        Logging.disable_logging(Logging.Warn)
+        symlink("target", link_path)
+        true
+    catch err
+        err isa Base.IOError || rethrow()
+        false
+    finally
+        Logging.disable_logging(log_level-1)
+        rm(link_path, force=true)
+    end
+end
+
 include("header.jl")
 include("create.jl")
 include("extract.jl")
@@ -134,12 +153,14 @@ function list(
 end
 
 """
-    extract([ predicate, ] tarball, [ dir ]; [ skeleton ]) -> dir
+    extract([ predicate, ] tarball, [ dir ];
+            [ skeleton, ] [ copy_symlinks ]) -> dir
 
-        predicate :: Header --> Bool
-        tarball   :: Union{AbstractString, AbstractCmd, IO}
-        dir       :: AbstractString
-        skeleton  :: Union{AbstractString, AbstractCmd, IO}
+        predicate     :: Header --> Bool
+        tarball       :: Union{AbstractString, AbstractCmd, IO}
+        dir           :: AbstractString
+        skeleton      :: Union{AbstractString, AbstractCmd, IO}
+        copy_symlinks :: Bool
 
 Extract a tar archive ("tarball") located at the path `tarball` into the
 directory `dir`. If `tarball` is an IO object instead of a path, then the
@@ -158,12 +179,21 @@ If the `skeleton` keyword is passed then a "skeleton" of the extracted tarball
 is written to the file or IO handle given. This skeleton file can be used to
 recreate an identical tarball by passing the `skeleton` keyword to the `create`
 function. The `skeleton` and `predicate` arguments cannot be used together.
+
+If `copy_symlinks` is `true` then instead of extracting symbolic links as such,
+they will be extracted as copies of what they link to if they are internal to
+the tarball and if it is possible to do so. Non-internal symlinks, such as a
+link to `/etc/passwd` will not be copied. Symlinks which are in any way cyclic
+will also not be copied and will instead be skipped. By default, `extract` will
+detect whether symlinks can be created in `dir` or not and will automatically
+copy symlinks if they cannot be created.
 """
 function extract(
     predicate::Function,
     tarball::ArgRead,
     dir::Union{AbstractString, Nothing} = nothing;
     skeleton::Union{ArgWrite, Nothing} = nothing,
+    copy_symlinks::Union{Bool, Nothing} = nothing,
 )
     predicate === true_predicate || skeleton === nothing ||
         error("extract: predicate and skeleton cannot be used together")
@@ -172,8 +202,15 @@ function extract(
     check_extract_dir(dir)
     arg_read(tarball) do tar
         arg_mkdir(dir) do dir
+            if copy_symlinks === nothing
+                copy_symlinks = !can_symlink(dir)
+            end
             arg_write(skeleton) do skeleton
-                extract_tarball(predicate, tar, dir, skeleton=skeleton)
+                extract_tarball(
+                    predicate, tar, dir,
+                    skeleton = skeleton,
+                    copy_symlinks = copy_symlinks,
+                )
             end
         end
     end
@@ -183,8 +220,13 @@ function extract(
     tarball::ArgRead,
     dir::Union{AbstractString, Nothing} = nothing;
     skeleton::Union{ArgWrite, Nothing} = nothing,
+    copy_symlinks::Union{Bool, Nothing} = nothing,
 )
-    extract(true_predicate, tarball, dir, skeleton=skeleton)
+    extract(
+        true_predicate, tarball, dir,
+        skeleton = skeleton,
+        copy_symlinks = copy_symlinks,
+    )
 end
 
 """

src/extract.jl

@@ -43,8 +43,9 @@ function extract_tarball(
     root::String;
     buf::Vector{UInt8} = Vector{UInt8}(undef, DEFAULT_BUFFER_SIZE),
     skeleton::IO = devnull,
+    copy_symlinks::Bool = false,
 )
-    read_tarball(predicate, tar; buf=buf, skeleton=skeleton) do hdr, parts
+    paths = read_tarball(predicate, tar; buf=buf, skeleton=skeleton) do hdr, parts
         # get the file system version of the path
         sys_path = reduce(joinpath, init=root, parts)
         # delete anything that's there already
@@ -60,7 +61,7 @@ function extract_tarball(
         if hdr.type == :directory
             mkdir(sys_path)
         elseif hdr.type == :symlink
-            symlink(hdr.link, sys_path)
+            copy_symlinks || symlink(hdr.link, sys_path)
         elseif hdr.type == :file
             read_data(tar, sys_path, size=hdr.size, buf=buf)
             # set executable bit if necessary
@@ -74,6 +75,89 @@ function extract_tarball(
             error("unsupported tarball entry type: $(hdr.type)")
         end
     end
+    copy_symlinks || return
+
+    # resolve the internal targets of symlinks
+    for (path, what) in paths
+        what isa AbstractString || continue
+        target = link_target(paths, path, what)
+        paths[path] = something(target, :symlink)
+    end
+
+    # follow chains of symlinks
+    follow(seen::Vector, what::Symbol) =
+        what == :symlink ? what : seen[end]
+    follow(seen::Vector, what::String) =
+        what in seen ? :symlink : follow(push!(seen, what), paths[what])
+    for (path, what) in paths
+        what isa AbstractString || continue
+        paths[path] = follow([path], what)
+    end
+
+    # copies that need to be made
+    copies = Pair{String,String}[]
+    for (path, what) in paths
+        what isa AbstractString || continue
+        push!(copies, path => what)
+    end
+    sort!(copies, by=last)
+
+    while !isempty(copies)
+        i = 1
+        while i ≤ length(copies)
+            path, what = copies[i]
+            # check if source is complete yet
+            if any(startswith(p, "$what/") for (p, w) in copies)
+                # `what` is an incomplete directory
+                # need to wait for source to be complete
+                i += 1
+            else
+                # source complete, can copy now
+                deleteat!(copies, i)
+                src = reduce(joinpath, init=root, split(what, '/'))
+                dst = reduce(joinpath, init=root, split(path, '/'))
+                cp(src, dst)
+            end
+        end
+    end
+end
+
+# resolve symlink target or nothing if not valid
+function link_target(
+    paths::Dict{String,Union{String,Symbol}},
+    path::AbstractString,
+    link::AbstractString,
+)
+    first(link) == '/' && return
+    path_parts = split(path, r"/+")
+    link_parts = split(link, r"/+")
+    pop!(path_parts)
+    part = nothing # remember the last part
+    while !isempty(link_parts)
+        part = popfirst!(link_parts)
+        part in ("", ".") && continue
+        if part == ".."
+            isempty(path_parts) && return
+            pop!(path_parts)
+        else
+            push!(path_parts, part)
+            prefix = join(path_parts, '/')
+            prefix in keys(paths) || return
+            isempty(link_parts) && break
+            what = paths[prefix]
+            if what isa AbstractString
+                prefix = link_target(paths, prefix, what)
+                path_parts = split(prefix, '/')
+            end
+        end
+    end
+    isempty(path_parts) && return
+    target = join(path_parts, '/')
+    # if link ends in `/` or `.` target must be a directory
+    part in ("", ".") && paths[target] != :directory && return
+    # can't copy a circular link to a prefix of itself
+    (path == target || startswith(path, "$target/")) && return
+    return target
 end
 
 function git_tree_hash(
@@ -210,8 +294,9 @@ function read_tarball(
     skeleton::IO = devnull,
 )
     write_skeleton_header(skeleton, buf=buf)
+    # symbols for path types except symlinks store the link
+    paths = Dict{String,Union{Symbol,String}}()
     globals = Dict{String,String}()
-    links = Set{String}()
     while !eof(tar)
         hdr = read_header(tar, globals, buf=buf, tee=skeleton)
         hdr === nothing && break
@@ -226,18 +311,15 @@ function read_tarball(
         for part in split(hdr.path, '/')
             (isempty(part) || part == ".") && continue
             # check_header doesn't allow ".." in path
-            path in links && error("""
+            get(paths, path, nothing) isa String && error("""
             Refusing to extract path with symlink prefix, possible attack
+             * path to extract: $(repr(hdr.path))
              * symlink prefix: $(repr(path))
-             * extracted path: $(repr(hdr.path))
             """)
+            isempty(path) || (paths[path] = :directory)
             path = isempty(path) ? part : "$path/$part"
         end
-        if hdr.type == :symlink
-            push!(links, path)
-        else
-            delete!(links, path)
-        end
+        paths[path] = hdr.type == :symlink ? hdr.link : hdr.type
         before = applicable(position, tar) ? position(tar) : 0
         callback(hdr, split(path, '/', keepempty=false))
         applicable(position, tar) || continue
@@ -246,6 +328,7 @@ function read_tarball(
         advanced == expected ||
             error("callback read $advanced bytes instead of $expected")
     end
+    return paths
 end
 
 function read_header(