From b1b1f2fa9e648a011256176e0b26b22880f2a4f6 Mon Sep 17 00:00:00 2001 From: pmeinhardt Date: Sat, 12 Dec 2020 07:59:54 +0100 Subject: [PATCH 01/54] Drop SCP --- CHANGELOG.md | 4 +- lib/sshkit.ex | 31 +--- lib/sshkit/scp.ex | 62 ------- lib/sshkit/scp/command.ex | 34 ---- lib/sshkit/scp/download.ex | 236 -------------------------- lib/sshkit/scp/upload.ex | 247 ---------------------------- test/sshkit/scp/command_test.exs | 70 -------- test/sshkit/scp/upload_test.exs | 215 ------------------------ test/sshkit/scp_functional_test.exs | 121 -------------- 9 files changed, 5 insertions(+), 1015 deletions(-) delete mode 100644 lib/sshkit/scp.ex delete mode 100644 lib/sshkit/scp/command.ex delete mode 100644 lib/sshkit/scp/download.ex delete mode 100644 lib/sshkit/scp/upload.ex delete mode 100644 test/sshkit/scp/command_test.exs delete mode 100644 test/sshkit/scp/upload_test.exs delete mode 100644 test/sshkit/scp_functional_test.exs diff --git a/CHANGELOG.md b/CHANGELOG.md index b8a56aa3..06875cea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,10 +4,12 @@ Presented in reverse chronological order. ## master -https://github.com/bitcrowd/sshkit.ex/compare/v0.2.0...HEAD +https://github.com/bitcrowd/sshkit.ex/compare/v0.3.0...HEAD +* Drop SCP support, create alternative file transfer implementation + ### Deprecations: diff --git a/lib/sshkit.ex b/lib/sshkit.ex index 718a40e8..b8d71f84 100644 --- a/lib/sshkit.ex +++ b/lib/sshkit.ex @@ -18,7 +18,6 @@ defmodule SSHKit do ``` """ - alias SSHKit.SCP alias SSHKit.SSH alias SSHKit.Context @@ -352,20 +351,7 @@ defmodule SSHKit do ``` """ def upload(context, source, options \\ []) do - options = Keyword.put(options, :map_cmd, &Context.build(context, &1)) - - target = Keyword.get(options, :as, Path.basename(source)) - - run = fn host -> - {:ok, res} = - SSH.connect(host.name, host.options, fn conn -> - SCP.upload(conn, source, target, options) - end) - - res - end - - Enum.map(context.hosts, run) + # TODO end @doc ~S""" @@ -400,19 +386,6 @@ defmodule SSHKit do ``` """ def download(context, source, options \\ []) do - options = Keyword.put(options, :map_cmd, &Context.build(context, &1)) - - target = Keyword.get(options, :as, Path.basename(source)) - - run = fn host -> - {:ok, res} = - SSH.connect(host.name, host.options, fn conn -> - SCP.download(conn, source, target, options) - end) - - res - end - - Enum.map(context.hosts, run) + # TODO end end diff --git a/lib/sshkit/scp.ex b/lib/sshkit/scp.ex deleted file mode 100644 index d0ad1ded..00000000 --- a/lib/sshkit/scp.ex +++ /dev/null @@ -1,62 +0,0 @@ -defmodule SSHKit.SCP do - @moduledoc ~S""" - Provides convenience functions for transferring files or directory trees to - or from a remote host via SCP. - - Built on top of `SSHKit.SSH`. - - ## Common options - - These options are available for both uploads and downloads: - - * `:verbose` - let the remote scp process be verbose, default `false` - * `:recursive` - set to `true` for copying directories, default `false` - * `:preserve` - preserve timestamps, default `false` - * `:timeout` - timeout in milliseconds, default `:infinity` - - ## Examples - - ``` - {:ok, conn} = SSHKit.SSH.connect("eg.io", user: "me") - :ok = SSHKit.SCP.upload(conn, ".", "/home/code/phx", recursive: true) - :ok = SSHKit.SSH.close(conn) - ``` - """ - - alias SSHKit.SCP.Download - alias SSHKit.SCP.Upload - - @doc """ - Uploads a local file or directory to a remote host. - - ## Options - - See `SSHKit.SCP.Upload.transfer/4`. - - ## Example - - ``` - :ok = SSHKit.SCP.upload(conn, ".", "/home/code/sshkit", recursive: true) - ``` - """ - def upload(connection, source, target, options \\ []) do - Upload.transfer(connection, source, target, options) - end - - @doc """ - Downloads a file or directory from a remote host. - - ## Options - - See `SSHKit.SCP.Download.transfer/4`. - - ## Example - - ``` - :ok = SSHKit.SCP.download(conn, "/home/code/sshkit", "downloads", recursive: true) - ``` - """ - def download(connection, source, target, options \\ []) do - Download.transfer(connection, source, target, options) - end -end diff --git a/lib/sshkit/scp/command.ex b/lib/sshkit/scp/command.ex deleted file mode 100644 index 5a3d9a11..00000000 --- a/lib/sshkit/scp/command.ex +++ /dev/null @@ -1,34 +0,0 @@ -defmodule SSHKit.SCP.Command do - @moduledoc false - - import SSHKit.Utils - - @flags [verbose: "-v", preserve: "-p", recursive: "-r"] - - def build(direction, path, options \\ []) - - def build(:upload, path, options) do - scp("-t", path, options) - end - - def build(:download, path, options) do - scp("-f", path, options) - end - - defp scp(mode, path, options) do - "scp #{mode}" |> flag(options) |> at(path) |> String.trim() - end - - defp flag(command, options) do - flags = - @flags - |> Enum.filter(fn {key, _} -> Keyword.get(options, key, false) end) - |> Enum.map(fn {_, flag} -> flag end) - - Enum.join([command] ++ flags, " ") - end - - defp at(command, path) do - "#{command} #{shellescape(path)}" - end -end diff --git a/lib/sshkit/scp/download.ex b/lib/sshkit/scp/download.ex deleted file mode 100644 index 516a760c..00000000 --- a/lib/sshkit/scp/download.ex +++ /dev/null @@ -1,236 +0,0 @@ -defmodule SSHKit.SCP.Download do - @moduledoc """ - Helper module used by SSHKit.SCP.download/4. - """ - - require Bitwise - - alias SSHKit.SCP.Command - alias SSHKit.SSH - - @doc """ - Downloads a file or directory from a remote host. - - ## Options - - * `:verbose` - let the remote scp process be verbose, default `false` - * `:recursive` - set to `true` for copying directories, default `false` - * `:preserve` - preserve timestamps, default `false` - * `:timeout` - timeout in milliseconds, default `:infinity` - - ## Example - - ``` - :ok = SSHKit.SCP.Download.transfer(conn, "/home/code/sshkit", "downloads", recursive: true) - ``` - """ - def transfer(connection, source, target, options \\ []) do - start(connection, source, Path.expand(target), options) - end - - defp start(connection, source, target, options) do - timeout = Keyword.get(options, :timeout, :infinity) - map_cmd = Keyword.get(options, :map_cmd, & &1) - command = map_cmd.(Command.build(:download, source, options)) - handler = connection_handler(options) - - ini = {:next, target, [], %{}, <<>>} - SSH.run(connection, command, timeout: timeout, acc: {:cont, <<0>>, ini}, fun: handler) - end - - defp connection_handler(options) do - fn message, state -> - case message do - {:data, _, 0, data} -> - process_data(state, data, options) - - {:exit_status, _, status} -> - exited(options, state, status) - - {:eof, _} -> - eof(options, state) - - {:closed, _} -> - closed(options, state) - end - end - end - - defp process_data(state, data, options) do - case state do - {:next, path, stack, attrs, buffer} -> - next(options, path, stack, attrs, buffer <> data) - - {:read, path, stack, attrs, buffer} -> - read(options, path, stack, attrs, buffer <> data) - end - end - - defp next(options, path, stack, attrs, buffer) do - if String.last(buffer) == "\n" do - case dirparse(buffer) do - {"T", mtime, _, atime, _} -> time(options, path, stack, attrs, mtime, atime) - {"C", mode, len, name} -> regular(options, path, stack, attrs, mode, len, name) - {"D", mode, _, name} -> directory(options, path, stack, attrs, mode, name) - {"E"} -> up(options, path, stack) - _ -> {:halt, {:error, "Invalid SCP directive received: #{buffer}"}} - end - else - {:cont, {:next, path, stack, attrs, buffer}} - end - end - - defp time(_, path, stack, attrs, mtime, atime) do - attrs = Map.merge(attrs, %{atime: atime, mtime: mtime}) - {:cont, <<0>>, {:next, path, stack, attrs, <<>>}} - end - - defp directory(options, path, stack, attrs, mode, name) do - target = if File.dir?(path), do: Path.join(path, name), else: path - - preserve? = Keyword.get(options, :preserve, false) - exists? = File.exists?(target) - - stat = if exists?, do: File.stat!(target), else: nil - - if exists? do - :ok = File.chmod!(target, Bitwise.bor(stat.mode, 0o700)) - else - :ok = File.mkdir!(target) - end - - mode = if exists? && !preserve?, do: stat.mode, else: mode - attrs = Map.put(attrs, :mode, mode) - - {:cont, <<0>>, {:next, target, [attrs | stack], %{}, <<>>}} - end - - defp regular(options, path, stack, attrs, mode, length, name) do - target = if File.dir?(path), do: Path.join(path, name), else: path - - preserve? = Keyword.get(options, :preserve, false) - exists? = File.exists?(target) - - stat = if exists?, do: File.stat!(target), else: nil - - if exists? do - :ok = File.chmod!(target, Bitwise.bor(stat.mode, 0o200)) - end - - device = File.open!(target, [:write, :binary]) - - mode = if exists? && !preserve?, do: stat.mode, else: mode - - attrs = - attrs - |> Map.put(:mode, mode) - |> Map.put(:device, device) - |> Map.put(:length, length) - |> Map.put(:written, 0) - - {:cont, <<0>>, {:read, target, stack, attrs, <<>>}} - end - - defp read(options, path, stack, attrs, buffer) do - %{device: device, length: length, written: written} = attrs - - {buffer, written} = - if written < length do - count = min(byte_size(buffer), length - written) - <> = buffer - :ok = IO.binwrite(device, chunk) - {rest, written + count} - else - {buffer, written} - end - - if written == length && buffer == <<0>> do - :ok = File.close(device) - - :ok = File.chmod!(path, attrs[:mode]) - - if Keyword.get(options, :preserve, false) do - :ok = touch!(path, attrs[:atime], attrs[:mtime]) - end - - {:cont, <<0>>, {:next, Path.dirname(path), stack, %{}, <<>>}} - else - {:cont, {:read, path, stack, Map.put(attrs, :written, written), <<>>}} - end - end - - defp up(options, path, [attrs | rest]) do - :ok = File.chmod!(path, attrs[:mode]) - - if Keyword.get(options, :preserve, false) do - :ok = touch!(path, attrs[:atime], attrs[:mtime]) - end - - {:cont, <<0>>, {:next, Path.dirname(path), rest, %{}, <<>>}} - end - - defp exited(_, {_, _, [], _, _}, status) do - {:cont, {:done, status}} - end - - defp exited(_, {_, _, _, _, _}, status) do - {:halt, {:error, "SCP exited before completing the transfer (#{status})"}} - end - - defp eof(_, state) do - {:cont, state} - end - - defp closed(_, {:done, 0}) do - {:cont, :ok} - end - - defp closed(_, {:done, status}) do - {:cont, {:error, "SCP exited with non-zero exit code #{status}"}} - end - - defp closed(_, _) do - {:cont, {:error, "SCP channel closed before completing the transfer"}} - end - - @epoch :calendar.datetime_to_gregorian_seconds({{1970, 1, 1}, {0, 0, 0}}) - - defp touch!(path, atime, mtime) do - atime = :calendar.gregorian_seconds_to_datetime(@epoch + atime) - mtime = :calendar.gregorian_seconds_to_datetime(@epoch + mtime) - {:ok, file_info} = File.stat(path) - :ok = File.write_stat(path, %{file_info | mtime: mtime, atime: atime}, [:posix]) - end - - @tfmt ~S"(T)(0|[1-9]\d*) (0|[1-9]\d{0,5}) (0|[1-9]\d*) (0|[1-9]\d{0,5})" - @ffmt ~S"(C|D)([0-7]{4}) (0|[1-9]\d*) ([^/]+)" - @efmt ~S"(E)" - - @dfmt ~r/\A(?|#{@efmt}|#{@tfmt}|#{@ffmt})\n\z/ - - defp dirparse(value) do - case Regex.run(@dfmt, value, capture: :all_but_first) do - ["T", mtime, mtus, atime, atus] -> - {"T", dec(mtime), dec(mtus), dec(atime), dec(atus)} - - [chr, _, _, name] when chr in ["C", "D"] and name in ["/", "..", "."] -> - nil - - ["C", mode, len, name] -> - {"C", oct(mode), dec(len), name} - - ["D", mode, len, name] -> - {"D", oct(mode), dec(len), name} - - ["E"] -> - {"E"} - - nil -> - nil - end - end - - defp int(value, base), do: String.to_integer(value, base) - defp dec(value), do: int(value, 10) - defp oct(value), do: int(value, 8) -end diff --git a/lib/sshkit/scp/upload.ex b/lib/sshkit/scp/upload.ex deleted file mode 100644 index e7830a7b..00000000 --- a/lib/sshkit/scp/upload.ex +++ /dev/null @@ -1,247 +0,0 @@ -defmodule SSHKit.SCP.Upload do - @moduledoc """ - Helper module used by SSHKit.SCP.upload/4. - """ - - require Bitwise - - alias SSHKit.SCP.Command - alias SSHKit.SSH - - defstruct [:source, :target, :state, :handler, options: []] - - @doc """ - Uploads a local file or directory to a remote host. - - ## Options - - * `:verbose` - let the remote scp process be verbose, default `false` - * `:recursive` - set to `true` for copying directories, default `false` - * `:preserve` - preserve timestamps, default `false` - * `:timeout` - timeout in milliseconds, default `:infinity` - - ## Example - - ``` - :ok = SSHKit.SCP.Upload.transfer(conn, ".", "/home/code/sshkit", recursive: true) - ``` - """ - def transfer(connection, source, target, options \\ []) do - source - |> init(target, options) - |> exec(connection) - end - - @doc """ - Configures the upload of a local file or directory to a remote host. - - ## Options - - * `:verbose` - let the remote scp process be verbose, default `false` - * `:recursive` - set to `true` for copying directories, default `false` - * `:preserve` - preserve timestamps, default `false` - * `:timeout` - timeout in milliseconds, default `:infinity` - - ## Example - - ``` - iex(1)> SSHKit.SCP.Upload.init(".", "/home/code/sshkit", recursive: true) - %SSHKit.SCP.Upload{ - handler: #Function<1.78222439/2 in SSHKit.SCP.Upload.connection_handler/1>, - options: [recursive: true], - source: "/Users/sshkit/code/sshkit.ex", - state: {:next, "/Users/sshkit/code", [["sshkit.ex"]], []}, - target: "/home/code/sshkit" - } - ``` - """ - def init(source, target, options \\ []) do - source = Path.expand(source) - state = {:next, Path.dirname(source), [[Path.basename(source)]], []} - handler = connection_handler(options) - %__MODULE__{source: source, target: target, state: state, handler: handler, options: options} - end - - @doc """ - Executes an upload of a local file or directory to a remote host. - - ## Example - - ``` - :ok = SSHKit.SCP.Upload.exec(upload, conn) - ``` - """ - def exec(upload = %{source: source, options: options}, connection) do - recursive = Keyword.get(options, :recursive, false) - - if !recursive && File.dir?(source) do - {:error, "SCP option :recursive not specified, but local file is a directory (#{source})"} - else - start(upload, connection) - end - end - - defp start(%{target: target, state: state, handler: handler, options: options}, connection) do - timeout = Keyword.get(options, :timeout, :infinity) - map_cmd = Keyword.get(options, :map_cmd, & &1) - command = map_cmd.(Command.build(:upload, target, options)) - ssh = Keyword.get(options, :ssh, SSH) - ssh.run(connection, command, timeout: timeout, acc: {:cont, state}, fun: handler) - end - - @normal 0 - @warning 1 - @fatal 2 - defp connection_handler(options) do - fn message, state -> - case message do - {:data, _, 0, <<@warning, data::binary>>} -> - warning(options, state, data) - - {:data, _, 0, <<@fatal, data::binary>>} -> - fatal(options, state, data) - - {:data, _, 0, <<@normal>>} -> - handle_data(state, options) - - {:data, _, 0, data} -> - handle_error_data(state, options, data) - - {:data, _, 1, data} -> - fatal(options, state, data) - - {:exit_status, _, status} -> - exited(options, state, status) - - {:eof, _} -> - eof(options, state) - - {:closed, _} -> - closed(options, state) - end - end - end - - defp handle_data(state, options) do - case state do - {:next, cwd, stack, errs} -> - next(options, cwd, stack, errs) - - {:directory, name, stat, cwd, stack, errs} -> - directory(options, name, stat, cwd, stack, errs) - - {:regular, name, stat, cwd, stack, errs} -> - regular(options, name, stat, cwd, stack, errs) - - {:write, name, stat, cwd, stack, errs} -> - write(options, name, stat, cwd, stack, errs) - end - end - - defp handle_error_data(state, options, data) do - case state do - {:warning, state, buffer} -> warning(options, state, buffer <> data) - {:fatal, state, buffer} -> fatal(options, state, buffer <> data) - end - end - - defp next(_, _, [[]], errs) do - {:cont, :eof, {:done, nil, errs}} - end - - defp next(_, cwd, [[] | dirs], errs) do - {:cont, 'E\n', {:next, Path.dirname(cwd), dirs, errs}} - end - - defp next(options, cwd, [[name | rest] | dirs], errs) do - path = Path.join(cwd, name) - stat = File.stat!(path, time: :posix) - - stack = - case stat.type do - :directory -> [File.ls!(path) | [rest | dirs]] - :regular -> [rest | dirs] - end - - if Keyword.get(options, :preserve, false) do - time(options, stat.type, name, stat, cwd, stack, errs) - else - case stat.type do - :directory -> directory(options, name, stat, cwd, stack, errs) - :regular -> regular(options, name, stat, cwd, stack, errs) - end - end - end - - defp time(_, type, name, stat, cwd, stack, errs) do - directive = 'T#{stat.mtime} 0 #{stat.atime} 0\n' - {:cont, directive, {type, name, stat, cwd, stack, errs}} - end - - defp directory(_, name, stat, cwd, stack, errs) do - directive = 'D#{modefmt(stat.mode)} 0 #{name}\n' - {:cont, directive, {:next, Path.join(cwd, name), stack, errs}} - end - - defp regular(_, name, stat, cwd, stack, errs) do - directive = 'C#{modefmt(stat.mode)} #{stat.size} #{name}\n' - {:cont, directive, {:write, name, stat, cwd, stack, errs}} - end - - defp write(_, name, _, cwd, stack, errs) do - fs = File.stream!(Path.join(cwd, name), [], 16_384) - {:cont, Stream.concat(fs, [<<0>>]), {:next, cwd, stack, errs}} - end - - defp exited(_, {:done, nil, errs}, status) do - {:cont, {:done, status, errs}} - end - - defp exited(_, {_, _, _, errs}, status) do - {:halt, - {:error, "SCP exited before completing the transfer (#{status}): #{Enum.join(errs, ", ")}"}} - end - - defp eof(_, state) do - {:cont, state} - end - - defp closed(_, {:done, 0, _}) do - {:cont, :ok} - end - - defp closed(_, {:done, status, errs}) do - {:cont, {:error, "SCP exited with non-zero exit code #{status}: #{Enum.join(errs, ", ")}"}} - end - - defp closed(_, _) do - {:cont, {:error, "SCP channel closed before completing the transfer"}} - end - - defp warning(options, {name, _file, _stat, cwd, stack, errs}, buffer) do - warning(options, {name, cwd, stack, errs}, buffer) - end - - defp warning(options, state = {_name, cwd, stack, errs}, buffer) do - if String.last(buffer) == "\n" do - next(options, cwd, stack, errs ++ [String.trim(buffer)]) - else - {:cont, {:warning, state, buffer}} - end - end - - defp fatal(_, state, buffer) do - if String.last(buffer) == "\n" do - {:halt, {:error, String.trim(buffer)}} - else - {:cont, {:fatal, state, buffer}} - end - end - - defp modefmt(value) do - value - |> Bitwise.band(0o7777) - |> Integer.to_string(8) - |> String.pad_leading(4, "0") - end -end diff --git a/test/sshkit/scp/command_test.exs b/test/sshkit/scp/command_test.exs deleted file mode 100644 index 6a6a17a6..00000000 --- a/test/sshkit/scp/command_test.exs +++ /dev/null @@ -1,70 +0,0 @@ -defmodule SSHKit.SCP.CommandTest do - use ExUnit.Case, async: true - - import SSHKit.Utils, only: [shellescape: 1] - import SSHKit.SCP.Command, only: [build: 3] - - @path "/home/test/code" - @escaped shellescape(@path) - - describe "build/3 (:upload)" do - test "constructs basic upload commands" do - assert build(:upload, @path, []) == "scp -t #{@escaped}" - end - - test "constructs verbose upload commands" do - assert build(:upload, @path, verbose: true) == "scp -t -v #{@escaped}" - end - - test "constructs preserving upload commands" do - assert build(:upload, @path, preserve: true) == "scp -t -p #{@escaped}" - end - - test "constructs recursive upload commands" do - assert build(:upload, @path, recursive: true) == "scp -t -r #{@escaped}" - end - - test "constructs verbose and preserving upload commands" do - assert build(:upload, @path, verbose: true, preserve: true) == "scp -t -v -p #{@escaped}" - end - - test "constructs verbose and recursive upload commands" do - assert build(:upload, @path, verbose: true, recursive: true) == "scp -t -v -r #{@escaped}" - end - - test "constructs preserving and recursive upload commands" do - assert build(:upload, @path, preserve: true, recursive: true) == "scp -t -p -r #{@escaped}" - end - end - - describe "build/3 (:download)" do - test "constructs basic download commands" do - assert build(:download, @path, []) == "scp -f #{@escaped}" - end - - test "constructs verbose download commands" do - assert build(:download, @path, verbose: true) == "scp -f -v #{@escaped}" - end - - test "constructs preserving download commands" do - assert build(:download, @path, preserve: true) == "scp -f -p #{@escaped}" - end - - test "constructs recursive download commands" do - assert build(:download, @path, recursive: true) == "scp -f -r #{@escaped}" - end - - test "constructs verbose and preserving download commands" do - assert build(:download, @path, verbose: true, preserve: true) == "scp -f -v -p #{@escaped}" - end - - test "constructs verbose and recursive download commands" do - assert build(:download, @path, verbose: true, recursive: true) == "scp -f -v -r #{@escaped}" - end - - test "constructs preserving and recursive download commands" do - assert build(:download, @path, preserve: true, recursive: true) == - "scp -f -p -r #{@escaped}" - end - end -end diff --git a/test/sshkit/scp/upload_test.exs b/test/sshkit/scp/upload_test.exs deleted file mode 100644 index 593250e2..00000000 --- a/test/sshkit/scp/upload_test.exs +++ /dev/null @@ -1,215 +0,0 @@ -defmodule SSHKit.SCP.UploadTest do - use ExUnit.Case, async: true - import Mox - - alias SSHKit.SCP.Command - alias SSHKit.SCP.Upload - alias SSHKit.SSHMock - - @source "test/fixtures/local_dir" - @target "/home/test/code" - - describe "init/3" do - test "returns a new upload struct" do - upload = Upload.init(@source, @target) - source_expanded = @source |> Path.expand() - assert %Upload{source: ^source_expanded, target: @target} = upload - end - - test "returns a new upload struct with options" do - options = [recursive: true] - upload = Upload.init(@source, @target, options) - assert %Upload{options: ^options} = upload - end - - test "upload struct has initial state" do - %Upload{state: state} = Upload.init(@source, @target) - current_directory = @source |> Path.expand() |> Path.dirname() - assert state == {:next, current_directory, [["local_dir"]], []} - end - - test "upload struct has a handler function" do - %Upload{handler: handler} = Upload.init(@source, @target) - assert is_function(handler) - end - end - - describe "exec/2" do - setup do - {:ok, conn: %SSHKit.SSH.Connection{}} - end - - test "allows modifying the executed scp command", %{conn: conn} do - upload = - Upload.init(@source, @target, map_cmd: &"(( #{&1} ))", ssh: SSHMock, recursive: true) - - SSHMock - |> expect(:run, fn _, command, _ -> - assert command == "(( #{Command.build(:upload, upload.target, recursive: true)} ))" - {:ok, :success} - end) - - assert {:ok, :success} = Upload.exec(upload, conn) - end - - test "returns error when trying to upload a directory non-recursively", %{conn: conn} do - upload = Upload.init(@source, @target, recursive: false) - assert {:error, _msg} = Upload.exec(upload, conn) - end - - test "uses the provided timeout option", %{conn: conn} do - upload = Upload.init(@source, @target, recursive: true, timeout: 55, ssh: SSHMock) - - SSHMock - |> expect(:run, fn _, _, timeout: timeout, acc: {:cont, _}, fun: _ -> - assert timeout == 55 - {:ok, :success} - end) - - assert {:ok, :success} = Upload.exec(upload, conn) - end - - test "performs an upload", %{conn: conn} do - upload = Upload.init(@source, @target, recursive: true, ssh: SSHMock) - - SSHMock - |> expect(:run, fn connection, - command, - timeout: timeout, - acc: {:cont, state}, - fun: handler -> - assert connection == conn - assert command == Command.build(:upload, upload.target, upload.options) - assert timeout == :infinity - assert state == upload.state - assert handler == upload.handler - {:ok, :success} - end) - - assert {:ok, :success} = Upload.exec(upload, conn) - end - end - - describe "exec handler" do - setup do - channel = %SSHKit.SSH.Channel{} - ack_message = {:data, channel, 0, <<0>>} - {:ok, upload: Upload.init(@source, @target), ack: ack_message, channel: channel} - end - - test "recurses into directories", %{upload: upload, ack: ack} do - %Upload{handler: handler, state: {:next, cwd, [["local_dir"]], []} = state} = upload - next_path = Path.join(cwd, "local_dir") - - assert {:cont, 'D0755 0 local_dir\n', {:next, ^next_path, [["other.txt"], []], []}} = - handler.(ack, state) - end - - test "create files in the current directory", %{upload: %Upload{handler: handler}, ack: ack} do - source_expanded = @source |> Path.expand() - state = {:next, source_expanded, [["other.txt"], []], []} - - assert {:cont, 'C0644 61 other.txt\n', - {:write, "other.txt", %File.Stat{}, ^source_expanded, [[], []], []}} = - handler.(ack, state) - end - - test "writes files in the current directory", %{upload: %Upload{handler: handler}, ack: ack} do - source_expanded = @source |> Path.expand() |> Path.join("local_dir") - state = {:write, "other.txt", %File.Stat{}, source_expanded, [[], []], []} - fs = File.stream!(Path.join(source_expanded, "other.txt"), [], 16_384) - write_state = {:cont, Stream.concat(fs, [<<0>>]), {:next, source_expanded, [[], []], []}} - - assert write_state == handler.(ack, state) - end - - test "moves upwards in the directory hierachy", %{upload: %Upload{handler: handler}, ack: ack} do - source_dir = @source |> Path.expand() |> Path.join("local_dir") - source_expanded = @source |> Path.expand() - state = {:next, source_dir, [[], []], []} - - assert {:cont, 'E\n', {:next, ^source_expanded, [[]], []}} = handler.(ack, state) - end - - test "finalizes the upload", %{upload: %Upload{handler: handler}, ack: ack, channel: channel} do - source_expanded = @source |> Path.expand() - state = {:next, source_expanded, [[]], []} - - assert {:cont, :eof, done_state} = handler.(ack, state) - assert done_state == {:done, nil, []} - - exit_msg = {:exit_status, channel, 0} - assert {:cont, exit_state} = handler.(exit_msg, done_state) - assert exit_state == {:done, 0, []} - - eof_msg = {:eof, channel} - assert {:cont, eof_state} = handler.(eof_msg, exit_state) - assert eof_state == {:done, 0, []} - - closed_msg = {:closed, channel} - assert {:cont, :ok} == handler.(closed_msg, eof_state) - end - - test "aggregates warnings in the state", %{ - upload: %Upload{handler: handler, state: state}, - channel: channel - } do - error_msg = "error part 1 error part 2 error part 3" - - msg1 = {:data, channel, 0, <<1, "error part 1 ">>} - state1 = {:warning, state, "error part 1 "} - assert {:cont, state1} == handler.(msg1, state) - - msg2 = {:data, channel, 0, <<"error part 2 ">>} - state2 = {:warning, state, "error part 1 error part 2 "} - assert {:cont, state2} == handler.(msg2, state1) - - msg3 = {:data, channel, 0, <<"error part 3\n">>} - assert {:cont, _directive, {_name, _cwd, _stack, [^error_msg]}} = handler.(msg3, state2) - end - - test "proceeds with next file in stack after warning", %{ - upload: %Upload{handler: handler, state: state}, - channel: channel - } do - {name, cwd, _stack, _errs} = state - msg = {:data, channel, 0, <<1, "error message\n">>} - - assert {:cont, 'D0755 0 local_dir\n', - {name, cwd <> "/local_dir", [["other.txt"], []], ["error message"]}} == - handler.(msg, state) - end - - test "collects warning when in write state", %{ - upload: %Upload{handler: handler, state: state}, - channel: channel - } do - {name, cwd, stack, _errs} = state - - state = {:write, "local.txt", %{}, cwd, stack, []} - msg = {:data, channel, 0, <<1, "error message\n">>} - - assert {:cont, 'D0755 0 local_dir\n', - {name, cwd <> "/local_dir", [["other.txt"], []], ["error message"]}} == - handler.(msg, state) - end - - test "aggregates connection errors in the state and halts", %{ - upload: %Upload{handler: handler, state: state}, - channel: channel - } do - error_msg = "error part 1 error part 2 error part 3" - - msg1 = {:data, channel, 0, <<2, "error part 1 ">>} - state1 = {:fatal, state, "error part 1 "} - assert {:cont, state1} == handler.(msg1, state) - - msg2 = {:data, channel, 0, <<"error part 2 ">>} - state2 = {:fatal, state, "error part 1 error part 2 "} - assert {:cont, state2} == handler.(msg2, state1) - - msg3 = {:data, channel, 0, <<"error part 3\n">>} - assert {:halt, {:error, error_msg}} == handler.(msg3, state2) - end - end -end diff --git a/test/sshkit/scp_functional_test.exs b/test/sshkit/scp_functional_test.exs deleted file mode 100644 index 56acdc93..00000000 --- a/test/sshkit/scp_functional_test.exs +++ /dev/null @@ -1,121 +0,0 @@ -defmodule SSHKit.SCPFunctionalTest do - use SSHKit.FunctionalCase, async: true - - alias SSHKit.SCP - alias SSHKit.SSH - - @bootconf [user: "me", password: "pass"] - - describe "upload/4" do - @tag boot: [@bootconf] - test "sends a file", %{hosts: [host]} do - local = "test/fixtures/local.txt" - remote = "file.txt" - - SSH.connect(host.name, host.options, fn conn -> - assert :ok = SCP.upload(conn, local, remote) - assert verify_transfer(conn, local, remote) - end) - end - - @tag boot: [@bootconf] - test "recursive: true", %{hosts: [host]} do - local = "test/fixtures" - remote = "/home/#{host.options[:user]}/destination" - - SSH.connect(host.name, host.options, fn conn -> - assert :ok = SCP.upload(conn, local, remote, recursive: true) - assert verify_transfer(conn, local, remote) - end) - end - - @tag boot: [@bootconf] - test "preserve: true", %{hosts: [host]} do - local = "test/fixtures/local.txt" - remote = "file.txt" - - SSH.connect(host.name, host.options, fn conn -> - assert :ok = SCP.upload(conn, local, remote, preserve: true) - assert verify_mode(conn, local, remote) - assert verify_mtime(conn, local, remote) - end) - end - - @tag boot: [@bootconf] - test "recursive: true, preserve: true", %{hosts: [host]} do - local = "test/fixtures/" - remote = "/home/#{host.options[:user]}/destination" - - SSH.connect(host.name, host.options, fn conn -> - assert :ok = SCP.upload(conn, local, remote, recursive: true, preserve: true) - assert verify_mode(conn, local, remote) - assert verify_mtime(conn, local, remote) - end) - end - - @tag boot: [@bootconf] - test "uploads to nonexistent target directory (recursive: true)", %{hosts: [host]} do - local = "test/fixtures/local_dir" - remote = "/some/nonexistent/destination" - - SSH.connect(host.name, host.options, fn conn -> - assert {:error, msg} = SCP.upload(conn, local, remote, recursive: true) - assert msg =~ "scp: /some/nonexistent/destination: No such file or directory" - end) - end - end - - describe "download/4" do - @tag boot: [@bootconf] - test "gets a file", %{hosts: [host]} do - remote = "/fixtures/remote.txt" - local = create_local_tmp_path() - on_exit(fn -> File.rm(local) end) - - SSH.connect(host.name, host.options, fn conn -> - assert :ok = SCP.download(conn, remote, local) - assert verify_transfer(conn, local, remote) - end) - end - - @tag boot: [@bootconf] - test "recursive: true", %{hosts: [host]} do - remote = "/fixtures" - local = create_local_tmp_path() - on_exit(fn -> File.rm_rf(local) end) - - SSH.connect(host.name, host.options, fn conn -> - assert :ok = SCP.download(conn, remote, local, recursive: true) - assert verify_transfer(conn, local, remote) - end) - end - - @tag boot: [@bootconf] - test "preserve: true", %{hosts: [host]} do - remote = "/fixtures/remote.txt" - local = create_local_tmp_path() - on_exit(fn -> File.rm(local) end) - - SSH.connect(host.name, host.options, fn conn -> - assert :ok = SCP.download(conn, remote, local, preserve: true) - assert verify_mode(conn, local, remote) - assert verify_atime(conn, local, remote) - assert verify_mtime(conn, local, remote) - end) - end - - @tag boot: [@bootconf] - test "recursive: true, preserve: true", %{hosts: [host]} do - remote = "/fixtures" - local = create_local_tmp_path() - on_exit(fn -> File.rm_rf(local) end) - - SSH.connect(host.name, host.options, fn conn -> - assert :ok = SCP.download(conn, remote, local, recursive: true, preserve: true) - assert verify_mode(conn, local, remote) - assert verify_atime(conn, local, remote) - assert verify_mtime(conn, local, remote) - end) - end - end -end From 15af968de076702043209e9e00dc67bc46abc484 Mon Sep 17 00:00:00 2001 From: pmeinhardt Date: Sat, 12 Dec 2020 08:22:46 +0100 Subject: [PATCH 02/54] Remove unnecessary config directory --- config/config.exs | 1 - 1 file changed, 1 deletion(-) delete mode 100644 config/config.exs diff --git a/config/config.exs b/config/config.exs deleted file mode 100644 index becde769..00000000 --- a/config/config.exs +++ /dev/null @@ -1 +0,0 @@ -import Config From 779c1c688b1a6178de6afe43c6d989b755c02f23 Mon Sep 17 00:00:00 2001 From: pmeinhardt Date: Sat, 12 Dec 2020 08:32:24 +0100 Subject: [PATCH 03/54] Update mix.exs --- mix.exs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/mix.exs b/mix.exs index 1b06f818..f66f01d6 100644 --- a/mix.exs +++ b/mix.exs @@ -1,13 +1,14 @@ defmodule SSHKit.Mixfile do use Mix.Project - @version "0.3.0" + @name "sshkit" + @version "1.0.0" @source "https://github.com/bitcrowd/sshkit.ex" def project do [ app: :sshkit, - name: "sshkit", + name: @name, version: @version, elixir: "~> 1.5", elixirc_paths: elixirc_paths(Mix.env()), From 56a664eead7a3f9772056ba4a0f1d02b838225e1 Mon Sep 17 00:00:00 2001 From: pmeinhardt Date: Sat, 12 Dec 2020 10:31:21 +0100 Subject: [PATCH 04/54] Draft top-level interface (work in progress) --- CHANGELOG.md | 12 +++++++++-- README.md | 61 +++++++++++++++++++++------------------------------- 2 files changed, 34 insertions(+), 39 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 06875cea..4dafdb50 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,8 +8,6 @@ https://github.com/bitcrowd/sshkit.ex/compare/v0.3.0...HEAD -* Drop SCP support, create alternative file transfer implementation - ### Deprecations: @@ -29,6 +27,16 @@ https://github.com/bitcrowd/sshkit.ex/compare/v0.3.0...HEAD +## `1.0.0` (??) + +* Implement new top-level package API + * Explicitly establish, re-use and close connections + * Enable streaming data to and from a remote + * Make context optional +* Replace SCP with an alternative file transfer implementation + +https://github.com/bitcrowd/sshkit.ex/compare/v0.3.0...v1.0.0 + ## `0.3.0` (2019-10-28) https://github.com/bitcrowd/sshkit.ex/compare/v0.2.0...v0.3.0 diff --git a/README.md b/README.md index 597434f5..0d5b754d 100644 --- a/README.md +++ b/README.md @@ -14,58 +14,45 @@ SSHKit is designed to enable server task automation in a structured and repeatab ```elixir hosts = ["1.eg.io", {"2.eg.io", port: 2222}] +{:ok, conn} = SSHKit.connect(hosts) + +{:ok, _} = SSHKit.run(conn, "apt-get update -y") + +{:ok, } = SSHKit.stream(chan) + context = - SSHKit.context(hosts) + SSHKit.context() |> SSHKit.path("/var/www/phx") |> SSHKit.user("deploy") |> SSHKit.group("deploy") |> SSHKit.umask("022") |> SSHKit.env(%{"NODE_ENV" => "production"}) -[:ok, :ok] = SSHKit.upload(context, ".", recursive: true) -[{:ok, _, 0}, {:ok, _, 0}] = SSHKit.run(context, "yarn install") +{:ok, _} = SSHKit.upload(conn, ".", recursive: true, context: context) + +# TODO: Receive upload status messages + +{:ok, chan} = SSHKit.run(conn, "yarn install", context: context) + +# TODO: Showcase streaming interface + +:ok = SSHKit.close(conn) ``` The [`SSHKit`](https://hexdocs.pm/sshkit/SSHKit.html) module documentation has more guidance and examples for the DSL. -If you need more control, take a look at the [`SSHKit.SSH`](https://hexdocs.pm/sshkit/SSHKit.SSH.html) and [`SSHKit.SCP`](https://hexdocs.pm/sshkit/SSHKit.SCP.html) modules. - ## Installation Just add `sshkit` to your list of dependencies in `mix.exs`: ```elixir def deps do - [{:sshkit, "~> 0.1"}] + [{:sshkit, "~> 1.0"}] end ``` SSHKit should be automatically started unless the `:applications` key is set inside `def application` in your `mix.exs`. In such cases, you need to [remove the `:applications` key in favor of `:extra_applications`](https://elixir-lang.org/blog/2017/01/05/elixir-v1-4-0-released/#application-inference). -## Modules - -SSHKit consists of three core modules: - -``` -+--------------------+ -| SSHKit | -+--------------------+ -| | SSHKit.SCP | -| +------------+ -| SSHKit.SSH | -+--------------------+ -``` - -1. [**`SSHKit.SSH`**](https://hexdocs.pm/sshkit/SSHKit.SSH.html) provides convenience functions for working with SSH connections and for executing commands on remote hosts. - -2. [**`SSHKit.SCP`**](https://hexdocs.pm/sshkit/SSHKit.SCP.html) provides convenience functions for transferring files or entire directory trees to or from a remote host via SCP. It is built on top of `SSHKit.SSH`. - -3. [**`SSHKit`**](https://hexdocs.pm/sshkit/SSHKit.html) provides the main API for automating tasks on remote hosts in a structured way. It uses both `SSH` and `SCP` to implement its functionality. - -Additional modules, e.g. for custom client key handling, are available as separate packages: - -* [**`ssh_client_key_api`**](https://hex.pm/packages/ssh_client_key_api): An Elixir implementation for the Erlang `ssh_client_key_api` behavior, to make it easier to specify SSH keys and `known_hosts` files independently of any particular user's home directory. - ## Testing As usual, to run all tests, use: @@ -74,7 +61,7 @@ As usual, to run all tests, use: mix test ``` -Apart from unit tests, we also have [functional tests](https://en.wikipedia.org/wiki/Functional_testing). These check SSHKit functionality against real SSH server implementations running inside Docker containers. Therefore, you need to have [Docker](https://www.docker.com/) installed. +Apart from unit tests, we also have [functional tests](https://en.wikipedia.org/wiki/Functional_testing). These check SSHKit against real SSH server implementations running inside Docker containers. Therefore, you need to have [Docker](https://www.docker.com/) installed. All functional tests are tagged as such. Hence, if you wish to skip them: @@ -121,9 +108,9 @@ SSHKit source code is released under the MIT License. Check the [LICENSE][license] file for more information. - [issues]: https://github.com/bitcrowd/sshkit.ex/issues - [pulls]: https://github.com/bitcrowd/sshkit.ex/pulls - [docs]: https://hexdocs.pm/sshkit - [changelog]: ./CHANGELOG.md - [license]: ./LICENSE - [writing-docs]: https://hexdocs.pm/elixir/writing-documentation.html +[issues]: https://github.com/bitcrowd/sshkit.ex/issues +[pulls]: https://github.com/bitcrowd/sshkit.ex/pulls +[docs]: https://hexdocs.pm/sshkit +[changelog]: ./CHANGELOG.md +[license]: ./LICENSE +[writing-docs]: https://hexdocs.pm/elixir/writing-documentation.html From dbb68fdc8f511b12328eb737284dbe563f93bf59 Mon Sep 17 00:00:00 2001 From: pmeinhardt Date: Sat, 12 Dec 2020 18:39:55 +0100 Subject: [PATCH 05/54] Remove host info from SSHKit.Context --- lib/sshkit/context.ex | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/sshkit/context.ex b/lib/sshkit/context.ex index 0a4b6ca2..64cc615d 100644 --- a/lib/sshkit/context.ex +++ b/lib/sshkit/context.ex @@ -2,7 +2,6 @@ defmodule SSHKit.Context do @moduledoc """ A context encapsulates the environment for the execution of a task. That is: - * hosts to run the task on, see `SSHKit.context/2` * working directory to start in, see `SSHKit.path/2` * user to run as, see `SSHKit.user/2` * group, see `SSHKit.group/2` @@ -15,7 +14,7 @@ defmodule SSHKit.Context do import SSHKit.Utils - defstruct hosts: [], env: nil, path: nil, umask: nil, user: nil, group: nil + defstruct env: nil, path: nil, umask: nil, user: nil, group: nil @doc """ Compiles an executable command string for running the given `command` From 1b6b1b184761a45774884d395f579ef205f0b83a Mon Sep 17 00:00:00 2001 From: pmeinhardt Date: Sun, 13 Dec 2020 09:56:03 +0100 Subject: [PATCH 06/54] Continue draft for the new top-level API --- examples/stream.exs | 34 ++++++++ lib/sshkit.ex | 197 ++++++++++---------------------------------- lib/sshkit/ssh.ex | 36 +------- 3 files changed, 80 insertions(+), 187 deletions(-) create mode 100644 examples/stream.exs diff --git a/examples/stream.exs b/examples/stream.exs new file mode 100644 index 00000000..affaa815 --- /dev/null +++ b/examples/stream.exs @@ -0,0 +1,34 @@ +# TODO: Timeouts? +# TODO: Multiple hosts? + +{:ok, conn} = SSHKit.connect("127.0.0.1", port: 2222, user: "deploy", password: "deploy", silently_accept_hosts: true) + +{:ok, chan} = SSHKit.run(conn, ~S(echo "Who's there?"; read msg; echo "Hello $msg!")) + +{:ok, out1} = SSHKit.stream(chan, "", fn {:stdout, ^chan, data}, buffer -> + tag = if String.ends_with?(data, "\n"), do: :halt, else: :cont + {tag, buffer <> data} +end) + +out1 +|> String.trim() +|> IO.puts() + +:ok = SSHKit.send(chan, "SSHKit\n") + +{:ok, out2} = SSHKit.stream(chan, "", fn + {:stdout, ^chan, data}, buffer -> + {:cont, buffer <> data} + + {:closed, ^chan}, buffer -> + {:halt, buffer} + + _, buffer -> + {:cont, buffer} +end) + +out2 +|> String.trim() +|> IO.puts() + +:ok = SSHKit.close(conn) diff --git a/lib/sshkit.ex b/lib/sshkit.ex index b8d71f84..e33ef285 100644 --- a/lib/sshkit.ex +++ b/lib/sshkit.ex @@ -6,7 +6,7 @@ defmodule SSHKit do hosts = ["1.eg.io", {"2.eg.io", port: 2222}] context = - SSHKit.context(hosts) + SSHKit.context() |> SSHKit.path("/var/www/phx") |> SSHKit.user("deploy") |> SSHKit.group("deploy") @@ -24,123 +24,67 @@ defmodule SSHKit do alias SSHKit.Host @doc """ - Produces an `SSHKit.Host` struct holding the information - needed to connect to a (remote) host. + TODO - ## Examples - - You can pass a map with hostname and options: - - ``` - host = SSHKit.host(%{name: "name.io", options: [port: 2222]}) - - # This means, that if you pass in a host struct, - # you'll get the same result. In particular: - host == SSHKit.host(host) - ``` - - …or, alternatively, a tuple with hostname and options: - - ``` - host = SSHKit.host({"name.io", port: 2222}) - ``` - - See `host/2` for additional details and examples. + Takes one or more (remote) hosts and creates an execution context in which + remote commands can be run. Accepts any form of host specification also + accepted by `host/1` and `host/2`, i.e. binaries, maps and 2-tuples. """ - def host(%{name: name, options: options}) do - %Host{name: name, options: options} + def connect(host, options \\ []) do + SSH.connect(host, options) end - def host({name, options}) do - %Host{name: name, options: options} + def close(conn) do + SSH.close(conn) end - @doc """ - Produces an `SSHKit.Host` struct holding the information - needed to connect to a (remote) host. - - ## Examples - - In its most basic version, you just pass a hostname and all other options - will use the defaults: + def run(conn, command, options \\ []) do + SSH.run(conn, command, options) + end - ``` - host = SSHKit.host("name.io") - ``` + def send(chan, type \\ 0, data) do + SSH.Channel.send(chan, type, data) + end - If you wish to provide additional host options, e.g. a non-standard port, - you can pass a keyword list as the second argument: + def stream(chan, acc, fun) do + {:ok, msg} = SSH.Channel.recv(chan) # TODO: timeout? - ``` - host = SSHKit.host("name.io", port: 2222) - ``` + next = + case msg do + {:exit_signal, ^chan, signal, message, lang} -> + fun.({:signal, chan, signal, message, lang}, acc) - One or many of these hosts can then be used to create an execution context - in which commands can be executed: + {:exit_status, ^chan, status} -> + fun.({:exit, chan, status}, acc) - ``` - host - |> SSHKit.context() - |> SSHKit.run("echo \"That was fun\"") - ``` + {:data, ^chan, 0, data} -> + fun.({:stdout, chan, data}, acc) - See `host/1` for additional ways of specifying host details. - """ - def host(host, options \\ []) + {:data, ^chan, 1, data} -> + fun.({:stderr, chan, data}, acc) - def host(name, options) when is_binary(name) do - %Host{name: name, options: options} - end + {:eof, ^chan} -> + fun.({:eof, chan}, acc) - def host(%{name: name, options: options}, defaults) do - %Host{name: name, options: Keyword.merge(defaults, options)} - end + {:closed, ^chan} -> + fun.({:closed, chan}, acc) + end - def host({name, options}, defaults) do - %Host{name: name, options: Keyword.merge(defaults, options)} + case next do + {:cont, acc} -> stream(chan, acc, fun) + {:halt, acc} -> {:ok, acc} + other -> other + end end @doc """ - Takes one or more (remote) hosts and creates an execution context in which - remote commands can be run. Accepts any form of host specification also - accepted by `host/1` and `host/2`, i.e. binaries, maps and 2-tuples. + Creates an execution context in which remote commands can be run. See `path/2`, `user/2`, `group/2`, `umask/2`, and `env/2` for details on how to derive variations of a context. - - ## Example - - Create an execution context for two hosts. Commands issued in this context - will be executed on both hosts. - - ``` - hosts = ["10.0.0.1", "10.0.0.2"] - context = SSHKit.context(hosts) - ``` - - Create a context for hosts with different connection options: - - ``` - hosts = [{"10.0.0.3", port: 2223}, %{name: "10.0.0.4", options: [port: 2224]}] - context = SSHKit.context(hosts) - ``` - - Any shared options can be specified in the second argument. - Here we add a user and port for all hosts. - - ``` - hosts = ["10.0.0.1", "10.0.0.2"] - options = [user: "admin", port: 2222] - context = SSHKit.context(hosts, options) - ``` """ - def context(hosts, defaults \\ []) do - hosts = - hosts - |> List.wrap() - |> Enum.map(&host(&1, defaults)) - - %Context{hosts: hosts} + def context() do + %Context{} end @doc """ @@ -153,10 +97,9 @@ defmodule SSHKit do Create `/var/www/app/config.json`: ``` - "10.0.0.1" - |> SSHKit.context() + SSHKit.context() |> SSHKit.path("/var/www/app") - |> SSHKit.run("touch config.json") + |> SSHKit.run(conn, "touch config.json") ``` """ def path(context, path) do @@ -263,62 +206,6 @@ defmodule SSHKit do %Context{context | env: map} end - @doc ~S""" - Executes a command in the given context. - - Returns a list of tuples, one fore each host in the context. - - The resulting tuples have the form `{:ok, output, exit_code}` – - as returned by `SSHKit.SSH.run/3`: - - * `exit_code` is the number with which the executed command returned. - - If everything went well, that usually is `0`. - - * `output` is a keyword list of the output collected from the command. - - It has the form: - - ``` - [ - stdout: "output on standard out", - stderr: "output on standard error", - stdout: "some more normal output", - … - ] - ``` - - ## Example - - Run a command and verify its output: - - ``` - [{:ok, output, 0}] = - "example.io" - |> SSHKit.context() - |> SSHKit.run("echo \"Hello World!\"") - - stdout = - output - |> Keyword.get_values(:stdout) - |> Enum.join() - - assert "Hello World!\n" == stdout - ``` - """ - def run(context, command) do - cmd = Context.build(context, command) - - run = fn host -> - {:ok, conn} = SSH.connect(host.name, host.options) - res = SSH.run(conn, cmd) - :ok = SSH.close(conn) - res - end - - Enum.map(context.hosts, run) - end - @doc ~S""" Upload a file or files to the given context. diff --git a/lib/sshkit/ssh.ex b/lib/sshkit/ssh.ex index 0f03cc09..c54d7ed1 100644 --- a/lib/sshkit/ssh.ex +++ b/lib/sshkit/ssh.ex @@ -109,9 +109,8 @@ defmodule SSHKit.SSH do ## Options + * TODO: disambiguate channel opening and exec options * `:timeout` - maximum wait time between messages, defaults to `:infinity` - * `:fun` - handler function passed to `SSHKit.SSH.Channel.loop/4` - * `:acc` - initial accumulator value used in the loop Any other options will be passed on to `SSHKit.SSH.Channel.open/2` when creating the channel for executing the command. @@ -125,46 +124,19 @@ defmodule SSHKit.SSH do """ @callback run(Connection.t(), binary(), keyword()) :: any() def run(connection, command, options \\ []) do - {acc, options} = Keyword.pop(options, :acc, {:cont, {[], nil}}) - {fun, options} = Keyword.pop(options, :fun, &capture/2) - timeout = Keyword.get(options, :timeout, :infinity) with {:ok, channel} <- Channel.open(connection, options) do case Channel.exec(channel, command, timeout) do :success -> - channel - |> Channel.loop(timeout, acc, fun) - |> elem(1) + {:ok, channel} :failure -> {:error, :failure} - err -> - err + error -> + error end end end - - defp capture(message, acc = {buffer, status}) do - next = - case message do - {:data, _, 0, data} -> - {[{:stdout, data} | buffer], status} - - {:data, _, 1, data} -> - {[{:stderr, data} | buffer], status} - - {:exit_status, _, code} -> - {buffer, code} - - {:closed, _} -> - {:ok, Enum.reverse(buffer), status} - - _ -> - acc - end - - {:cont, next} - end end From 93aeeb45e9a5220b306197aa5ead15703e20e24d Mon Sep 17 00:00:00 2001 From: pmeinhardt Date: Tue, 15 Dec 2020 00:08:15 +0100 Subject: [PATCH 07/54] Return a Stream from SSHKit.stream/1 --- examples/stream.exs | 38 +++++++++++++++------------------ lib/sshkit.ex | 52 ++++++++++++++++++++++++++------------------- 2 files changed, 47 insertions(+), 43 deletions(-) diff --git a/examples/stream.exs b/examples/stream.exs index affaa815..8c118df9 100644 --- a/examples/stream.exs +++ b/examples/stream.exs @@ -1,34 +1,30 @@ -# TODO: Timeouts? # TODO: Multiple hosts? {:ok, conn} = SSHKit.connect("127.0.0.1", port: 2222, user: "deploy", password: "deploy", silently_accept_hosts: true) {:ok, chan} = SSHKit.run(conn, ~S(echo "Who's there?"; read msg; echo "Hello $msg!")) -{:ok, out1} = SSHKit.stream(chan, "", fn {:stdout, ^chan, data}, buffer -> - tag = if String.ends_with?(data, "\n"), do: :halt, else: :cont - {tag, buffer <> data} -end) - -out1 -|> String.trim() -|> IO.puts() +IO.write("> ") -:ok = SSHKit.send(chan, "SSHKit\n") +chan +|> SSHKit.stream() +|> Stream.map(fn {:stdout, ^chan, chunk} -> chunk end) +|> Stream.each(&IO.write/1) +|> Stream.take_while(fn chunk -> !String.ends_with?(chunk, "\n") end) +|> Stream.run() -{:ok, out2} = SSHKit.stream(chan, "", fn - {:stdout, ^chan, data}, buffer -> - {:cont, buffer <> data} +IO.write("< SSHKit\n") - {:closed, ^chan}, buffer -> - {:halt, buffer} +:ok = SSHKit.send(chan, "SSHKit\n") - _, buffer -> - {:cont, buffer} -end) +IO.write("> ") -out2 -|> String.trim() -|> IO.puts() +# TODO: Timeouts? +chan +|> SSHKit.stream() +|> Stream.filter(fn msg -> elem(msg, 0) == :stdout end) +|> Stream.map(fn {:stdout, ^chan, chunk} -> chunk end) +|> Stream.each(&IO.write/1) +|> Stream.run() :ok = SSHKit.close(conn) diff --git a/lib/sshkit.ex b/lib/sshkit.ex index e33ef285..1d1ed835 100644 --- a/lib/sshkit.ex +++ b/lib/sshkit.ex @@ -46,35 +46,43 @@ defmodule SSHKit do SSH.Channel.send(chan, type, data) end - def stream(chan, acc, fun) do - {:ok, msg} = SSH.Channel.recv(chan) # TODO: timeout? + def stream(chan) do + Stream.unfold(:cont, fn + :cont -> + {:ok, msg} = SSH.Channel.recv(chan) # TODO: timeout? - next = - case msg do - {:exit_signal, ^chan, signal, message, lang} -> - fun.({:signal, chan, signal, message, lang}, acc) + value = + case msg do + {:exit_signal, ^chan, signal, message, lang} -> + {:signal, chan, signal, message, lang} - {:exit_status, ^chan, status} -> - fun.({:exit, chan, status}, acc) + {:exit_status, ^chan, status} -> + {:exit, chan, status} - {:data, ^chan, 0, data} -> - fun.({:stdout, chan, data}, acc) + {:data, ^chan, 0, data} -> + {:stdout, chan, data} - {:data, ^chan, 1, data} -> - fun.({:stderr, chan, data}, acc) + {:data, ^chan, 1, data} -> + {:stderr, chan, data} - {:eof, ^chan} -> - fun.({:eof, chan}, acc) + {:eof, ^chan} -> + {:eof, chan} - {:closed, ^chan} -> - fun.({:closed, chan}, acc) - end + {:closed, ^chan} -> + {:closed, chan} + end - case next do - {:cont, acc} -> stream(chan, acc, fun) - {:halt, acc} -> {:ok, acc} - other -> other - end + next = + case value do + {:closed, _} -> :halt + _ -> :cont + end + + {value, next} + + :halt -> + nil + end) end @doc """ From 307cbcb009881849b982775c122a70af26a3fd47 Mon Sep 17 00:00:00 2001 From: pmeinhardt Date: Tue, 15 Dec 2020 00:25:45 +0100 Subject: [PATCH 08/54] Update example variable naming --- examples/stream.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/stream.exs b/examples/stream.exs index 8c118df9..7a259f5a 100644 --- a/examples/stream.exs +++ b/examples/stream.exs @@ -2,7 +2,7 @@ {:ok, conn} = SSHKit.connect("127.0.0.1", port: 2222, user: "deploy", password: "deploy", silently_accept_hosts: true) -{:ok, chan} = SSHKit.run(conn, ~S(echo "Who's there?"; read msg; echo "Hello $msg!")) +{:ok, chan} = SSHKit.run(conn, ~S(echo "Who's there?"; read name; echo "Hello $name")) IO.write("> ") From 750a5af8e112a5dcdb0caf7a86ba2612d95b982e Mon Sep 17 00:00:00 2001 From: pmeinhardt Date: Tue, 15 Dec 2020 00:35:21 +0100 Subject: [PATCH 09/54] Drop Channel.loop/4, obsoleted by SSHKit.stream/1 --- lib/sshkit/ssh/channel.ex | 115 -------------------------------------- 1 file changed, 115 deletions(-) diff --git a/lib/sshkit/ssh/channel.ex b/lib/sshkit/ssh/channel.ex index aea8954a..64264853 100644 --- a/lib/sshkit/ssh/channel.ex +++ b/lib/sshkit/ssh/channel.ex @@ -195,119 +195,4 @@ defmodule SSHKit.SSH.Channel do def adjust(channel, size) when is_integer(size) do channel.impl.adjust_window(channel.connection.ref, channel.id, size) end - - @doc """ - Executes the user default shell at server side. - - Returns `:ok`. - - For more details, see [`:ssh_connection.shell/2`](http://erlang.org/doc/man/ssh_connection.html#shell-2). - """ - def shell(channel) do - channel.impl.shell(channel.connection.ref, channel.id) - end - - @doc """ - Loops over channel messages until the channel is closed, or looping is stopped - explicitly. - - Expects an accumulator on each call that determines how to proceed: - - 1. `{:cont, acc}` - - The loop will wait for an inbound message. It will then pass the message and - current `acc` to the looping function. `fun`'s return value is the - accumulator for the next cycle. - - 2. `{:cont, message, acc}` - - Sends a message to the remote end of the channel before waiting for a - message as outlined in the `{:cont, acc}` case above. `message` may be one - of the following: - - * `{0, data}` or `{1, data}` - sends normal or stderr data to the remote - * `data` - is a shortcut for `{0, data}` - * `:eof` - sends EOF - - 3. `{:halt, acc}` - - Terminates the loop, returning `{:halted, acc}`. - - 4. `{:suspend, acc}` - - Suspends the loop, returning `{:suspended, acc, continuation}`. - `continuation` is a function that accepts a new accumulator value and that, - when called, will resume the loop. - - `timeout` specifies the maximum wait time for receiving and sending individual - messages. - - Once the final `{:closed, channel}` message is received, the loop will - terminate and return `{:done, acc}`. The channel will be closed if it has - not been closed before. - """ - def loop(channel, timeout \\ :infinity, acc, fun) - - def loop(channel, timeout, {:cont, msg, acc}, fun) do - case lsend(channel, msg, timeout) do - :ok -> loop(channel, timeout, {:cont, acc}, fun) - err -> halt(channel, err) - end - end - - def loop(channel, timeout, {:cont, acc}, fun) do - case recv(channel, timeout) do - {:ok, msg} -> - if elem(msg, 0) == :closed do - {_, acc} = fun.(msg, acc) - done(channel, acc) - else - :ok = ljust(channel, msg) - loop(channel, timeout, fun.(msg, acc), fun) - end - - err -> - halt(channel, err) - end - end - - def loop(channel, _, {:halt, acc}, _) do - halt(channel, acc) - end - - def loop(channel, timeout, {:suspend, acc}, fun) do - suspend(channel, acc, fun, timeout) - end - - defp halt(channel, acc) do - :ok = close(channel) - :ok = flush(channel) - {:halted, acc} - end - - defp suspend(channel, acc, fun, timeout) do - {:suspended, acc, &loop(channel, timeout, &1, fun)} - end - - defp done(_, acc) do - {:done, acc} - end - - defp lsend(_, nil, _), do: :ok - - defp lsend(channel, :eof, _), do: eof(channel) - - defp lsend(channel, {type, data}, timeout) do - send(channel, type, data, timeout) - end - - defp lsend(channel, data, timeout) do - send(channel, 0, data, timeout) - end - - defp ljust(channel, {:data, _, _, data}) do - adjust(channel, byte_size(data)) - end - - defp ljust(_, _), do: :ok end From 0c5e3a669a7dea0d62d056101b6ed57c11ebb21a Mon Sep 17 00:00:00 2001 From: pmeinhardt Date: Tue, 15 Dec 2020 08:59:53 +0100 Subject: [PATCH 10/54] Example of parallel remote tasks --- examples/parallel.exs | 53 +++++++++++++++++++++++++++++++++++++++++++ examples/stream.exs | 2 -- 2 files changed, 53 insertions(+), 2 deletions(-) create mode 100644 examples/parallel.exs diff --git a/examples/parallel.exs b/examples/parallel.exs new file mode 100644 index 00000000..feb8a6ea --- /dev/null +++ b/examples/parallel.exs @@ -0,0 +1,53 @@ +defaults = [user: "deploy", password: "deploy", silently_accept_hosts: true] + +hosts = + [{"127.0.0.1", port: 2222}, {"127.0.0.1", port: 2223}] + |> Enum.map(fn {name, options} -> {name, Keyword.merge(defaults, options)} end) + +conns = Enum.map(hosts, fn {name, options} -> + {:ok, conn} = SSHKit.connect(name, options) + conn +end) + +label = fn conn -> Enum.join([conn.host, conn.port], ":") end + +tasks = + Enum.map(conns, fn conn -> + Task.async(fn -> + {:ok, chan} = SSHKit.run(conn, "uptime") + + chan + |> SSHKit.stream() + |> Enum.reduce(nil, fn + {:stdout, chan, output}, acc -> + IO.write("[#{label.(chan.connection)}] (stdout) #{output}") + acc + + {:stderr, chan, output}, acc -> + IO.write("[#{label.(chan.connection)}] (stderr) #{output}") + acc + + {:exit, _, status}, _ -> + status + + _, acc -> + acc + end) + end) + end) + +okay? = fn status -> status == 0 end + +results = Enum.map(tasks, &Task.await/1) + +unless Enum.all?(results, okay?) do + results + |> Enum.with_index() + |> Enum.filter(fn {status, _} -> !okay?.(status) end) + |> Enum.each(fn {status, index} -> + conn = Enum.at(conns, index) + IO.puts("[#{label.(conn)}] exited with status #{status}") + end) +end + +:ok = Enum.each(conns, &SSHKit.close/1) diff --git a/examples/stream.exs b/examples/stream.exs index 7a259f5a..4c876990 100644 --- a/examples/stream.exs +++ b/examples/stream.exs @@ -1,5 +1,3 @@ -# TODO: Multiple hosts? - {:ok, conn} = SSHKit.connect("127.0.0.1", port: 2222, user: "deploy", password: "deploy", silently_accept_hosts: true) {:ok, chan} = SSHKit.run(conn, ~S(echo "Who's there?"; read name; echo "Hello $name")) From 43f273454e811ed3fc4ed2b771167c6083366ab0 Mon Sep 17 00:00:00 2001 From: pmeinhardt Date: Thu, 17 Dec 2020 08:35:19 +0100 Subject: [PATCH 11/54] Prototype new upload implementation using SFTP --- examples/upload.exs | 5 +++ lib/sshkit.ex | 11 +++-- lib/sshkit/upload.ex | 105 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 118 insertions(+), 3 deletions(-) create mode 100644 examples/upload.exs create mode 100644 lib/sshkit/upload.ex diff --git a/examples/upload.exs b/examples/upload.exs new file mode 100644 index 00000000..ed12658c --- /dev/null +++ b/examples/upload.exs @@ -0,0 +1,5 @@ +{:ok, conn} = SSHKit.connect("127.0.0.1", port: 2222, user: "deploy", password: "deploy", silently_accept_hosts: true) + +:ok = SSHKit.upload(conn, "test/fixtures", "/tmp/fixtures", recursive: true) + +:ok = SSHKit.close(conn) diff --git a/lib/sshkit.ex b/lib/sshkit.ex index 1d1ed835..5719aa6e 100644 --- a/lib/sshkit.ex +++ b/lib/sshkit.ex @@ -22,6 +22,7 @@ defmodule SSHKit do alias SSHKit.Context alias SSHKit.Host + alias SSHKit.Upload @doc """ TODO @@ -245,8 +246,12 @@ defmodule SSHKit do |> SSHKit.upload("local.txt", as: "remote.txt") ``` """ - def upload(context, source, options \\ []) do - # TODO + def upload(conn, source, target, options \\ []) do + upload = Upload.init(source, target, options) + + with {:ok, upload} <- Upload.start(upload, conn) do + Upload.loop(upload) + end end @doc ~S""" @@ -280,7 +285,7 @@ defmodule SSHKit do |> SSHKit.download("remote.txt", as: "local.txt") ``` """ - def download(context, source, options \\ []) do + def download(conn, source, options \\ []) do # TODO end end diff --git a/lib/sshkit/upload.ex b/lib/sshkit/upload.ex new file mode 100644 index 00000000..f6e78953 --- /dev/null +++ b/lib/sshkit/upload.ex @@ -0,0 +1,105 @@ +defmodule SSHKit.Upload do + @moduledoc """ + TODO + """ + + defstruct [:source, :target, :options, :cwd, :stack, :channel] + + def init(source, target, options \\ []) do + %__MODULE__{source: Path.expand(source), target: target, options: options} + end + + def start(%__MODULE__{} = upload, connection) do + with {:ok, upload} <- prepare(upload) do + {:ok, channel} = :ssh_sftp.start_channel(connection.ref) # accepts options like timeout… http://erlang.org/doc/man/ssh_sftp.html#start_channel-1 + {:ok, %{upload | channel: channel}} + end + end + + defp prepare(%__MODULE__{source: source, options: options} = upload) do + # TODO: Support globs, https://hexdocs.pm/elixir/Path.html#wildcard/2 + if !Keyword.get(options, :recursive, false) && File.dir?(source) do + {:error, "Option :recursive not specified, but local file is a directory (#{source})"} # TODO: Better error + else + {:ok, %{upload | cwd: Path.dirname(source), stack: [[Path.basename(source)]]}} + end + end + + def stop(%__MODULE__{channel: nil} = upload), do: {:ok, upload} + def stop(%__MODULE__{channel: channel} = upload) do + with :ok <- :ssh_sftp.stop_channel(channel) do + {:ok, %{upload | channel: nil}} + end + end + + # TODO: Handle unstarted uploads w/o channel, cwd, stack… and provide helpful error? + + def continue(%__MODULE__{stack: []} = upload) do + {:ok, upload} + end + + def continue(%__MODULE__{stack: [[] | paths]} = upload) do + {:ok, %{upload | cwd: Path.dirname(upload.cwd), stack: paths}} + end + + def continue(%__MODULE__{stack: [[name | rest] | paths]} = upload) do + path = Path.join(upload.cwd, name) + relpath = Path.relative_to(path, Path.expand(upload.source)) + relpath = if relpath == path, do: ".", else: relpath + + remote = + upload.target + |> Path.join(relpath) + |> Path.expand() + + with {:ok, stat} <- File.stat(path, time: :posix) do + # TODO: Set timestamps… if :preserve option is true, http://erlang.org/doc/man/ssh_sftp.html#write_file_info-3 + + channel = upload.channel + + case stat.type do + :directory -> + # TODO: Timeouts + :ok = :ssh_sftp.make_dir(channel, remote) + {:ok, names} = File.ls(path) + {:ok, %{upload | cwd: path, stack: [names | [rest | paths]]}} + + :regular -> + # TODO: Timeouts + {:ok, handle} = :ssh_sftp.open(channel, remote, [:write, :binary]) + + path + |> File.stream!([], 16_384) + |> Stream.each(fn data -> :ok = :ssh_sftp.write(channel, handle, data) end) + |> Stream.run() + + :ok = :ssh_sftp.close(channel, handle) + {:ok, %{upload | stack: [rest | paths]}} + + :symlink -> + nil + + _ -> + {:error, {:unkown_file_type, path}} + end + end + end + + # TODO: Make `loop` return a stream? Possibly rename to "stream" then + def loop(%__MODULE__{stack: []}) do + :ok + end + + def loop(%__MODULE__{} = upload) do + case continue(upload) do + {:ok, upload} -> + loop(upload) + + error -> + error + end + end + + def done?(%__MODULE__{stack: []}), do: true + def done?(%__MODULE__{}), do: false +end From 998abb50d4fb753b4894d7a2078cb4b9e52586fd Mon Sep 17 00:00:00 2001 From: pmeinhardt Date: Fri, 18 Dec 2020 00:30:14 +0100 Subject: [PATCH 12/54] Rename SSHKit.stream to SSHKit.stream! Taking an element from the stream may result in an exception, if receiving data from the remote fails, e.g. because of a timeout. Therefore using "stream!" seems more appropriate and is in line with Elixir's `File.stream!` from the standard library. --- examples/parallel.exs | 2 +- examples/stream.exs | 4 ++-- lib/sshkit.ex | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/examples/parallel.exs b/examples/parallel.exs index feb8a6ea..037989a1 100644 --- a/examples/parallel.exs +++ b/examples/parallel.exs @@ -17,7 +17,7 @@ tasks = {:ok, chan} = SSHKit.run(conn, "uptime") chan - |> SSHKit.stream() + |> SSHKit.stream!() |> Enum.reduce(nil, fn {:stdout, chan, output}, acc -> IO.write("[#{label.(chan.connection)}] (stdout) #{output}") diff --git a/examples/stream.exs b/examples/stream.exs index 4c876990..f2d67d77 100644 --- a/examples/stream.exs +++ b/examples/stream.exs @@ -5,7 +5,7 @@ IO.write("> ") chan -|> SSHKit.stream() +|> SSHKit.stream!() |> Stream.map(fn {:stdout, ^chan, chunk} -> chunk end) |> Stream.each(&IO.write/1) |> Stream.take_while(fn chunk -> !String.ends_with?(chunk, "\n") end) @@ -19,7 +19,7 @@ IO.write("> ") # TODO: Timeouts? chan -|> SSHKit.stream() +|> SSHKit.stream!() |> Stream.filter(fn msg -> elem(msg, 0) == :stdout end) |> Stream.map(fn {:stdout, ^chan, chunk} -> chunk end) |> Stream.each(&IO.write/1) diff --git a/lib/sshkit.ex b/lib/sshkit.ex index 5719aa6e..406d8856 100644 --- a/lib/sshkit.ex +++ b/lib/sshkit.ex @@ -47,7 +47,7 @@ defmodule SSHKit do SSH.Channel.send(chan, type, data) end - def stream(chan) do + def stream!(chan) do Stream.unfold(:cont, fn :cont -> {:ok, msg} = SSH.Channel.recv(chan) # TODO: timeout? From e565ec327aac2074c48a7c0a1e36158c893e8d7e Mon Sep 17 00:00:00 2001 From: pmeinhardt Date: Fri, 18 Dec 2020 05:46:21 +0100 Subject: [PATCH 13/54] Improve error handling in upload implementation --- lib/sshkit/upload.ex | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/lib/sshkit/upload.ex b/lib/sshkit/upload.ex index f6e78953..d361f676 100644 --- a/lib/sshkit/upload.ex +++ b/lib/sshkit/upload.ex @@ -60,24 +60,22 @@ defmodule SSHKit.Upload do case stat.type do :directory -> # TODO: Timeouts - :ok = :ssh_sftp.make_dir(channel, remote) - {:ok, names} = File.ls(path) - {:ok, %{upload | cwd: path, stack: [names | [rest | paths]]}} + with :ok <- :ssh_sftp.make_dir(channel, remote), + {:ok, names} <- File.ls(path) do + {:ok, %{upload | cwd: path, stack: [names | [rest | paths]]}} + end :regular -> # TODO: Timeouts - {:ok, handle} = :ssh_sftp.open(channel, remote, [:write, :binary]) - - path - |> File.stream!([], 16_384) - |> Stream.each(fn data -> :ok = :ssh_sftp.write(channel, handle, data) end) - |> Stream.run() - - :ok = :ssh_sftp.close(channel, handle) - {:ok, %{upload | stack: [rest | paths]}} + with {:ok, handle} <- :ssh_sftp.open(channel, remote, [:write, :binary]), + :ok <- write(path, channel, handle), + :ok = :ssh_sftp.close(channel, handle) do + {:ok, %{upload | stack: [rest | paths]}} + end :symlink -> - nil + # TODO: http://erlang.org/doc/man/ssh_sftp.html#make_symlink-3 + raise "not yet implemented" _ -> {:error, {:unkown_file_type, path}} @@ -85,6 +83,13 @@ defmodule SSHKit.Upload do end end + defp write(path, channel, handle) do + path + |> File.stream!([], 16_384) + |> Stream.map(fn data -> :ssh_sftp.write(channel, handle, data) end) + |> Enum.find(:ok, &(&1 != :ok)) + end + # TODO: Make `loop` return a stream? Possibly rename to "stream" then def loop(%__MODULE__{stack: []}) do :ok From a6bbbc21da7a702bbc1d08317e36009e3c66445b Mon Sep 17 00:00:00 2001 From: pmeinhardt Date: Sat, 19 Dec 2020 19:52:50 +0100 Subject: [PATCH 14/54] Move all Context functions into SSHKit.Context --- README.md | 51 ++++++++---- lib/sshkit.ex | 129 ------------------------------- lib/sshkit/context.ex | 159 ++++++++++++++++++++++++++++++++++---- lib/sshkit/ssh/channel.ex | 2 +- 4 files changed, 181 insertions(+), 160 deletions(-) diff --git a/README.md b/README.md index 0d5b754d..2cbb31dd 100644 --- a/README.md +++ b/README.md @@ -12,29 +12,50 @@ SSHKit is an Elixir toolkit for performing tasks on one or more servers, built o SSHKit is designed to enable server task automation in a structured and repeatable way, e.g. in the context of deployment tools: ```elixir -hosts = ["1.eg.io", {"2.eg.io", port: 2222}] +{:ok, conn} = SSHKit.connect("eg.io", port: 2222) +{:ok, chan} = SSHKit.run(conn, "apt-get update -y") +:ok = SSHKit.flush(chan) +:ok = SSHKit.close(conn) +``` -{:ok, conn} = SSHKit.connect(hosts) +```elixir +{:ok, conn} = SSHKit.connect("eg.io", port: 2222) -{:ok, _} = SSHKit.run(conn, "apt-get update -y") +context = + SSHKit.Context.new() + |> SSHKit.Context.path("/var/www/phx") + |> SSHKit.Context.user("deploy") + |> SSHKit.Context.group("deploy") + |> SSHKit.Context.umask("022") + |> SSHKit.Context.env(%{"NODE_ENV" => "production"}) + +# TODO: Track/report upload progress +{:ok, _} = SSHKit.upload(conn, ".", recursive: true, context: context) -{:ok, } = SSHKit.stream(chan) +{:ok, chan} = SSHKit.run(conn, "yarn install", context: context) -context = - SSHKit.context() - |> SSHKit.path("/var/www/phx") - |> SSHKit.user("deploy") - |> SSHKit.group("deploy") - |> SSHKit.umask("022") - |> SSHKit.env(%{"NODE_ENV" => "production"}) +status = + chan + |> SSHKit.stream!() + |> Enum.reduce(nil, fn + {:stdout, ^chan, data}, status -> + IO.write(:stdio, data) + status -{:ok, _} = SSHKit.upload(conn, ".", recursive: true, context: context) + {:stderr, ^chan, data}, status -> + IO.write(:stderr, data) + status -# TODO: Receive upload status messages + {:exited, ^chan, status}, _ -> + status -{:ok, chan} = SSHKit.run(conn, "yarn install", context: context) + _, status -> + status + end) -# TODO: Showcase streaming interface +if status != 0 do + IO.write(:stderr, "Non-zero exit code #{status}") +end :ok = SSHKit.close(conn) ``` diff --git a/lib/sshkit.ex b/lib/sshkit.ex index 406d8856..04b3d565 100644 --- a/lib/sshkit.ex +++ b/lib/sshkit.ex @@ -86,135 +86,6 @@ defmodule SSHKit do end) end - @doc """ - Creates an execution context in which remote commands can be run. - - See `path/2`, `user/2`, `group/2`, `umask/2`, and `env/2` - for details on how to derive variations of a context. - """ - def context() do - %Context{} - end - - @doc """ - Changes the working directory commands are executed in for the given context. - - Returns a new, derived context for easy chaining. - - ## Example - - Create `/var/www/app/config.json`: - - ``` - SSHKit.context() - |> SSHKit.path("/var/www/app") - |> SSHKit.run(conn, "touch config.json") - ``` - """ - def path(context, path) do - %Context{context | path: path} - end - - @doc """ - Changes the file creation mode mask affecting default file and directory - permissions. - - Returns a new, derived context for easy chaining. - - ## Example - - Create `precious.txt`, readable and writable only for the logged-in user: - - ``` - "10.0.0.1" - |> SSHKit.context() - |> SSHKit.umask("077") - |> SSHKit.run("touch precious.txt") - ``` - """ - def umask(context, mask) do - %Context{context | umask: mask} - end - - @doc """ - Specifies the user under whose name commands are executed. - That user might be different than the user with which - ssh connects to the remote host. - - Returns a new, derived context for easy chaining. - - ## Example - - All commands executed in the created `context` will run as `deploy_user`, - although we use the `login_user` to log in to the remote host: - - ``` - context = - {"10.0.0.1", port: 3000, user: "login_user", password: "secret"} - |> SSHKit.context() - |> SSHKit.user("deploy_user") - ``` - """ - def user(context, name) do - %Context{context | user: name} - end - - @doc """ - Specifies the group commands are executed with. - - Returns a new, derived context for easy chaining. - - ## Example - - All commands executed in the created `context` will run in group `www`: - - ``` - context = - "10.0.0.1" - |> SSHKit.context() - |> SSHKit.group("www") - ``` - """ - def group(context, name) do - %Context{context | group: name} - end - - @doc """ - Defines new environment variables or overrides existing ones - for a given context. - - Returns a new, derived context for easy chaining. - - ## Examples - - Setting `NODE_ENV=production`: - - ``` - context = - "10.0.0.1" - |> SSHKit.context() - |> SSHKit.env(%{"NODE_ENV" => "production"}) - - # Run the npm start script with NODE_ENV=production - SSHKit.run(context, "npm start") - ``` - - Modifying the `PATH`: - - ``` - context = - "10.0.0.1" - |> SSHKit.context() - |> SSHKit.env(%{"PATH" => "$HOME/.rbenv/shims:$PATH"}) - - # Execute the rbenv-installed ruby to print its version - SSHKit.run(context, "ruby --version") - ``` - """ - def env(context, map) do - %Context{context | env: map} - end - @doc ~S""" Upload a file or files to the given context. diff --git a/lib/sshkit/context.ex b/lib/sshkit/context.ex index 64cc615d..4784a35c 100644 --- a/lib/sshkit/context.ex +++ b/lib/sshkit/context.ex @@ -16,6 +16,135 @@ defmodule SSHKit.Context do defstruct env: nil, path: nil, umask: nil, user: nil, group: nil + @doc """ + Creates an execution context in which remote commands can be run. + + See `path/2`, `user/2`, `group/2`, `umask/2`, and `env/2` + for details on how to derive variations of a context. + """ + def new do + %__MODULE__{} + end + + @doc """ + Changes the working directory commands are executed in for the given context. + + Returns a new, derived context for easy chaining. + + ## Example + + Create `/var/www/app/config.json`: + + ``` + SSHKit.context() + |> SSHKit.path("/var/www/app") + |> SSHKit.run(conn, "touch config.json") + ``` + """ + def path(context, path) do + %__MODULE__{context | path: path} + end + + @doc """ + Changes the file creation mode mask affecting default file and directory + permissions. + + Returns a new, derived context for easy chaining. + + ## Example + + Create `precious.txt`, readable and writable only for the logged-in user: + + ``` + "10.0.0.1" + |> SSHKit.context() + |> SSHKit.umask("077") + |> SSHKit.run("touch precious.txt") + ``` + """ + def umask(context, mask) do + %__MODULE__{context | umask: mask} + end + + @doc """ + Specifies the user under whose name commands are executed. + That user might be different than the user with which + ssh connects to the remote host. + + Returns a new, derived context for easy chaining. + + ## Example + + All commands executed in the created `context` will run as `deploy_user`, + although we use the `login_user` to log in to the remote host: + + ``` + context = + {"10.0.0.1", port: 3000, user: "login_user", password: "secret"} + |> SSHKit.context() + |> SSHKit.user("deploy_user") + ``` + """ + def user(context, name) do + %__MODULE__{context | user: name} + end + + @doc """ + Specifies the group commands are executed with. + + Returns a new, derived context for easy chaining. + + ## Example + + All commands executed in the created `context` will run in group `www`: + + ``` + context = + "10.0.0.1" + |> SSHKit.context() + |> SSHKit.group("www") + ``` + """ + def group(context, name) do + %__MODULE__{context | group: name} + end + + @doc """ + Defines new environment variables or overrides existing ones + for a given context. + + Returns a new, derived context for easy chaining. + + ## Examples + + Setting `NODE_ENV=production`: + + ``` + context = + "10.0.0.1" + |> SSHKit.context() + |> SSHKit.env(%{"NODE_ENV" => "production"}) + + # Run the npm start script with NODE_ENV=production + SSHKit.run(context, "npm start") + ``` + + Modifying the `PATH`: + + ``` + context = + "10.0.0.1" + |> SSHKit.context() + |> SSHKit.env(%{"PATH" => "$HOME/.rbenv/shims:$PATH"}) + + # Execute the rbenv-installed ruby to print its version + SSHKit.run(context, "ruby --version") + ``` + """ + def env(context, map) do + %__MODULE__{context | env: map} + end + @doc """ Compiles an executable command string for running the given `command` in the provided `context`. @@ -30,34 +159,34 @@ defmodule SSHKit.Context do """ def build(context, command) do "/usr/bin/env #{command}" - |> export(context.env) - |> sudo(context.user, context.group) - |> umask(context.umask) - |> cd(context.path) + |> add(:export, context.env) + |> add(:sudo, context.user, context.group) + |> add(:umask, context.umask) + |> add(:cd, context.path) end - defp sudo(command, nil, nil), do: command + defp add(command, :sudo, nil, nil), do: command - defp sudo(command, username, nil), + defp add(command, :sudo, username, nil), do: "sudo -H -n -u #{username} -- sh -c #{shellquote(command)}" - defp sudo(command, nil, groupname), + defp add(command, :sudo, nil, groupname), do: "sudo -H -n -g #{groupname} -- sh -c #{shellquote(command)}" - defp sudo(command, username, groupname), + defp add(command, :sudo, username, groupname), do: "sudo -H -n -u #{username} -g #{groupname} -- sh -c #{shellquote(command)}" - defp export(command, nil), do: command - defp export(command, env) when env == %{}, do: command + defp add(command, :export, nil), do: command + defp add(command, :export, env) when env == %{}, do: command - defp export(command, env) do + defp add(command, :export, env) do exports = Enum.map_join(env, " ", fn {name, value} -> "#{name}=\"#{value}\"" end) "(export #{exports} && #{command})" end - defp umask(command, nil), do: command - defp umask(command, mask), do: "umask #{mask} && #{command}" + defp add(command, :umask, nil), do: command + defp add(command, :umask, mask), do: "umask #{mask} && #{command}" - defp cd(command, nil), do: command - defp cd(command, path), do: "cd #{path} && #{command}" + defp add(command, :cd, nil), do: command + defp add(command, :cd, path), do: "cd #{path} && #{command}" end diff --git a/lib/sshkit/ssh/channel.ex b/lib/sshkit/ssh/channel.ex index 64264853..8a6262fd 100644 --- a/lib/sshkit/ssh/channel.ex +++ b/lib/sshkit/ssh/channel.ex @@ -145,7 +145,7 @@ defmodule SSHKit.SSH.Channel do ## Messages - The message tuples returned by `recv/3` correspond to the underlying Erlang + The message tuples returned by `recv/2` correspond to the underlying Erlang channel messages with the channel id replaced by the SSHKit channel struct: * `{:data, channel, type, data}` From 5e64c04405b8fddafa86f8477e85d3a4a39276e1 Mon Sep 17 00:00:00 2001 From: pmeinhardt Date: Sun, 20 Dec 2020 00:43:55 +0100 Subject: [PATCH 15/54] Lift Connection & Channel modules up - Move `Connection` and `Channel` modules into the top-level namespace - Smaller fixes and cleanup on `Upload` module - Rename `SSHKit.run/3` to `SSHKit.exec/3`, pull up implementation - Add a few type specs - Update examples --- examples/parallel.exs | 2 +- examples/stream.exs | 2 +- lib/sshkit.ex | 56 +++++++++--- lib/sshkit/{ssh => }/channel.ex | 14 ++- lib/sshkit/{ssh => }/connection.ex | 13 ++- lib/sshkit/download.ex | 42 +++++++++ lib/sshkit/ssh.ex | 142 ----------------------------- lib/sshkit/upload.ex | 4 +- 8 files changed, 103 insertions(+), 172 deletions(-) rename lib/sshkit/{ssh => }/channel.ex (94%) rename lib/sshkit/{ssh => }/connection.ex (89%) create mode 100644 lib/sshkit/download.ex delete mode 100644 lib/sshkit/ssh.ex diff --git a/examples/parallel.exs b/examples/parallel.exs index 037989a1..15c72d17 100644 --- a/examples/parallel.exs +++ b/examples/parallel.exs @@ -14,7 +14,7 @@ label = fn conn -> Enum.join([conn.host, conn.port], ":") end tasks = Enum.map(conns, fn conn -> Task.async(fn -> - {:ok, chan} = SSHKit.run(conn, "uptime") + {:ok, chan} = SSHKit.exec(conn, "uptime") chan |> SSHKit.stream!() diff --git a/examples/stream.exs b/examples/stream.exs index f2d67d77..eaa4dac8 100644 --- a/examples/stream.exs +++ b/examples/stream.exs @@ -1,6 +1,6 @@ {:ok, conn} = SSHKit.connect("127.0.0.1", port: 2222, user: "deploy", password: "deploy", silently_accept_hosts: true) -{:ok, chan} = SSHKit.run(conn, ~S(echo "Who's there?"; read name; echo "Hello $name")) +{:ok, chan} = SSHKit.exec(conn, ~S(echo "Who's there?"; read name; echo -n "Hello"; sleep 3; echo " $name")) IO.write("> ") diff --git a/lib/sshkit.ex b/lib/sshkit.ex index 04b3d565..2fc9e4ae 100644 --- a/lib/sshkit.ex +++ b/lib/sshkit.ex @@ -18,9 +18,10 @@ defmodule SSHKit do ``` """ - alias SSHKit.SSH - + alias SSHKit.Channel + alias SSHKit.Connection alias SSHKit.Context + alias SSHKit.Download alias SSHKit.Host alias SSHKit.Upload @@ -31,26 +32,56 @@ defmodule SSHKit do remote commands can be run. Accepts any form of host specification also accepted by `host/1` and `host/2`, i.e. binaries, maps and 2-tuples. """ + @spec connect(binary(), keyword()) :: {:ok, Connection.t()} | {:error, any()} def connect(host, options \\ []) do - SSH.connect(host, options) + Connection.open(host, options) end + @spec close(Connection.t()) :: :ok def close(conn) do - SSH.close(conn) + Connection.close(conn) end - def run(conn, command, options \\ []) do - SSH.run(conn, command, options) + # TODO: Accept :stdout (default) and :stderr (1) for type parameter? + # TODO: Add `send(chan, :eof)`? + def send(chan, type \\ 0, data) do + Channel.send(chan, type, data) end - def send(chan, type \\ 0, data) do - SSH.Channel.send(chan, type, data) + # TODO: Do we need to expose lower-level channel operations here? + # + # * Send `eof`? + # * Subsystem + # * ppty + # * … + # + # Seems like `send` and `eof` should be enough for the intended high-level use cases. + # If more fine-grained control is needed, feel free to reach for the `SSHKit.Channel` module. + + @spec exec(Connection.t(), binary(), keyword()) :: {:ok, Channel.t()} | {:error, any()} + def exec(conn, command, options \\ []) do + timeout = Keyword.get(options, :timeout, :infinity) + + with {:ok, chan} <- Channel.open(conn, options) do + case Channel.exec(chan, command, timeout) do + :success -> + {:ok, chan} + + :failure -> + {:error, :failure} + + error -> + error + end + end end def stream!(chan) do Stream.unfold(:cont, fn :cont -> - {:ok, msg} = SSH.Channel.recv(chan) # TODO: timeout? + {:ok, msg} = Channel.recv(chan) # TODO: timeout?, TODO: handle {:error, reason} and raise custom error struct? + + # TODO: Adjust channel window size? value = case msg do @@ -120,8 +151,11 @@ defmodule SSHKit do def upload(conn, source, target, options \\ []) do upload = Upload.init(source, target, options) - with {:ok, upload} <- Upload.start(upload, conn) do - Upload.loop(upload) + # TODO: Close SFTP channel (Upload.stop/1) if there is an error + with {:ok, upload} <- Upload.start(upload, conn), + {:ok, upload} <- Upload.loop(upload), + {:ok, _} <- Upload.stop(upload) do + :ok end end diff --git a/lib/sshkit/ssh/channel.ex b/lib/sshkit/channel.ex similarity index 94% rename from lib/sshkit/ssh/channel.ex rename to lib/sshkit/channel.ex index 8a6262fd..423445d6 100644 --- a/lib/sshkit/ssh/channel.ex +++ b/lib/sshkit/channel.ex @@ -1,16 +1,14 @@ -defmodule SSHKit.SSH.Channel do +defmodule SSHKit.Channel do @moduledoc """ - Defines a `SSHKit.SSH.Channel` struct representing a connection channel. + Defines a `SSHKit.Channel` struct representing a connection channel. A channel struct has the following fields: - * `connection` - the underlying `SSHKit.SSH.Connection` + * `connection` - the underlying `SSHKit.Connection` * `type` - the type of the channel, i.e. `:session` * `id` - the unique channel id """ - alias SSHKit.SSH.Channel - defstruct [:connection, :type, :id, impl: :ssh_connection] @doc """ @@ -34,13 +32,13 @@ defmodule SSHKit.SSH.Channel do impl = Keyword.get(options, :impl, :ssh_connection) case impl.session_channel(connection.ref, ini_window_size, max_packet_size, timeout) do - {:ok, id} -> {:ok, build(connection, id, impl)} + {:ok, id} -> {:ok, new(connection, id, impl)} err -> err end end - defp build(connection, id, impl) do - %Channel{connection: connection, type: :session, id: id, impl: impl} + defp new(connection, id, impl) do + %__MODULE__{connection: connection, type: :session, id: id, impl: impl} end @doc """ diff --git a/lib/sshkit/ssh/connection.ex b/lib/sshkit/connection.ex similarity index 89% rename from lib/sshkit/ssh/connection.ex rename to lib/sshkit/connection.ex index 0e15b3ec..7f9a81c4 100644 --- a/lib/sshkit/ssh/connection.ex +++ b/lib/sshkit/connection.ex @@ -1,6 +1,6 @@ -defmodule SSHKit.SSH.Connection do +defmodule SSHKit.Connection do @moduledoc """ - Defines a `SSHKit.SSH.Connection` struct representing a host connection. + Defines a `SSHKit.Connection` struct representing a host connection. A connection struct has the following fields: @@ -10,7 +10,6 @@ defmodule SSHKit.SSH.Connection do * `ref` - the underlying `:ssh` connection ref """ - alias SSHKit.SSH.Connection alias SSHKit.Utils defstruct [:host, :port, :options, :ref, impl: :ssh] @@ -55,7 +54,7 @@ defmodule SSHKit.SSH.Connection do impl = details[:impl] case impl.connect(host, port, opts, timeout) do - {:ok, ref} -> {:ok, build(host, port, opts, ref, impl)} + {:ok, ref} -> {:ok, new(host, port, opts, ref, impl)} err -> err end end @@ -76,8 +75,8 @@ defmodule SSHKit.SSH.Connection do {connect_options, impl_options} end - defp build(host, port, options, ref, impl) do - %Connection{host: host, port: port, options: options, ref: ref, impl: impl} + defp new(host, port, options, ref, impl) do + %__MODULE__{host: host, port: port, options: options, ref: ref, impl: impl} end @doc """ @@ -97,7 +96,7 @@ defmodule SSHKit.SSH.Connection do The timeout value of the original connection is discarded. Other connection options are reused and may be overridden. - Uses `open/2`. + Uses `SSHKit.Connection.open/2`. Returns `{:ok, conn}` or `{:error, reason}`. """ diff --git a/lib/sshkit/download.ex b/lib/sshkit/download.ex new file mode 100644 index 00000000..9c0bf2f3 --- /dev/null +++ b/lib/sshkit/download.ex @@ -0,0 +1,42 @@ +defmodule SSHKit.Download do + @moduledoc """ + TODO + """ + + defstruct [:source, :target, :options, :cwd, :stack, :channel] + + def init(source, target, options \\ []) do + %__MODULE__{source: source, target: Path.expand(target), options: options} + end + + def start(%__MODULE__{} = download, connection) do + end + + def stop(%__MODULE__{channel: nil} = download), do: {:ok, download} + def stop(%__MODULE__{channel: channel} = download) do + with :ok <- :ssh_sftp.stop_channel(channel) do + {:ok, %{download | channel: nil}} + end + end + + def continue(%__MODULE__{stack: []} = download) do + {:ok, download} + end + + def loop(%__MODULE__{stack: []} = download) do + {:ok, download} + end + + def loop(%__MODULE__{} = download) do + case continue(download) do + {:ok, download} -> + loop(download) + + error -> + error + end + end + + def done?(%__MODULE__{stack: []}), do: true + def done?(%__MODULE__{}), do: false +end diff --git a/lib/sshkit/ssh.ex b/lib/sshkit/ssh.ex deleted file mode 100644 index c54d7ed1..00000000 --- a/lib/sshkit/ssh.ex +++ /dev/null @@ -1,142 +0,0 @@ -defmodule SSHKit.SSH do - @moduledoc ~S""" - Provides convenience functions for working with SSH connections - and executing commands on remote hosts. - - ## Examples - - ``` - {:ok, conn} = SSHKit.SSH.connect("eg.io", user: "me") - {:ok, output, status} = SSHKit.SSH.run(conn, "uptime") - :ok = SSHKit.SSH.close(conn) - - Enum.each(output, fn - {:stdout, data} -> IO.write(data) - {:stderr, data} -> IO.write([IO.ANSI.red, data, IO.ANSI.reset]) - end) - - IO.puts("$?: #{status}") - ``` - """ - - alias SSHKit.SSH.Channel - alias SSHKit.SSH.Connection - - @doc """ - Establishes a connection to an SSH server. - - Uses `SSHKit.SSH.Connection.open/2` to open a connection. - - `options_or_function` can either be a list of options or a function. - If it is a list, it is considered to be a list of options as described in - `SSHKit.SSH.Connection.open/2`. If it is a function, then it is equivalent to - calling `connect(host, [], options_or_function)`. - - See the documentation for `connect/3` for more information on this function. - - ## Example - - ``` - {:ok, conn} = SSHKit.SSH.connect("eg.io", port: 2222, user: "me", timeout: 1000) - ``` - """ - @callback connect(binary(), keyword() | fun()) :: {:ok, Connection.t()} | {:error, any()} - def connect(host, options_or_function \\ []) - def connect(host, function) when is_function(function), do: connect(host, [], function) - def connect(host, options) when is_list(options), do: Connection.open(host, options) - - @doc """ - Similar to `connect/2` but expects a function as its last argument. - - The connection is opened, given to the function as an argument and - automatically closed after the function returns, regardless of any - errors raised while executing the function. - - Returns `{:ok, function_result}` in case of success, - `{:error, reason}` otherwise. - - ## Examples - - ``` - SSH.connect("eg.io", port: 2222, user: "me", fn conn -> - SCP.upload(conn, "list.txt") - end) - ``` - - See `SSHKit.SSH.Connection.open/2` for the list of available `options`. - """ - def connect(host, options, function) do - case connect(host, options) do - {:ok, conn} -> - try do - {:ok, function.(conn)} - after - :ok = close(conn) - end - - other -> - other - end - end - - @doc """ - Closes an SSH connection. - - Uses `SSHKit.SSH.Connection.close/1` to close the connection. - - ## Example - - ``` - :ok = SSHKit.SSH.close(conn) - ``` - """ - @callback close(Connection.t()) :: :ok - def close(connection) do - Connection.close(connection) - end - - @doc """ - Executes a command on the remote and aggregates incoming messages. - - Using the default handler, returns `{:ok, output, status}` or `{:error, - reason}`. By default, command output is captured into a list of tuples of the - form `{:stdout, data}` or `{:stderr, data}`. - - A custom handler function can be provided to handle channel messages. - - For further details on handling incoming messages, - see `SSHKit.SSH.Channel.loop/4`. - - ## Options - - * TODO: disambiguate channel opening and exec options - * `:timeout` - maximum wait time between messages, defaults to `:infinity` - - Any other options will be passed on to `SSHKit.SSH.Channel.open/2` when - creating the channel for executing the command. - - ## Example - - ``` - {:ok, output, status} = SSHKit.SSH.run(conn, "uptime") - IO.inspect(output) - ``` - """ - @callback run(Connection.t(), binary(), keyword()) :: any() - def run(connection, command, options \\ []) do - timeout = Keyword.get(options, :timeout, :infinity) - - with {:ok, channel} <- Channel.open(connection, options) do - case Channel.exec(channel, command, timeout) do - :success -> - {:ok, channel} - - :failure -> - {:error, :failure} - - error -> - error - end - end - end -end diff --git a/lib/sshkit/upload.ex b/lib/sshkit/upload.ex index d361f676..f7a7274e 100644 --- a/lib/sshkit/upload.ex +++ b/lib/sshkit/upload.ex @@ -91,8 +91,8 @@ defmodule SSHKit.Upload do end # TODO: Make `loop` return a stream? Possibly rename to "stream" then - def loop(%__MODULE__{stack: []}) do - :ok + def loop(%__MODULE__{stack: []} = upload) do + {:ok, upload} end def loop(%__MODULE__{} = upload) do From 98a103dbaaebea7490bada89dfa7c9d9b8366524 Mon Sep 17 00:00:00 2001 From: pmeinhardt Date: Sun, 20 Dec 2020 03:31:39 +0100 Subject: [PATCH 16/54] Make send more symmetric with recv & stream! Use :stdout and :stderr instead of 0 and 1. --- lib/sshkit.ex | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/lib/sshkit.ex b/lib/sshkit.ex index 2fc9e4ae..1a107abf 100644 --- a/lib/sshkit.ex +++ b/lib/sshkit.ex @@ -42,12 +42,6 @@ defmodule SSHKit do Connection.close(conn) end - # TODO: Accept :stdout (default) and :stderr (1) for type parameter? - # TODO: Add `send(chan, :eof)`? - def send(chan, type \\ 0, data) do - Channel.send(chan, type, data) - end - # TODO: Do we need to expose lower-level channel operations here? # # * Send `eof`? @@ -76,6 +70,11 @@ defmodule SSHKit do end end + @spec send(Channel.t(), :stdout | :stderr, any()) :: :ok | {:error, any()} + def send(chan, type \\ :stdout, data) + def send(chan, :stdout, data), do: Channel.send(chan, 0, data) + def send(chan, :stderr, data), do: Channel.send(chan, 1, data) + def stream!(chan) do Stream.unfold(:cont, fn :cont -> From fb4c603f741b8668678ebd0b4b0173a33328ae72 Mon Sep 17 00:00:00 2001 From: pmeinhardt Date: Sun, 20 Dec 2020 03:54:15 +0100 Subject: [PATCH 17/54] Add timeout parameter to send/4 --- lib/sshkit.ex | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/sshkit.ex b/lib/sshkit.ex index 1a107abf..5dea73e7 100644 --- a/lib/sshkit.ex +++ b/lib/sshkit.ex @@ -32,7 +32,7 @@ defmodule SSHKit do remote commands can be run. Accepts any form of host specification also accepted by `host/1` and `host/2`, i.e. binaries, maps and 2-tuples. """ - @spec connect(binary(), keyword()) :: {:ok, Connection.t()} | {:error, any()} + @spec connect(binary(), keyword()) :: {:ok, Connection.t()} | {:error, term()} def connect(host, options \\ []) do Connection.open(host, options) end @@ -52,7 +52,7 @@ defmodule SSHKit do # Seems like `send` and `eof` should be enough for the intended high-level use cases. # If more fine-grained control is needed, feel free to reach for the `SSHKit.Channel` module. - @spec exec(Connection.t(), binary(), keyword()) :: {:ok, Channel.t()} | {:error, any()} + @spec exec(Connection.t(), binary(), keyword()) :: {:ok, Channel.t()} | {:error, term()} def exec(conn, command, options \\ []) do timeout = Keyword.get(options, :timeout, :infinity) @@ -70,10 +70,10 @@ defmodule SSHKit do end end - @spec send(Channel.t(), :stdout | :stderr, any()) :: :ok | {:error, any()} - def send(chan, type \\ :stdout, data) - def send(chan, :stdout, data), do: Channel.send(chan, 0, data) - def send(chan, :stderr, data), do: Channel.send(chan, 1, data) + @spec send(Channel.t(), :stdout | :stderr, term(), timeout()) :: :ok | {:error, term()} + def send(chan, type \\ :stdout, data, timeout \\ :infinity) + def send(chan, :stdout, data, timeout), do: Channel.send(chan, 0, data, timeout) + def send(chan, :stderr, data, timeout), do: Channel.send(chan, 1, data, timeout) def stream!(chan) do Stream.unfold(:cont, fn From ab6ecef5a252a2cdece8e72e782a12e1aca4d6d1 Mon Sep 17 00:00:00 2001 From: pmeinhardt Date: Sun, 20 Dec 2020 09:12:46 +0100 Subject: [PATCH 18/54] Add a convenient way of sending `eof` on a channel --- lib/sshkit.ex | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/sshkit.ex b/lib/sshkit.ex index 5dea73e7..c43b4e43 100644 --- a/lib/sshkit.ex +++ b/lib/sshkit.ex @@ -70,6 +70,11 @@ defmodule SSHKit do end end + @spec send(Channel.t(), :eof) :: :ok | {:error, term()} + def send(chan, :eof) do + Channel.eof(chan) + end + @spec send(Channel.t(), :stdout | :stderr, term(), timeout()) :: :ok | {:error, term()} def send(chan, type \\ :stdout, data, timeout \\ :infinity) def send(chan, :stdout, data, timeout), do: Channel.send(chan, 0, data, timeout) From 4a6f5c6126c2433bcd06bb47774b20f4a15e7a71 Mon Sep 17 00:00:00 2001 From: pmeinhardt Date: Sun, 20 Dec 2020 09:59:46 +0100 Subject: [PATCH 19/54] Make "conn" argument naming consistent --- lib/sshkit/channel.ex | 10 +++++----- lib/sshkit/connection.ex | 10 +++++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/lib/sshkit/channel.ex b/lib/sshkit/channel.ex index 423445d6..14632649 100644 --- a/lib/sshkit/channel.ex +++ b/lib/sshkit/channel.ex @@ -25,20 +25,20 @@ defmodule SSHKit.Channel do * `:initial_window_size` - defaults to 128 KiB * `:max_packet_size` - defaults to 32 KiB """ - def open(connection, options \\ []) do + def open(conn, options \\ []) do timeout = Keyword.get(options, :timeout, :infinity) ini_window_size = Keyword.get(options, :initial_window_size, 128 * 1024) max_packet_size = Keyword.get(options, :max_packet_size, 32 * 1024) impl = Keyword.get(options, :impl, :ssh_connection) - case impl.session_channel(connection.ref, ini_window_size, max_packet_size, timeout) do - {:ok, id} -> {:ok, new(connection, id, impl)} + case impl.session_channel(conn.ref, ini_window_size, max_packet_size, timeout) do + {:ok, id} -> {:ok, new(conn, id, impl)} err -> err end end - defp new(connection, id, impl) do - %__MODULE__{connection: connection, type: :session, id: id, impl: impl} + defp new(conn, id, impl) do + %__MODULE__{connection: conn, type: :session, id: id, impl: impl} end @doc """ diff --git a/lib/sshkit/connection.ex b/lib/sshkit/connection.ex index 7f9a81c4..b1558a21 100644 --- a/lib/sshkit/connection.ex +++ b/lib/sshkit/connection.ex @@ -100,13 +100,13 @@ defmodule SSHKit.Connection do Returns `{:ok, conn}` or `{:error, reason}`. """ - def reopen(connection, options \\ []) do + def reopen(conn, options \\ []) do options = - connection.options - |> Keyword.put(:port, connection.port) - |> Keyword.put(:impl, connection.impl) + conn.options + |> Keyword.put(:port, conn.port) + |> Keyword.put(:impl, conn.impl) |> Keyword.merge(options) - open(connection.host, options) + open(conn.host, options) end end From c08120f6186e6856df5e0a2b4f83ccfc40ed5b17 Mon Sep 17 00:00:00 2001 From: pmeinhardt Date: Sun, 20 Dec 2020 14:45:30 +0100 Subject: [PATCH 20/54] Add internal SSHKit.SFTP.Channel module The `SSHKit.SFTP.Channel` struct keeps a reference of the connection, so connection information can be accessed, same as the `SSHKit.Channel`. The swappable `:impl` field allows replacing `:ssh_sftp` with a mock. Also: Add more type annotations. --- lib/sshkit/channel.ex | 11 ++++----- lib/sshkit/connection.ex | 3 ++- lib/sshkit/context.ex | 2 ++ lib/sshkit/download.ex | 10 +++++--- lib/sshkit/host.ex | 2 ++ lib/sshkit/sftp/channel.ex | 48 ++++++++++++++++++++++++++++++++++++++ lib/sshkit/upload.ex | 38 +++++++++++++++++++----------- 7 files changed, 90 insertions(+), 24 deletions(-) create mode 100644 lib/sshkit/sftp/channel.ex diff --git a/lib/sshkit/channel.ex b/lib/sshkit/channel.ex index 14632649..01f374f8 100644 --- a/lib/sshkit/channel.ex +++ b/lib/sshkit/channel.ex @@ -11,6 +11,8 @@ defmodule SSHKit.Channel do defstruct [:connection, :type, :id, impl: :ssh_connection] + @type t() :: %__MODULE__{} + @doc """ Opens a channel on an SSH connection. @@ -31,9 +33,8 @@ defmodule SSHKit.Channel do max_packet_size = Keyword.get(options, :max_packet_size, 32 * 1024) impl = Keyword.get(options, :impl, :ssh_connection) - case impl.session_channel(conn.ref, ini_window_size, max_packet_size, timeout) do - {:ok, id} -> {:ok, new(conn, id, impl)} - err -> err + with {:ok, id} <- impl.session_channel(conn.ref, ini_window_size, max_packet_size, timeout) do + {:ok, new(conn, id, impl)} end end @@ -52,9 +53,7 @@ defmodule SSHKit.Channel do :success | :failure | {:error, reason :: String.t()} def subsystem(channel, subsystem, options \\ []) do timeout = Keyword.get(options, :timeout, :infinity) - impl = Keyword.get(options, :impl, :ssh_connection) - - impl.subsystem(channel.connection.ref, channel.id, to_charlist(subsystem), timeout) + channel.impl.subsystem(channel.connection.ref, channel.id, to_charlist(subsystem), timeout) end @doc """ diff --git a/lib/sshkit/connection.ex b/lib/sshkit/connection.ex index b1558a21..1a097c1c 100644 --- a/lib/sshkit/connection.ex +++ b/lib/sshkit/connection.ex @@ -12,9 +12,10 @@ defmodule SSHKit.Connection do alias SSHKit.Utils + # TODO: Add :tag allowing arbitrary data to be attached? defstruct [:host, :port, :options, :ref, impl: :ssh] - @type t :: __MODULE__ + @type t() :: %__MODULE__{} @default_impl_options [user_interaction: false] @default_connect_options [port: 22, timeout: :infinity, impl: :ssh] diff --git a/lib/sshkit/context.ex b/lib/sshkit/context.ex index 4784a35c..21e4896e 100644 --- a/lib/sshkit/context.ex +++ b/lib/sshkit/context.ex @@ -16,6 +16,8 @@ defmodule SSHKit.Context do defstruct env: nil, path: nil, umask: nil, user: nil, group: nil + @type t() :: %__MODULE__{} + @doc """ Creates an execution context in which remote commands can be run. diff --git a/lib/sshkit/download.ex b/lib/sshkit/download.ex index 9c0bf2f3..408819bd 100644 --- a/lib/sshkit/download.ex +++ b/lib/sshkit/download.ex @@ -3,18 +3,22 @@ defmodule SSHKit.Download do TODO """ + alias SSHKit.SFTP.Channel + defstruct [:source, :target, :options, :cwd, :stack, :channel] + @type t() :: %__MODULE__{} + def init(source, target, options \\ []) do %__MODULE__{source: source, target: Path.expand(target), options: options} end - def start(%__MODULE__{} = download, connection) do + def start(%__MODULE__{options: options} = download, conn) do end def stop(%__MODULE__{channel: nil} = download), do: {:ok, download} - def stop(%__MODULE__{channel: channel} = download) do - with :ok <- :ssh_sftp.stop_channel(channel) do + def stop(%__MODULE__{channel: chan} = download) do + with :ok <- Channel.stop(chan) do {:ok, %{download | channel: nil}} end end diff --git a/lib/sshkit/host.ex b/lib/sshkit/host.ex index aba058ad..5f954f60 100644 --- a/lib/sshkit/host.ex +++ b/lib/sshkit/host.ex @@ -12,4 +12,6 @@ defmodule SSHKit.Host do ``` """ defstruct [:name, :options] + + @type t() :: %__MODULE__{} end diff --git a/lib/sshkit/sftp/channel.ex b/lib/sshkit/sftp/channel.ex new file mode 100644 index 00000000..c2143ab0 --- /dev/null +++ b/lib/sshkit/sftp/channel.ex @@ -0,0 +1,48 @@ +defmodule SSHKit.SFTP.Channel do + @moduledoc false + + alias SSHKit.Connection + + defstruct [:connection, :id, impl: :ssh_sftp] + + @type t() :: %__MODULE__{} + @type handle() :: term() + + @spec start(Connection.t(), keyword()) :: {:ok, t()} | {:error, term()} + def start(conn, options \\ []) do + {impl, options} = Keyword.pop(options, :impl, :ssh_sftp) + + with {:ok, id} <- impl.start_channel(conn.ref, options) do + {:ok, new(conn, id, impl)} + end + end + + defp new(conn, id, impl) do + %__MODULE__{connection: conn, id: id, impl: impl} + end + + @spec stop(t()) :: :ok + def stop(chan) do + chan.impl.stop_channel(chan.id) + end + + @spec mkdir(t(), binary(), timeout()) :: :ok | {:error, term()} + def mkdir(chan, name, timeout \\ :infinity) do + chan.impl.make_dir(chan.id, name, timeout) + end + + @spec open(t(), binary(), [:read | :write | :append | :binary | :raw], timeout()) :: {:ok, handle()} | {:error, term()} + def open(chan, name, mode, timeout \\ :infinity) do + chan.impl.open(chan.id, name, mode, timeout) + end + + @spec close(t(), handle(), timeout()) :: :ok | {:error, term()} + def close(chan, handle, timeout \\ :infinity) do + chan.impl.close(chan.id, handle, timeout) + end + + @spec write(t(), handle(), iodata(), timeout()) :: :ok, {:error, term()} + def write(chan, handle, data, timeout \\ :infinity) do + chan.impl.write(chan.id, handle, data, timeout) + end +end diff --git a/lib/sshkit/upload.ex b/lib/sshkit/upload.ex index f7a7274e..a9e6f870 100644 --- a/lib/sshkit/upload.ex +++ b/lib/sshkit/upload.ex @@ -3,16 +3,26 @@ defmodule SSHKit.Upload do TODO """ + alias SSHKit.SFTP.Channel + defstruct [:source, :target, :options, :cwd, :stack, :channel] + @type t() :: %__MODULE__{} + def init(source, target, options \\ []) do %__MODULE__{source: Path.expand(source), target: target, options: options} end - def start(%__MODULE__{} = upload, connection) do - with {:ok, upload} <- prepare(upload) do - {:ok, channel} = :ssh_sftp.start_channel(connection.ref) # accepts options like timeout… http://erlang.org/doc/man/ssh_sftp.html#start_channel-1 - {:ok, %{upload | channel: channel}} + def start(%__MODULE__{options: options} = upload, conn) do + # accepts options like timeout… http://erlang.org/doc/man/ssh_sftp.html#start_channel-1 + channel_options = + options + |> Keyword.get(:start, []) + |> Keyword.put_new(:timeout, Keyword.get(options, :timeout, :infinity)) + + with {:ok, upload} <- prepare(upload), + {:ok, chan} <- Channel.start(conn, channel_options) do + {:ok, %{upload | channel: chan}} end end @@ -26,8 +36,8 @@ defmodule SSHKit.Upload do end def stop(%__MODULE__{channel: nil} = upload), do: {:ok, upload} - def stop(%__MODULE__{channel: channel} = upload) do - with :ok <- :ssh_sftp.stop_channel(channel) do + def stop(%__MODULE__{channel: chan} = upload) do + with :ok <- Channel.stop(chan) do {:ok, %{upload | channel: nil}} end end @@ -44,7 +54,7 @@ defmodule SSHKit.Upload do def continue(%__MODULE__{stack: [[name | rest] | paths]} = upload) do path = Path.join(upload.cwd, name) - relpath = Path.relative_to(path, Path.expand(upload.source)) + relpath = Path.relative_to(path, upload.source) relpath = if relpath == path, do: ".", else: relpath remote = @@ -55,21 +65,21 @@ defmodule SSHKit.Upload do with {:ok, stat} <- File.stat(path, time: :posix) do # TODO: Set timestamps… if :preserve option is true, http://erlang.org/doc/man/ssh_sftp.html#write_file_info-3 - channel = upload.channel + chan = upload.channel case stat.type do :directory -> # TODO: Timeouts - with :ok <- :ssh_sftp.make_dir(channel, remote), + with :ok <- Channel.mkdir(chan, remote), {:ok, names} <- File.ls(path) do {:ok, %{upload | cwd: path, stack: [names | [rest | paths]]}} end :regular -> # TODO: Timeouts - with {:ok, handle} <- :ssh_sftp.open(channel, remote, [:write, :binary]), - :ok <- write(path, channel, handle), - :ok = :ssh_sftp.close(channel, handle) do + with {:ok, handle} <- Channel.open(chan, remote, [:write, :binary]), + :ok <- write(path, chan, handle), + :ok = Channel.close(chan, handle) do {:ok, %{upload | stack: [rest | paths]}} end @@ -83,10 +93,10 @@ defmodule SSHKit.Upload do end end - defp write(path, channel, handle) do + defp write(path, chan, handle) do path |> File.stream!([], 16_384) - |> Stream.map(fn data -> :ssh_sftp.write(channel, handle, data) end) + |> Stream.map(fn data -> Channel.write(chan, handle, data) end) |> Enum.find(:ok, &(&1 != :ok)) end From bd00f151bb5cb4a22ae199ed9f41f3bcee273c00 Mon Sep 17 00:00:00 2001 From: pmeinhardt Date: Sun, 20 Dec 2020 16:42:56 +0100 Subject: [PATCH 21/54] Drop idea of supporting globs for file transfers Glob expansion is not directly supported by the `:ssh_sftp` module, implementation would differ a lot between uploads (expand locally) and downloads (expand on remote) and, lastly, it's not quite clear what the semantics should be in case of conflicting files names. I.e. would globbed files be uploaded into the target directory, "flat"? If needed by code using SSHKit, globbing support with specific semantics can be implemented as a series of SSHKit uploads, one for each matched source path. --- lib/sshkit/upload.ex | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/sshkit/upload.ex b/lib/sshkit/upload.ex index a9e6f870..75c7ce9b 100644 --- a/lib/sshkit/upload.ex +++ b/lib/sshkit/upload.ex @@ -27,7 +27,6 @@ defmodule SSHKit.Upload do end defp prepare(%__MODULE__{source: source, options: options} = upload) do - # TODO: Support globs, https://hexdocs.pm/elixir/Path.html#wildcard/2 if !Keyword.get(options, :recursive, false) && File.dir?(source) do {:error, "Option :recursive not specified, but local file is a directory (#{source})"} # TODO: Better error else From 3ac5bf5c9841d853e833b782d3da2d6280f09dab Mon Sep 17 00:00:00 2001 From: pmeinhardt Date: Sun, 20 Dec 2020 16:49:18 +0100 Subject: [PATCH 22/54] Bump file-reading chunk size --- lib/sshkit/upload.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/sshkit/upload.ex b/lib/sshkit/upload.ex index 75c7ce9b..b563a858 100644 --- a/lib/sshkit/upload.ex +++ b/lib/sshkit/upload.ex @@ -94,7 +94,7 @@ defmodule SSHKit.Upload do defp write(path, chan, handle) do path - |> File.stream!([], 16_384) + |> File.stream!([], 65_536) |> Stream.map(fn data -> Channel.write(chan, handle, data) end) |> Enum.find(:ok, &(&1 != :ok)) end From 20898cb65bb6f103594f88b2411092f59ac2215a Mon Sep 17 00:00:00 2001 From: pmeinhardt Date: Mon, 21 Dec 2020 04:46:04 +0100 Subject: [PATCH 23/54] Safer streaming interface Ensure channels are always closed at the end of enumeration and that any pending channel messages are flushed. --- examples/parallel.exs | 38 +++++++++++--------------- examples/stream.exs | 36 ++++++++++++------------- lib/sshkit.ex | 63 +++++++++++++++++++------------------------ 3 files changed, 60 insertions(+), 77 deletions(-) diff --git a/examples/parallel.exs b/examples/parallel.exs index 15c72d17..b76c0d93 100644 --- a/examples/parallel.exs +++ b/examples/parallel.exs @@ -14,40 +14,32 @@ label = fn conn -> Enum.join([conn.host, conn.port], ":") end tasks = Enum.map(conns, fn conn -> Task.async(fn -> - {:ok, chan} = SSHKit.exec(conn, "uptime") - - chan - |> SSHKit.stream!() + conn + |> SSHKit.exec!("uptime") |> Enum.reduce(nil, fn - {:stdout, chan, output}, acc -> + {:stdout, chan, output}, status -> IO.write("[#{label.(chan.connection)}] (stdout) #{output}") - acc + status - {:stderr, chan, output}, acc -> + {:stderr, chan, output}, status -> IO.write("[#{label.(chan.connection)}] (stderr) #{output}") - acc + status {:exit, _, status}, _ -> status - _, acc -> - acc + _, status -> + status end) end) end) -okay? = fn status -> status == 0 end - -results = Enum.map(tasks, &Task.await/1) - -unless Enum.all?(results, okay?) do - results - |> Enum.with_index() - |> Enum.filter(fn {status, _} -> !okay?.(status) end) - |> Enum.each(fn {status, index} -> - conn = Enum.at(conns, index) - IO.puts("[#{label.(conn)}] exited with status #{status}") - end) -end +tasks +|> Enum.map(&Task.await/1) +|> Enum.filter(&(&1 != 0)) +|> Enum.zip(conns) +|> Enum.each(fn {status, conn} -> + IO.puts("[#{label.(conn)}] exited with status #{status}") +end) :ok = Enum.each(conns, &SSHKit.close/1) diff --git a/examples/stream.exs b/examples/stream.exs index eaa4dac8..d39d1a08 100644 --- a/examples/stream.exs +++ b/examples/stream.exs @@ -1,28 +1,28 @@ {:ok, conn} = SSHKit.connect("127.0.0.1", port: 2222, user: "deploy", password: "deploy", silently_accept_hosts: true) -{:ok, chan} = SSHKit.exec(conn, ~S(echo "Who's there?"; read name; echo -n "Hello"; sleep 3; echo " $name")) +stream = SSHKit.exec!(conn, ~S(echo "Who's there?"; read name; echo -n "Hello"; sleep 3; echo " $name.")) -IO.write("> ") +:ok = IO.write("> ") -chan -|> SSHKit.stream!() -|> Stream.map(fn {:stdout, ^chan, chunk} -> chunk end) -|> Stream.each(&IO.write/1) -|> Stream.take_while(fn chunk -> !String.ends_with?(chunk, "\n") end) -|> Stream.run() +code = Enum.reduce(stream, nil, fn + {:stdout, chan, chunk}, status -> + :ok = IO.write("#{chunk}") -IO.write("< SSHKit\n") + if String.ends_with?(chunk, "?\n") do + :ok = SSHKit.send(chan, "SSHKit\n") + :ok = IO.write("< SSHKit\n") + :ok = IO.write("> ") + end -:ok = SSHKit.send(chan, "SSHKit\n") + status -IO.write("> ") + {:exit, _, status}, _ -> + status -# TODO: Timeouts? -chan -|> SSHKit.stream!() -|> Stream.filter(fn msg -> elem(msg, 0) == :stdout end) -|> Stream.map(fn {:stdout, ^chan, chunk} -> chunk end) -|> Stream.each(&IO.write/1) -|> Stream.run() + _, status -> + status +end) + +:ok = IO.puts("? #{code}") :ok = SSHKit.close(conn) diff --git a/lib/sshkit.ex b/lib/sshkit.ex index c43b4e43..20c22640 100644 --- a/lib/sshkit.ex +++ b/lib/sshkit.ex @@ -52,37 +52,16 @@ defmodule SSHKit do # Seems like `send` and `eof` should be enough for the intended high-level use cases. # If more fine-grained control is needed, feel free to reach for the `SSHKit.Channel` module. - @spec exec(Connection.t(), binary(), keyword()) :: {:ok, Channel.t()} | {:error, term()} - def exec(conn, command, options \\ []) do - timeout = Keyword.get(options, :timeout, :infinity) - - with {:ok, chan} <- Channel.open(conn, options) do - case Channel.exec(chan, command, timeout) do - :success -> - {:ok, chan} - - :failure -> - {:error, :failure} - - error -> - error - end - end - end - - @spec send(Channel.t(), :eof) :: :ok | {:error, term()} - def send(chan, :eof) do - Channel.eof(chan) - end - - @spec send(Channel.t(), :stdout | :stderr, term(), timeout()) :: :ok | {:error, term()} - def send(chan, type \\ :stdout, data, timeout \\ :infinity) - def send(chan, :stdout, data, timeout), do: Channel.send(chan, 0, data, timeout) - def send(chan, :stderr, data, timeout), do: Channel.send(chan, 1, data, timeout) - - def stream!(chan) do - Stream.unfold(:cont, fn - :cont -> + @spec exec!(Connection.t(), binary(), keyword()) :: Enumerable.t() + def exec!(conn, command, options \\ []) do + # TODO: Separate options for open/exec/recv + Stream.resource( + fn -> + {:ok, chan} = Channel.open(conn, options) # TODO: handle {:error, reason} and raise custom error struct? + :success = Channel.exec(chan, command) # TODO: timeout?, TODO: Handle :failure and {:error, reason} and raise custom error struct? + chan + end, + fn chan -> {:ok, msg} = Channel.recv(chan) # TODO: timeout?, TODO: handle {:error, reason} and raise custom error struct? # TODO: Adjust channel window size? @@ -111,16 +90,28 @@ defmodule SSHKit do next = case value do {:closed, _} -> :halt - _ -> :cont + _ -> [value] end - {value, next} + {next, chan} + end, + fn chan -> + :ok = Channel.close(chan) + :ok = Channel.flush(chan) + end + ) + end - :halt -> - nil - end) + @spec send(Channel.t(), :eof) :: :ok | {:error, term()} + def send(chan, :eof) do + Channel.eof(chan) end + @spec send(Channel.t(), :stdout | :stderr, term(), timeout()) :: :ok | {:error, term()} + def send(chan, type \\ :stdout, data, timeout \\ :infinity) + def send(chan, :stdout, data, timeout), do: Channel.send(chan, 0, data, timeout) + def send(chan, :stderr, data, timeout), do: Channel.send(chan, 1, data, timeout) + @doc ~S""" Upload a file or files to the given context. From bb07c16303327f55fddbc81a3fc3a7ba17410b99 Mon Sep 17 00:00:00 2001 From: pmeinhardt Date: Mon, 21 Dec 2020 05:38:48 +0100 Subject: [PATCH 24/54] Update context documentation --- lib/sshkit/context.ex | 105 +++++++++++++++++++++++++++++------------- 1 file changed, 72 insertions(+), 33 deletions(-) diff --git a/lib/sshkit/context.ex b/lib/sshkit/context.ex index 21e4896e..e15ad948 100644 --- a/lib/sshkit/context.ex +++ b/lib/sshkit/context.ex @@ -2,14 +2,14 @@ defmodule SSHKit.Context do @moduledoc """ A context encapsulates the environment for the execution of a task. That is: - * working directory to start in, see `SSHKit.path/2` - * user to run as, see `SSHKit.user/2` - * group, see `SSHKit.group/2` - * file creation mode mask, see `SSHKit.umask/2` - * environment variables, see `SSHKit.env/2` + * working directory to start in, see `SSHKit.Context.path/2` + * user to run as, see `SSHKit.Context.user/2` + * group, see `SSHKit.Context.group/2` + * file creation mode mask, see `SSHKit.Context.umask/2` + * environment variables, see `SSHKit.Context.env/2` A context can then be used to run commands, upload or download files: - See `SSHKit.run/2`, `SSHKit.upload/3` and `SSHKit.download/3`. + See `SSHKit.exec!/3`, `SSHKit.upload/4` and `SSHKit.download/4`. """ import SSHKit.Utils @@ -38,9 +38,17 @@ defmodule SSHKit.Context do Create `/var/www/app/config.json`: ``` - SSHKit.context() - |> SSHKit.path("/var/www/app") - |> SSHKit.run(conn, "touch config.json") + {:ok, conn} = SSHKit.connect("10.0.0.1") + + ctx = + SSHKit.Context.new() + |> SSHKit.Context.path("/var/www/app") + + conn + |> SSHKit.exec!("touch config.json", context: ctx) + |> Stream.run() + + :ok = SSHKit.close(conn) ``` """ def path(context, path) do @@ -58,10 +66,17 @@ defmodule SSHKit.Context do Create `precious.txt`, readable and writable only for the logged-in user: ``` - "10.0.0.1" - |> SSHKit.context() - |> SSHKit.umask("077") - |> SSHKit.run("touch precious.txt") + {:ok, conn} = SSHKit.connect("10.0.0.1") + + ctx = + SSHKit.Context.new() + |> SSHKit.Context.umask("077") + + conn + |> SSHKit.exec!("touch precious.txt", context: ctx) + |> Stream.run() + + :ok = SSHKit.close(conn) ``` """ def umask(context, mask) do @@ -70,8 +85,8 @@ defmodule SSHKit.Context do @doc """ Specifies the user under whose name commands are executed. - That user might be different than the user with which - ssh connects to the remote host. + That user might be different from the user with which + you connect to the remote host. Returns a new, derived context for easy chaining. @@ -81,10 +96,19 @@ defmodule SSHKit.Context do although we use the `login_user` to log in to the remote host: ``` - context = - {"10.0.0.1", port: 3000, user: "login_user", password: "secret"} - |> SSHKit.context() - |> SSHKit.user("deploy_user") + {:ok, conn} = SSHKit.connect("10.0.0.1", user: "login_user", password: "secret") + + ctx = + SSHKit.Context.new() + |> SSHKit.Context.user("deploy_user") + + conn + |> SSHKit.exec!("whoami", context: ctx) + |> Stream.filter(&(elem(&1) == :stdout)) + |> Stream.each(&IO.puts/1) + |> Stream.run() + + :ok = SSHKit.close(conn) ``` """ def user(context, name) do @@ -101,10 +125,19 @@ defmodule SSHKit.Context do All commands executed in the created `context` will run in group `www`: ``` - context = - "10.0.0.1" - |> SSHKit.context() - |> SSHKit.group("www") + {:ok, conn} = SSHKit.connect("10.0.0.1") + + ctx = + SSHKit.Context.new() + |> SSHKit.Context.group("www") + + conn + |> SSHKit.exec!("id -gn", context: ctx) + |> Stream.filter(&(elem(&1, 0) == :stdout)) + |> Stream.each(&IO.write/1) + |> Stream.run() + + :ok = SSHKit.close(conn) ``` """ def group(context, name) do @@ -122,25 +155,31 @@ defmodule SSHKit.Context do Setting `NODE_ENV=production`: ``` - context = - "10.0.0.1" - |> SSHKit.context() - |> SSHKit.env(%{"NODE_ENV" => "production"}) + {:ok, conn} = SSHKit.connect("10.0.0.1") + + ctx = + SSHKit.Context.new() + |> SSHKit.Context.env(%{"NODE_ENV" => "production"}) # Run the npm start script with NODE_ENV=production - SSHKit.run(context, "npm start") + conn + |> SSHKit.exec!("npm start", context: ctx) + |> Stream.run() + + :ok = SSHKit.close(conn) ``` Modifying the `PATH`: ``` - context = - "10.0.0.1" - |> SSHKit.context() - |> SSHKit.env(%{"PATH" => "$HOME/.rbenv/shims:$PATH"}) + ctx = + SSHKit.Context.new() + |> SSHKit.Context.env(%{"PATH" => "$HOME/.rbenv/shims:$PATH"}) # Execute the rbenv-installed ruby to print its version - SSHKit.run(context, "ruby --version") + conn + |> SSHKit.exec!(conn, "ruby --version", context: ctx) + |> Stream.run() ``` """ def env(context, map) do From ef07831671ddc6ed793f1241a7d92a2bead14076 Mon Sep 17 00:00:00 2001 From: pmeinhardt Date: Mon, 21 Dec 2020 13:47:45 +0100 Subject: [PATCH 25/54] =?UTF-8?q?Add=20`SSHKit.run!/3`=20to=20run=20comman?= =?UTF-8?q?ds=20easily=E2=80=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit …and capture their output. --- lib/sshkit.ex | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/lib/sshkit.ex b/lib/sshkit.ex index 20c22640..8ed55bb9 100644 --- a/lib/sshkit.ex +++ b/lib/sshkit.ex @@ -112,6 +112,24 @@ defmodule SSHKit do def send(chan, :stdout, data, timeout), do: Channel.send(chan, 0, data, timeout) def send(chan, :stderr, data, timeout), do: Channel.send(chan, 1, data, timeout) + @spec run!(Connection.t(), binary(), keyword()) :: [{:stdout, binary()} | {:stderr, binary()}] + def run!(conn, command, options \\ []) do + stream = exec!(conn, command, options) + + {status, output} = Enum.reduce(stream, {nil, []}, fn + {:stdout, _, data}, {status, output} -> {status, [{:stdout, data} | output]} + {:stderr, _, data}, {status, output} -> {status, [{:stderr, data} | output]} + {:exit, _, status}, {_, output} -> {status, output} + _, acc -> acc + end) + + output = Enum.reverse(output) + + if status != 0, do: raise "Non-zero exit code: #{status}" # TODO: Proper file struct? + + output + end + @doc ~S""" Upload a file or files to the given context. From a136449c72d9edfb0b407039944bf79eb253c74d Mon Sep 17 00:00:00 2001 From: pmeinhardt Date: Mon, 21 Dec 2020 20:54:37 +0100 Subject: [PATCH 26/54] Add example simply running commands --- examples/command.exs | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 examples/command.exs diff --git a/examples/command.exs b/examples/command.exs new file mode 100644 index 00000000..b9917be3 --- /dev/null +++ b/examples/command.exs @@ -0,0 +1,7 @@ +{:ok, conn} = SSHKit.connect("127.0.0.1", port: 2222, user: "deploy", password: "deploy", silently_accept_hosts: true) + +conn +|> SSHKit.run!(~S(uname -a && ssh -V)) +|> Enum.each(fn {type, data} -> IO.write("#{type}: #{data}") end) + +:ok = SSHKit.close(conn) From 658355e7b7e59de14d5e9411f2327ade6ae6aa40 Mon Sep 17 00:00:00 2001 From: pmeinhardt Date: Mon, 21 Dec 2020 21:54:11 +0100 Subject: [PATCH 27/54] Remove unused SSHKit.Host struct/module --- lib/sshkit.ex | 2 -- lib/sshkit/host.ex | 17 ----------------- 2 files changed, 19 deletions(-) delete mode 100644 lib/sshkit/host.ex diff --git a/lib/sshkit.ex b/lib/sshkit.ex index 8ed55bb9..d1ebda69 100644 --- a/lib/sshkit.ex +++ b/lib/sshkit.ex @@ -20,9 +20,7 @@ defmodule SSHKit do alias SSHKit.Channel alias SSHKit.Connection - alias SSHKit.Context alias SSHKit.Download - alias SSHKit.Host alias SSHKit.Upload @doc """ diff --git a/lib/sshkit/host.ex b/lib/sshkit/host.ex deleted file mode 100644 index 5f954f60..00000000 --- a/lib/sshkit/host.ex +++ /dev/null @@ -1,17 +0,0 @@ -defmodule SSHKit.Host do - @moduledoc ~S""" - Provides the data structure holding the information - about how to connect to a host. - - ## Examples - - ``` - %SSHKit.Host{name: "3.eg.io", options: [port: 2223]} - |> SSHKit.context() - |> SSHKit.run("touch base") - ``` - """ - defstruct [:name, :options] - - @type t() :: %__MODULE__{} -end From 7cedfcd849bf40bd2354b0c2453fece0560a96dc Mon Sep 17 00:00:00 2001 From: pmeinhardt Date: Mon, 21 Dec 2020 21:55:18 +0100 Subject: [PATCH 28/54] Clean up SSHKit.run!/3 typespec --- lib/sshkit.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/sshkit.ex b/lib/sshkit.ex index d1ebda69..580bbe90 100644 --- a/lib/sshkit.ex +++ b/lib/sshkit.ex @@ -110,7 +110,7 @@ defmodule SSHKit do def send(chan, :stdout, data, timeout), do: Channel.send(chan, 0, data, timeout) def send(chan, :stderr, data, timeout), do: Channel.send(chan, 1, data, timeout) - @spec run!(Connection.t(), binary(), keyword()) :: [{:stdout, binary()} | {:stderr, binary()}] + @spec run!(Connection.t(), binary(), keyword()) :: [{:stdout | :stderr, binary()}] def run!(conn, command, options \\ []) do stream = exec!(conn, command, options) From 067f0035a741fb1f064f086e70bc7064521b0231 Mon Sep 17 00:00:00 2001 From: pmeinhardt Date: Mon, 21 Dec 2020 23:24:46 +0100 Subject: [PATCH 29/54] Experiment with another design for file transfers --- examples/upload.exs | 5 ++++- lib/sshkit.ex | 34 ++++++++++++++-------------------- lib/sshkit/sftp/channel.ex | 2 +- lib/sshkit/transfer.ex | 30 ++++++++++++++++++++++++++++++ lib/sshkit/upload.ex | 23 ++++------------------- 5 files changed, 53 insertions(+), 41 deletions(-) create mode 100644 lib/sshkit/transfer.ex diff --git a/examples/upload.exs b/examples/upload.exs index ed12658c..87a4dd42 100644 --- a/examples/upload.exs +++ b/examples/upload.exs @@ -1,5 +1,8 @@ {:ok, conn} = SSHKit.connect("127.0.0.1", port: 2222, user: "deploy", password: "deploy", silently_accept_hosts: true) -:ok = SSHKit.upload(conn, "test/fixtures", "/tmp/fixtures", recursive: true) +:ok = + conn + |> SSHKit.upload!("test/fixtures", "/tmp/fixtures", recursive: true) + |> Stream.run() :ok = SSHKit.close(conn) diff --git a/lib/sshkit.ex b/lib/sshkit.ex index 580bbe90..edcee431 100644 --- a/lib/sshkit.ex +++ b/lib/sshkit.ex @@ -21,6 +21,7 @@ defmodule SSHKit do alias SSHKit.Channel alias SSHKit.Connection alias SSHKit.Download + alias SSHKit.Transfer alias SSHKit.Upload @doc """ @@ -40,16 +41,6 @@ defmodule SSHKit do Connection.close(conn) end - # TODO: Do we need to expose lower-level channel operations here? - # - # * Send `eof`? - # * Subsystem - # * ppty - # * … - # - # Seems like `send` and `eof` should be enough for the intended high-level use cases. - # If more fine-grained control is needed, feel free to reach for the `SSHKit.Channel` module. - @spec exec!(Connection.t(), binary(), keyword()) :: Enumerable.t() def exec!(conn, command, options \\ []) do # TODO: Separate options for open/exec/recv @@ -100,6 +91,16 @@ defmodule SSHKit do ) end + # TODO: Do we need to expose lower-level channel operations here? + # + # * Send `eof`? + # * Subsystem + # * ppty + # * … + # + # Seems like `send` and `eof` should be enough for the intended high-level use cases. + # If more fine-grained control is needed, feel free to reach for the `SSHKit.Channel` module. + @spec send(Channel.t(), :eof) :: :ok | {:error, term()} def send(chan, :eof) do Channel.eof(chan) @@ -159,15 +160,8 @@ defmodule SSHKit do |> SSHKit.upload("local.txt", as: "remote.txt") ``` """ - def upload(conn, source, target, options \\ []) do - upload = Upload.init(source, target, options) - - # TODO: Close SFTP channel (Upload.stop/1) if there is an error - with {:ok, upload} <- Upload.start(upload, conn), - {:ok, upload} <- Upload.loop(upload), - {:ok, _} <- Upload.stop(upload) do - :ok - end + def upload!(conn, source, target, options \\ []) do + Transfer.stream!(conn, Upload.init(source, target, options)) end @doc ~S""" @@ -201,7 +195,7 @@ defmodule SSHKit do |> SSHKit.download("remote.txt", as: "local.txt") ``` """ - def download(conn, source, options \\ []) do + def download!(conn, source, target, options \\ []) do # TODO end end diff --git a/lib/sshkit/sftp/channel.ex b/lib/sshkit/sftp/channel.ex index c2143ab0..f6aa07f2 100644 --- a/lib/sshkit/sftp/channel.ex +++ b/lib/sshkit/sftp/channel.ex @@ -41,7 +41,7 @@ defmodule SSHKit.SFTP.Channel do chan.impl.close(chan.id, handle, timeout) end - @spec write(t(), handle(), iodata(), timeout()) :: :ok, {:error, term()} + @spec write(t(), handle(), iodata(), timeout()) :: :ok | {:error, term()} def write(chan, handle, data, timeout \\ :infinity) do chan.impl.write(chan.id, handle, data, timeout) end diff --git a/lib/sshkit/transfer.ex b/lib/sshkit/transfer.ex new file mode 100644 index 00000000..acaa3e47 --- /dev/null +++ b/lib/sshkit/transfer.ex @@ -0,0 +1,30 @@ +defmodule SSHKit.Transfer do + @moduledoc false + + alias SSHKit.Connection + + @spec stream!(Connection.t(), struct(), keyword()) :: Enumerable.t() + def stream!(conn, transfer, options \\ []) do + module = mod(transfer) + + Stream.resource( + fn -> + {:ok, transfer} = module.start(transfer, conn) + transfer + end, + fn transfer -> + if module.done?(transfer) do + {:halt, transfer} + else + {:ok, transfer} = module.continue(transfer) + {[transfer], transfer} + end + end, + fn transfer -> + {:ok, transfer} = module.stop(transfer) + end + ) + end + + defp mod(%{__struct__: name}), do: name +end diff --git a/lib/sshkit/upload.ex b/lib/sshkit/upload.ex index b563a858..edf99299 100644 --- a/lib/sshkit/upload.ex +++ b/lib/sshkit/upload.ex @@ -15,13 +15,13 @@ defmodule SSHKit.Upload do def start(%__MODULE__{options: options} = upload, conn) do # accepts options like timeout… http://erlang.org/doc/man/ssh_sftp.html#start_channel-1 - channel_options = + start_options = options |> Keyword.get(:start, []) |> Keyword.put_new(:timeout, Keyword.get(options, :timeout, :infinity)) with {:ok, upload} <- prepare(upload), - {:ok, chan} <- Channel.start(conn, channel_options) do + {:ok, chan} <- Channel.start(conn, start_options) do {:ok, %{upload | channel: chan}} end end @@ -99,21 +99,6 @@ defmodule SSHKit.Upload do |> Enum.find(:ok, &(&1 != :ok)) end - # TODO: Make `loop` return a stream? Possibly rename to "stream" then - def loop(%__MODULE__{stack: []} = upload) do - {:ok, upload} - end - - def loop(%__MODULE__{} = upload) do - case continue(upload) do - {:ok, upload} -> - loop(upload) - - error -> - error - end - end - - def done?(%__MODULE__{stack: []}), do: true - def done?(%__MODULE__{}), do: false + def done?(%{stack: []}), do: true + def done?(%{}), do: false end From d1675b1402f0be09be42dd40cb465575442d1803 Mon Sep 17 00:00:00 2001 From: pmeinhardt Date: Tue, 22 Dec 2020 13:13:38 +0100 Subject: [PATCH 30/54] Update tests, remove outdated ones for now --- test/sshkit/{ssh => }/channel_test.exs | 107 +------ test/sshkit/{ssh => }/connection_test.exs | 10 +- test/sshkit/context_test.exs | 48 ++- test/sshkit/ssh/channel_functional_test.exs | 6 +- test/sshkit/ssh_functional_test.exs | 22 -- test/sshkit/ssh_test.exs | 186 ------------ test/sshkit_functional_test.exs | 319 +------------------- test/sshkit_test.exs | 166 +--------- test/support/mocks.ex | 10 +- 9 files changed, 65 insertions(+), 809 deletions(-) rename test/sshkit/{ssh => }/channel_test.exs (73%) rename test/sshkit/{ssh => }/connection_test.exs (97%) delete mode 100644 test/sshkit/ssh_functional_test.exs delete mode 100644 test/sshkit/ssh_test.exs diff --git a/test/sshkit/ssh/channel_test.exs b/test/sshkit/channel_test.exs similarity index 73% rename from test/sshkit/ssh/channel_test.exs rename to test/sshkit/channel_test.exs index f5bdf264..6756ff5a 100644 --- a/test/sshkit/ssh/channel_test.exs +++ b/test/sshkit/channel_test.exs @@ -1,11 +1,11 @@ -defmodule SSHKit.SSH.ChannelTest do +defmodule SSHKit.ChannelTest do use ExUnit.Case, async: true - import Mox - import SSHKit.SSH.Channel + import Mox + import SSHKit.Channel - alias SSHKit.SSH.Channel - alias SSHKit.SSH.Connection + alias SSHKit.Channel + alias SSHKit.Connection setup do Mox.verify_on_exit!() @@ -290,103 +290,6 @@ defmodule SSHKit.SSH.ChannelTest do end end - describe "shell/2" do - test "set shell to server side", %{chan: chan, impl: impl} do - impl - |> expect(:shell, fn connection_ref, channel_id -> - assert connection_ref == chan.connection.ref - assert channel_id == chan.id - :ok - end) - - assert shell(chan) == :ok - end - end - - describe "loop/4" do - test "loops over channel messages until channel is closed", %{conn: conn, chan: chan} do - Enum.each(0..2, &Kernel.send(self(), {:ssh_cm, conn.ref, {:msg, chan.id, &1}})) - Kernel.send(self(), {:ssh_cm, conn.ref, {:closed, chan.id}}) - assert Enum.count(messages(self())) == 4 - - fun = fn - {:msg, _, index}, _ -> {:cont, index} - {:closed, _}, acc -> {:cont, acc} - end - - assert loop(chan, 0, {:cont, -1}, fun) == {:done, 2} - assert messages(self()) == [] - end - - test "allows sending messages to the remote", %{conn: conn, chan: chan, impl: impl} do - impl - |> expect(:send_eof, fn _, _ -> :ok end) - |> expect(:send, sends(chan, 0, "plain", 200, :ok)) - |> expect(:send, sends(chan, 0, "normal", 200, :ok)) - |> expect(:send, sends(chan, 1, "error", 200, :ok)) - - Enum.each(0..4, &Kernel.send(self(), {:ssh_cm, conn.ref, {:msg, chan.id, &1}})) - Kernel.send(self(), {:ssh_cm, conn.ref, {:closed, chan.id}}) - - msgs = [nil, :eof, "plain", {0, "normal"}, {1, "error"}] - - fun = fn - {:msg, _, index}, _ -> {:cont, Enum.at(msgs, index), index} - {:closed, _}, acc -> {:cont, acc} - end - - assert loop(chan, 200, {:cont, -1}, fun) == {:done, 4} - assert messages(self()) == [] - end - - test "allows suspending the loop", %{conn: conn, chan: chan} do - Enum.each(0..1, &Kernel.send(self(), {:ssh_cm, conn.ref, {:msg, chan.id, &1}})) - Kernel.send(self(), {:ssh_cm, conn.ref, {:closed, chan.id}}) - - fun = fn - {:msg, _, _}, acc when acc < 5 -> {:suspend, acc + 1} - {:msg, _, _}, acc -> {:cont, acc * 3} - {:closed, _}, acc -> {:cont, acc} - end - - {:suspended, 1, continue} = loop(chan, 0, {:cont, 0}, fun) - - assert continue.({:cont, 5}) == {:done, 15} - assert messages(self()) == [] - end - - test "allows halting the loop", %{conn: conn, chan: chan, impl: impl} do - impl |> expect(:close, fn _, _ -> :ok end) - - Enum.each(0..5, &Kernel.send(self(), {:ssh_cm, conn.ref, {:msg, chan.id, &1}})) - - fun = fn - _, acc when acc < 2 -> {:cont, acc + 1} - _, acc -> {:halt, acc} - end - - assert loop(chan, 0, {:cont, 0}, fun) == {:halted, 2} - # remaining messages flushed - assert messages(self()) == [] - end - - test "returns an error if next message is not received in time", %{chan: chan, impl: impl} do - impl |> expect(:close, fn _, _ -> :ok end) - res = loop(chan, 0, {:cont, []}, &[&1 | &2]) - assert res == {:halted, {:error, :timeout}} - end - - test "returns an error if message sending fails", %{chan: chan, impl: impl} do - impl - |> expect(:send, sends(chan, 0, "data", 100, {:error, :timeout})) - |> expect(:close, fn _, _ -> :ok end) - - res = loop(chan, 100, {:cont, "data", []}, &[&1 | &2]) - - assert res == {:halted, {:error, :timeout}} - end - end - defp sends(chan, expected_type, expected_data, expected_timeout, res) do fn connection_ref, channel_id, type, data, timeout -> assert connection_ref == chan.connection.ref diff --git a/test/sshkit/ssh/connection_test.exs b/test/sshkit/connection_test.exs similarity index 97% rename from test/sshkit/ssh/connection_test.exs rename to test/sshkit/connection_test.exs index 0cfdc203..759c0698 100644 --- a/test/sshkit/ssh/connection_test.exs +++ b/test/sshkit/connection_test.exs @@ -1,11 +1,11 @@ -defmodule SSHKit.SSH.ConnectionTest do +defmodule SSHKit.ConnectionTest do use ExUnit.Case, async: true - import Mox - import SSHKit.SSH.Connection + import Mox + import SSHKit.Connection - alias SSHKit.SSH.Connection - alias SSHKit.SSH.Connection.ImplMock + alias SSHKit.Connection + alias SSHKit.Connection.ImplMock setup do Mox.verify_on_exit!() diff --git a/test/sshkit/context_test.exs b/test/sshkit/context_test.exs index 5317a58a..5cc49d3b 100644 --- a/test/sshkit/context_test.exs +++ b/test/sshkit/context_test.exs @@ -3,9 +3,53 @@ defmodule SSHKit.ContextTest do alias SSHKit.Context - @empty %Context{hosts: []} + @empty %Context{} - doctest Context + describe "new/0" do + test "returns a new context" do + context = Context.new() + assert context == @empty + end + end + + describe "path/2" do + test "sets the path for the context" do + context = Context.path(@empty, "/var/www/app") + assert context.path == "/var/www/app" + end + end + + describe "umask/2" do + test "sets the file permission mask for the context" do + context = Context.umask(@empty, "077") + assert context.umask == "077" + end + end + + describe "user/2" do + test "sets the user for the context" do + context = Context.user(@empty, "meg") + assert context.user == "meg" + end + end + + describe "group/2" do + test "sets the group for the context" do + context = Context.group(@empty, "stripes") + assert context.group == "stripes" + end + end + + describe "env/2" do + test "overwrites existing env" do + context = + @empty + |> Context.env(%{"NODE_ENV" => "production"}) + |> Context.env(%{"CI" => "true"}) + + assert context.env == %{"CI" => "true"} + end + end describe "build/2" do test "with user" do diff --git a/test/sshkit/ssh/channel_functional_test.exs b/test/sshkit/ssh/channel_functional_test.exs index c8a8544f..b96c1aef 100644 --- a/test/sshkit/ssh/channel_functional_test.exs +++ b/test/sshkit/ssh/channel_functional_test.exs @@ -1,16 +1,16 @@ -defmodule SSHKit.SSH.ChannelFunctionalTest do +defmodule SSHKit.ChannelFunctionalTest do @moduledoc false use SSHKit.FunctionalCase, async: true - alias SSHKit.SSH.Channel + alias SSHKit.Channel @bootconf [user: "me", password: "pass"] describe "Channel.subsystem/3" do @tag boot: [@bootconf] test "with user", %{hosts: [host]} do - {:ok, conn} = SSHKit.SSH.connect(host.name, host.options) + {:ok, conn} = SSHKit.connect(host.name, host.options) {:ok, channel} = Channel.open(conn) :success = Channel.subsystem(channel, "greeting-subsystem") diff --git a/test/sshkit/ssh_functional_test.exs b/test/sshkit/ssh_functional_test.exs deleted file mode 100644 index da316551..00000000 --- a/test/sshkit/ssh_functional_test.exs +++ /dev/null @@ -1,22 +0,0 @@ -defmodule SSHKit.SSHFunctionalTest do - use SSHKit.FunctionalCase, async: true - - alias SSHKit.SSH - - @bootconf [user: "me", password: "pass"] - - @tag boot: [@bootconf] - test "opens a connection with username and password, runs a command", %{hosts: [host]} do - {:ok, conn} = SSH.connect(host.name, host.options) - {:ok, data, status} = SSH.run(conn, "id -un") - assert [stdout: "#{host.options[:user]}\n"] == data - assert 0 = status - end - - @tag boot: [@bootconf] - test "opens a connection and runs a command in a lambda function", %{hosts: [host]} do - fun = fn conn -> SSH.run(conn, "id -un") end - result = {:ok, [stdout: "#{host.options[:user]}\n"], 0} - assert SSH.connect(host.name, host.options, fun) == {:ok, result} - end -end diff --git a/test/sshkit/ssh_test.exs b/test/sshkit/ssh_test.exs deleted file mode 100644 index 9c3fb66c..00000000 --- a/test/sshkit/ssh_test.exs +++ /dev/null @@ -1,186 +0,0 @@ -defmodule SSHKit.SSHTest do - use ExUnit.Case, async: true - import Mox - - import SSHKit.SSH - - alias SSHKit.SSH.Channel - alias SSHKit.SSH.Connection - - @host "test.io" - @user "me" - - setup do - Mox.verify_on_exit!() - end - - describe "connect/2" do - setup do - {:ok, impl: Connection.ImplMock} - end - - test "opens a connection with the given options and keeps it open", %{impl: impl} do - impl - |> expect(:connect, fn host, port, opts, timeout -> - assert host == 'test.io' - assert port == 2222 - assert opts == [user_interaction: false, user: 'me'] - assert timeout == :infinity - {:ok, :connection_ref} - end) - - {:ok, conn} = connect(@host, user: @user, port: 2222, impl: impl) - - assert conn == %Connection{ - host: 'test.io', - options: [user_interaction: false, user: 'me'], - port: 2222, - ref: :connection_ref, - impl: impl - } - end - - test "returns an error if connection cannot be opened", %{impl: impl} do - impl |> expect(:connect, fn _, _, _, _ -> {:error, :timeout} end) - assert connect(@host, impl: impl) == {:error, :timeout} - end - - test "returns an error if no host given" do - assert connect(nil) == {:error, "No host given."} - end - - test "returns an error if options not provided as list" do - options = %{user: @user, password: "secret"} - assert_raise FunctionClauseError, fn -> connect(@host, options) end - end - end - - describe "connect/3" do - setup do - {:ok, impl: Connection.ImplMock} - end - - test "executes function on open connection", %{impl: impl} do - impl - |> expect(:connect, fn _, _, _, _ -> {:ok, :opened_connection_ref} end) - |> expect(:close, fn :opened_connection_ref -> :ok end) - - fun = fn conn -> - assert conn.ref == :opened_connection_ref - 42 - end - - assert connect(@host, [impl: impl], fun) == {:ok, 42} - end - - test "closes connection although function errored", %{impl: impl} do - impl - |> expect(:connect, fn _, _, _, _ -> {:ok, :opened_connection_ref} end) - |> expect(:close, fn :opened_connection_ref -> :ok end) - - fun = fn _ -> raise(RuntimeError, message: "error") end - - assert_raise RuntimeError, "error", fn -> - connect(@host, [impl: impl], fun) - end - end - - test "returns connection errors", %{impl: impl} do - impl |> expect(:connect, fn _, _, _, _ -> {:error, :timeout} end) - fun = fn _ -> flunk("should never be called") end - assert connect(@host, [impl: impl], fun) == {:error, :timeout} - end - end - - describe "close/1" do - setup do - {:ok, impl: Connection.ImplMock} - end - - test "closes the connection", %{impl: impl} do - conn = %SSHKit.SSH.Connection{ - host: 'test.io', - port: 22, - options: [user_interaction: false], - ref: :connection_ref, - impl: impl - } - - impl - |> expect(:close, fn ref -> - assert ref == conn.ref - :ok - end) - - assert close(conn) == :ok - end - end - - describe "run/3" do - setup do - conn = %Connection{ref: :cref, impl: Connection.ImplMock} - {:ok, conn: conn, impl: Channel.ImplMock} - end - - test "captures output and exit status by default", %{conn: conn, impl: impl} do - impl - |> expect(:session_channel, fn :cref, _, _, :infinity -> {:ok, 11} end) - |> expect(:exec, fn :cref, 11, 'try', :infinity -> :success end) - |> expect(:adjust_window, 2, fn :cref, 11, _ -> :ok end) - - send(self(), {:ssh_cm, conn.ref, {:data, 11, 0, "out"}}) - send(self(), {:ssh_cm, conn.ref, {:data, 11, 1, "err"}}) - send(self(), {:ssh_cm, conn.ref, {:exit_status, 11, 127}}) - send(self(), {:ssh_cm, conn.ref, {:closed, 11}}) - - assert run(conn, "try", impl: impl) == {:ok, [stdout: "out", stderr: "err"], 127} - end - - test "accepts a custom handler function and accumulator", %{conn: conn, impl: impl} do - impl - |> expect(:session_channel, fn :cref, _, _, _ -> {:ok, 31} end) - |> expect(:exec, fn :cref, 31, 'cmd', _ -> :success end) - |> expect(:send, fn :cref, 31, 0, "PING", _ -> :ok end) - |> expect(:adjust_window, fn :cref, 31, _ -> :ok end) - |> expect(:close, fn :cref, 31 -> :ok end) - - send(self(), {:ssh_cm, conn.ref, {:data, 31, 0, "PONG"}}) - - ini = {:cont, "PING", ["START"]} - - fun = fn msg, acc -> - case msg do - {:data, _, 0, data} -> {:halt, [data | acc]} - _ -> {:cont, "NOPE", ["FAILED" | acc]} - end - end - - assert run(conn, "cmd", acc: ini, fun: fun, impl: impl) == ["PONG", "START"] - end - - test "accepts a timeout value", %{conn: conn, impl: impl} do - impl - |> expect(:session_channel, fn _, _, _, 500 -> {:ok, 61} end) - |> expect(:exec, fn _, _, _, 500 -> :success end) - |> expect(:send, fn _, _, _, _, 500 -> {:error, :timeout} end) - |> expect(:close, fn _, _ -> :ok end) - - acc = {:cont, "INIT", {[], nil}} - - assert run(conn, "cmd", acc: acc, timeout: 500, impl: impl) == {:error, :timeout} - end - - test "returns an error if channel cannot be opened", %{conn: conn, impl: impl} do - impl |> expect(:session_channel, fn _, _, _, _ -> {:error, :timeout} end) - assert run(conn, "uptime", impl: impl) == {:error, :timeout} - end - - test "returns an error if command fails to execute", %{conn: conn, impl: impl} do - impl - |> expect(:session_channel, fn _, _, _, _ -> {:ok, 13} end) - |> expect(:exec, fn _, _, _, _ -> :failure end) - - assert run(conn, "uptime", impl: impl) == {:error, :failure} - end - end -end diff --git a/test/sshkit_functional_test.exs b/test/sshkit_functional_test.exs index 244720bd..f91ec2a7 100644 --- a/test/sshkit_functional_test.exs +++ b/test/sshkit_functional_test.exs @@ -3,322 +3,5 @@ defmodule SSHKitFunctionalTest do use SSHKit.FunctionalCase, async: true - @bootconf [user: "me", password: "pass"] - - describe "run/2" do - @tag boot: [@bootconf] - test "connects as the login user and runs commands", %{hosts: [host]} do - [{:ok, output, 0}] = - host - |> SSHKit.context() - |> SSHKit.run("id -un") - - name = String.trim(stdout(output)) - assert name == host.options[:user] - end - - @tag boot: [@bootconf] - test "runs commands and returns their output and exit status", %{hosts: [host]} do - context = SSHKit.context(host) - - [{:ok, output, status}] = SSHKit.run(context, "pwd") - assert status == 0 - assert stdout(output) == "/home/me\n" - - [{:ok, output, status}] = SSHKit.run(context, "ls nonexistent") - assert status == 1 - assert stderr(output) =~ "ls: nonexistent: No such file or directory" - - [{:ok, output, status}] = SSHKit.run(context, "nonexistent") - assert status == 127 - assert stderr(output) =~ "'nonexistent': No such file or directory" - end - - @tag boot: [@bootconf] - test "with env", %{hosts: [host]} do - [{:ok, output, status}] = - host - |> SSHKit.context() - |> SSHKit.env(%{"PATH" => "$HOME/.rbenv/shims:$PATH", "NODE_ENV" => "production"}) - |> SSHKit.run("env") - - assert status == 0 - - output = stdout(output) - assert output =~ ~r/^NODE_ENV=production$/m - assert output =~ ~r/^PATH=\/home\/me\/.rbenv\/shims:.*$/m - end - - @tag boot: [@bootconf] - test "with umask", %{hosts: [host]} do - context = - host - |> SSHKit.context() - |> SSHKit.umask("077") - - [{:ok, _, 0}] = SSHKit.run(context, "mkdir my_dir") - [{:ok, _, 0}] = SSHKit.run(context, "touch my_file") - - [{:ok, output, status}] = SSHKit.run(context, "ls -la") - - assert status == 0 - - output = stdout(output) - assert output =~ ~r/^drwx--S---\s+2\s+me\s+me\s+4096.+\smy_dir$/m - assert output =~ ~r/^-rw-------\s+1\s+me\s+me\s+0.+\smy_file$/m - end - - @tag boot: [@bootconf] - test "with path", %{hosts: [host]} do - context = - host - |> SSHKit.context() - |> SSHKit.path("/var/log") - - [{:ok, output, status}] = SSHKit.run(context, "pwd") - - assert status == 0 - assert stdout(output) == "/var/log\n" - end - - @tag boot: [@bootconf] - test "with user", %{hosts: [host]} do - add_user_to_group!(host, host.options[:user], "passwordless-sudoers") - adduser!(host, "despicable_me") - - context = - host - |> SSHKit.context() - |> SSHKit.user("despicable_me") - - [{:ok, output, status}] = SSHKit.run(context, "id -un") - - assert status == 0 - assert stdout(output) == "despicable_me\n" - end - - @tag boot: [@bootconf] - test "with group", %{hosts: [host]} do - add_user_to_group!(host, host.options[:user], "passwordless-sudoers") - - adduser!(host, "gru") - addgroup!(host, "villains") - add_user_to_group!(host, "gru", "villains") - - context = - host - |> SSHKit.context() - |> SSHKit.user("gru") - |> SSHKit.group("villains") - - [{:ok, output, status}] = SSHKit.run(context, "id -gn") - - assert status == 0 - assert stdout(output) == "villains\n" - end - - @tag boot: [@bootconf] - test "with path, umask, user, group and env", %{hosts: [host]} do - add_user_to_group!(host, host.options[:user], "passwordless-sudoers") - - adduser!(host, "stuart") - addgroup!(host, "minions") - add_user_to_group!(host, "stuart", "minions") - - context = - host - |> SSHKit.context() - |> SSHKit.path("/tmp") - |> SSHKit.user("stuart") - |> SSHKit.group("minions") - |> SSHKit.umask("077") - |> SSHKit.env(%{"INSTRUMENT" => "super-mega ukulele"}) - - [{:ok, output, status}] = SSHKit.run(context, "echo $INSTRUMENT > bag") - - assert status == 0 - assert output == [] - - info = exec!(host, "ls", ["-l", "/tmp/bag"]) - assert info =~ ~r/^-rw-------\s+1\s+stuart\s+minions\s+.+\s+\/tmp\/bag$/m - - content = exec!(host, "cat", ["/tmp/bag"]) - assert content == "super-mega ukulele" - end - end - - describe "upload/3" do - @describetag boot: [@bootconf, @bootconf] - - test "uploads a file", %{hosts: hosts} do - local = "test/fixtures/local.txt" - - context = SSHKit.context(hosts) - - assert [:ok, :ok] = SSHKit.upload(context, local) - assert verify_transfer(context, local, Path.basename(local)) - end - - test "uploads a file to a directory that does not exist", %{hosts: hosts} do - local = "test/fixtures/local.txt" - - context = - hosts - |> SSHKit.context() - |> SSHKit.path("/otp/releases") - - assert [ - error: "sh: cd: line 1: can't cd to /otp/releases", - error: "sh: cd: line 1: can't cd to /otp/releases" - ] = SSHKit.upload(context, local) - end - - test "uploads a file to a directory we have no access to", %{hosts: hosts} do - local = "test/fixtures/local.txt" - - context = - hosts - |> SSHKit.context() - |> SSHKit.path("/") - - assert [ - error: "SCP exited with non-zero exit code 1: scp: local.txt: Permission denied", - error: "SCP exited with non-zero exit code 1: scp: local.txt: Permission denied" - ] = SSHKit.upload(context, local) - end - - test "recursive: true", %{hosts: [host | _] = hosts} do - local = "test/fixtures" - remote = "/home/#{host.options[:user]}/fixtures" - - context = SSHKit.context(hosts) - - assert [:ok, :ok] = SSHKit.upload(context, local, recursive: true) - assert verify_transfer(context, local, remote) - end - - test "preserve: true", %{hosts: hosts} do - local = "test/fixtures/local.txt" - remote = Path.basename(local) - - context = SSHKit.context(hosts) - - assert [:ok, :ok] = SSHKit.upload(context, local, preserve: true) - assert verify_transfer(context, local, remote) - assert verify_mode(context, local, remote) - assert verify_mtime(context, local, remote) - end - - test "recursive: true, preserve: true", %{hosts: [host | _] = hosts} do - local = "test/fixtures" - remote = "/home/#{host.options[:user]}/fixtures" - - context = SSHKit.context(hosts) - - assert [:ok, :ok] = SSHKit.upload(context, local, recursive: true, preserve: true) - assert verify_transfer(context, local, remote) - assert verify_mode(context, local, remote) - assert verify_mtime(context, local, remote) - end - - test "with context", %{hosts: hosts} do - local = "test/fixtures" - # path relative to context path - remote = "target" - - context = - hosts - |> SSHKit.context() - |> SSHKit.path("/tmp") - - assert [:ok, :ok] = - SSHKit.upload(context, local, recursive: true, preserve: true, as: remote) - - assert verify_transfer(context, local, Path.join(context.path, remote)) - assert verify_mode(context, local, Path.join(context.path, remote)) - assert verify_mtime(context, local, Path.join(context.path, remote)) - end - end - - describe "download/3" do - @describetag boot: [@bootconf] - - setup do - tmpdir = create_local_tmp_path() - - :ok = File.mkdir!(tmpdir) - on_exit(fn -> File.rm_rf(tmpdir) end) - - {:ok, tmpdir: tmpdir} - end - - test "gets a file", %{hosts: hosts, tmpdir: tmpdir} do - remote = "/fixtures/remote.txt" - local = Path.join(tmpdir, Path.basename(remote)) - - context = SSHKit.context(hosts) - - assert [:ok] = SSHKit.download(context, remote, as: local) - assert verify_transfer(context, local, remote) - end - - test "recursive: true", %{hosts: hosts, tmpdir: tmpdir} do - remote = "/fixtures" - local = Path.join(tmpdir, "fixtures") - - context = SSHKit.context(hosts) - - assert [:ok] = SSHKit.download(context, remote, recursive: true, as: local) - assert verify_transfer(context, local, remote) - end - - test "preserve: true", %{hosts: hosts, tmpdir: tmpdir} do - remote = "/fixtures/remote.txt" - local = Path.join(tmpdir, Path.basename(remote)) - - context = SSHKit.context(hosts) - - assert [:ok] = SSHKit.download(context, remote, preserve: true, as: local) - assert verify_mode(context, local, remote) - assert verify_atime(context, local, remote) - assert verify_mtime(context, local, remote) - end - - test "recursive: true, preserve: true", %{hosts: hosts, tmpdir: tmpdir} do - remote = "/fixtures" - local = Path.join(tmpdir, "fixtures") - - context = SSHKit.context(hosts) - - assert [:ok] = SSHKit.download(context, remote, recursive: true, preserve: true, as: local) - assert verify_mode(context, local, remote) - assert verify_atime(context, local, remote) - assert verify_mtime(context, local, remote) - end - - test "with context", %{hosts: hosts, tmpdir: tmpdir} do - # path relative to context path - remote = "fixtures" - local = Path.join(tmpdir, "fixtures") - - context = - hosts - |> SSHKit.context() - |> SSHKit.path("/") - - assert [:ok] = SSHKit.download(context, remote, recursive: true, preserve: true, as: local) - assert verify_transfer(context, local, Path.join(context.path, remote)) - assert verify_mode(context, local, Path.join(context.path, remote)) - assert verify_mtime(context, local, Path.join(context.path, remote)) - end - end - - defp stdio(output, type) do - output - |> Keyword.get_values(type) - |> Enum.join() - end - - def stdout(output), do: stdio(output, :stdout) - def stderr(output), do: stdio(output, :stderr) + # TODO end diff --git a/test/sshkit_test.exs b/test/sshkit_test.exs index 318f6e25..36012d3b 100644 --- a/test/sshkit_test.exs +++ b/test/sshkit_test.exs @@ -1,169 +1,5 @@ defmodule SSHKitTest do use ExUnit.Case, async: true - alias SSHKit.Context - alias SSHKit.Host - - @empty %Context{hosts: []} - - describe "host/2" do - test "creates a host from a hostname (binary) and options" do - assert SSHKit.host("10.0.0.1", user: "me") == %Host{name: "10.0.0.1", options: [user: "me"]} - end - - test "creates a host from a map with :name and :options" do - input = %{name: "10.0.0.1", options: [user: "me"]} - assert SSHKit.host(input) == %Host{name: "10.0.0.1", options: [user: "me"]} - end - - test "creates a host from a tuple" do - input = {"10.0.0.1", user: "me"} - assert SSHKit.host(input) == %Host{name: "10.0.0.1", options: [user: "me"]} - end - end - - describe "context/2" do - test "creates a context from a single hostname (binary)" do - context = SSHKit.context("10.0.0.1") - hosts = [%Host{name: "10.0.0.1", options: []}] - assert context == %Context{hosts: hosts} - end - - test "creates a context from a single map with :name and :options" do - context = SSHKit.context(%{name: "10.0.0.1", options: [user: "me"]}) - hosts = [%Host{name: "10.0.0.1", options: [user: "me"]}] - assert context == %Context{hosts: hosts} - end - - test "creates a context from a single tuple" do - context = SSHKit.context({"10.0.0.1", user: "me"}) - hosts = [%Host{name: "10.0.0.1", options: [user: "me"]}] - assert context == %Context{hosts: hosts} - end - - test "creates a context from a list of hostnames (binaries)" do - context = SSHKit.context(["10.0.0.1", "10.0.0.2"]) - - hosts = [ - %Host{name: "10.0.0.1", options: []}, - %Host{name: "10.0.0.2", options: []} - ] - - assert context == %Context{hosts: hosts} - end - - test "creates a context from a list of maps with :name and :options" do - context = - SSHKit.context([ - %{name: "10.0.0.1", options: [user: "me"]}, - %{name: "10.0.0.2", options: []} - ]) - - hosts = [ - %Host{name: "10.0.0.1", options: [user: "me"]}, - %Host{name: "10.0.0.2", options: []} - ] - - assert context == %Context{hosts: hosts} - end - - test "creates a context from a list of tuples" do - context = - SSHKit.context([ - {"10.0.0.1", [user: "me"]}, - {"10.0.0.2", []} - ]) - - hosts = [ - %Host{name: "10.0.0.1", options: [user: "me"]}, - %Host{name: "10.0.0.2", options: []} - ] - - assert context == %Context{hosts: hosts} - end - - test "creates a context from a mixed list" do - context = - SSHKit.context([ - %Host{name: "10.0.0.4", options: [user: "me"]}, - %{name: "10.0.0.3", options: [password: "three"]}, - {"10.0.0.2", port: 2222}, - "10.0.0.1" - ]) - - hosts = [ - %Host{name: "10.0.0.4", options: [user: "me"]}, - %Host{name: "10.0.0.3", options: [password: "three"]}, - %Host{name: "10.0.0.2", options: [port: 2222]}, - %Host{name: "10.0.0.1", options: []} - ] - - assert context == %Context{hosts: hosts} - end - - test "includes default options" do - context = - [{"10.0.0.1", user: "me"}, "10.0.0.2"] - |> SSHKit.context(port: 2222) - - hosts = [ - %Host{name: "10.0.0.1", options: [port: 2222, user: "me"]}, - %Host{name: "10.0.0.2", options: [port: 2222]} - ] - - assert context == %Context{hosts: hosts} - end - - test "default options do not override host options" do - context = - [{"10.0.0.1", user: "other"}, "10.0.0.2"] - |> SSHKit.context(user: "me") - - hosts = [ - %Host{name: "10.0.0.1", options: [user: "other"]}, - %Host{name: "10.0.0.2", options: [user: "me"]} - ] - - assert context == %Context{hosts: hosts} - end - end - - describe "path/2" do - test "sets the path for the context" do - context = SSHKit.path(@empty, "/var/www/app") - assert context.path == "/var/www/app" - end - end - - describe "umask/2" do - test "sets the file permission mask for the context" do - context = SSHKit.umask(@empty, "077") - assert context.umask == "077" - end - end - - describe "user/2" do - test "sets the user for the context" do - context = SSHKit.user(@empty, "meg") - assert context.user == "meg" - end - end - - describe "group/2" do - test "sets the group for the context" do - context = SSHKit.group(@empty, "stripes") - assert context.group == "stripes" - end - end - - describe "env/2" do - test "overwrites existing env" do - context = - @empty - |> SSHKit.env(%{"NODE_ENV" => "production"}) - |> SSHKit.env(%{"CI" => "true"}) - - assert context.env == %{"CI" => "true"} - end - end + # TODO end diff --git a/test/support/mocks.ex b/test/support/mocks.ex index af25536a..88c6afe1 100644 --- a/test/support/mocks.ex +++ b/test/support/mocks.ex @@ -1,4 +1,4 @@ -defmodule SSHKit.SSH.Connection.Impl do +defmodule SSHKit.Connection.Impl do @moduledoc false @type conn :: any() @@ -7,9 +7,9 @@ defmodule SSHKit.SSH.Connection.Impl do @callback close(conn) :: :ok end -Mox.defmock(SSHKit.SSH.Connection.ImplMock, for: SSHKit.SSH.Connection.Impl) +Mox.defmock(SSHKit.Connection.ImplMock, for: SSHKit.Connection.Impl) -defmodule SSHKit.SSH.Channel.Impl do +defmodule SSHKit.Channel.Impl do @moduledoc false @type conn :: any() @@ -31,6 +31,4 @@ defmodule SSHKit.SSH.Channel.Impl do @callback shell(conn, chan) :: :ok | {:error, :closed} end -Mox.defmock(SSHKit.SSH.Channel.ImplMock, for: SSHKit.SSH.Channel.Impl) - -Mox.defmock(SSHKit.SSHMock, for: SSHKit.SSH) +Mox.defmock(SSHKit.Channel.ImplMock, for: SSHKit.Channel.Impl) From d2d3d9868e5e07c313b5b324627a3e8dd678c596 Mon Sep 17 00:00:00 2001 From: pmeinhardt Date: Tue, 22 Dec 2020 13:22:11 +0100 Subject: [PATCH 31/54] Add convenience aliases to Docker image Note that you will need to run a login shell so /etc/profile is loaded. If you are starting the image directly, use: docker run -it --rm sshkit-test-sshd /bin/sh -l The "-l" flag will make `sh` behave like a login shell. --- test/support/docker/Dockerfile | 18 +++++++++--------- test/support/docker/etc/profile.d/aliases.sh | 3 +++ 2 files changed, 12 insertions(+), 9 deletions(-) create mode 100644 test/support/docker/etc/profile.d/aliases.sh diff --git a/test/support/docker/Dockerfile b/test/support/docker/Dockerfile index 02b0a38e..cbfe08d1 100644 --- a/test/support/docker/Dockerfile +++ b/test/support/docker/Dockerfile @@ -1,38 +1,38 @@ FROM alpine:3.7 # Set up an Alpine Linux machine running an SSH server. -# Autogenerate missing host keys. - RUN apk add --update --no-cache openssh sudo + +# Autogenerate missing host keys. RUN ssh-keygen -A # Create the skeleton directory, used for creating new users. - RUN mkdir -p /etc/skel/.ssh RUN chmod 700 /etc/skel/.ssh - RUN touch /etc/skel/.ssh/authorized_keys RUN chmod 600 /etc/skel/.ssh/authorized_keys # Allow members of group "wheel" to execute any command. - RUN echo "%wheel ALL=(ALL) ALL" >> /etc/sudoers.d/wheel # Allow passwordless sudo for users in the "passwordless-sudoers" group. # Users in this group may run commands as any user in any group. - RUN addgroup -S passwordless-sudoers RUN echo "%passwordless-sudoers ALL=(ALL:ALL) NOPASSWD: ALL" >> /etc/sudoers.d/yolo -# Add fixture data for tests +# Add configuration for a better developer experience. +COPY etc/profile.d/* /etc/profile.d + +# Add fixture data for tests. COPY fixtures /fixtures -# Set up a subsystem +# Set up a subsystem. RUN echo "Subsystem greeting-subsystem echo Hello, who am I talking to?; while :; do read varname; echo It\'s nice to meet you \$varname; done" >> /etc/ssh/sshd_config -# Run SSH daemon and expose the standard SSH port. +# Expose the standard SSH port. EXPOSE 22 +# Run SSH daemon. CMD ["/usr/sbin/sshd", "-D", "-e"] # For debugging, let sshd be more verbose: diff --git a/test/support/docker/etc/profile.d/aliases.sh b/test/support/docker/etc/profile.d/aliases.sh new file mode 100644 index 00000000..faca8f5b --- /dev/null +++ b/test/support/docker/etc/profile.d/aliases.sh @@ -0,0 +1,3 @@ +alias la='ls -1A' +alias ll='ls -al' +alias l='ls -1' From 204dd9ff6ff5947367f74b418144471820c194aa Mon Sep 17 00:00:00 2001 From: pmeinhardt Date: Tue, 22 Dec 2020 13:23:03 +0100 Subject: [PATCH 32/54] Update test instructions for Docker setup --- test/test_helper.exs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/test/test_helper.exs b/test/test_helper.exs index 1b890505..713ad3cd 100644 --- a/test/test_helper.exs +++ b/test/test_helper.exs @@ -4,13 +4,15 @@ included = Application.get_env(:ex_unit, :include) unless :functional in excluded && !(:functional in included) do unless Docker.ready?() do IO.puts(""" - It seems like Docker isn't running? + It seems like Docker isn't available? Please check: 1. Docker is installed: `docker version` - 2. On OS X and Windows: `docker-machine start` - 3. Environment is set up: `eval $(docker-machine env)` + 2. Docker is running: `docker info` + + Learn more about Docker: + https://www.docker.com/ """) exit({:shutdown, 1}) From 141fd6557f073dcf440fdaf66481d5b0ca8021e4 Mon Sep 17 00:00:00 2001 From: pmeinhardt Date: Tue, 22 Dec 2020 13:52:29 +0100 Subject: [PATCH 33/54] Upgrade credo config and fix issues --- lib/sshkit/sftp/channel.ex | 3 +- lib/sshkit/upload.ex | 2 +- test/support/functional_assertion_helpers.ex | 113 +++++++++---------- test/support/functional_case_helpers.ex | 2 +- 4 files changed, 56 insertions(+), 64 deletions(-) diff --git a/lib/sshkit/sftp/channel.ex b/lib/sshkit/sftp/channel.ex index f6aa07f2..e379959f 100644 --- a/lib/sshkit/sftp/channel.ex +++ b/lib/sshkit/sftp/channel.ex @@ -31,7 +31,8 @@ defmodule SSHKit.SFTP.Channel do chan.impl.make_dir(chan.id, name, timeout) end - @spec open(t(), binary(), [:read | :write | :append | :binary | :raw], timeout()) :: {:ok, handle()} | {:error, term()} + @spec open(t(), binary(), [:read | :write | :append | :binary | :raw], timeout()) :: + {:ok, handle()} | {:error, term()} def open(chan, name, mode, timeout \\ :infinity) do chan.impl.open(chan.id, name, mode, timeout) end diff --git a/lib/sshkit/upload.ex b/lib/sshkit/upload.ex index edf99299..47b7996e 100644 --- a/lib/sshkit/upload.ex +++ b/lib/sshkit/upload.ex @@ -78,7 +78,7 @@ defmodule SSHKit.Upload do # TODO: Timeouts with {:ok, handle} <- Channel.open(chan, remote, [:write, :binary]), :ok <- write(path, chan, handle), - :ok = Channel.close(chan, handle) do + :ok <- Channel.close(chan, handle) do {:ok, %{upload | stack: [rest | paths]}} end diff --git a/test/support/functional_assertion_helpers.ex b/test/support/functional_assertion_helpers.ex index 9e8fd66f..d5fbab1d 100644 --- a/test/support/functional_assertion_helpers.ex +++ b/test/support/functional_assertion_helpers.ex @@ -1,75 +1,66 @@ defmodule SSHKit.FunctionalAssertionHelpers do @moduledoc false - import ExUnit.Assertions + # TODO - alias SSHKit.Context - alias SSHKit.SSH + # import ExUnit.Assertions - def verify_transfer(conn, local, remote) do - command = &"find #{&1} -type f -exec #{&2} {} \\; | sort | awk '{print $1}' | xargs" + # alias SSHKit.Context - compare_command_output( - conn, - command.(local, SystemCommands.shasum_cmd()), - command.(remote, "sha1sum") - ) - end + # def verify_transfer(conn, local, remote) do + # command = &"find #{&1} -type f -exec #{&2} {} \\; | sort | awk '{print $1}' | xargs" + # compare_command_output(conn, + # command.(local, SystemCommands.shasum_cmd()), + # command.(remote, "sha1sum") + # ) + # end - def verify_mode(conn, local, remote) do - command = &"find #{&1} -type f -exec ls -l {} + | awk '{print $1 $5}' | sort |xargs" + # def verify_mode(conn, local, remote) do + # command = &"find #{&1} -type f -exec ls -l {} + | awk '{print $1 $5}' | sort |xargs" + # compare_command_output(conn, + # command.(local), + # command.(remote) + # ) + # end - compare_command_output( - conn, - command.(local), - command.(remote) - ) - end + # def verify_atime(conn, local, remote) do + # command = &"env find #{&1} -type f -exec #{&2} {} \\; | cut -f1,2 | sort | xargs" + # compare_command_output(conn, + # command.(local, SystemCommands.stat_cmd()), + # command.(remote, "stat -c '%s\t%X\t%Y'") + # ) + # end - def verify_atime(conn, local, remote) do - command = &"env find #{&1} -type f -exec #{&2} {} \\; | cut -f1,2 | sort | xargs" + # def verify_mtime(conn, local, remote) do + # command = &"env find #{&1} -type f -exec #{&2} {} \\; | cut -f1,3 | sort | xargs" + # compare_command_output(conn, + # command.(local, SystemCommands.stat_cmd()), + # command.(remote, "stat -c '%s\t%X\t%Y'") + # ) + # end - compare_command_output( - conn, - command.(local, SystemCommands.stat_cmd()), - command.(remote, "stat -c '%s\t%X\t%Y'") - ) - end + # def compare_command_output(%Context{} = context, local, remote) do + # Enum.map(context.hosts, + # fn(h) -> + # SSH.connect h.name, h.options, fn(conn) -> + # compare_command_output(conn, local, remote) + # end + # end) + # end - def verify_mtime(conn, local, remote) do - command = &"env find #{&1} -type f -exec #{&2} {} \\; | cut -f1,3 | sort | xargs" + # def compare_command_output(conn, local, remote) do + # local_output = local |> String.to_charlist |> :os.cmd |> to_string + # {:ok, [stdout: remote_output], 0} = SSH.run(conn, remote) + # assert local_output == remote_output + # end - compare_command_output( - conn, - command.(local, SystemCommands.stat_cmd()), - command.(remote, "stat -c '%s\t%X\t%Y'") - ) - end + # def create_local_tmp_path do + # rand = + # 16 + # |> :crypto.strong_rand_bytes() + # |> Base.url_encode64() + # |> binary_part(0, 16) - def compare_command_output(context = %Context{}, local, remote) do - Enum.map( - context.hosts, - fn h -> - SSH.connect(h.name, h.options, fn conn -> - compare_command_output(conn, local, remote) - end) - end - ) - end - - def compare_command_output(conn, local, remote) do - local_output = local |> String.to_charlist() |> :os.cmd() |> to_string - {:ok, [stdout: remote_output], 0} = SSH.run(conn, remote) - assert local_output == remote_output - end - - def create_local_tmp_path do - rand = - 16 - |> :crypto.strong_rand_bytes() - |> Base.url_encode64() - |> binary_part(0, 16) - - Path.join(System.tmp_dir(), "sshkit-test-#{rand}") - end + # Path.join(System.tmp_dir(), "sshkit-test-#{rand}") + # end end diff --git a/test/support/functional_case_helpers.ex b/test/support/functional_case_helpers.ex index 047f3025..8a28aac4 100644 --- a/test/support/functional_case_helpers.ex +++ b/test/support/functional_case_helpers.ex @@ -1,7 +1,7 @@ defmodule SSHKit.FunctionalCaseHelpers do @moduledoc false - def exec!(_host = %{id: id}, command, args \\ []) do + def exec!(%{id: id} = _host, command, args \\ []) do Docker.exec!([], id, command, args) end From 10e183016ef1d939c85b58c0f8e8aa3c9e6465da Mon Sep 17 00:00:00 2001 From: pmeinhardt Date: Tue, 22 Dec 2020 15:08:22 +0100 Subject: [PATCH 34/54] Add `mix format` config, run format check for CI --- .formatter.exs | 2 +- .github/workflows/ci.yml | 5 +++-- lib/sshkit.ex | 26 ++++++++++++++++---------- lib/sshkit/download.ex | 1 + lib/sshkit/upload.ex | 4 +++- 5 files changed, 24 insertions(+), 14 deletions(-) diff --git a/.formatter.exs b/.formatter.exs index d2cda26e..7f59b1f0 100644 --- a/.formatter.exs +++ b/.formatter.exs @@ -1,4 +1,4 @@ # Used by "mix format" [ - inputs: ["{mix,.formatter}.exs", "{config,lib,test}/**/*.{ex,exs}"] + inputs: ["{mix,.formatter}.exs", "{lib,test}/**/*.{ex,exs}"] ] diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 371a4e5a..cd07aeef 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,8 +23,9 @@ jobs: - name: Check mix format run: mix format --check-formatted - - name: Compile with warnings as errors - run: mix compile --warnings-as-errors + # TODO: Enable check before finalizing release + # - name: Compile with warnings as errors + # run: mix compile --warnings-as-errors analysis: name: Run static code analysis diff --git a/lib/sshkit.ex b/lib/sshkit.ex index edcee431..6455d275 100644 --- a/lib/sshkit.ex +++ b/lib/sshkit.ex @@ -46,12 +46,16 @@ defmodule SSHKit do # TODO: Separate options for open/exec/recv Stream.resource( fn -> - {:ok, chan} = Channel.open(conn, options) # TODO: handle {:error, reason} and raise custom error struct? - :success = Channel.exec(chan, command) # TODO: timeout?, TODO: Handle :failure and {:error, reason} and raise custom error struct? + # TODO: handle {:error, reason} and raise custom error struct? + {:ok, chan} = Channel.open(conn, options) + + # TODO: timeout?, TODO: Handle :failure and {:error, reason} and raise custom error struct? + :success = Channel.exec(chan, command) chan end, fn chan -> - {:ok, msg} = Channel.recv(chan) # TODO: timeout?, TODO: handle {:error, reason} and raise custom error struct? + # TODO: timeout?, TODO: handle {:error, reason} and raise custom error struct? + {:ok, msg} = Channel.recv(chan) # TODO: Adjust channel window size? @@ -115,16 +119,18 @@ defmodule SSHKit do def run!(conn, command, options \\ []) do stream = exec!(conn, command, options) - {status, output} = Enum.reduce(stream, {nil, []}, fn - {:stdout, _, data}, {status, output} -> {status, [{:stdout, data} | output]} - {:stderr, _, data}, {status, output} -> {status, [{:stderr, data} | output]} - {:exit, _, status}, {_, output} -> {status, output} - _, acc -> acc - end) + {status, output} = + Enum.reduce(stream, {nil, []}, fn + {:stdout, _, data}, {status, output} -> {status, [{:stdout, data} | output]} + {:stderr, _, data}, {status, output} -> {status, [{:stderr, data} | output]} + {:exit, _, status}, {_, output} -> {status, output} + _, acc -> acc + end) output = Enum.reverse(output) - if status != 0, do: raise "Non-zero exit code: #{status}" # TODO: Proper file struct? + # TODO: Proper file struct? + if status != 0, do: raise("Non-zero exit code: #{status}") output end diff --git a/lib/sshkit/download.ex b/lib/sshkit/download.ex index 408819bd..18d8e7e2 100644 --- a/lib/sshkit/download.ex +++ b/lib/sshkit/download.ex @@ -17,6 +17,7 @@ defmodule SSHKit.Download do end def stop(%__MODULE__{channel: nil} = download), do: {:ok, download} + def stop(%__MODULE__{channel: chan} = download) do with :ok <- Channel.stop(chan) do {:ok, %{download | channel: nil}} diff --git a/lib/sshkit/upload.ex b/lib/sshkit/upload.ex index 47b7996e..48409791 100644 --- a/lib/sshkit/upload.ex +++ b/lib/sshkit/upload.ex @@ -28,13 +28,15 @@ defmodule SSHKit.Upload do defp prepare(%__MODULE__{source: source, options: options} = upload) do if !Keyword.get(options, :recursive, false) && File.dir?(source) do - {:error, "Option :recursive not specified, but local file is a directory (#{source})"} # TODO: Better error + # TODO: Better error + {:error, "Option :recursive not specified, but local file is a directory (#{source})"} else {:ok, %{upload | cwd: Path.dirname(source), stack: [[Path.basename(source)]]}} end end def stop(%__MODULE__{channel: nil} = upload), do: {:ok, upload} + def stop(%__MODULE__{channel: chan} = upload) do with :ok <- Channel.stop(chan) do {:ok, %{upload | channel: nil}} From 38cea2cea64a81c609d0ebd7c8dd9c88f3581912 Mon Sep 17 00:00:00 2001 From: pmeinhardt Date: Tue, 22 Dec 2020 17:19:54 +0100 Subject: [PATCH 35/54] Move channel functional test to correct location --- test/sshkit/{ssh => }/channel_functional_test.exs | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename test/sshkit/{ssh => }/channel_functional_test.exs (100%) diff --git a/test/sshkit/ssh/channel_functional_test.exs b/test/sshkit/channel_functional_test.exs similarity index 100% rename from test/sshkit/ssh/channel_functional_test.exs rename to test/sshkit/channel_functional_test.exs From efd660c2ec8fb6048c5d5e7ea826250f73f5add2 Mon Sep 17 00:00:00 2001 From: pmeinhardt Date: Tue, 22 Dec 2020 17:39:38 +0100 Subject: [PATCH 36/54] Update how we set up mocking Remove `:impl` option from calls. Instead configure the module to use - implementation or mock - in the application environment. --- config/config.exs | 7 ++ lib/sshkit/channel.ex | 28 +++--- lib/sshkit/connection.ex | 31 +++---- lib/sshkit/context.ex | 2 +- lib/sshkit/sftp/channel.ex | 25 +++--- test/sshkit/channel_test.exs | 147 ++++++++++++++------------------ test/sshkit/connection_test.exs | 106 +++++++++-------------- test/support/functional_case.ex | 8 ++ test/support/mocks.ex | 83 +++++++++++++----- 9 files changed, 230 insertions(+), 207 deletions(-) create mode 100644 config/config.exs diff --git a/config/config.exs b/config/config.exs new file mode 100644 index 00000000..4c091556 --- /dev/null +++ b/config/config.exs @@ -0,0 +1,7 @@ +use Mix.Config + +if Mix.env() == :test do + config :sshkit, :ssh, MockErlangSsh + config :sshkit, :ssh_connection, MockErlangSshConnection + config :sshkit, :ssh_sftp, MockErlangSshSftp +end diff --git a/lib/sshkit/channel.ex b/lib/sshkit/channel.ex index 01f374f8..dc87f759 100644 --- a/lib/sshkit/channel.ex +++ b/lib/sshkit/channel.ex @@ -9,10 +9,13 @@ defmodule SSHKit.Channel do * `id` - the unique channel id """ - defstruct [:connection, :type, :id, impl: :ssh_connection] + defstruct [:connection, :type, :id] @type t() :: %__MODULE__{} + # credo:disable-for-next-line + @core Application.get_env(:sshkit, :ssh_connection, :ssh_connection) + @doc """ Opens a channel on an SSH connection. @@ -31,15 +34,14 @@ defmodule SSHKit.Channel do timeout = Keyword.get(options, :timeout, :infinity) ini_window_size = Keyword.get(options, :initial_window_size, 128 * 1024) max_packet_size = Keyword.get(options, :max_packet_size, 32 * 1024) - impl = Keyword.get(options, :impl, :ssh_connection) - with {:ok, id} <- impl.session_channel(conn.ref, ini_window_size, max_packet_size, timeout) do - {:ok, new(conn, id, impl)} + with {:ok, id} <- @core.session_channel(conn.ref, ini_window_size, max_packet_size, timeout) do + {:ok, new(conn, id)} end end - defp new(conn, id, impl) do - %__MODULE__{connection: conn, type: :session, id: id, impl: impl} + defp new(conn, id) do + %__MODULE__{connection: conn, type: :session, id: id} end @doc """ @@ -53,7 +55,7 @@ defmodule SSHKit.Channel do :success | :failure | {:error, reason :: String.t()} def subsystem(channel, subsystem, options \\ []) do timeout = Keyword.get(options, :timeout, :infinity) - channel.impl.subsystem(channel.connection.ref, channel.id, to_charlist(subsystem), timeout) + @core.subsystem(channel.connection.ref, channel.id, to_charlist(subsystem), timeout) end @doc """ @@ -64,7 +66,7 @@ defmodule SSHKit.Channel do For more details, see [`:ssh_connection.close/2`](http://erlang.org/doc/man/ssh_connection.html#close-2). """ def close(channel) do - channel.impl.close(channel.connection.ref, channel.id) + @core.close(channel.connection.ref, channel.id) end @doc """ @@ -86,7 +88,7 @@ defmodule SSHKit.Channel do end def exec(channel, command, timeout) do - channel.impl.exec(channel.connection.ref, channel.id, command, timeout) + @core.exec(channel.connection.ref, channel.id, command, timeout) end @doc """ @@ -97,7 +99,7 @@ defmodule SSHKit.Channel do For more details, see [`:ssh_connection.ptty_alloc/4`](http://erlang.org/doc/man/ssh_connection.html#ptty_alloc-4). """ def ptty(channel, options \\ [], timeout \\ :infinity) do - channel.impl.ptty_alloc(channel.connection.ref, channel.id, options, timeout) + @core.ptty_alloc(channel.connection.ref, channel.id, options, timeout) end @doc """ @@ -112,7 +114,7 @@ defmodule SSHKit.Channel do def send(channel, type \\ 0, data, timeout \\ :infinity) def send(channel, type, data, timeout) when is_binary(data) or is_list(data) do - channel.impl.send(channel.connection.ref, channel.id, type, data, timeout) + @core.send(channel.connection.ref, channel.id, type, data, timeout) end def send(channel, type, data, timeout) do @@ -130,7 +132,7 @@ defmodule SSHKit.Channel do For more details, see [`:ssh_connection.send_eof/2`](http://erlang.org/doc/man/ssh_connection.html#send_eof-2). """ def eof(channel) do - channel.impl.send_eof(channel.connection.ref, channel.id) + @core.send_eof(channel.connection.ref, channel.id) end @doc """ @@ -190,6 +192,6 @@ defmodule SSHKit.Channel do For more details, see [`:ssh_connection.adjust_window/3`](http://erlang.org/doc/man/ssh_connection.html#adjust_window-3). """ def adjust(channel, size) when is_integer(size) do - channel.impl.adjust_window(channel.connection.ref, channel.id, size) + @core.adjust_window(channel.connection.ref, channel.id, size) end end diff --git a/lib/sshkit/connection.ex b/lib/sshkit/connection.ex index 1a097c1c..b613999b 100644 --- a/lib/sshkit/connection.ex +++ b/lib/sshkit/connection.ex @@ -13,12 +13,15 @@ defmodule SSHKit.Connection do alias SSHKit.Utils # TODO: Add :tag allowing arbitrary data to be attached? - defstruct [:host, :port, :options, :ref, impl: :ssh] + defstruct [:host, :port, :options, :ref] @type t() :: %__MODULE__{} - @default_impl_options [user_interaction: false] - @default_connect_options [port: 22, timeout: :infinity, impl: :ssh] + # credo:disable-for-next-line + @core Application.get_env(:sshkit, :ssh, :ssh) + + @default_ssh_options [user_interaction: false] + @default_connect_options [port: 22, timeout: :infinity] @doc """ Opens a connection to an SSH server. @@ -52,32 +55,31 @@ defmodule SSHKit.Connection do port = details[:port] timeout = details[:timeout] - impl = details[:impl] - case impl.connect(host, port, opts, timeout) do - {:ok, ref} -> {:ok, new(host, port, opts, ref, impl)} + case @core.connect(host, port, opts, timeout) do + {:ok, ref} -> {:ok, new(host, port, opts, ref)} err -> err end end defp extract(options) do connect_option_keys = Keyword.keys(@default_connect_options) - {connect_options, impl_options} = Keyword.split(options, connect_option_keys) + {connect_options, ssh_options} = Keyword.split(options, connect_option_keys) connect_options = @default_connect_options |> Keyword.merge(connect_options) - impl_options = - @default_impl_options - |> Keyword.merge(impl_options) + ssh_options = + @default_ssh_options + |> Keyword.merge(ssh_options) |> Utils.charlistify() - {connect_options, impl_options} + {connect_options, ssh_options} end - defp new(host, port, options, ref, impl) do - %__MODULE__{host: host, port: port, options: options, ref: ref, impl: impl} + defp new(host, port, options, ref) do + %__MODULE__{host: host, port: port, options: options, ref: ref} end @doc """ @@ -88,7 +90,7 @@ defmodule SSHKit.Connection do For details, see [`:ssh.close/1`](http://erlang.org/doc/man/ssh.html#close-1). """ def close(conn) do - conn.impl.close(conn.ref) + @core.close(conn.ref) end @doc """ @@ -105,7 +107,6 @@ defmodule SSHKit.Connection do options = conn.options |> Keyword.put(:port, conn.port) - |> Keyword.put(:impl, conn.impl) |> Keyword.merge(options) open(conn.host, options) diff --git a/lib/sshkit/context.ex b/lib/sshkit/context.ex index e15ad948..1abcad78 100644 --- a/lib/sshkit/context.ex +++ b/lib/sshkit/context.ex @@ -14,7 +14,7 @@ defmodule SSHKit.Context do import SSHKit.Utils - defstruct env: nil, path: nil, umask: nil, user: nil, group: nil + defstruct [:env, :path, :umask, :user, :group] @type t() :: %__MODULE__{} diff --git a/lib/sshkit/sftp/channel.ex b/lib/sshkit/sftp/channel.ex index e379959f..825a8bb3 100644 --- a/lib/sshkit/sftp/channel.ex +++ b/lib/sshkit/sftp/channel.ex @@ -3,47 +3,48 @@ defmodule SSHKit.SFTP.Channel do alias SSHKit.Connection - defstruct [:connection, :id, impl: :ssh_sftp] + defstruct [:connection, :id] @type t() :: %__MODULE__{} @type handle() :: term() + # credo:disable-for-next-line + @core Application.get_env(:sshkit, :ssh_sftp, :ssh_sftp) + @spec start(Connection.t(), keyword()) :: {:ok, t()} | {:error, term()} def start(conn, options \\ []) do - {impl, options} = Keyword.pop(options, :impl, :ssh_sftp) - - with {:ok, id} <- impl.start_channel(conn.ref, options) do - {:ok, new(conn, id, impl)} + with {:ok, id} <- @core.start_channel(conn.ref, options) do + {:ok, new(conn, id)} end end - defp new(conn, id, impl) do - %__MODULE__{connection: conn, id: id, impl: impl} + defp new(conn, id) do + %__MODULE__{connection: conn, id: id} end @spec stop(t()) :: :ok def stop(chan) do - chan.impl.stop_channel(chan.id) + @core.stop_channel(chan.id) end @spec mkdir(t(), binary(), timeout()) :: :ok | {:error, term()} def mkdir(chan, name, timeout \\ :infinity) do - chan.impl.make_dir(chan.id, name, timeout) + @core.make_dir(chan.id, name, timeout) end @spec open(t(), binary(), [:read | :write | :append | :binary | :raw], timeout()) :: {:ok, handle()} | {:error, term()} def open(chan, name, mode, timeout \\ :infinity) do - chan.impl.open(chan.id, name, mode, timeout) + @core.open(chan.id, name, mode, timeout) end @spec close(t(), handle(), timeout()) :: :ok | {:error, term()} def close(chan, handle, timeout \\ :infinity) do - chan.impl.close(chan.id, handle, timeout) + @core.close(chan.id, handle, timeout) end @spec write(t(), handle(), iodata(), timeout()) :: :ok | {:error, term()} def write(chan, handle, data, timeout \\ :infinity) do - chan.impl.write(chan.id, handle, data, timeout) + @core.write(chan.id, handle, data, timeout) end end diff --git a/test/sshkit/channel_test.exs b/test/sshkit/channel_test.exs index 6756ff5a..03c03141 100644 --- a/test/sshkit/channel_test.exs +++ b/test/sshkit/channel_test.exs @@ -7,19 +7,23 @@ defmodule SSHKit.ChannelTest do alias SSHKit.Channel alias SSHKit.Connection - setup do - Mox.verify_on_exit!() + @core MockErlangSshConnection + + setup :verify_on_exit! - conn = %Connection{ref: :test_connection, impl: Connection.ImplMock} - chan = %Channel{connection: conn, type: :session, id: 1, impl: Channel.ImplMock} + setup do + conn = %Connection{ref: :test_connection} + chan = %Channel{connection: conn, type: :session, id: 1} - {:ok, conn: conn, chan: chan, impl: Channel.ImplMock} + {:ok, conn: conn, chan: chan} end describe "open/2" do - test "opens a channel on a connection", %{conn: conn, impl: impl} do - impl - |> expect(:session_channel, fn connection_ref, ini_window_size, max_packet_size, timeout -> + test "opens a channel on a connection", %{conn: conn} do + expect(@core, :session_channel, fn connection_ref, + ini_window_size, + max_packet_size, + timeout -> assert connection_ref == conn.ref assert ini_window_size == 128 * 1024 assert max_packet_size == 32 * 1024 @@ -27,36 +31,29 @@ defmodule SSHKit.ChannelTest do {:ok, 0} end) - {:ok, chan} = open(conn, impl: impl) + {:ok, chan} = open(conn) - assert chan == %Channel{ - connection: conn, - type: :session, - id: 0, - impl: impl - } + assert chan == %Channel{connection: conn, type: :session, id: 0} end - test "opens a channel with a specific timeout", %{conn: conn, impl: impl} do - impl - |> expect(:session_channel, fn _, _, _, timeout -> + test "opens a channel with a specific timeout", %{conn: conn} do + expect(@core, :session_channel, fn _, _, _, timeout -> assert timeout == 3000 {:ok, 0} end) - {:ok, _} = open(conn, timeout: 3000, impl: impl) + {:ok, _} = open(conn, timeout: 3000) end - test "returns an error if channel cannot be opened", %{conn: conn, impl: impl} do - impl |> expect(:session_channel, fn _, _, _, _ -> {:error, :timeout} end) - assert open(conn, impl: impl) == {:error, :timeout} + test "returns an error if channel cannot be opened", %{conn: conn} do + expect(@core, :session_channel, fn _, _, _, _ -> {:error, :timeout} end) + assert open(conn) == {:error, :timeout} end end describe "subsystem/3" do - test "requests a subsystem", %{chan: chan, impl: impl} do - impl - |> expect(:subsystem, fn connection_ref, channel_id, subsystem, timeout -> + test "requests a subsystem", %{chan: chan} do + expect(@core, :subsystem, fn connection_ref, channel_id, subsystem, timeout -> assert connection_ref == chan.connection.ref assert channel_id == chan.id assert subsystem == 'example-subsystem' @@ -64,42 +61,32 @@ defmodule SSHKit.ChannelTest do :success end) - :success = subsystem(chan, "example-subsystem", impl: impl) + assert :success == subsystem(chan, "example-subsystem") end - test "requests a subsystem with a specific timeout", %{chan: chan, impl: impl} do - impl - |> expect(:subsystem, fn _, _, _, timeout -> + test "requests a subsystem with a specific timeout", %{chan: chan} do + expect(@core, :subsystem, fn _, _, _, timeout -> assert timeout == 3000 :success end) - :success = subsystem(chan, "example-subsystem", timeout: 3000, impl: impl) + assert :success == subsystem(chan, "example-subsystem", timeout: 3000) end - test "returns a failure if the subsystem could not be initialized", %{chan: chan, impl: impl} do - impl - |> expect(:subsystem, fn _, _, _, _ -> - :failure - end) - - :failure = subsystem(chan, "example-subsystem", impl: impl) + test "returns a failure if the subsystem could not be initialized", %{chan: chan} do + expect(@core, :subsystem, fn _, _, _, _ -> :failure end) + assert :failure = subsystem(chan, "example-subsystem") end - test "returns an error if the initialization times out", %{chan: chan, impl: impl} do - impl - |> expect(:subsystem, fn _, _, _, _ -> - {:error, :timeout} - end) - - {:error, :timeout} = subsystem(chan, "example-subsystem", impl: impl) + test "returns an error if the initialization times out", %{chan: chan} do + expect(@core, :subsystem, fn _, _, _, _ -> {:error, :timeout} end) + assert {:error, :timeout} == subsystem(chan, "example-subsystem") end end describe "close/1" do - test "closes the channel", %{chan: chan, impl: impl} do - impl - |> expect(:close, fn connection_ref, channel_id -> + test "closes the channel", %{chan: chan} do + expect(@core, :close, fn connection_ref, channel_id -> assert connection_ref == chan.connection.ref assert channel_id == chan.id :ok @@ -110,9 +97,8 @@ defmodule SSHKit.ChannelTest do end describe "exec/3" do - test "executes a command (binary) over a channel", %{chan: chan, impl: impl} do - impl - |> expect(:exec, fn connection_ref, channel_id, command, timeout -> + test "executes a command (binary) over a channel", %{chan: chan} do + expect(@core, :exec, fn connection_ref, channel_id, command, timeout -> assert connection_ref == chan.connection.ref assert channel_id == chan.id assert command == 'cmd arg1 arg2' @@ -123,9 +109,8 @@ defmodule SSHKit.ChannelTest do assert exec(chan, "cmd arg1 arg2") == :success end - test "executes a command (charlist) over a channel", %{chan: chan, impl: impl} do - impl - |> expect(:exec, fn connection_ref, channel_id, command, timeout -> + test "executes a command (charlist) over a channel", %{chan: chan} do + expect(@core, :exec, fn connection_ref, channel_id, command, timeout -> assert connection_ref == chan.connection.ref assert channel_id == chan.id assert command == 'cmd arg1 arg2' @@ -136,9 +121,8 @@ defmodule SSHKit.ChannelTest do assert exec(chan, 'cmd arg1 arg2') == :success end - test "executes a command with a specific timeout", %{chan: chan, impl: impl} do - impl - |> expect(:exec, fn _, _, _, timeout -> + test "executes a command with a specific timeout", %{chan: chan} do + expect(@core, :exec, fn _, _, _, timeout -> assert timeout == 4000 {:ok, 0} end) @@ -146,21 +130,20 @@ defmodule SSHKit.ChannelTest do {:ok, _} = exec(chan, "cmd", 4000) end - test "executes a failing command", %{chan: chan, impl: impl} do - impl |> expect(:exec, fn _, _, _, _ -> :failure end) + test "executes a failing command", %{chan: chan} do + expect(@core, :exec, fn _, _, _, _ -> :failure end) assert exec(chan, "cmd") == :failure end - test "returns an error if command cannot be executed", %{chan: chan, impl: impl} do - impl |> expect(:exec, fn _, _, _, _ -> {:error, :closed} end) + test "returns an error if command cannot be executed", %{chan: chan} do + expect(@core, :exec, fn _, _, _, _ -> {:error, :closed} end) assert exec(chan, "cmd") == {:error, :closed} end end describe "ptty/4" do - test "allocates ptty", %{chan: chan, impl: impl} do - impl - |> expect(:ptty_alloc, fn connection_ref, channel_id, options, timeout -> + test "allocates ptty", %{chan: chan} do + expect(@core, :ptty_alloc, fn connection_ref, channel_id, options, timeout -> assert connection_ref == chan.connection.ref assert channel_id == chan.id assert options == [] @@ -173,20 +156,20 @@ defmodule SSHKit.ChannelTest do end describe "send/4" do - test "send binary data across channel", %{chan: chan, impl: impl} do - impl |> expect(:send, sends(chan, 0, "binary data", :infinity, :ok)) + test "send binary data across channel", %{chan: chan} do + expect(@core, :send, sends(chan, 0, "binary data", :infinity, :ok)) assert Channel.send(chan, "binary data") == :ok end - test "sends charlist data across channel", %{chan: chan, impl: impl} do - impl |> expect(:send, sends(chan, 0, 'charlist data', :infinity, :ok)) + test "sends charlist data across channel", %{chan: chan} do + expect(@core, :send, sends(chan, 0, 'charlist data', :infinity, :ok)) assert Channel.send(chan, 'charlist data') == :ok end - test "sends stream data across channel", %{chan: chan, impl: impl} do + test "sends stream data across channel", %{chan: chan} do data = 0..2 |> Stream.map(&Integer.to_string/1) - impl + @core |> expect(:send, sends(chan, 0, "0", :infinity, :ok)) |> expect(:send, sends(chan, 0, "1", :infinity, :ok)) |> expect(:send, sends(chan, 0, "2", :infinity, :ok)) @@ -194,31 +177,30 @@ defmodule SSHKit.ChannelTest do assert Channel.send(chan, data) == :ok end - test "returns an error streaming data fails", %{chan: chan, impl: impl} do + test "returns an error streaming data fails", %{chan: chan} do data = 0..2 |> Stream.map(&Integer.to_string/1) - impl + @core |> expect(:send, sends(chan, 0, "0", :infinity, :ok)) |> expect(:send, sends(chan, 0, "1", :infinity, {:error, :timeout})) assert Channel.send(chan, data) == {:error, :timeout} end - test "returns an error when channel not open", %{chan: chan, impl: impl} do - impl |> expect(:send, fn _, _, _, _, _ -> {:error, :closed} end) + test "returns an error when channel not open", %{chan: chan} do + expect(@core, :send, fn _, _, _, _, _ -> {:error, :closed} end) assert Channel.send(chan, "data") == {:error, :closed} end - test "returns an error when channel times out", %{chan: chan, impl: impl} do - impl |> expect(:send, fn _, _, _, _, _ -> {:error, :timeout} end) + test "returns an error when channel times out", %{chan: chan} do + expect(@core, :send, fn _, _, _, _, _ -> {:error, :timeout} end) assert Channel.send(chan, "data") == {:error, :timeout} end end describe "eof/1" do - test "sends EOF to open channel", %{chan: chan, impl: impl} do - impl - |> expect(:send_eof, fn connection_ref, channel_id -> + test "sends EOF to open channel", %{chan: chan} do + expect(@core, :send_eof, fn connection_ref, channel_id -> assert connection_ref == chan.connection.ref assert channel_id == chan.id :ok @@ -227,8 +209,8 @@ defmodule SSHKit.ChannelTest do assert eof(chan) == :ok end - test "returns an error when channel not open", %{chan: chan, impl: impl} do - impl |> expect(:send_eof, fn _, _ -> {:error, :closed} end) + test "returns an error when channel not open", %{chan: chan} do + expect(@core, :send_eof, fn _, _ -> {:error, :closed} end) assert eof(chan) == {:error, :closed} end end @@ -277,9 +259,8 @@ defmodule SSHKit.ChannelTest do end end - test "adjusts the window size", %{chan: chan, impl: impl} do - impl - |> expect(:adjust_window, fn connection_ref, channel_id, size -> + test "adjusts the window size", %{chan: chan} do + expect(@core, :adjust_window, fn connection_ref, channel_id, size -> assert connection_ref == chan.connection.ref assert channel_id == chan.id assert size == 4096 diff --git a/test/sshkit/connection_test.exs b/test/sshkit/connection_test.exs index 759c0698..7b7f2e9e 100644 --- a/test/sshkit/connection_test.exs +++ b/test/sshkit/connection_test.exs @@ -5,17 +5,14 @@ defmodule SSHKit.ConnectionTest do import SSHKit.Connection alias SSHKit.Connection - alias SSHKit.Connection.ImplMock - setup do - Mox.verify_on_exit!() - {:ok, impl: ImplMock} - end + @core MockErlangSsh + + setup :verify_on_exit! describe "open/2" do - test "opens a connection", %{impl: impl} do - impl - |> expect(:connect, fn host, port, opts, timeout -> + test "opens a connection" do + expect(@core, :connect, fn host, port, opts, timeout -> assert host == 'test.io' assert port == 22 assert opts == [user_interaction: false] @@ -23,20 +20,18 @@ defmodule SSHKit.ConnectionTest do {:ok, :connection_ref} end) - {:ok, conn} = open("test.io", impl: impl) + {:ok, conn} = open("test.io") assert conn == %Connection{ host: 'test.io', port: 22, options: [user_interaction: false], - ref: :connection_ref, - impl: impl + ref: :connection_ref } end - test "opens a connection on a different port with user and password", %{impl: impl} do - impl - |> expect(:connect, fn _, port, opts, _ -> + test "opens a connection on a different port with user and password" do + expect(@core, :connect, fn _, port, opts, _ -> assert port == 666 assert opts[:user] == 'me' assert opts[:password] == 'secret' @@ -44,27 +39,25 @@ defmodule SSHKit.ConnectionTest do {:ok, :ref_with_port_user_pass} end) - {:ok, conn} = open("test.io", port: 666, user: "me", password: "secret", impl: impl) + {:ok, conn} = open("test.io", port: 666, user: "me", password: "secret") assert conn == %Connection{ host: 'test.io', port: 666, options: [user_interaction: false, user: 'me', password: 'secret'], - ref: :ref_with_port_user_pass, - impl: impl + ref: :ref_with_port_user_pass } end - test "opens a connection with user interaction option set to true", %{impl: impl} do - impl - |> expect(:connect, fn _, _, opts, _ -> + test "opens a connection with user interaction option set to true" do + expect(@core, :connect, fn _, _, opts, _ -> assert opts[:user] == 'me' assert opts[:password] == 'secret' assert opts[:user_interaction] == true {:ok, :ref_with_user_interaction} end) - options = [user: "me", password: "secret", user_interaction: true, impl: impl] + options = [user: "me", password: "secret", user_interaction: true] {:ok, conn} = open("test.io", options) @@ -72,66 +65,59 @@ defmodule SSHKit.ConnectionTest do host: 'test.io', options: [user: 'me', password: 'secret', user_interaction: true], port: 22, - ref: :ref_with_user_interaction, - impl: impl + ref: :ref_with_user_interaction } end - test "opens a connection with a specific timeout", %{impl: impl} do - impl - |> expect(:connect, fn _, _, _, timeout -> + test "opens a connection with a specific timeout" do + expect(@core, :connect, fn _, _, _, timeout -> assert timeout == 3000 {:ok, :ref} end) - {:ok, _} = open("test.io", timeout: 3000, impl: impl) + {:ok, _} = open("test.io", timeout: 3000) end - test "removes options irrelevant for connect/4", %{impl: impl} do - impl - |> expect(:connect, fn _, _, opts, _ -> + test "removes options irrelevant for connect/4" do + expect(@core, :connect, fn _, _, opts, _ -> option_keys = Keyword.keys(opts) refute :port in option_keys refute :timeout in option_keys - refute :impl in option_keys {:ok, :ref} end) - options = [port: 666, timeout: 1000, user: "me", password: "secret", impl: impl] + options = [port: 666, timeout: 1000, user: "me", password: "secret"] {:ok, _} = open("test.io", options) end - test "converts host to charlist", %{impl: impl} do - impl - |> expect(:connect, fn host, _, _, _ -> + test "converts host to charlist" do + expect(@core, :connect, fn host, _, _, _ -> assert host == 'test.io' {:ok, :ref} end) - {:ok, _} = open("test.io", impl: impl) + {:ok, _} = open("test.io") end - test "converts option values to charlists", %{impl: impl} do - impl - |> expect(:connect, fn _, _, opts, _ -> + test "converts option values to charlists" do + expect(@core, :connect, fn _, _, opts, _ -> assert {:user, 'me'} in opts assert {:password, 'secret'} in opts {:ok, :ref} end) - {:ok, _} = open("test.io", user: "me", password: "secret", impl: impl) + {:ok, _} = open("test.io", user: "me", password: "secret") end - test "returns an error when connection cannot be opened", %{impl: impl} do - impl - |> expect(:connect, fn _, _, _, _ -> + test "returns an error when connection cannot be opened" do + expect(@core, :connect, fn _, _, _, _ -> {:error, :failed} end) - assert open("test.io", impl: impl) == {:error, :failed} + assert open("test.io") == {:error, :failed} end test "returns an error if no host is given" do @@ -140,9 +126,8 @@ defmodule SSHKit.ConnectionTest do end describe "close/1" do - test "closes a connection", %{impl: impl} do - impl - |> expect(:close, fn ref -> + test "closes a connection" do + expect(@core, :close, fn ref -> assert ref == :connection_ref :ok end) @@ -151,8 +136,7 @@ defmodule SSHKit.ConnectionTest do host: 'foo.io', port: 22, options: [user_interaction: false], - ref: :connection_ref, - impl: impl + ref: :connection_ref } assert close(conn) == :ok @@ -160,17 +144,15 @@ defmodule SSHKit.ConnectionTest do end describe "reopen/2" do - test "opens a new connection with the same options as the existing connection", %{impl: impl} do + test "opens a new connection with the same options as the existing connection" do conn = %Connection{ host: 'test.io', port: 22, options: [user_interaction: false, user: 'me'], - ref: :connection_ref, - impl: impl + ref: :connection_ref } - impl - |> expect(:connect, fn host, port, opts, _ -> + expect(@core, :connect, fn host, port, opts, _ -> assert host == conn.host assert port == conn.port assert opts == conn.options @@ -182,17 +164,15 @@ defmodule SSHKit.ConnectionTest do assert reopen(conn) == {:ok, new_conn} end - test "reopens a connection on new port", %{impl: impl} do + test "reopens a connection on new port" do conn = %Connection{ host: 'test.io', port: 22, options: [user_interaction: false, user: 'me'], - ref: :connection_ref, - impl: impl + ref: :connection_ref } - impl - |> expect(:connect, fn _, port, _, _ -> + expect(@core, :connect, fn _, port, _, _ -> assert port == 666 {:ok, :new_connection_ref} end) @@ -202,17 +182,15 @@ defmodule SSHKit.ConnectionTest do assert reopen(conn, port: 666) == {:ok, new_conn} end - test "errors when unable to open connection", %{impl: impl} do + test "errors when unable to open connection" do conn = %Connection{ host: 'test.io', port: 22, options: [user_interaction: false], - ref: :sandbox, - impl: impl + ref: :sandbox } - impl - |> expect(:connect, fn _, _, _, _ -> + expect(@core, :connect, fn _, _, _, _ -> {:error, :failed} end) diff --git a/test/support/functional_case.ex b/test/support/functional_case.ex index eae15632..9ad843ea 100644 --- a/test/support/functional_case.ex +++ b/test/support/functional_case.ex @@ -15,6 +15,14 @@ defmodule SSHKit.FunctionalCase do import SSHKit.FunctionalAssertionHelpers @moduletag :functional + + setup do + # Stub mocks with implementations delegating to the proper Erlang modules + Mox.stub_with(MockErlangSsh, ErlangSsh) + Mox.stub_with(MockErlangSshConnection, ErlangSshConnection) + Mox.stub_with(MockErlangSshSftp, ErlangSshSftp) + :ok + end end end diff --git a/test/support/mocks.ex b/test/support/mocks.ex index 88c6afe1..03f439c7 100644 --- a/test/support/mocks.ex +++ b/test/support/mocks.ex @@ -1,34 +1,79 @@ -defmodule SSHKit.Connection.Impl do +defmodule ErlangSshBehaviour do @moduledoc false - @type conn :: any() + @type conn() :: term() - @callback connect(binary(), integer(), keyword(), timeout()) :: {:ok, conn} | {:error, any()} - @callback close(conn) :: :ok + @callback connect(binary(), integer(), keyword(), timeout()) :: {:ok, conn()} | {:error, term()} + @callback close(conn()) :: :ok end -Mox.defmock(SSHKit.Connection.ImplMock, for: SSHKit.Connection.Impl) +defmodule ErlangSsh do + @moduledoc false + + @behaviour ErlangSshBehaviour + + defdelegate connect(host, port, options, timeout), to: :ssh + defdelegate close(conn), to: :ssh +end + +Mox.defmock(MockErlangSsh, for: ErlangSshBehaviour) -defmodule SSHKit.Channel.Impl do +defmodule ErlangSshConnectionBehaviour do @moduledoc false - @type conn :: any() - @type chan :: integer() + @type conn() :: term() + @type chan() :: integer() - @callback session_channel(conn, integer(), integer(), timeout()) :: - {:ok, chan} | {:error, any()} - @callback subsystem(conn, chan, charlist(), keyword()) :: + @callback session_channel(conn(), integer(), integer(), timeout()) :: + {:ok, chan()} | {:error, term()} + @callback subsystem(conn(), chan(), charlist(), timeout()) :: :success | :failure | {:error, :timeout} | {:error, :closed} - @callback close(conn, chan) :: :ok - @callback exec(conn, chan, binary(), timeout()) :: + @callback close(conn(), chan()) :: :ok + @callback exec(conn(), chan(), binary(), timeout()) :: :success | :failure | {:error, :timeout} | {:error, :closed} - @callback ptty_alloc(conn, chan, keyword(), timeout()) :: + @callback ptty_alloc(conn(), chan(), keyword(), timeout()) :: :success | :failure | {:error, :timeout} | {:error, :closed} - @callback send(conn, chan, 0..1, binary(), timeout()) :: + @callback send(conn(), chan(), 0..1, binary(), timeout()) :: :ok | {:error, :timeout} | {:error, :closed} - @callback send_eof(conn, chan) :: :ok | {:error, :closed} - @callback adjust_window(conn, chan, integer()) :: :ok - @callback shell(conn, chan) :: :ok | {:error, :closed} + @callback send_eof(conn(), chan()) :: :ok | {:error, :closed} + @callback adjust_window(conn(), chan(), integer()) :: :ok +end + +defmodule ErlangSshConnection do + @moduledoc false + + @behaviour ErlangSshConnectionBehaviour + + defdelegate session_channel(conn, initial_window_size, max_packet_size, timeout), + to: :ssh_connection + + defdelegate subsystem(conn, chan, name, timeout), to: :ssh_connection + defdelegate close(conn, chan), to: :ssh_connection + defdelegate exec(conn, chan, command, timeout), to: :ssh_connection + defdelegate ptty_alloc(conn, chan, keyword, timeout), to: :ssh_connection + defdelegate send(conn, chan, type, data, timeout), to: :ssh_connection + defdelegate send_eof(conn, chan), to: :ssh_connection + defdelegate adjust_window(conn, chan, size), to: :ssh_connection +end + +Mox.defmock(MockErlangSshConnection, for: ErlangSshConnectionBehaviour) + +defmodule ErlangSshSftpBehaviour do + @moduledoc false + + @type conn() :: term() + @type chan() :: pid() + + # TODO + @callback start_channel(conn(), keyword()) :: {:ok, chan()} | {:error, term()} +end + +defmodule ErlangSshSftp do + @moduledoc false + + @behaviour ErlangSshSftpBehaviour + + defdelegate start_channel(conn, options), to: :ssh_sftp end -Mox.defmock(SSHKit.Channel.ImplMock, for: SSHKit.Channel.Impl) +Mox.defmock(MockErlangSshSftp, for: ErlangSshSftpBehaviour) From dcacc9074df6fd7c0476ae51c4ee0d9313476602 Mon Sep 17 00:00:00 2001 From: pmeinhardt Date: Fri, 25 Dec 2020 01:38:28 +0100 Subject: [PATCH 37/54] Generate code required to mock Erlang modules --- lib/sshkit/connection.ex | 2 +- test/support/functional_case.ex | 4 +- test/support/gen.ex | 61 +++++++++++++++++++++++ test/support/mocks.ex | 86 ++++----------------------------- 4 files changed, 75 insertions(+), 78 deletions(-) create mode 100644 test/support/gen.ex diff --git a/lib/sshkit/connection.ex b/lib/sshkit/connection.ex index b613999b..760ea624 100644 --- a/lib/sshkit/connection.ex +++ b/lib/sshkit/connection.ex @@ -5,7 +5,7 @@ defmodule SSHKit.Connection do A connection struct has the following fields: * `host` - the name or IP of the remote host - * `port` - the port to connect to + * `port` - the port connected to * `options` - additional connection options * `ref` - the underlying `:ssh` connection ref """ diff --git a/test/support/functional_case.ex b/test/support/functional_case.ex index 9ad843ea..a316fe98 100644 --- a/test/support/functional_case.ex +++ b/test/support/functional_case.ex @@ -17,7 +17,9 @@ defmodule SSHKit.FunctionalCase do @moduletag :functional setup do - # Stub mocks with implementations delegating to the proper Erlang modules + # Stub mocks with implementations delegating to the original Erlang + # modules, essentially "unmocking" them unless explicit expectations + # are set up. Mox.stub_with(MockErlangSsh, ErlangSsh) Mox.stub_with(MockErlangSshConnection, ErlangSshConnection) Mox.stub_with(MockErlangSshSftp, ErlangSshSftp) diff --git a/test/support/gen.ex b/test/support/gen.ex new file mode 100644 index 00000000..42899f30 --- /dev/null +++ b/test/support/gen.ex @@ -0,0 +1,61 @@ +defmodule Gen do + @moduledoc false + + @doc """ + Generates a behaviour based on an existing module. + + Mox requires a behaviour to be defined in order to create a mock. To mock + core modules in tests - e.g. :ssh, :ssh_connection and :ssh_sftp - we need + behaviours mirroring their public API. + """ + def defbehaviour(name, target) when is_atom(name) and is_atom(target) do + info = moduledoc("Generated behaviour for #{inspect(target)}.") + + body = + for {fun, arity} <- functions(target) do + args = 0..arity |> Enum.map(fn _ -> {:term, [], []} end) |> tl() + + quote do + @callback unquote(fun)(unquote_splicing(args)) :: term() + end + end + + Module.create(name, info ++ body, Macro.Env.location(__ENV__)) + end + + @doc """ + Generates a module delegating all function calls to another module. + + Mox requires modules used for stubbing to implement the mocked behaviour. To + mock core modules without behaviour definitions, we generate stand-in modules + which delegate + """ + def defdelegated(name, target, options \\ []) + when is_atom(name) and is_atom(target) and is_list(options) do + info = + moduledoc("Generated stand-in module for #{inspect(target)}.") ++ + behaviour(Keyword.get(options, :behaviour)) + + body = + for {fun, arity} <- functions(target) do + args = Macro.generate_arguments(arity, name) + + quote do + defdelegate unquote(fun)(unquote_splicing(args)), to: unquote(target) + end + end + + Module.create(name, info ++ body, Macro.Env.location(__ENV__)) + end + + defp functions(module) do + exports = module.module_info(:exports) + Keyword.drop(exports, ~w[__info__ module_info]a) + end + + defp moduledoc(nil), do: [] + defp moduledoc(docstr), do: [quote(do: @moduledoc(unquote(docstr)))] + + defp behaviour(nil), do: [] + defp behaviour(name), do: [quote(do: @behaviour(unquote(name)))] +end diff --git a/test/support/mocks.ex b/test/support/mocks.ex index 03f439c7..703790f6 100644 --- a/test/support/mocks.ex +++ b/test/support/mocks.ex @@ -1,79 +1,13 @@ -defmodule ErlangSshBehaviour do - @moduledoc false +require Gen - @type conn() :: term() +Gen.defbehaviour(ErlangSsh.Behaviour, :ssh) +Gen.defdelegated(ErlangSsh, :ssh, behaviour: ErlangSsh.Behaviour) +Mox.defmock(MockErlangSsh, for: ErlangSsh.Behaviour) - @callback connect(binary(), integer(), keyword(), timeout()) :: {:ok, conn()} | {:error, term()} - @callback close(conn()) :: :ok -end +Gen.defbehaviour(ErlangSshConnection.Behaviour, :ssh_connection) +Gen.defdelegated(ErlangSshConnection, :ssh_connection, behaviour: ErlangSshConnection.Behaviour) +Mox.defmock(MockErlangSshConnection, for: ErlangSshConnection.Behaviour) -defmodule ErlangSsh do - @moduledoc false - - @behaviour ErlangSshBehaviour - - defdelegate connect(host, port, options, timeout), to: :ssh - defdelegate close(conn), to: :ssh -end - -Mox.defmock(MockErlangSsh, for: ErlangSshBehaviour) - -defmodule ErlangSshConnectionBehaviour do - @moduledoc false - - @type conn() :: term() - @type chan() :: integer() - - @callback session_channel(conn(), integer(), integer(), timeout()) :: - {:ok, chan()} | {:error, term()} - @callback subsystem(conn(), chan(), charlist(), timeout()) :: - :success | :failure | {:error, :timeout} | {:error, :closed} - @callback close(conn(), chan()) :: :ok - @callback exec(conn(), chan(), binary(), timeout()) :: - :success | :failure | {:error, :timeout} | {:error, :closed} - @callback ptty_alloc(conn(), chan(), keyword(), timeout()) :: - :success | :failure | {:error, :timeout} | {:error, :closed} - @callback send(conn(), chan(), 0..1, binary(), timeout()) :: - :ok | {:error, :timeout} | {:error, :closed} - @callback send_eof(conn(), chan()) :: :ok | {:error, :closed} - @callback adjust_window(conn(), chan(), integer()) :: :ok -end - -defmodule ErlangSshConnection do - @moduledoc false - - @behaviour ErlangSshConnectionBehaviour - - defdelegate session_channel(conn, initial_window_size, max_packet_size, timeout), - to: :ssh_connection - - defdelegate subsystem(conn, chan, name, timeout), to: :ssh_connection - defdelegate close(conn, chan), to: :ssh_connection - defdelegate exec(conn, chan, command, timeout), to: :ssh_connection - defdelegate ptty_alloc(conn, chan, keyword, timeout), to: :ssh_connection - defdelegate send(conn, chan, type, data, timeout), to: :ssh_connection - defdelegate send_eof(conn, chan), to: :ssh_connection - defdelegate adjust_window(conn, chan, size), to: :ssh_connection -end - -Mox.defmock(MockErlangSshConnection, for: ErlangSshConnectionBehaviour) - -defmodule ErlangSshSftpBehaviour do - @moduledoc false - - @type conn() :: term() - @type chan() :: pid() - - # TODO - @callback start_channel(conn(), keyword()) :: {:ok, chan()} | {:error, term()} -end - -defmodule ErlangSshSftp do - @moduledoc false - - @behaviour ErlangSshSftpBehaviour - - defdelegate start_channel(conn, options), to: :ssh_sftp -end - -Mox.defmock(MockErlangSshSftp, for: ErlangSshSftpBehaviour) +Gen.defbehaviour(ErlangSshSftp.Behaviour, :ssh_sftp) +Gen.defdelegated(ErlangSshSftp, :ssh_sftp, behaviour: ErlangSshSftp.Behaviour) +Mox.defmock(MockErlangSshSftp, for: ErlangSshSftp.Behaviour) From caaa8eb634ea811eabcbd1d78dc4502f9897ecce Mon Sep 17 00:00:00 2001 From: pmeinhardt Date: Fri, 25 Dec 2020 08:59:00 +0100 Subject: [PATCH 38/54] Add typespecs for `Upload` --- examples/upload.exs | 2 +- lib/sshkit/transfer.ex | 2 +- lib/sshkit/upload.ex | 32 +++++++++++++++++++++----------- 3 files changed, 23 insertions(+), 13 deletions(-) diff --git a/examples/upload.exs b/examples/upload.exs index 87a4dd42..e69c6a87 100644 --- a/examples/upload.exs +++ b/examples/upload.exs @@ -2,7 +2,7 @@ :ok = conn - |> SSHKit.upload!("test/fixtures", "/tmp/fixtures", recursive: true) + |> SSHKit.upload!("test/fixtures", "/tmp/fixtures") |> Stream.run() :ok = SSHKit.close(conn) diff --git a/lib/sshkit/transfer.ex b/lib/sshkit/transfer.ex index acaa3e47..632c278c 100644 --- a/lib/sshkit/transfer.ex +++ b/lib/sshkit/transfer.ex @@ -16,7 +16,7 @@ defmodule SSHKit.Transfer do if module.done?(transfer) do {:halt, transfer} else - {:ok, transfer} = module.continue(transfer) + {:ok, transfer} = module.step(transfer) {[transfer], transfer} end end, diff --git a/lib/sshkit/upload.ex b/lib/sshkit/upload.ex index 48409791..138a2282 100644 --- a/lib/sshkit/upload.ex +++ b/lib/sshkit/upload.ex @@ -3,16 +3,19 @@ defmodule SSHKit.Upload do TODO """ + alias SSHKit.Connection alias SSHKit.SFTP.Channel defstruct [:source, :target, :options, :cwd, :stack, :channel] @type t() :: %__MODULE__{} + @spec init(binary(), binary(), keyword()) :: t() def init(source, target, options \\ []) do %__MODULE__{source: Path.expand(source), target: target, options: options} end + @spec start(t(), Connection.t()) :: {:ok, t()} | {:error, term()} def start(%__MODULE__{options: options} = upload, conn) do # accepts options like timeout… http://erlang.org/doc/man/ssh_sftp.html#start_channel-1 start_options = @@ -20,21 +23,23 @@ defmodule SSHKit.Upload do |> Keyword.get(:start, []) |> Keyword.put_new(:timeout, Keyword.get(options, :timeout, :infinity)) - with {:ok, upload} <- prepare(upload), + with {:ok, upload} <- preflight(upload), {:ok, chan} <- Channel.start(conn, start_options) do {:ok, %{upload | channel: chan}} end end - defp prepare(%__MODULE__{source: source, options: options} = upload) do - if !Keyword.get(options, :recursive, false) && File.dir?(source) do - # TODO: Better error - {:error, "Option :recursive not specified, but local file is a directory (#{source})"} - else + defp preflight(%__MODULE__{source: source, options: options} = upload) do + if File.exists?(source) do {:ok, %{upload | cwd: Path.dirname(source), stack: [[Path.basename(source)]]}} + else + {:error, :enoent} end end + @spec stop(t()) :: {:ok, t()} | {:error, term()} + def stop(upload) + def stop(%__MODULE__{channel: nil} = upload), do: {:ok, upload} def stop(%__MODULE__{channel: chan} = upload) do @@ -45,15 +50,18 @@ defmodule SSHKit.Upload do # TODO: Handle unstarted uploads w/o channel, cwd, stack… and provide helpful error? - def continue(%__MODULE__{stack: []} = upload) do + @spec step(t()) :: {:ok, t()} | {:error, term()} + def step(upload) + + def step(%__MODULE__{stack: []} = upload) do {:ok, upload} end - def continue(%__MODULE__{stack: [[] | paths]} = upload) do + def step(%__MODULE__{stack: [[] | paths]} = upload) do {:ok, %{upload | cwd: Path.dirname(upload.cwd), stack: paths}} end - def continue(%__MODULE__{stack: [[name | rest] | paths]} = upload) do + def step(%__MODULE__{stack: [[name | rest] | paths]} = upload) do path = Path.join(upload.cwd, name) relpath = Path.relative_to(path, upload.source) relpath = if relpath == path, do: ".", else: relpath @@ -101,6 +109,8 @@ defmodule SSHKit.Upload do |> Enum.find(:ok, &(&1 != :ok)) end - def done?(%{stack: []}), do: true - def done?(%{}), do: false + @spec done?(t()) :: boolean() + def done?(upload) + def done?(%__MODULE__{stack: []}), do: true + def done?(%__MODULE__{}), do: false end From d70c169c0589771e29a95a43fd63f299335270af Mon Sep 17 00:00:00 2001 From: pmeinhardt Date: Sat, 26 Dec 2020 15:21:38 +0100 Subject: [PATCH 39/54] Add typespecs for `Connection` and `Channel` --- lib/sshkit.ex | 2 +- lib/sshkit/channel.ex | 17 ++++++++++++++--- lib/sshkit/connection.ex | 9 ++++----- test/sshkit/connection_test.exs | 4 ---- 4 files changed, 19 insertions(+), 13 deletions(-) diff --git a/lib/sshkit.ex b/lib/sshkit.ex index 6455d275..ff21b6ee 100644 --- a/lib/sshkit.ex +++ b/lib/sshkit.ex @@ -129,7 +129,7 @@ defmodule SSHKit do output = Enum.reverse(output) - # TODO: Proper file struct? + # TODO: Proper error struct? if status != 0, do: raise("Non-zero exit code: #{status}") output diff --git a/lib/sshkit/channel.ex b/lib/sshkit/channel.ex index dc87f759..97721403 100644 --- a/lib/sshkit/channel.ex +++ b/lib/sshkit/channel.ex @@ -9,6 +9,8 @@ defmodule SSHKit.Channel do * `id` - the unique channel id """ + alias SSHKit.Connection + defstruct [:connection, :type, :id] @type t() :: %__MODULE__{} @@ -30,6 +32,7 @@ defmodule SSHKit.Channel do * `:initial_window_size` - defaults to 128 KiB * `:max_packet_size` - defaults to 32 KiB """ + @spec open(Connection.t(), keyword()) :: {:ok, t()} | {:error, term()} def open(conn, options \\ []) do timeout = Keyword.get(options, :timeout, :infinity) ini_window_size = Keyword.get(options, :initial_window_size, 128 * 1024) @@ -51,8 +54,7 @@ defmodule SSHKit.Channel do For more details, see [`:ssh_connection.subsystem/4`](http://erlang.org/doc/man/ssh_connection.html#subsystem-4). """ - @spec subsystem(channel :: struct(), subsystem :: String.t(), options :: list()) :: - :success | :failure | {:error, reason :: String.t()} + @spec subsystem(t(), binary(), keyword()) :: :success | :failure | {:error, term()} def subsystem(channel, subsystem, options \\ []) do timeout = Keyword.get(options, :timeout, :infinity) @core.subsystem(channel.connection.ref, channel.id, to_charlist(subsystem), timeout) @@ -65,6 +67,7 @@ defmodule SSHKit.Channel do For more details, see [`:ssh_connection.close/2`](http://erlang.org/doc/man/ssh_connection.html#close-2). """ + @spec close(t()) :: :ok def close(channel) do @core.close(channel.connection.ref, channel.id) end @@ -81,6 +84,7 @@ defmodule SSHKit.Channel do `loop/4` may be used to process any channel messages received as a result of executing `command` on the remote. """ + @spec exec(t(), binary() | charlist(), timeout()) :: :success | :failure | {:error, term()} def exec(channel, command, timeout \\ :infinity) def exec(channel, command, timeout) when is_binary(command) do @@ -94,10 +98,11 @@ defmodule SSHKit.Channel do @doc """ Allocates PTTY. - Returns `:success`. + Returns `:success`, `:failure` or `{:error, reason}`. For more details, see [`:ssh_connection.ptty_alloc/4`](http://erlang.org/doc/man/ssh_connection.html#ptty_alloc-4). """ + @spec ptty(t(), keyword(), timeout()) :: :success | :failure | {:error, term()} def ptty(channel, options \\ [], timeout \\ :infinity) do @core.ptty_alloc(channel.connection.ref, channel.id, options, timeout) end @@ -111,6 +116,8 @@ defmodule SSHKit.Channel do For more details, see [`:ssh_connection.send/5`](http://erlang.org/doc/man/ssh_connection.html#send-5). """ + @spec send(t(), non_neg_integer(), term(), timeout()) :: + :ok | {:error, :timeout} | {:error, :closed} def send(channel, type \\ 0, data, timeout \\ :infinity) def send(channel, type, data, timeout) when is_binary(data) or is_list(data) do @@ -131,6 +138,7 @@ defmodule SSHKit.Channel do For more details, see [`:ssh_connection.send_eof/2`](http://erlang.org/doc/man/ssh_connection.html#send_eof-2). """ + @spec eof(t()) :: :ok | {:error, term()} def eof(channel) do @core.send_eof(channel.connection.ref, channel.id) end @@ -155,6 +163,7 @@ defmodule SSHKit.Channel do For more details, see [`:ssh_connection`](http://erlang.org/doc/man/ssh_connection.html). """ + @spec recv(t(), timeout()) :: {:ok, tuple()} | {:error, term()} def recv(channel, timeout \\ :infinity) do ref = channel.connection.ref id = channel.id @@ -173,6 +182,7 @@ defmodule SSHKit.Channel do Returns `:ok`. """ + @spec flush(t(), timeout()) :: :ok def flush(channel, timeout \\ 0) do ref = channel.connection.ref id = channel.id @@ -191,6 +201,7 @@ defmodule SSHKit.Channel do For more details, see [`:ssh_connection.adjust_window/3`](http://erlang.org/doc/man/ssh_connection.html#adjust_window-3). """ + @spec adjust(t(), non_neg_integer()) :: :ok def adjust(channel, size) when is_integer(size) do @core.adjust_window(channel.connection.ref, channel.id, size) end diff --git a/lib/sshkit/connection.ex b/lib/sshkit/connection.ex index 760ea624..bea89041 100644 --- a/lib/sshkit/connection.ex +++ b/lib/sshkit/connection.ex @@ -40,17 +40,14 @@ defmodule SSHKit.Connection do Returns `{:ok, conn}` on success, `{:error, reason}` otherwise. """ + @spec open(binary() | charlist(), keyword()) :: {:ok, t()} | {:error, term()} def open(host, options \\ []) - def open(nil, _) do - {:error, "No host given."} - end - def open(host, options) when is_binary(host) do open(to_charlist(host), options) end - def open(host, options) do + def open(host, options) when is_list(host) do {details, opts} = extract(options) port = details[:port] @@ -89,6 +86,7 @@ defmodule SSHKit.Connection do For details, see [`:ssh.close/1`](http://erlang.org/doc/man/ssh.html#close-1). """ + @spec close(t()) :: :ok def close(conn) do @core.close(conn.ref) end @@ -103,6 +101,7 @@ defmodule SSHKit.Connection do Returns `{:ok, conn}` or `{:error, reason}`. """ + @spec reopen(term(), keyword()) :: {:ok, t()} | {:error, term()} def reopen(conn, options \\ []) do options = conn.options diff --git a/test/sshkit/connection_test.exs b/test/sshkit/connection_test.exs index 7b7f2e9e..e699c399 100644 --- a/test/sshkit/connection_test.exs +++ b/test/sshkit/connection_test.exs @@ -119,10 +119,6 @@ defmodule SSHKit.ConnectionTest do assert open("test.io") == {:error, :failed} end - - test "returns an error if no host is given" do - assert open(nil) == {:error, "No host given."} - end end describe "close/1" do From 3176874c542564f020f764ef35d4c58c7dea7c91 Mon Sep 17 00:00:00 2001 From: pmeinhardt Date: Sat, 26 Dec 2020 22:13:24 +0100 Subject: [PATCH 40/54] Update `Context` tests --- test/sshkit/context_test.exs | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/test/sshkit/context_test.exs b/test/sshkit/context_test.exs index 5cc49d3b..e1b54bbf 100644 --- a/test/sshkit/context_test.exs +++ b/test/sshkit/context_test.exs @@ -13,28 +13,28 @@ defmodule SSHKit.ContextTest do end describe "path/2" do - test "sets the path for the context" do + test "sets the path" do context = Context.path(@empty, "/var/www/app") assert context.path == "/var/www/app" end end describe "umask/2" do - test "sets the file permission mask for the context" do + test "sets the file permission mask" do context = Context.umask(@empty, "077") assert context.umask == "077" end end describe "user/2" do - test "sets the user for the context" do + test "sets the user" do context = Context.user(@empty, "meg") assert context.user == "meg" end end describe "group/2" do - test "sets the group for the context" do + test "sets the group" do context = Context.group(@empty, "stripes") assert context.group == "stripes" end @@ -126,6 +126,11 @@ defmodule SSHKit.ContextTest do assert command == "cd /var/www && /usr/bin/env ls -l" end + test "without any options" do + command = @empty |> Context.build("uptime") + assert command == "/usr/bin/env uptime" + end + test "with all options" do command = @empty From 68cb2ed97ed3d97cc2bb3ec6544fc75ebb6e241e Mon Sep 17 00:00:00 2001 From: pmeinhardt Date: Sat, 26 Dec 2020 22:28:06 +0100 Subject: [PATCH 41/54] Add `:context` option to `exec!/3` and `run!/3` --- examples/context.exs | 13 +++++++++++++ lib/sshkit.ex | 10 ++++++++++ 2 files changed, 23 insertions(+) create mode 100644 examples/context.exs diff --git a/examples/context.exs b/examples/context.exs new file mode 100644 index 00000000..66fe8ee1 --- /dev/null +++ b/examples/context.exs @@ -0,0 +1,13 @@ +{:ok, conn} = SSHKit.connect("127.0.0.1", port: 2222, user: "deploy", password: "deploy", silently_accept_hosts: true) + +context = + SSHKit.Context.new() + |> SSHKit.Context.path("/tmp") + |> SSHKit.Context.umask("007") + |> SSHKit.Context.env(%{"X" => "Y"}) + +conn +|> SSHKit.run!(~S(echo $X && pwd && umask && id -un && id -gn), context: context) +|> Enum.each(fn {type, data} -> IO.write("#{type}: #{data}") end) + +:ok = SSHKit.close(conn) diff --git a/lib/sshkit.ex b/lib/sshkit.ex index ff21b6ee..267fdedc 100644 --- a/lib/sshkit.ex +++ b/lib/sshkit.ex @@ -20,6 +20,7 @@ defmodule SSHKit do alias SSHKit.Channel alias SSHKit.Connection + alias SSHKit.Context alias SSHKit.Download alias SSHKit.Transfer alias SSHKit.Upload @@ -43,6 +44,10 @@ defmodule SSHKit do @spec exec!(Connection.t(), binary(), keyword()) :: Enumerable.t() def exec!(conn, command, options \\ []) do + {context, options} = Keyword.pop(options, :context, Context.new()) + + command = Context.build(context, command) + # TODO: Separate options for open/exec/recv Stream.resource( fn -> @@ -115,6 +120,11 @@ defmodule SSHKit do def send(chan, :stdout, data, timeout), do: Channel.send(chan, 0, data, timeout) def send(chan, :stderr, data, timeout), do: Channel.send(chan, 1, data, timeout) + @doc """ + TODO + + Accepts the same options as `exec!/3`. + """ @spec run!(Connection.t(), binary(), keyword()) :: [{:stdout | :stderr, binary()}] def run!(conn, command, options \\ []) do stream = exec!(conn, command, options) From dda6ac1e5f35219996cda855ffe1870b42e77903 Mon Sep 17 00:00:00 2001 From: pmeinhardt Date: Wed, 30 Dec 2020 00:23:48 +0100 Subject: [PATCH 42/54] Add simple context test case --- test/sshkit/context_test.exs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test/sshkit/context_test.exs b/test/sshkit/context_test.exs index e1b54bbf..e0fb9fe6 100644 --- a/test/sshkit/context_test.exs +++ b/test/sshkit/context_test.exs @@ -41,6 +41,11 @@ defmodule SSHKit.ContextTest do end describe "env/2" do + test "sets the env" do + context = Context.env(@empty, %{"NODE_ENV" => "production"}) + assert context.env == %{"NODE_ENV" => "production"} + end + test "overwrites existing env" do context = @empty From cc763bb3f7105ce1c55619956cfc0c6af0d97658 Mon Sep 17 00:00:00 2001 From: pmeinhardt Date: Wed, 30 Dec 2020 06:20:49 +0100 Subject: [PATCH 43/54] Use underlying OTP channel message tags --- examples/parallel.exs | 2 +- examples/stream.exs | 2 +- lib/sshkit.ex | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/examples/parallel.exs b/examples/parallel.exs index b76c0d93..54ba0f5b 100644 --- a/examples/parallel.exs +++ b/examples/parallel.exs @@ -25,7 +25,7 @@ tasks = IO.write("[#{label.(chan.connection)}] (stderr) #{output}") status - {:exit, _, status}, _ -> + {:exit_status, _, status}, _ -> status _, status -> diff --git a/examples/stream.exs b/examples/stream.exs index d39d1a08..06fbba40 100644 --- a/examples/stream.exs +++ b/examples/stream.exs @@ -16,7 +16,7 @@ code = Enum.reduce(stream, nil, fn status - {:exit, _, status}, _ -> + {:exit_status, _, status}, _ -> status _, status -> diff --git a/lib/sshkit.ex b/lib/sshkit.ex index 267fdedc..9f5b66a3 100644 --- a/lib/sshkit.ex +++ b/lib/sshkit.ex @@ -67,10 +67,10 @@ defmodule SSHKit do value = case msg do {:exit_signal, ^chan, signal, message, lang} -> - {:signal, chan, signal, message, lang} + {:exit_signal, chan, signal, message, lang} {:exit_status, ^chan, status} -> - {:exit, chan, status} + {:exit_status, chan, status} {:data, ^chan, 0, data} -> {:stdout, chan, data} @@ -131,9 +131,9 @@ defmodule SSHKit do {status, output} = Enum.reduce(stream, {nil, []}, fn + {:exit_status, _, status}, {_, output} -> {status, output} {:stdout, _, data}, {status, output} -> {status, [{:stdout, data} | output]} {:stderr, _, data}, {status, output} -> {status, [{:stderr, data} | output]} - {:exit, _, status}, {_, output} -> {status, output} _, acc -> acc end) From a14269fcc5c40f0d2c7147b2ed3878147c61e0a2 Mon Sep 17 00:00:00 2001 From: pmeinhardt Date: Wed, 30 Dec 2020 13:33:31 +0100 Subject: [PATCH 44/54] Add info/1 function for debugging connections --- lib/sshkit/connection.ex | 19 ++++++++++++++++- test/sshkit/connection_test.exs | 38 +++++++++++++++++++++++++++------ 2 files changed, 50 insertions(+), 7 deletions(-) diff --git a/lib/sshkit/connection.ex b/lib/sshkit/connection.ex index bea89041..0ef0d1fd 100644 --- a/lib/sshkit/connection.ex +++ b/lib/sshkit/connection.ex @@ -101,7 +101,7 @@ defmodule SSHKit.Connection do Returns `{:ok, conn}` or `{:error, reason}`. """ - @spec reopen(term(), keyword()) :: {:ok, t()} | {:error, term()} + @spec reopen(t(), keyword()) :: {:ok, t()} | {:error, term()} def reopen(conn, options \\ []) do options = conn.options @@ -110,4 +110,21 @@ defmodule SSHKit.Connection do open(conn.host, options) end + + @doc """ + Returns information about a connection. + + For OTP versions prior to 21.1, only `:client_version`, `:server_version`, + `:user`, `:peer` and `:sockname` are available. + + For details, see [`:ssh.connection_info/1`](http://erlang.org/doc/man/ssh.html#connection_info-1). + """ + @spec info(t()) :: keyword() + def info(conn) do + if function_exported?(@core, :connection_info, 1) do + @core.connection_info(conn.ref) + else + @core.connection_info(conn.ref, [:client_version, :server_version, :user, :peer, :sockname]) + end + end end diff --git a/test/sshkit/connection_test.exs b/test/sshkit/connection_test.exs index e699c399..94914dee 100644 --- a/test/sshkit/connection_test.exs +++ b/test/sshkit/connection_test.exs @@ -129,7 +129,7 @@ defmodule SSHKit.ConnectionTest do end) conn = %Connection{ - host: 'foo.io', + host: 'test.io', port: 22, options: [user_interaction: false], ref: :connection_ref @@ -140,7 +140,7 @@ defmodule SSHKit.ConnectionTest do end describe "reopen/2" do - test "opens a new connection with the same options as the existing connection" do + test "opens a new connection with the same options as an existing connection" do conn = %Connection{ host: 'test.io', port: 22, @@ -160,7 +160,7 @@ defmodule SSHKit.ConnectionTest do assert reopen(conn) == {:ok, new_conn} end - test "reopens a connection on new port" do + test "reopens a connection on a new port" do conn = %Connection{ host: 'test.io', port: 22, @@ -178,12 +178,12 @@ defmodule SSHKit.ConnectionTest do assert reopen(conn, port: 666) == {:ok, new_conn} end - test "errors when unable to open connection" do + test "errors when unable to open a connection" do conn = %Connection{ host: 'test.io', port: 22, - options: [user_interaction: false], - ref: :sandbox + options: [], + ref: :connection_ref } expect(@core, :connect, fn _, _, _, _ -> @@ -193,4 +193,30 @@ defmodule SSHKit.ConnectionTest do assert reopen(conn) == {:error, :failed} end end + + describe "info/1" do + test "returns information about a connection" do + if function_exported?(@core, :connection_info, 1) do + expect(@core, :connection_info, fn ref -> + assert ref == :connection_ref + [info: :test] + end) + else + expect(@core, :connection_info, fn ref, keys -> + assert ref == :connection_ref + assert keys == [:client_version, :server_version, :user, :peer, :sockname] + [info: :test] + end) + end + + conn = %Connection{ + host: 'test.io', + port: 22, + options: [], + ref: :connection_ref + } + + assert info(conn) == [info: :test] + end + end end From e54d1ecd07e7df926c93dd07f2306fdfb7289ea4 Mon Sep 17 00:00:00 2001 From: pmeinhardt Date: Wed, 30 Dec 2020 16:08:35 +0100 Subject: [PATCH 45/54] Re-order functions in `Channel` module --- lib/sshkit/channel.ex | 50 +++++++++++++++++++++---------------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/lib/sshkit/channel.ex b/lib/sshkit/channel.ex index 97721403..ae4e7790 100644 --- a/lib/sshkit/channel.ex +++ b/lib/sshkit/channel.ex @@ -47,19 +47,6 @@ defmodule SSHKit.Channel do %__MODULE__{connection: conn, type: :session, id: id} end - @doc """ - Activates a subsystem on a channel. - - Returns `:success`, `:failure` or `{:error, reason}`. - - For more details, see [`:ssh_connection.subsystem/4`](http://erlang.org/doc/man/ssh_connection.html#subsystem-4). - """ - @spec subsystem(t(), binary(), keyword()) :: :success | :failure | {:error, term()} - def subsystem(channel, subsystem, options \\ []) do - timeout = Keyword.get(options, :timeout, :infinity) - @core.subsystem(channel.connection.ref, channel.id, to_charlist(subsystem), timeout) - end - @doc """ Closes an SSH channel. @@ -95,6 +82,19 @@ defmodule SSHKit.Channel do @core.exec(channel.connection.ref, channel.id, command, timeout) end + @doc """ + Activates a subsystem on a channel. + + Returns `:success`, `:failure` or `{:error, reason}`. + + For more details, see [`:ssh_connection.subsystem/4`](http://erlang.org/doc/man/ssh_connection.html#subsystem-4). + """ + @spec subsystem(t(), binary(), keyword()) :: :success | :failure | {:error, term()} + def subsystem(channel, subsystem, options \\ []) do + timeout = Keyword.get(options, :timeout, :infinity) + @core.subsystem(channel.connection.ref, channel.id, to_charlist(subsystem), timeout) + end + @doc """ Allocates PTTY. @@ -177,6 +177,18 @@ defmodule SSHKit.Channel do end end + @doc """ + Adjusts the flow control window. + + Returns `:ok`. + + For more details, see [`:ssh_connection.adjust_window/3`](http://erlang.org/doc/man/ssh_connection.html#adjust_window-3). + """ + @spec adjust(t(), non_neg_integer()) :: :ok + def adjust(channel, size) when is_integer(size) do + @core.adjust_window(channel.connection.ref, channel.id, size) + end + @doc """ Flushes any pending messages for the given channel. @@ -193,16 +205,4 @@ defmodule SSHKit.Channel do timeout -> :ok end end - - @doc """ - Adjusts the flow control window. - - Returns `:ok`. - - For more details, see [`:ssh_connection.adjust_window/3`](http://erlang.org/doc/man/ssh_connection.html#adjust_window-3). - """ - @spec adjust(t(), non_neg_integer()) :: :ok - def adjust(channel, size) when is_integer(size) do - @core.adjust_window(channel.connection.ref, channel.id, size) - end end From 1fcfc0e9d0e072e77fa9df07f4240d1274aa653a Mon Sep 17 00:00:00 2001 From: pmeinhardt Date: Thu, 31 Dec 2020 12:02:01 +0100 Subject: [PATCH 46/54] Experiment with tar pipe for file transfer Does not respect all the context options at the moment. --- examples/tarpipe.exs | 69 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 examples/tarpipe.exs diff --git a/examples/tarpipe.exs b/examples/tarpipe.exs new file mode 100644 index 00000000..9fa63f7e --- /dev/null +++ b/examples/tarpipe.exs @@ -0,0 +1,69 @@ +{:ok, conn} = SSHKit.connect("127.0.0.1", port: 2222, user: "deploy", password: "deploy", silently_accept_hosts: true) + +ctx = + SSHKit.Context.new() + |> SSHKit.Context.path("/tmp") + |> SSHKit.Context.user("other") + |> SSHKit.Context.group("other") + +defmodule Xfer do + # https://github.com/erlang/otp/blob/OTP-23.2.1/lib/ssh/src/ssh.hrl + def to_binary(data) when is_list(data) do + :erlang.iolist_to_binary(data) + catch + _ -> :unicode.characters_to_binary(data) + end + + def to_binary(data) when is_binary(data) do + data + end +end + +:ok = + with {:ok, chan} <- SSHKit.Channel.open(conn, []) do + # command = SSHKit.Context.build(ctx, "tar -x") + command = "tar -x -C #{ctx.path}" + IO.puts(command) + + case SSHKit.Channel.exec(chan, command) do + :success -> + {:ok, tar} = :erl_tar.init(self(), :write, fn + :position, {_, position} -> + # IO.write("tar position: #{inspect(position)}") + {:ok, 0} + + :write, {_, data} -> + :ok = SSHKit.Channel.send(chan, Xfer.to_binary(data)) + :ok + + :close, _ -> + :ok = SSHKit.Channel.eof(chan) + :ok + end) + + source = "test/fixtures" + # :ok = :erl_tar.add(tar, to_charlist(source), to_charlist(source)) + + with {:ok, names} <- File.ls(source) do + Enum.each(names, fn name -> + path = Path.join(source, name) + + with {:ok, stat} <- File.stat(path, time: :posix) do + IO.puts("#{stat.type}: #{path}") + + :ok = :erl_tar.add(tar, to_charlist(path), to_charlist(path), atime: stat.atime, mtime: stat.mtime, ctime: stat.ctime) + end + end) + end + + :ok = :erl_tar.close(tar) + + :failure -> + {:error, :failure} + + other -> + other + end + end + +:ok = SSHKit.close(conn) From 59114efaefed6a608b42d8fd8ee5af1990438756 Mon Sep 17 00:00:00 2001 From: pmeinhardt Date: Thu, 31 Dec 2020 12:21:15 +0100 Subject: [PATCH 47/54] Use context for tar pipe --- examples/tarpipe.exs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/examples/tarpipe.exs b/examples/tarpipe.exs index 9fa63f7e..e3a74b14 100644 --- a/examples/tarpipe.exs +++ b/examples/tarpipe.exs @@ -21,12 +21,13 @@ end :ok = with {:ok, chan} <- SSHKit.Channel.open(conn, []) do - # command = SSHKit.Context.build(ctx, "tar -x") - command = "tar -x -C #{ctx.path}" - IO.puts(command) + command = SSHKit.Context.build(ctx, "tar -x") case SSHKit.Channel.exec(chan, command) do :success -> + # In case of failed upload, check command output: + # IO.inspect(SSHKit.Channel.recv(chan)) + {:ok, tar} = :erl_tar.init(self(), :write, fn :position, {_, position} -> # IO.write("tar position: #{inspect(position)}") From dd52dd9e1e63592b00698c7a567d59b1da133104 Mon Sep 17 00:00:00 2001 From: pmeinhardt Date: Thu, 31 Dec 2020 12:24:13 +0100 Subject: [PATCH 48/54] Simplify tar pipe example Apparently `:erl_tar.add/3` recursively adds files and subdirectories. --- examples/tarpipe.exs | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/examples/tarpipe.exs b/examples/tarpipe.exs index e3a74b14..e84d37d5 100644 --- a/examples/tarpipe.exs +++ b/examples/tarpipe.exs @@ -43,19 +43,8 @@ end end) source = "test/fixtures" - # :ok = :erl_tar.add(tar, to_charlist(source), to_charlist(source)) - with {:ok, names} <- File.ls(source) do - Enum.each(names, fn name -> - path = Path.join(source, name) - - with {:ok, stat} <- File.stat(path, time: :posix) do - IO.puts("#{stat.type}: #{path}") - - :ok = :erl_tar.add(tar, to_charlist(path), to_charlist(path), atime: stat.atime, mtime: stat.mtime, ctime: stat.ctime) - end - end) - end + :ok = :erl_tar.add(tar, to_charlist(source), to_charlist(source)) :ok = :erl_tar.close(tar) From c261aed586faaa6cf5b703c87c5491cb999017e0 Mon Sep 17 00:00:00 2001 From: pmeinhardt Date: Thu, 31 Dec 2020 12:51:16 +0100 Subject: [PATCH 49/54] Make `:erl_tar.init/3` arguments clearer --- examples/tarpipe.exs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/examples/tarpipe.exs b/examples/tarpipe.exs index e84d37d5..7421c088 100644 --- a/examples/tarpipe.exs +++ b/examples/tarpipe.exs @@ -5,6 +5,7 @@ ctx = |> SSHKit.Context.path("/tmp") |> SSHKit.Context.user("other") |> SSHKit.Context.group("other") + |> SSHKit.Context.umask("0077") defmodule Xfer do # https://github.com/erlang/otp/blob/OTP-23.2.1/lib/ssh/src/ssh.hrl @@ -28,16 +29,16 @@ end # In case of failed upload, check command output: # IO.inspect(SSHKit.Channel.recv(chan)) - {:ok, tar} = :erl_tar.init(self(), :write, fn - :position, {_, position} -> + {:ok, tar} = :erl_tar.init(chan, :write, fn + :position, {^chan, position} -> # IO.write("tar position: #{inspect(position)}") {:ok, 0} - :write, {_, data} -> + :write, {^chan, data} -> :ok = SSHKit.Channel.send(chan, Xfer.to_binary(data)) :ok - :close, _ -> + :close, ^chan -> :ok = SSHKit.Channel.eof(chan) :ok end) From e889b264434ac777345cd1ec6efb4bc7f1285aaa Mon Sep 17 00:00:00 2001 From: pmeinhardt Date: Thu, 31 Dec 2020 18:38:12 +0100 Subject: [PATCH 50/54] Fix path nesting for tar pipe example --- examples/tarpipe.exs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/examples/tarpipe.exs b/examples/tarpipe.exs index 7421c088..30cd978a 100644 --- a/examples/tarpipe.exs +++ b/examples/tarpipe.exs @@ -20,6 +20,8 @@ defmodule Xfer do end end +source = "test/fixtures" + :ok = with {:ok, chan} <- SSHKit.Channel.open(conn, []) do command = SSHKit.Context.build(ctx, "tar -x") @@ -35,6 +37,7 @@ end {:ok, 0} :write, {^chan, data} -> + # TODO: Send data in chunks based on channel window size? :ok = SSHKit.Channel.send(chan, Xfer.to_binary(data)) :ok @@ -43,9 +46,7 @@ end :ok end) - source = "test/fixtures" - - :ok = :erl_tar.add(tar, to_charlist(source), to_charlist(source)) + :ok = :erl_tar.add(tar, to_charlist(source), to_charlist(Path.basename(source)), []) :ok = :erl_tar.close(tar) From 23029f9324e473d927a56c6860ae2dc0faf1a894 Mon Sep 17 00:00:00 2001 From: pmeinhardt Date: Wed, 3 Feb 2021 22:11:44 +0100 Subject: [PATCH 51/54] Iterate on tarpipe upload implementation --- examples/tarpipe.exs | 109 +++++++++++++++++++++++++++---------------- 1 file changed, 70 insertions(+), 39 deletions(-) diff --git a/examples/tarpipe.exs b/examples/tarpipe.exs index 30cd978a..01c6d30d 100644 --- a/examples/tarpipe.exs +++ b/examples/tarpipe.exs @@ -3,59 +3,90 @@ ctx = SSHKit.Context.new() |> SSHKit.Context.path("/tmp") - |> SSHKit.Context.user("other") - |> SSHKit.Context.group("other") + # |> SSHKit.Context.user("other") + # |> SSHKit.Context.group("other") |> SSHKit.Context.umask("0077") -defmodule Xfer do - # https://github.com/erlang/otp/blob/OTP-23.2.1/lib/ssh/src/ssh.hrl - def to_binary(data) when is_list(data) do - :erlang.iolist_to_binary(data) - catch - _ -> :unicode.characters_to_binary(data) - end +defmodule TP do + def upload!(conn, source, dest, opts \\ []) do + ctx = Keyword.get(opts, :context, SSHKit.Context.new()) - def to_binary(data) when is_binary(data) do - data - end -end + Stream.resource( + fn -> + {:ok, chan} = SSHKit.Channel.open(conn, []) + command = SSHKit.Context.build(ctx, "tar -x") + :success = SSHKit.Channel.exec(chan, command) -source = "test/fixtures" + owner = self() -:ok = - with {:ok, chan} <- SSHKit.Channel.open(conn, []) do - command = SSHKit.Context.build(ctx, "tar -x") + tarpipe = spawn(fn -> + {:ok, tar} = :erl_tar.init(chan, :write, fn + :position, {^chan, position} -> + # IO.inspect(position, label: "position") + {:ok, 0} - case SSHKit.Channel.exec(chan, command) do - :success -> - # In case of failed upload, check command output: - # IO.inspect(SSHKit.Channel.recv(chan)) + :write, {^chan, data} -> + # TODO: Send data in chunks based on channel window size? + # IO.inspect(data, label: "write") + # In case of failing upload, check command output: + # IO.inspect(SSHKit.Channel.recv(chan, 0)) + chunk = to_binary(data) - {:ok, tar} = :erl_tar.init(chan, :write, fn - :position, {^chan, position} -> - # IO.write("tar position: #{inspect(position)}") - {:ok, 0} + receive do + :cont -> + :ok = SSHKit.Channel.send(chan, chunk) + end + send(owner, {:write, chan, self(), chunk}) + :ok - :write, {^chan, data} -> - # TODO: Send data in chunks based on channel window size? - :ok = SSHKit.Channel.send(chan, Xfer.to_binary(data)) - :ok + :close, ^chan -> + # IO.puts("close") + :ok = SSHKit.Channel.eof(chan) + send(owner, {:close, chan, self()}) + :ok + end) - :close, ^chan -> - :ok = SSHKit.Channel.eof(chan) - :ok + :ok = :erl_tar.add(tar, to_charlist(source), to_charlist(Path.basename(source)), []) + :ok = :erl_tar.close(tar) end) - :ok = :erl_tar.add(tar, to_charlist(source), to_charlist(Path.basename(source)), []) + {chan, tarpipe} + end, + fn {chan, tarpipe} -> + send(tarpipe, :cont) - :ok = :erl_tar.close(tar) + receive do + {:write, ^chan, ^tarpipe, data} -> + {[{:write, chan, data}], {chan, tarpipe}} - :failure -> - {:error, :failure} + {:close, ^chan, ^tarpipe} -> + {:halt, {chan, tarpipe}} + end + end, + fn {chan, tarpipe} -> + :ok = SSHKit.Channel.close(chan) + :ok = SSHKit.Channel.flush(chan) + end + ) + end - other -> - other - end + # https://github.com/erlang/otp/blob/OTP-23.2.1/lib/ssh/src/ssh.hrl + def to_binary(data) when is_list(data) do + :erlang.iolist_to_binary(data) + catch + _ -> :unicode.characters_to_binary(data) end + def to_binary(data) when is_binary(data) do + data + end +end + +stream = TP.upload!(conn, "test/fixtures", "upload", context: ctx) + +Enum.each(stream, fn + {:write, chan, data} -> + IO.puts("Upload, sent #{byte_size(data)} bytes") +end) + :ok = SSHKit.close(conn) From 77f66dbfc34b356ee527bff646b261cc0a5c518c Mon Sep 17 00:00:00 2001 From: pmeinhardt Date: Wed, 3 Feb 2021 22:49:03 +0100 Subject: [PATCH 52/54] Iterate on tarpipe implementation --- examples/tarpipe.exs | 47 +++++++++++++++++++++++++++++--------------- 1 file changed, 31 insertions(+), 16 deletions(-) diff --git a/examples/tarpipe.exs b/examples/tarpipe.exs index 01c6d30d..047ca2d6 100644 --- a/examples/tarpipe.exs +++ b/examples/tarpipe.exs @@ -13,13 +13,16 @@ defmodule TP do Stream.resource( fn -> - {:ok, chan} = SSHKit.Channel.open(conn, []) - command = SSHKit.Context.build(ctx, "tar -x") - :success = SSHKit.Channel.exec(chan, command) - owner = self() tarpipe = spawn(fn -> + {:ok, chan} = SSHKit.Channel.open(conn, []) + command = SSHKit.Context.build(ctx, "tar -x") + :success = SSHKit.Channel.exec(chan, command) + + # TODO: What if command immediately exits or does not exist? + # IO.inspect(SSHKit.Channel.recv(chan, 1000)) + {:ok, tar} = :erl_tar.init(chan, :write, fn :position, {^chan, position} -> # IO.inspect(position, label: "position") @@ -27,16 +30,18 @@ defmodule TP do :write, {^chan, data} -> # TODO: Send data in chunks based on channel window size? - # IO.inspect(data, label: "write") + IO.inspect(data, label: "write") # In case of failing upload, check command output: # IO.inspect(SSHKit.Channel.recv(chan, 0)) chunk = to_binary(data) receive do :cont -> - :ok = SSHKit.Channel.send(chan, chunk) + case SSHKit.Channel.send(chan, chunk) do + :ok -> send(owner, {:write, chan, self(), chunk}) + other -> send(owner, {:error, chan, self(), other}) + end end - send(owner, {:write, chan, self(), chunk}) :ok :close, ^chan -> @@ -48,24 +53,34 @@ defmodule TP do :ok = :erl_tar.add(tar, to_charlist(source), to_charlist(Path.basename(source)), []) :ok = :erl_tar.close(tar) + + :ok = SSHKit.Channel.close(chan) end) - {chan, tarpipe} + tarpipe end, - fn {chan, tarpipe} -> + fn tarpipe -> send(tarpipe, :cont) receive do - {:write, ^chan, ^tarpipe, data} -> - {[{:write, chan, data}], {chan, tarpipe}} + {:write, chan, ^tarpipe, data} -> + {[{:write, chan, data}], tarpipe} + + {:close, chan, ^tarpipe} -> + {:halt, tarpipe} - {:close, ^chan, ^tarpipe} -> - {:halt, {chan, tarpipe}} + {:error, chan, ^tarpipe, error} -> + IO.inspect(error, label: "received error") + {:halt, tarpipe} end + + # case Tarpipe.proceed(tarpipe) do + # {:write, …} -> {[], tarpipe} + # {:error, …} -> raise + # end end, - fn {chan, tarpipe} -> - :ok = SSHKit.Channel.close(chan) - :ok = SSHKit.Channel.flush(chan) + fn tarpipe -> + nil # :ok = Tarpipe.close(tarpipe) end ) end From 07a4f5b9f6061f25c7191a777d4db817e5c949e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20Kn=C3=B6pfle?= Date: Wed, 11 Jan 2023 13:29:15 +0100 Subject: [PATCH 53/54] Enable warnings as errors --- .github/workflows/ci.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cd07aeef..371a4e5a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,9 +23,8 @@ jobs: - name: Check mix format run: mix format --check-formatted - # TODO: Enable check before finalizing release - # - name: Compile with warnings as errors - # run: mix compile --warnings-as-errors + - name: Compile with warnings as errors + run: mix compile --warnings-as-errors analysis: name: Run static code analysis From af9880ec76f4c78af2543a6cd2185d848c494299 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20Kn=C3=B6pfle?= Date: Wed, 11 Jan 2023 13:30:29 +0100 Subject: [PATCH 54/54] Fixup config --- .formatter.exs | 2 +- config/config.exs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.formatter.exs b/.formatter.exs index 7f59b1f0..d2cda26e 100644 --- a/.formatter.exs +++ b/.formatter.exs @@ -1,4 +1,4 @@ # Used by "mix format" [ - inputs: ["{mix,.formatter}.exs", "{lib,test}/**/*.{ex,exs}"] + inputs: ["{mix,.formatter}.exs", "{config,lib,test}/**/*.{ex,exs}"] ] diff --git a/config/config.exs b/config/config.exs index 4c091556..ced864e8 100644 --- a/config/config.exs +++ b/config/config.exs @@ -1,4 +1,4 @@ -use Mix.Config +import Config if Mix.env() == :test do config :sshkit, :ssh, MockErlangSsh