diff --git a/gems/aws-sdk-s3/CHANGELOG.md b/gems/aws-sdk-s3/CHANGELOG.md index ad3779f590a..8cb3e74f16d 100644 --- a/gems/aws-sdk-s3/CHANGELOG.md +++ b/gems/aws-sdk-s3/CHANGELOG.md @@ -1,6 +1,12 @@ Unreleased Changes ------------------ +* Issue - When multipart stream uploader fails to complete multipart upload, it calls abort multipart upload. + +* Issue - For `Aws::S3::Object` class, the following methods have been deprecated: `download_file`, `upload_file` and `upload_stream`. Use `Aws::S3::TransferManager` instead. + +* Feature - Add `Aws::S3::TransferManager`, a S3 transfer utility that provides upload/download capabilities with automatic multipart handling, progress tracking, and handling of large files. + 1.196.1 (2025-08-05) ------------------ diff --git a/gems/aws-sdk-s3/lib/aws-sdk-s3/customizations.rb b/gems/aws-sdk-s3/lib/aws-sdk-s3/customizations.rb index e21e4bc6bf9..528ba42fc15 100644 --- a/gems/aws-sdk-s3/lib/aws-sdk-s3/customizations.rb +++ b/gems/aws-sdk-s3/lib/aws-sdk-s3/customizations.rb @@ -18,14 +18,18 @@ module S3 autoload :ObjectMultipartCopier, 'aws-sdk-s3/object_multipart_copier' autoload :PresignedPost, 'aws-sdk-s3/presigned_post' autoload :Presigner, 'aws-sdk-s3/presigner' + autoload :TransferManager, 'aws-sdk-s3/transfer_manager' # s3 express session auth autoload :ExpressCredentials, 'aws-sdk-s3/express_credentials' autoload :ExpressCredentialsProvider, 'aws-sdk-s3/express_credentials_provider' # s3 access grants auth - autoload :AccessGrantsCredentials, 'aws-sdk-s3/access_grants_credentials' autoload :AccessGrantsCredentialsProvider, 'aws-sdk-s3/access_grants_credentials_provider' + + # testing transfer manager + autoload :DirectoryUploader, 'aws-sdk-s3/directory_uploader' + autoload :TransferManager, 'aws-sdk-s3/transfer_manager' end end diff --git a/gems/aws-sdk-s3/lib/aws-sdk-s3/customizations/object.rb b/gems/aws-sdk-s3/lib/aws-sdk-s3/customizations/object.rb index 5f3b80a9b7a..6409a0fef8b 100644 --- a/gems/aws-sdk-s3/lib/aws-sdk-s3/customizations/object.rb +++ b/gems/aws-sdk-s3/lib/aws-sdk-s3/customizations/object.rb @@ -398,6 +398,7 @@ def upload_stream(options = {}, &block) end true end + deprecated(:upload_stream, use: 'Aws::S3::TransferManager#upload_stream', version: 'next major version') # Uploads a file from disk to the current object in S3. # @@ -465,6 +466,7 @@ def upload_file(source, options = {}) yield response if block_given? true end + deprecated(:upload_file, use: 'Aws::S3::TransferManager#upload_file', version: 'next major version') # Downloads a file in S3 to a path on disk. # @@ -534,6 +536,7 @@ def download_file(destination, options = {}) end true end + deprecated(:download_file, use: 'Aws::S3::TransferManager#download_file', version: 'next major version') class Collection < Aws::Resources::Collection alias_method :delete, :batch_delete! diff --git a/gems/aws-sdk-s3/lib/aws-sdk-s3/directory_uploader.rb b/gems/aws-sdk-s3/lib/aws-sdk-s3/directory_uploader.rb new file mode 100644 index 00000000000..a4e11dc5249 --- /dev/null +++ b/gems/aws-sdk-s3/lib/aws-sdk-s3/directory_uploader.rb @@ -0,0 +1,145 @@ +# frozen_string_literal: true + +require 'find' +require 'set' +require 'thread' + +module Aws + module S3 + # @api private + class DirectoryUploader + def initialize(options = {}) + @client = options[:client] || Client.new + @thread_count = options[:thread_count] || 10 + @executor = options[:executor] + end + + # @return [Client] + attr_reader :client + + def upload(source, options = {}) + raise ArgumentError, 'Invalid directory' unless Dir.exist?(source) + + upload_opts = options.dup + @source = source + @recursive = upload_opts.delete(:recursive) || false + @follow_symlinks = upload_opts.delete(:follow_symlinks) || false + @s3_prefix = upload_opts.delete(:s3_prefix) || nil + @s3_delimiter = upload_opts.delete(:s3_delimiter) || '/' + @filter_callback = upload_opts.delete(:filter_callback) || nil + + uploader = FileUploader.new( + multipart_threshold: upload_opts.delete(:multipart_threshold), + client: @client, + executor: @executor + ) + @file_queue = SizedQueue.new(5) # TODO: random number for now, intended to relive backpressure + @disable_queue = false + + _producer = Thread.new do + if @recursive + stream_recursive_files + else + stream_direct_files + end + + # signals queue being done + if @executor + @file_queue << :done + else + @thread_count.times { @file_queue << :done } + end + end + + if @executor + upload_with_executor(uploader, upload_opts) + else + threads = [] + @thread_count.times do + thread = Thread.new do + return if @disable_queue + + while (file = @file_queue.shift) != :done + path = File.join(@source, file) + # TODO: key to consider s3_prefix and custom delimiter + uploader.upload(path, upload_opts.merge(key: file)) + end + nil + rescue StandardError => e # TODO: handle failure policies + @disable_queue = true + e + end + threads << thread + end + threads.map(&:value).compact + end + end + + private + + def upload_with_executor(uploader, upload_opts) + total_files = 0 + completion_queue = Queue.new + errors = [] + while (file = @file_queue.shift) != :done + total_files += 1 + @executor.post(file) do |f| + begin + next if @disable_queue + + path = File.join(@source, f) + # TODO: key to consider s3_prefix and custom delimiter + uploader.upload(path, upload_opts.merge(key: f)) + rescue StandardError => e # TODO: handle failure policies + @disable_queue = true + errors << e + end + ensure + completion_queue << :done + end + end + puts 'waiting for completion' + total_files.times { completion_queue.pop } + puts 'all done waiting!' + raise StandardError, 'directory upload failed' unless errors.empty? + end + + def stream_recursive_files + visited = Set.new + # TODO: add filter callback + Find.find(@source) do |p| + break if @disable_queue + + if !@follow_symlinks && File.symlink?(p) + Find.prune + next + end + + absolute_path = File.realpath(p) + if visited.include?(absolute_path) + Find.prune + next + end + + visited << absolute_path + + # TODO: if non-default s3_delimiter is used, validate here and fail + @file_queue << p.sub(%r{^#{Regexp.escape(@source)}/}, '') if File.file?(p) + end + end + + def stream_direct_files + # TODO: add filter callback4 + Dir.each_child(@source) do |entry| + break if @disable_queue + + path = File.join(@source, entry) + next if !@follow_symlinks && File.symlink?(path) + + # TODO: if non-default s3_delimiter is used, validate here and fail + @file_queue << entry if File.file?(path) + end + end + end + end +end diff --git a/gems/aws-sdk-s3/lib/aws-sdk-s3/file_uploader.rb b/gems/aws-sdk-s3/lib/aws-sdk-s3/file_uploader.rb index 7937de9bc66..587066551ea 100644 --- a/gems/aws-sdk-s3/lib/aws-sdk-s3/file_uploader.rb +++ b/gems/aws-sdk-s3/lib/aws-sdk-s3/file_uploader.rb @@ -7,7 +7,7 @@ module S3 # @api private class FileUploader - ONE_HUNDRED_MEGABYTES = 100 * 1024 * 1024 + DEFAULT_MULTIPART_THRESHOLD = 100 * 1024 * 1024 # @param [Hash] options # @option options [Client] :client @@ -15,15 +15,13 @@ class FileUploader def initialize(options = {}) @options = options @client = options[:client] || Client.new - @multipart_threshold = options[:multipart_threshold] || - ONE_HUNDRED_MEGABYTES + @multipart_threshold = options[:multipart_threshold] || DEFAULT_MULTIPART_THRESHOLD end # @return [Client] attr_reader :client - # @return [Integer] Files larger than or equal to this in bytes are uploaded - # using a {MultipartFileUploader}. + # @return [Integer] Files larger than or equal to this in bytes are uploaded using a {MultipartFileUploader}. attr_reader :multipart_threshold # @param [String, Pathname, File, Tempfile] source The file to upload. diff --git a/gems/aws-sdk-s3/lib/aws-sdk-s3/multipart_file_uploader.rb b/gems/aws-sdk-s3/lib/aws-sdk-s3/multipart_file_uploader.rb index 4c18ace70ab..fdd0732e73a 100644 --- a/gems/aws-sdk-s3/lib/aws-sdk-s3/multipart_file_uploader.rb +++ b/gems/aws-sdk-s3/lib/aws-sdk-s3/multipart_file_uploader.rb @@ -9,17 +9,11 @@ module S3 class MultipartFileUploader MIN_PART_SIZE = 5 * 1024 * 1024 # 5MB - MAX_PARTS = 10_000 - - THREAD_COUNT = 10 - + DEFAULT_THREAD_COUNT = 10 CREATE_OPTIONS = Set.new(Client.api.operation(:create_multipart_upload).input.shape.member_names) - COMPLETE_OPTIONS = Set.new(Client.api.operation(:complete_multipart_upload).input.shape.member_names) - UPLOAD_PART_OPTIONS = Set.new(Client.api.operation(:upload_part).input.shape.member_names) - CHECKSUM_KEYS = Set.new( Client.api.operation(:upload_part).input.shape.members.map do |n, s| n if s.location == 'header' && s.location_name.start_with?('x-amz-checksum-') @@ -27,10 +21,11 @@ class MultipartFileUploader ) # @option options [Client] :client - # @option options [Integer] :thread_count (THREAD_COUNT) + # @option options [Integer] :thread_count (DEFAULT_THREAD_COUNT) def initialize(options = {}) @client = options[:client] || Client.new - @thread_count = options[:thread_count] || THREAD_COUNT + @executor = options[:executor] + @thread_count = options[:thread_count] || DEFAULT_THREAD_COUNT end # @return [Client] @@ -72,7 +67,13 @@ def complete_upload(upload_id, parts, source, options) def upload_parts(upload_id, source, options) completed = PartList.new pending = PartList.new(compute_parts(upload_id, source, options)) - errors = upload_in_threads(pending, completed, options) + errors = + if @executor + puts "Executor route - using #{@executor}" + upload_in_executor(pending, completed, options) + else + upload_in_threads(pending, completed, options) + end if errors.empty? completed.to_a.sort_by { |part| part[:part_number] } else @@ -141,6 +142,62 @@ def upload_part_opts(options) end end + def upload_in_executor(pending, completed, options) + if (callback = options[:progress_callback]) + progress = MultipartProgress.new(pending, callback) + end + max_parts = pending.count + completion_queue = Queue.new + stop_work = false + errors = [] + counter = 0 + + puts "Submitting #{max_parts} tasks" + while (part = pending.shift) + counter += 1 + puts "Submitting #{counter} task to executor" + @executor.post(part) do |p| + if stop_work + puts 'Work stopped so skipping' + completion_queue << :done + next + end + + if progress + p[:on_chunk_sent] = + proc do |_chunk, bytes, _total| + progress.call(p[:part_number], bytes) + end + end + + begin + puts "Uploading #{p[:part_number]}" + + resp = @client.upload_part(p) + p[:body].close + completed_part = { etag: resp.etag, part_number: p[:part_number] } + algorithm = resp.context.params[:checksum_algorithm] + k = "checksum_#{algorithm.downcase}".to_sym + completed_part[k] = resp.send(k) + completed.push(completed_part) + rescue StandardError => e + puts "Encountered Error #{e}" + stop_work = true + errors << e + ensure + puts 'Adding to completion queue' + completion_queue << :done + end + nil + end + end + + puts "Waiting for #{counter} completion" + max_parts.times { completion_queue.pop } + puts "Done Waiting. Result: \n Completed:#{completed} \n Error: #{errors}" + errors + end + def upload_in_threads(pending, completed, options) threads = [] if (callback = options[:progress_callback]) @@ -195,6 +252,10 @@ def initialize(parts = []) @mutex = Mutex.new end + def count + @mutex.synchronize { @parts.count } + end + def push(part) @mutex.synchronize { @parts.push(part) } end diff --git a/gems/aws-sdk-s3/lib/aws-sdk-s3/multipart_stream_uploader.rb b/gems/aws-sdk-s3/lib/aws-sdk-s3/multipart_stream_uploader.rb index dccd224a22f..60a298c720b 100644 --- a/gems/aws-sdk-s3/lib/aws-sdk-s3/multipart_stream_uploader.rb +++ b/gems/aws-sdk-s3/lib/aws-sdk-s3/multipart_stream_uploader.rb @@ -9,33 +9,19 @@ module Aws module S3 # @api private class MultipartStreamUploader - # api private - PART_SIZE = 5 * 1024 * 1024 # 5MB - # api private - THREAD_COUNT = 10 - - # api private - TEMPFILE_PREIX = 'aws-sdk-s3-upload_stream'.freeze - - # @api private - CREATE_OPTIONS = - Set.new(Client.api.operation(:create_multipart_upload).input.shape.member_names) - - # @api private - UPLOAD_PART_OPTIONS = - Set.new(Client.api.operation(:upload_part).input.shape.member_names) - - # @api private - COMPLETE_UPLOAD_OPTIONS = - Set.new(Client.api.operation(:complete_multipart_upload).input.shape.member_names) + DEFAULT_PART_SIZE = 5 * 1024 * 1024 # 5MB + DEFAULT_THREAD_COUNT = 10 + CREATE_OPTIONS = Set.new(Client.api.operation(:create_multipart_upload).input.shape.member_names) + UPLOAD_PART_OPTIONS = Set.new(Client.api.operation(:upload_part).input.shape.member_names) + COMPLETE_UPLOAD_OPTIONS = Set.new(Client.api.operation(:complete_multipart_upload).input.shape.member_names) # @option options [Client] :client def initialize(options = {}) @client = options[:client] || Client.new @tempfile = options[:tempfile] - @part_size = options[:part_size] || PART_SIZE - @thread_count = options[:thread_count] || THREAD_COUNT + @part_size = options[:part_size] || DEFAULT_PART_SIZE + @thread_count = options[:thread_count] || DEFAULT_THREAD_COUNT end # @return [Client] @@ -43,7 +29,7 @@ def initialize(options = {}) # @option options [required,String] :bucket # @option options [required,String] :key - # @option options [Integer] :thread_count (THREAD_COUNT) + # @option options [Integer] :thread_count (DEFAULT_THREAD_COUNT) # @return [Seahorse::Client::Response] - the CompleteMultipartUploadResponse def upload(options = {}, &block) Aws::Plugins::UserAgent.metric('S3_TRANSFER') do @@ -61,11 +47,10 @@ def initiate_upload(options) def complete_upload(upload_id, parts, options) @client.complete_multipart_upload( - **complete_opts(options).merge( - upload_id: upload_id, - multipart_upload: { parts: parts } - ) + **complete_opts(options).merge(upload_id: upload_id, multipart_upload: { parts: parts }) ) + rescue StandardError => e + abort_upload(upload_id, options, [e]) end def upload_parts(upload_id, options, &block) @@ -74,9 +59,11 @@ def upload_parts(upload_id, options, &block) errors = begin IO.pipe do |read_pipe, write_pipe| threads = upload_in_threads( - read_pipe, completed, + read_pipe, + completed, upload_part_opts(options).merge(upload_id: upload_id), - thread_errors) + thread_errors + ) begin block.call(write_pipe) ensure @@ -85,62 +72,53 @@ def upload_parts(upload_id, options, &block) end threads.map(&:value).compact end - rescue => e + rescue StandardError => e thread_errors + [e] end + return ordered_parts(completed) if errors.empty? - if errors.empty? - Array.new(completed.size) { completed.pop }.sort_by { |part| part[:part_number] } - else - abort_upload(upload_id, options, errors) - end + abort_upload(upload_id, options, errors) end def abort_upload(upload_id, options, errors) - @client.abort_multipart_upload( - bucket: options[:bucket], - key: options[:key], - upload_id: upload_id - ) + @client.abort_multipart_upload(bucket: options[:bucket], key: options[:key], upload_id: upload_id) msg = "multipart upload failed: #{errors.map(&:message).join('; ')}" raise MultipartUploadError.new(msg, errors) - rescue MultipartUploadError => error - raise error - rescue => error - msg = "failed to abort multipart upload: #{error.message}. "\ + rescue MultipartUploadError => e + raise e + rescue StandardError => e + msg = "failed to abort multipart upload: #{e.message}. "\ "Multipart upload failed: #{errors.map(&:message).join('; ')}" - raise MultipartUploadError.new(msg, errors + [error]) + raise MultipartUploadError.new(msg, errors + [e]) end def create_opts(options) - CREATE_OPTIONS.inject({}) do |hash, key| + CREATE_OPTIONS.each_with_object({}) do |key, hash| hash[key] = options[key] if options.key?(key) - hash end end def upload_part_opts(options) - UPLOAD_PART_OPTIONS.inject({}) do |hash, key| + UPLOAD_PART_OPTIONS.each_with_object({}) do |key, hash| hash[key] = options[key] if options.key?(key) - hash end end def complete_opts(options) - COMPLETE_UPLOAD_OPTIONS.inject({}) do |hash, key| + COMPLETE_UPLOAD_OPTIONS.each_with_object({}) do |key, hash| hash[key] = options[key] if options.key?(key) - hash end end def read_to_part_body(read_pipe) return if read_pipe.closed? - temp_io = @tempfile ? Tempfile.new(TEMPFILE_PREIX) : StringIO.new(String.new) + + temp_io = @tempfile ? Tempfile.new('aws-sdk-s3-upload_stream') : StringIO.new(String.new) temp_io.binmode bytes_copied = IO.copy_stream(read_pipe, temp_io, @part_size) temp_io.rewind - if bytes_copied == 0 - if Tempfile === temp_io + if bytes_copied.zero? + if temp_io.is_a?(Tempfile) temp_io.close temp_io.unlink end @@ -155,48 +133,62 @@ def upload_in_threads(read_pipe, completed, options, thread_errors) part_number = 0 options.fetch(:thread_count, @thread_count).times.map do thread = Thread.new do - begin - loop do - body, thread_part_number = mutex.synchronize do - [read_to_part_body(read_pipe), part_number += 1] - end - break unless (body || thread_part_number == 1) - begin - part = options.merge( - body: body, - part_number: thread_part_number, - ) - resp = @client.upload_part(part) - completed_part = {etag: resp.etag, part_number: part[:part_number]} - - # get the requested checksum from the response - if part[:checksum_algorithm] - k = "checksum_#{part[:checksum_algorithm].downcase}".to_sym - completed_part[k] = resp[k] - end - completed.push(completed_part) - ensure - if Tempfile === body - body.close - body.unlink - elsif StringIO === body - body.string.clear - end - end + loop do + body, thread_part_number = mutex.synchronize do + [read_to_part_body(read_pipe), part_number += 1] end - nil - rescue => error - # keep other threads from uploading other parts - mutex.synchronize do - thread_errors.push(error) - read_pipe.close_read unless read_pipe.closed? + break unless body || thread_part_number == 1 + + begin + part = options.merge(body: body, part_number: thread_part_number) + resp = @client.upload_part(part) + completed_part = create_completed_part(resp, part) + completed.push(completed_part) + ensure + clear_body(body) end - error end + nil + rescue StandardError => e + # keep other threads from uploading other parts + mutex.synchronize do + thread_errors.push(e) + read_pipe.close_read unless read_pipe.closed? + end + e end thread end end + + def create_completed_part(resp, part) + completed_part = { etag: resp.etag, part_number: part[:part_number] } + return completed_part unless part[:checksum_algorithm] + + # get the requested checksum from the response + k = "checksum_#{part[:checksum_algorithm].downcase}".to_sym + completed_part[k] = resp[k] + completed_part + end + + def ordered_parts(parts) + sorted = [] + until parts.empty? + part = parts.pop + index = sorted.bsearch_index { |p| p[:part_number] >= part[:part_number] } || sorted.size + sorted.insert(index, part) + end + sorted + end + + def clear_body(body) + if body.is_a?(Tempfile) + body.close + body.unlink + elsif body.is_a?(StringIO) + body.string.clear + end + end end end end diff --git a/gems/aws-sdk-s3/lib/aws-sdk-s3/transfer_manager.rb b/gems/aws-sdk-s3/lib/aws-sdk-s3/transfer_manager.rb new file mode 100644 index 00000000000..0fb509534d7 --- /dev/null +++ b/gems/aws-sdk-s3/lib/aws-sdk-s3/transfer_manager.rb @@ -0,0 +1,262 @@ +# frozen_string_literal: true + +module Aws + module S3 + # A high-level S3 transfer utility that provides enhanced upload and download + # capabilities with automatic multipart handling, progress tracking, and + # handling of large files. The following features are supported: + # + # * upload a file with multipart upload + # * upload a stream with multipart upload + # * download a S3 object with multipart download + # * track transfer progress by using progress listener + # + class TransferManager + # @param [Hash] options + # @option options [S3::Client] :client (S3::Client.new) + # The S3 client to use for {TransferManager} operations. If not provided, a new default client + # will be created automatically. + def initialize(options = {}) + @client = options.delete(:client) || Client.new + @executor = options.delete(:executor) + end + + # @return [S3::Client] + attr_reader :client + + # @return [Object] executor + attr_reader :executor + + # Downloads a file in S3 to a path on disk. + # + # # small files (< 5MB) are downloaded in a single API call + # tm = TransferManager.new + # tm.download_file('/path/to/file', bucket: 'bucket', key: 'key') + # + # Files larger than 5MB are downloaded using multipart method: + # + # # large files are split into parts and the parts are downloaded in parallel + # tm.download_file('/path/to/large_file', bucket: 'bucket', key: 'key') + # + # You can provide a callback to monitor progress of the download: + # + # # bytes and part_sizes are each an array with 1 entry per part + # # part_sizes may not be known until the first bytes are retrieved + # progress = proc do |bytes, part_sizes, file_size| + # bytes.map.with_index do |b, i| + # puts "Part #{i + 1}: #{b} / #{part_sizes[i]}".join(' ') + "Total: #{100.0 * bytes.sum / file_size}%" + # end + # end + # tm.download_file('/path/to/file', bucket: 'bucket', key: 'key', progress_callback: progress) + # + # @param [String] destination + # Where to download the file to. + # + # @param [String] bucket + # The name of the S3 bucket to upload to. + # + # @param [String] key + # The object key name in S3 bucket. + # + # @param [Hash] options + # Additional options for {Client#get_object} and #{Client#head_object} may be provided. + # + # @option options [String] :mode ("auto") `"auto"`, `"single_request"` or `"get_range"` + # + # * `"auto"` mode is enabled by default, which performs `multipart_download` + # * `"single_request`" mode forces only 1 GET request is made in download + # * `"get_range"` mode requires `:chunk_size` parameter to configured in customizing each range size + # + # @option options [Integer] :chunk_size required in `"get_range"` mode. + # + # @option options [Integer] :thread_count (10) Customize threads used in the multipart download. + # + # @option options [String] :version_id The object version id used to retrieve the object. + # + # @see https://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectVersioning.html ObjectVersioning + # + # @option options [String] :checksum_mode ("ENABLED") + # When `"ENABLED"` and the object has a stored checksum, it will be used to validate the download and will + # raise an `Aws::Errors::ChecksumError` if checksum validation fails. You may provide a `on_checksum_validated` + # callback if you need to verify that validation occurred and which algorithm was used. + # To disable checksum validation, set `checksum_mode` to `"DISABLED"`. + # + # @option options [Callable] :on_checksum_validated + # Called each time a request's checksum is validated with the checksum algorithm and the + # response. For multipart downloads, this will be called for each part that is downloaded and validated. + # + # @option options [Proc] :progress_callback + # A Proc that will be called when each chunk of the download is received. It will be invoked with + # `bytes_read`, `part_sizes`, `file_size`. When the object is downloaded as parts (rather than by ranges), + # the `part_sizes` will not be known ahead of time and will be `nil` in the callback until the first bytes + # in the part are received. + # + # @raise [MultipartDownloadError] Raised when an object validation fails outside of service errors. + # + # @return [Boolean] Returns `true` when the file is downloaded without any errors. + # + # @see Client#get_object + # @see Client#head_object + def download_file(destination, bucket:, key:, **options) + downloader = FileDownloader.new(client: @client) + downloader.download(destination, options.merge(bucket: bucket, key: key)) + true + end + + # Uploads a file from disk to S3. + # + # # a small file are uploaded with PutObject API + # tm = TransferManager.new + # tm.upload_file('/path/to/small_file', bucket: 'bucket', key: 'key') + # + # Files larger than or equal to `:multipart_threshold` are uploaded using multipart upload APIs. + # + # # large files are automatically split into parts and the parts are uploaded in parallel + # tm.upload_file('/path/to/large_file', bucket: 'bucket', key: 'key') + # + # The response of the S3 upload API is yielded if a block given. + # + # # API response will have etag value of the file + # tm.upload_file('/path/to/file', bucket: 'bucket', key: 'key') do |response| + # etag = response.etag + # end + # + # You can provide a callback to monitor progress of the upload: + # + # # bytes and totals are each an array with 1 entry per part + # progress = proc do |bytes, totals| + # bytes.map.with_index do |b, i| + # puts "Part #{i + 1}: #{b} / #{totals[i]} " + "Total: #{100.0 * bytes.sum / totals.sum}%" + # end + # end + # tm.upload_file('/path/to/file', bucket: 'bucket', key: 'key', progress_callback: progress) + # + # @param [String, Pathname, File, Tempfile] source + # A file on the local file system that will be uploaded. This can either be a `String` or `Pathname` to the + # file, an open `File` object, or an open `Tempfile` object. If you pass an open `File` or `Tempfile` object, + # then you are responsible for closing it after the upload completes. When using an open Tempfile, rewind it + # before uploading or else the object will be empty. + # + # @param [String] bucket + # The name of the S3 bucket to upload to. + # + # @param [String] key + # The object key name for the uploaded file. + # + # @param [Hash] options + # Additional options for {Client#put_object} when file sizes below the multipart threshold. + # For files larger than the multipart threshold, options for {Client#create_multipart_upload}, + # {Client#complete_multipart_upload}, and {Client#upload_part} can be provided. + # + # @option options [Integer] :multipart_threshold (104857600) + # Files larger han or equal to `:multipart_threshold` are uploaded using the S3 multipart upload APIs. + # Default threshold is `100MB`. + # + # @option options [Integer] :thread_count (10) + # The number of parallel multipart uploads. This option is not used if the file is smaller than + # `:multipart_threshold`. + # + # @option options [Proc] :progress_callback (nil) + # A Proc that will be called when each chunk of the upload is sent. + # It will be invoked with `[bytes_read]` and `[total_sizes]`. + # + # @raise [MultipartUploadError] If an file is being uploaded in parts, and the upload can not be completed, + # then the upload is aborted and this error is raised. The raised error has a `#errors` method that + # returns the failures that caused the upload to be aborted. + # + # @return [Boolean] Returns `true` when the file is uploaded without any errors. + # + # @see Client#put_object + # @see Client#create_multipart_upload + # @see Client#complete_multipart_upload + # @see Client#upload_part + def upload_file(source, bucket:, key:, **options) + uploading_options = options.dup + uploader = FileUploader.new( + multipart_threshold: uploading_options.delete(:multipart_threshold), + client: @client, + executor: @executor + ) + response = uploader.upload(source, uploading_options.merge(bucket: bucket, key: key)) + yield response if block_given? + true + end + + # Uploads a stream in a streaming fashion to S3. + # + # Passed chunks automatically split into multipart upload parts and the parts are uploaded in parallel. + # This allows for streaming uploads that never touch the disk. + # + # **Note**: There are known issues in JRuby until jruby-9.1.15.0, so avoid using this with older JRuby versions. + # + # @example Streaming chunks of data + # tm = TransferManager.new + # tm.upload_stream(bucket: 'bucket', key: 'key') do |write_stream| + # 10.times { write_stream << 'foo' } + # end + # @example Streaming chunks of data + # tm.upload_stream(bucket: 'bucket', key: 'key') do |write_stream| + # IO.copy_stream(IO.popen('ls'), write_stream) + # end + # @example Streaming chunks of data + # tm.upload_stream(bucket: 'bucket', key: 'key') do |write_stream| + # IO.copy_stream(STDIN, write_stream) + # end + # + # @param [String] bucket + # The name of the S3 bucket to upload to. + # + # @param [String] key + # The object key name for the uploaded file. + # + # @param [Hash] options + # Additional options for {Client#create_multipart_upload}, {Client#complete_multipart_upload}, and + # {Client#upload_part} can be provided. + # + # @option options [Integer] :thread_count (10) + # The number of parallel multipart uploads. + # + # @option options [Boolean] :tempfile (false) + # Normally read data is stored in memory when building the parts in order to complete the underlying + # multipart upload. By passing `:tempfile => true`, the data read will be temporarily stored on disk reducing + # the memory footprint vastly. + # + # @option options [Integer] :part_size (5242880) + # Define how big each part size but the last should be. Default `:part_size` is `5 * 1024 * 1024`. + # + # @raise [MultipartUploadError] If an object is being uploaded in parts, and the upload can not be completed, + # then the upload is aborted and this error is raised. The raised error has a `#errors` method that returns + # the failures that caused the upload to be aborted. + # + # @return [Boolean] Returns `true` when the object is uploaded without any errors. + # + # @see Client#create_multipart_upload + # @see Client#complete_multipart_upload + # @see Client#upload_part + def upload_stream(bucket:, key:, **options, &block) + uploading_options = options.dup + uploader = MultipartStreamUploader.new( + client: @client, + thread_count: uploading_options.delete(:thread_count), + tempfile: uploading_options.delete(:tempfile), + part_size: uploading_options.delete(:part_size) + ) + uploader.upload(uploading_options.merge(bucket: bucket, key: key), &block) + true + end + + def upload_directory(source, options = {}) + upload_directory_opts = options.dup + directory_uploader = DirectoryUploader.new( + client: @client, + thread_count: upload_directory_opts.delete(:thread_count), + executor: @executor + ) + directory_uploader.upload(source, upload_directory_opts) + true # TODO: need to change depending on failure policy set + end + + def download_directory(source, options = {}); end + end + end +end diff --git a/gems/aws-sdk-s3/spec/file_downloader_spec.rb b/gems/aws-sdk-s3/spec/file_downloader_spec.rb new file mode 100644 index 00000000000..bd782e58bcf --- /dev/null +++ b/gems/aws-sdk-s3/spec/file_downloader_spec.rb @@ -0,0 +1,254 @@ +# frozen_string_literal: true + +require_relative 'spec_helper' +require 'tempfile' + +module Aws + module S3 + describe FileDownloader do + let(:client) { S3::Client.new(stub_responses: true) } + let(:subject) { FileDownloader.new(client: client) } + let(:tmpdir) { Dir.tmpdir } + + describe '#initialize' do + it 'constructs a default s3 client when not given' do + client = double('client') + expect(S3::Client).to receive(:new).and_return(client) + + downloader = FileDownloader.new + expect(downloader.client).to be(client) + end + end + + describe '#download', :jruby_flaky do + let(:path) { Tempfile.new('destination').path } + let(:one_meg) { 1024 * 1024 } + let(:single_params) { { bucket: 'bucket', key: 'single' } } + let(:parts_params) { { bucket: 'bucket', key: 'parts' } } + let(:range_params) { { bucket: 'bucket', key: 'range' } } + + before(:each) do + allow(Dir).to receive(:tmpdir).and_return(tmpdir) + client.stub_responses(:head_object, lambda { |context| + case context.params[:key] + when 'single' + { content_length: one_meg, parts_count: nil } + when 'parts' + resp = { content_length: 20 * one_meg, parts_count: nil } + resp[:parts_count] = 4 if context.params[:part_number] + resp + when 'range' + { content_length: 15 * one_meg, parts_count: nil } + end + }) + end + + it 'downloads a single object using Client#get_object' do + expect(client).to receive(:get_object).with(single_params.merge(response_target: path)).exactly(1).times + subject.download(path, single_params) + end + + it 'downloads a large object in parts' do + parts = 0 + client.stub_responses(:get_object, lambda do |_ctx| + parts += 1 + { body: 'body', content_range: 'bytes 0-3/4' } + end) + subject.download(path, parts_params) + expect(parts).to eq(4) + end + + it 'downloads a large object in ranges' do + client.stub_responses(:get_object, lambda { |context| + responses = { + 'bytes=0-5242879' => { body: 'body', content_range: 'bytes 0-5242879/15728640' }, + 'bytes=5242880-10485759' => { body: 'body', content_range: 'bytes 5242880-10485759/15728640' }, + 'bytes=10485760-15728639' => { body: 'body', content_range: 'bytes 10485760-15728639/15728640' } + } + responses[context.params[:range]] + }) + subject.download(path, range_params.merge(chunk_size: 5 * one_meg, mode: 'get_range')) + end + + it 'supports download object with version_id' do + params = single_params.merge(version_id: 'foo') + expect(client).to receive(:get_object).with(params.merge(response_target: path)).exactly(1).times + + subject.download(path, params) + end + + it 'calls on_checksum_validated on single object' do + client.stub_responses(:get_object, { body: 'body', checksum_sha1: 'Agg/RXngimEkJcDBoX7ket14O5Q=' }) + callback_data = { called: 0 } + mutex = Mutex.new + callback = proc do |_alg, _resp| + mutex.synchronize { callback_data[:called] += 1 } + end + + subject.download(path, single_params.merge(on_checksum_validated: callback)) + expect(callback_data[:called]).to eq(1) + end + + it 'calls on_checksum_validated on multipart object' do + callback_data = { called: 0 } + client.stub_responses( + :get_object, { body: 'body', content_range: 'bytes 0-3/4', checksum_sha1: 'Agg/RXngimEkJcDBoX7ket14O5Q=' } + ) + mutex = Mutex.new + callback = proc do |_alg, _resp| + mutex.synchronize { callback_data[:called] += 1 } + end + + subject.download(path, parts_params.merge(on_checksum_validated: callback)) + expect(callback_data[:called]).to eq(4) + end + + it 'supports disabling checksum_mode' do + client.stub_responses(:head_object, lambda { |context| + expect(context.params[:checksum_mode]).to eq('DISABLED') + { content_length: one_meg, parts_count: nil } + }) + client.stub_responses(:get_object, lambda { |context| + expect(context.params[:checksum_mode]).to eq('DISABLED') + { body: 'body' } + }) + + subject.download(path, single_params.merge(checksum_mode: 'DISABLED')) + end + + it 'downloads the file in range chunks' do + client.stub_responses(:get_object, lambda { |context| + ranges = context.params[:range].match(/bytes=(?\d+)-(?\d+)/) + expect(ranges[:end].to_i - ranges[:start].to_i + 1).to eq(one_meg) + { content_range: "bytes #{ranges[:start]}-#{ranges[:end]}/#{20 * one_meg}" } + }) + + subject.download(path, range_params.merge(chunk_size: one_meg)) + end + + context 'multipart progress' do + it 'reports progress for single object' do + small_file_size = 1024 + expect(client) + .to receive(:get_object) + .with(single_params.merge(response_target: path, on_chunk_received: instance_of(Proc))) do |args| + args[:on_chunk_received].call(Tempfile.new('small-file'), small_file_size, small_file_size) + end + + n_calls = 0 + callback = proc do |bytes, part_sizes, total| + expect(bytes).to eq([small_file_size]) + expect(part_sizes).to eq([small_file_size]) + expect(total).to eq(small_file_size) + n_calls += 1 + end + + subject.download(path, single_params.merge(progress_callback: callback)) + expect(n_calls).to eq(1) + end + + it 'reports progress for downloading a large object in parts' do + expect(client).to receive(:get_object).exactly(4).times do |args| + args[:on_chunk_received].call(Tempfile.new('large-file'), 4, 4) + client.stub_data(:get_object, body: StringIO.new('chunk'), content_range: 'bytes 0-3/4') + end + + n_calls = 0 + mutex = Mutex.new + callback = proc do |bytes, part_sizes, total| + mutex.synchronize do + expect(bytes.size).to eq(4) + expect(part_sizes.size).to eq(4) + expect(total).to eq(20 * one_meg) + n_calls += 1 + end + end + subject.download(path, parts_params.merge(progress_callback: callback)) + expect(n_calls).to eq(4) + end + end + + context 'error handling' do + it 'raises when checksum validation fails on single object' do + client.stub_responses(:get_object, { body: 'body', checksum_sha1: 'invalid' }) + expect { subject.download(path, single_params) }.to raise_error(Aws::Errors::ChecksumError) + end + + it 'raises when checksum validation fails on multipart object' do + client.stub_responses(:get_object, { body: 'body', checksum_sha1: 'invalid' }) + expect(Thread).to receive(:new).and_yield.and_return(double(value: nil)) + expect { subject.download(path, parts_params) }.to raise_error(Aws::Errors::ChecksumError) + end + + it 'raises when ETAG does not match during multipart get by ranges' do + client.stub_responses(:head_object, content_length: 15 * one_meg, parts_count: nil, etag: 'test-etag') + client.stub_responses(:get_object, lambda { |ctx| + expect(ctx.params[:if_match]).to eq('test-etag') + 'PreconditionFailed' + }) + expect(Thread).to receive(:new).and_yield.and_return(double(value: nil)) + expect { subject.download(path, range_params.merge(chunk_size: one_meg, mode: 'get_range')) } + .to raise_error(Aws::S3::Errors::PreconditionFailed) + end + + it 'raises when ETAG does not match during multipart get by parts' do + client.stub_responses(:head_object, { content_length: 20 * one_meg, etag: 'test-etag', parts_count: 4 }) + client.stub_responses(:get_object, lambda { |ctx| + expect(ctx.params[:if_match]).to eq('test-etag') + 'PreconditionFailed' + }) + + expect(Thread).to receive(:new).and_yield.and_return(double(value: nil)) + expect { subject.download(path, parts_params) }.to raise_error(Aws::S3::Errors::PreconditionFailed) + end + + it 'raises when given an invalid mode' do + expect { subject.download(path, parts_params.merge(mode: 'invalid_mode')) } + .to raise_error(ArgumentError, /Invalid mode invalid_mode provided/) + end + + it 'raises when given an "get_range" mode without :chunk_size' do + expect { subject.download(path, range_params.merge(mode: 'get_range')) } + .to raise_error(ArgumentError, /In get_range mode, :chunk_size must be provided/) + end + + it 'raises when given :chunk_size is larger than file size' do + expect { subject.download(path, range_params.merge(chunk_size: 50 * one_meg)) } + .to raise_error(ArgumentError, /:chunk_size shouldn't exceed total file size/) + end + + it 'raises when :on_checksum_validated is not callable' do + expect { subject.download(path, parts_params.merge(on_checksum_validated: 'string')) } + .to raise_error(ArgumentError, /:on_checksum_validated must be callable/) + end + + it 'raises when range validation fails' do + client.stub_responses(:get_object, { body: 'body', content_range: 'bytes 0-3/4' }) + expect(Thread).to receive(:new).and_yield.and_return(double(value: nil)) + expect { subject.download(path, range_params.merge(mode: 'get_range', chunk_size: one_meg)) } + .to raise_error(Aws::S3::MultipartDownloadError) + end + + it 'does not overwrite existing file when download fails' do + File.write(path, 'existing content') + + client.stub_responses(:get_object, lambda { |context| + responses = { + 'bytes=0-5242879' => { body: 'body', content_range: 'bytes 0-5242879/15728640' }, + 'bytes=5242880-10485759' => { body: 'body', content_range: 'bytes 5242880-10485759/15728640' }, + 'bytes=10485760-15728639' => { body: 'fake-range', content_range: 'bytes 10485800-15728639/15728640' } + } + responses[context.params[:range]] + }) + + expect(Thread).to receive(:new).and_yield.and_return(double(value: nil)) + expect { subject.download(path, range_params.merge(chunk_size: 5 * one_meg, mode: 'get_range')) } + .to raise_error(Aws::S3::MultipartDownloadError) + expect(File.exist?(path)).to be(true) + expect(File.read(path)).to eq('existing content') + end + end + end + end + end +end diff --git a/gems/aws-sdk-s3/spec/file_uploader_spec.rb b/gems/aws-sdk-s3/spec/file_uploader_spec.rb new file mode 100644 index 00000000000..64dbaf7cdd2 --- /dev/null +++ b/gems/aws-sdk-s3/spec/file_uploader_spec.rb @@ -0,0 +1,93 @@ +# frozen_string_literal: true + +require_relative 'spec_helper' +require 'tempfile' + +module Aws + module S3 + describe FileUploader do + let(:client) { S3::Client.new(stub_responses: true) } + let(:subject) { FileUploader.new(client: client) } + let(:params) { { bucket: 'bucket', key: 'key' } } + let(:one_meg) { 1024 * 1024 } + let(:one_mb) { '.' * one_meg } + + describe '#initialize' do + it 'constructs a default s3 client when not given' do + client = double('client') + expect(S3::Client).to receive(:new).and_return(client) + + uploader = MultipartFileUploader.new + expect(uploader.client).to be(client) + end + + it 'sets a default multipart threshold when not given' do + expect(subject.multipart_threshold).to be(FileUploader::DEFAULT_MULTIPART_THRESHOLD) + end + + it 'sets a custom multipart threshold' do + five_mb = 5 * one_meg + uploader = FileUploader.new(client: client, multipart_threshold: five_mb) + expect(uploader.multipart_threshold).to be(five_mb) + end + end + + describe '#upload' do + let(:ten_meg_file) do + Tempfile.new('ten-meg-file').tap do |f| + 10.times { f.write(one_mb) } + f.rewind + end + end + + it 'uploads a small file using Client#put_object' do + file = Tempfile.new('one-meg-file').tap do |f| + f.write(one_mb) + f.rewind + end + + expect(client).to receive(:put_object).with({ bucket: 'bucket', key: 'key', body: file }) + subject.upload(file, params) + end + + it 'delegates the large file to use multipart upload' do + large_file = Tempfile.new('one-hundred-seventeen-meg-file').tap do |f| + 117.times { f.write(one_mb) } + f.rewind + end + + expect_any_instance_of(MultipartFileUploader).to receive(:upload).with(large_file, params) + subject.upload(large_file, params) + end + + it 'reports progress for a small file' do + expect(client).to receive(:put_object).with( + params.merge(body: ten_meg_file, on_chunk_sent: instance_of(Proc)) + ) do |args| + args[:on_chunk_sent].call(ten_meg_file, ten_meg_file.size, ten_meg_file.size) + end + callback = proc do |bytes, totals| + expect(bytes).to eq([ten_meg_file.size]) + expect(totals).to eq([ten_meg_file.size]) + end + + subject.upload(ten_meg_file, params.merge(progress_callback: callback)) + end + + it 'accepts paths to files to upload' do + file = double('file') + expect(File).to receive(:open).with(ten_meg_file.path, 'rb').and_yield(file) + expect(client).to receive(:put_object).with(params.merge(body: file)) + + subject.upload(ten_meg_file.path, params) + end + + it 'does not fail when given :thread_count' do + expect(client).to receive(:put_object).with(params.merge(body: ten_meg_file)) + + subject.upload(ten_meg_file, params.merge(thread_count: 1)) + end + end + end + end +end diff --git a/gems/aws-sdk-s3/spec/multipart_file_uploader_spec.rb b/gems/aws-sdk-s3/spec/multipart_file_uploader_spec.rb new file mode 100644 index 00000000000..82e147986e3 --- /dev/null +++ b/gems/aws-sdk-s3/spec/multipart_file_uploader_spec.rb @@ -0,0 +1,160 @@ +# frozen_string_literal: true + +require_relative 'spec_helper' +require 'tempfile' + +module Aws + module S3 + describe MultipartFileUploader do + let(:client) { S3::Client.new(stub_responses: true) } + let(:subject) { MultipartFileUploader.new(client: client) } + let(:params) { { bucket: 'bucket', key: 'key' } } + + describe '#initialize' do + it 'constructs a default s3 client when not given' do + client = double('client') + expect(S3::Client).to receive(:new).and_return(client) + + uploader = MultipartFileUploader.new + expect(uploader.client).to be(client) + end + end + + describe '#upload' do + let(:one_mb) { '.' * 1024 * 1024 } + let(:large_file) do + Tempfile.new('one-hundred-seventeen-meg-file').tap do |f| + 117.times { f.write(one_mb) } + f.rewind + end + end + + it 'uses multipart APIs for objects >= 100MB' do + client.stub_responses(:create_multipart_upload, upload_id: 'id') + client.stub_responses(:upload_part, etag: 'etag', checksum_crc32: 'checksum') + expect(client).to receive(:complete_multipart_upload).with( + params.merge( + upload_id: 'id', + multipart_upload: { + parts: [ + { checksum_crc32: 'checksum', etag: 'etag', part_number: 1 }, + { checksum_crc32: 'checksum', etag: 'etag', part_number: 2 }, + { checksum_crc32: 'checksum', etag: 'etag', part_number: 3 }, + { checksum_crc32: 'checksum', etag: 'etag', part_number: 4 }, + { checksum_crc32: 'checksum', etag: 'etag', part_number: 5 }, + { checksum_crc32: 'checksum', etag: 'etag', part_number: 6 }, + { checksum_crc32: 'checksum', etag: 'etag', part_number: 7 }, + { checksum_crc32: 'checksum', etag: 'etag', part_number: 8 }, + { checksum_crc32: 'checksum', etag: 'etag', part_number: 9 }, + { checksum_crc32: 'checksum', etag: 'etag', part_number: 10 }, + { checksum_crc32: 'checksum', etag: 'etag', part_number: 11 }, + { checksum_crc32: 'checksum', etag: 'etag', part_number: 12 }, + { checksum_crc32: 'checksum', etag: 'etag', part_number: 13 }, + { checksum_crc32: 'checksum', etag: 'etag', part_number: 14 }, + { checksum_crc32: 'checksum', etag: 'etag', part_number: 15 }, + { checksum_crc32: 'checksum', etag: 'etag', part_number: 16 }, + { checksum_crc32: 'checksum', etag: 'etag', part_number: 17 }, + { checksum_crc32: 'checksum', etag: 'etag', part_number: 18 }, + { checksum_crc32: 'checksum', etag: 'etag', part_number: 19 }, + { checksum_crc32: 'checksum', etag: 'etag', part_number: 20 }, + { checksum_crc32: 'checksum', etag: 'etag', part_number: 21 }, + { checksum_crc32: 'checksum', etag: 'etag', part_number: 22 }, + { checksum_crc32: 'checksum', etag: 'etag', part_number: 23 }, + { checksum_crc32: 'checksum', etag: 'etag', part_number: 24 } + ] + }, + mpu_object_size: large_file.size + ) + ) + subject.upload(large_file, params.merge(content_type: 'text/plain')) + end + + it 'allows for full object checksums' do + expect(client).to receive(:create_multipart_upload) + .with(params.merge(checksum_algorithm: 'CRC32', checksum_type: 'FULL_OBJECT', content_type: 'text/plain')) + .and_call_original + expect(client).to receive(:upload_part) + .with(hash_not_including(checksum_crc32: anything)) + .exactly(24).times + .and_call_original + expect(client).to receive(:complete_multipart_upload) + .with(hash_including(checksum_type: 'FULL_OBJECT', checksum_crc32: 'checksum')) + .and_call_original + + subject.upload(large_file, params.merge(content_type: 'text/plain', checksum_crc32: 'checksum')) + end + + it 'reports progress for multipart uploads' do + allow(Thread).to receive(:new).and_yield.and_return(double(value: nil)) + client.stub_responses(:create_multipart_upload, upload_id: 'id') + client.stub_responses(:complete_multipart_upload) + expect(client).to receive(:upload_part).exactly(24).times do |args| + args[:on_chunk_sent].call(args[:body], args[:body].size, args[:body].size) + double(context: double(params: { checksum_algorithm: 'crc32' }), checksum_crc32: 'checksum', etag: 'etag') + end + callback = proc do |bytes, totals| + expect(bytes.size).to eq(24) + expect(totals.size).to eq(24) + end + + subject.upload(large_file, params.merge(content_type: 'text/plain', progress_callback: callback)) + end + + it 'raises when given a file smaller than 5MB' do + file = Tempfile.new('one-meg-file').tap do |f| + f.write(one_mb) + f.rewind + end + + expect { subject.upload(file, params) } + .to raise_error(ArgumentError, /unable to multipart upload files smaller than 5MB/) + end + + it 'automatically deletes failed multipart upload on error' do + allow_any_instance_of(FilePart).to receive(:read).and_return(nil) + client.stub_responses( + :upload_part, + [ + { etag: 'etag-1' }, + { etag: 'etag-2' }, + RuntimeError.new('part 3 failed'), + { etag: 'etag-4' } + ] + ) + + expect(client).to receive(:abort_multipart_upload).with(params.merge(upload_id: 'MultipartUploadId')) + expect { subject.upload(large_file, params) }.to raise_error(/multipart upload failed: part 3 failed/) + end + + it 'reports when it is unable to abort a failed multipart upload' do + allow(Thread).to receive(:new) do |_, &block| + double(value: block.call) + end + + client.stub_responses( + :upload_part, + [ + { etag: 'etag-1' }, + { etag: 'etag-2' }, + { etag: 'etag-3' }, + RuntimeError.new('part failed') + ] + ) + client.stub_responses(:abort_multipart_upload, [RuntimeError.new('network-error')]) + expect do + subject.upload(large_file, params) + end.to raise_error(/failed to abort multipart upload: network-error. Multipart upload failed: part failed/) + + end + + it 'aborts multipart upload when upload fails to complete' do + client.stub_responses(:complete_multipart_upload, RuntimeError.new('network-error')) + + expect(client).to receive(:abort_multipart_upload).with(params.merge(upload_id: 'MultipartUploadId')) + expect { subject.upload(large_file, params) }.to raise_error(Aws::S3::MultipartUploadError) + end + end + end + end +end + diff --git a/gems/aws-sdk-s3/spec/multipart_stream_uploader_spec.rb b/gems/aws-sdk-s3/spec/multipart_stream_uploader_spec.rb new file mode 100644 index 00000000000..e7c2860f5a7 --- /dev/null +++ b/gems/aws-sdk-s3/spec/multipart_stream_uploader_spec.rb @@ -0,0 +1,258 @@ +# frozen_string_literal: true + +require_relative 'spec_helper' +require 'tempfile' + +module Aws + module S3 + describe MultipartStreamUploader do + let(:client) { S3::Client.new(stub_responses: true) } + let(:subject) { MultipartStreamUploader.new(client: client) } + let(:params) { { bucket: 'bucket', key: 'key' } } + let(:one_mb) { '.' * 1024 * 1024 } + let(:seventeen_mb) { one_mb * 17 } + + describe '#initialize' do + it 'constructs a default s3 client when none provided' do + client = double('client') + expect(S3::Client).to receive(:new).and_return(client) + + uploader = MultipartStreamUploader.new + expect(uploader.client).to be(client) + end + end + + describe '#upload_stream', :jruby_flaky do + it 'can upload empty stream' do + client.stub_responses(:create_multipart_upload, upload_id: 'id') + client.stub_responses(:upload_part, etag: 'etag') + expected_params = params.merge( + upload_id: 'id', + multipart_upload: { parts: [{ etag: 'etag', part_number: 1 }] } + ) + expect(client).to receive(:complete_multipart_upload).with(expected_params).once + + subject.upload(params.merge(content_type: 'text/plain')) { |write_stream| write_stream << '' } + end + + it 'uses multipart APIs' do + client.stub_responses(:create_multipart_upload, upload_id: 'id') + client.stub_responses(:upload_part, etag: 'etag') + expected_params = params.merge( + upload_id: 'id', + multipart_upload: { + parts: [ + { etag: 'etag', part_number: 1 }, + { etag: 'etag', part_number: 2 }, + { etag: 'etag', part_number: 3 }, + { etag: 'etag', part_number: 4 } + ] + } + ) + expect(client).to receive(:complete_multipart_upload).with(expected_params).once + + subject.upload(params.merge(content_type: 'text/plain')) { |write_stream| write_stream << seventeen_mb } + end + + it 'uploads the correct parts' do + client.stub_responses(:create_multipart_upload, upload_id: 'id') + 4.times.each do |p| + expect(client) + .to receive(:upload_part) + .with(params.merge(upload_id: 'id', body: instance_of(StringIO), part_number: p + 1)) + .once + .and_return(double(:upload_part, etag: 'etag')) + end + + subject.upload(params.merge(content_type: 'text/plain')) { |write_stream| write_stream << seventeen_mb } + end + + it 'uploads the correct parts when input is chunked' do + client.stub_responses(:create_multipart_upload, upload_id: 'id') + client.stub_responses(:complete_multipart_upload) + 4.times.each do |p| + expect(client) + .to receive(:upload_part) + .with(params.merge(upload_id: 'id', body: instance_of(StringIO), part_number: p + 1)) + .once + .and_return(double(:upload_part, etag: 'etag')) + end + + subject.upload(params) do |write_stream| + 17.times { write_stream << one_mb } + end + end + + it 'passes stringios with correct contents to upload_part' do + client.stub_responses(:create_multipart_upload, upload_id: 'id') + client.stub_responses(:complete_multipart_upload) + result = [] + mutex = Mutex.new + allow(client).to receive(:upload_part) do |part| + mutex.synchronize { result << [part[:part_number], part[:body].read.size] } + end.and_return(double(:upload_part, etag: 'etag')) + + subject.upload(params) do |write_stream| + 17.times { write_stream << one_mb } + end + + expect(result.sort_by(&:first)).to eq( + [ + [1, 5 * 1024 * 1024], + [2, 5 * 1024 * 1024], + [3, 5 * 1024 * 1024], + [4, 2 * 1024 * 1024] + ] + ) + end + + it 'automatically deletes failed multipart upload on part processing error' do + client.stub_responses( + :upload_part, + [ + { etag: 'etag-1' }, + { etag: 'etag-2' }, + RuntimeError.new('part 3 failed'), + { etag: 'etag-4' } + ] + ) + expect(client).to receive(:abort_multipart_upload).with(params.merge(upload_id: 'MultipartUploadId')) + + expect do + subject.upload(params) do |write_stream| + write_stream << seventeen_mb + rescue Errno::EPIPE + # ignore + end + end.to raise_error(MultipartUploadError, /multipart upload failed: part 3 failed/) + end + + it 'automatically deletes failed multipart upload on stream read error' do + expect(client).to receive(:abort_multipart_upload).with(params.merge(upload_id: 'MultipartUploadId')) + + expect do + subject.upload(params) do |_write_stream| + raise 'something went wrong' + end + end.to raise_error(/something went wrong/) + end + + it 'reports when it is unable to abort a failed multipart upload' do + client.stub_responses( + :upload_part, + [ + { etag: 'etag-1' }, + { etag: 'etag-2' }, + { etag: 'etag-3' }, + RuntimeError.new('part failed') + ] + ) + client.stub_responses(:abort_multipart_upload, RuntimeError.new('network-error')) + + expect do + subject.upload(params) { |write_stream| write_stream << seventeen_mb } + end.to raise_error(S3::MultipartUploadError, /failed to abort multipart upload: network-error/) + end + + context 'when tempfile is true' do + let(:subject) { MultipartStreamUploader.new(client: client, tempfile: true) } + + it 'uses multipart APIs' do + client.stub_responses(:create_multipart_upload, upload_id: 'id') + client.stub_responses(:upload_part, etag: 'etag') + expected_params = params.merge( + upload_id: 'id', + multipart_upload: { + parts: [ + { etag: 'etag', part_number: 1 }, + { etag: 'etag', part_number: 2 }, + { etag: 'etag', part_number: 3 }, + { etag: 'etag', part_number: 4 } + ] + } + ) + expect(client).to receive(:complete_multipart_upload).with(expected_params).once + subject.upload(params.merge(content_type: 'text/plain')) { |write_stream| write_stream << seventeen_mb } + end + + it 'uploads the correct parts' do + client.stub_responses(:create_multipart_upload, upload_id: 'id') + 4.times.each do |p| + expect(client) + .to receive(:upload_part) + .with(params.merge(upload_id: 'id', body: instance_of(Tempfile), part_number: p + 1)) + .once + .and_return(double(:upload_part, etag: 'etag')) + end + + subject.upload(params) { |write_stream| write_stream << seventeen_mb } + end + + it 'uploads the correct parts when input is chunked' do + client.stub_responses(:create_multipart_upload, upload_id: 'id') + client.stub_responses(:complete_multipart_upload) + 4.times.each do |p| + expect(client) + .to receive(:upload_part) + .with(params.merge(upload_id: 'id', body: instance_of(Tempfile), part_number: p + 1)) + .once + .and_return(double(:upload_part, etag: 'etag')) + end + + subject.upload(params) do |write_stream| + 17.times { write_stream << one_mb } + end + end + + it 'automatically deletes failed multipart upload on part processing error' do + client.stub_responses( + :upload_part, + [ + { etag: 'etag-1' }, + { etag: 'etag-2' }, + RuntimeError.new('part 3 failed'), + { etag: 'etag-4' } + ] + ) + expect(client).to receive(:abort_multipart_upload).with(params.merge(upload_id: 'MultipartUploadId')) + + expect do + subject.upload(params.merge(tempfile: true)) do |write_stream| + write_stream << seventeen_mb + rescue Errno::EPIPE + # ignore + end + end.to raise_error(MultipartUploadError, /multipart upload failed: part 3 failed/) + end + + it 'automatically deletes failed multipart upload on stream read error' do + expect(client).to receive(:abort_multipart_upload).with(params.merge(upload_id: 'MultipartUploadId')) + + expect do + subject.upload(params) do |_write_stream| + raise 'something went wrong' + end + end.to raise_error(/something went wrong/) + end + + it 'reports when it is unable to abort a failed multipart upload' do + client.stub_responses( + :upload_part, + [ + { etag: 'etag-1' }, + { etag: 'etag-2' }, + { etag: 'etag-3' }, + RuntimeError.new('part failed') + ] + ) + client.stub_responses(:abort_multipart_upload, RuntimeError.new('network-error')) + + expect do + subject.upload(params) { |write_stream| write_stream << seventeen_mb } + end.to raise_error(S3::MultipartUploadError, /failed to abort multipart upload: network-error/) + end + end + end + end + end +end diff --git a/gems/aws-sdk-s3/spec/object/download_file_spec.rb b/gems/aws-sdk-s3/spec/object/download_file_spec.rb index b6bd29aec92..dda08d89fc0 100644 --- a/gems/aws-sdk-s3/spec/object/download_file_spec.rb +++ b/gems/aws-sdk-s3/spec/object/download_file_spec.rb @@ -7,236 +7,34 @@ module Aws module S3 describe Object do let(:client) { S3::Client.new(stub_responses: true) } - let(:tmpdir) { Dir.tmpdir } + let(:subject) { S3::Object.new(bucket_name: 'bucket', key: 'key', client: client) } describe '#download_file', :jruby_flaky do let(:path) { Tempfile.new('destination').path } - let(:small_obj) { S3::Object.new(bucket_name: 'bucket', key: 'small', client: client) } - let(:large_obj) { S3::Object.new(bucket_name: 'bucket', key: 'large', client: client) } - let(:single_obj) { S3::Object.new(bucket_name: 'bucket', key: 'single', client: client) } - let(:small_obj_params) { { bucket: 'bucket', key: 'small', response_target: path } } - let(:one_meg) { 1024 * 1024 } + let(:one_mb_size) { 1024 * 1024 } - before(:each) do - allow(Dir).to receive(:tmpdir).and_return(tmpdir) - client.stub_responses(:head_object, lambda { |context| - case context.params[:key] - when 'small' - { content_length: one_meg, parts_count: nil } - when 'large' - resp = { content_length: 20 * one_meg, parts_count: nil } - resp[:parts_count] = 4 if context.params[:part_number] - resp - when 'single' - { content_length: 15 * one_meg, parts_count: nil } - end - }) + before do + client.stub_responses(:head_object, content_length: one_mb_size, parts_count: nil) + client.stub_responses(:get_object, { body: 'hello-world' }) end - it 'downloads single part files in Client#get_object' do - expect(client).to receive(:get_object).with(small_obj_params).exactly(1).times - small_obj.download_file(path) + it 'returns true when download succeeds' do + expect(subject.download_file(path)).to be(true) + expect(File.read(path)).to eq('hello-world') end - it 'download larger files in parts' do - parts = 0 - client.stub_responses(:get_object, lambda { |_ctx| - parts += 1 - { body: 'body', content_range: 'bytes 0-3/4' } - }) - large_obj.download_file(path) - expect(parts).to eq(4) + it 'raises when download errors' do + client.stub_responses(:head_object, 'NoSuchKey') + expect { subject.download_file(path) }.to raise_error(Aws::S3::Errors::NoSuchKey) end - it 'download larger files in ranges' do - client.stub_responses(:get_object, lambda { |context| - responses = { - 'bytes=0-5242879' => { body: 'body', content_range: 'bytes 0-5242879/15728640' }, - 'bytes=5242880-10485759' => { body: 'body', content_range: 'bytes 5242880-10485759/15728640' }, - 'bytes=10485760-15728639' => { body: 'body', content_range: 'bytes 10485760-15728639/15728640' } - } - responses[context.params[:range]] - }) - single_obj.download_file(path, chunk_size: 5 * one_meg, mode: 'get_range') - end - - it 'supports download object with version_id' do - expect(client).to receive(:get_object).with(small_obj_params.merge(version_id: 'foo')).exactly(1).times - small_obj.download_file(path, version_id: 'foo') - end - - it 'calls on_checksum_validated on single part' do - client.stub_responses(:get_object, { body: 'body', checksum_sha1: 'Agg/RXngimEkJcDBoX7ket14O5Q=' }) - callback_data = { called: 0 } - mutex = Mutex.new - callback = proc do |_alg, _resp| - mutex.synchronize { callback_data[:called] += 1 } - end - - small_obj.download_file(path, on_checksum_validated: callback) - expect(callback_data[:called]).to eq(1) - end - - it 'calls on_checksum_validated on multipart' do - callback_data = { called: 0 } - client.stub_responses( - :get_object, { body: 'body', content_range: 'bytes 0-3/4', checksum_sha1: 'Agg/RXngimEkJcDBoX7ket14O5Q=' } - ) - mutex = Mutex.new - callback = proc do |_alg, _resp| - mutex.synchronize { callback_data[:called] += 1 } - end - - large_obj.download_file(path, on_checksum_validated: callback) - expect(callback_data[:called]).to eq(4) - end - - it 'supports disabling checksum_mode' do - client.stub_responses(:head_object, lambda { |context| - expect(context.params[:checksum_mode]).to eq('DISABLED') - { content_length: one_meg, parts_count: nil } - }) - client.stub_responses(:get_object, lambda { |context| - expect(context.params[:checksum_mode]).to eq('DISABLED') - { body: 'body' } - }) - - small_obj.download_file(path, checksum_mode: 'DISABLED') - end - - it 'downloads the file in range chunks' do - client.stub_responses(:get_object, lambda { |context| - ranges = context.params[:range].match(/bytes=(?\d+)-(?\d+)/) - expect(ranges[:end].to_i - ranges[:start].to_i + 1).to eq(one_meg) - { content_range: "bytes #{ranges[:start]}-#{ranges[:end]}/#{20 * one_meg}" } - }) - - large_obj.download_file(path, chunk_size: one_meg) - end - - context 'multipart progress' do - it 'reports progress for single part objects' do - small_file_size = 1024 - expect(client) - .to receive(:get_object) - .with(small_obj_params.merge(on_chunk_received: instance_of(Proc))) do |args| - args[:on_chunk_received].call(Tempfile.new('small-file'), small_file_size, small_file_size) - end - - n_calls = 0 - callback = proc do |bytes, part_sizes, total| - expect(bytes).to eq([small_file_size]) - expect(part_sizes).to eq([small_file_size]) - expect(total).to eq(small_file_size) - n_calls += 1 - end - - small_obj.download_file(path, progress_callback: callback) - expect(n_calls).to eq(1) - end - - it 'reports progress for files downloaded in parts' do - expect(client).to receive(:get_object).exactly(4).times do |args| - args[:on_chunk_received].call(Tempfile.new('large-file'), 4, 4) - client.stub_data(:get_object, body: StringIO.new('chunk'), content_range: 'bytes 0-3/4') - end - - n_calls = 0 - mutex = Mutex.new - callback = proc do |bytes, part_sizes, total| - mutex.synchronize do - expect(bytes.size).to eq(4) - expect(part_sizes.size).to eq(4) - expect(total).to eq(20 * one_meg) - n_calls += 1 - end - end - large_obj.download_file(path, progress_callback: callback) - expect(n_calls).to eq(4) - end - end - - context 'error handling' do - it 'raises an error when checksum validation fails on single part' do - client.stub_responses(:get_object, { body: 'body', checksum_sha1: 'invalid' }) - - expect { small_obj.download_file(path) }.to raise_error(Aws::Errors::ChecksumError) - end - - it 'raises an error when checksum validation fails on multipart' do - client.stub_responses(:get_object, { body: 'body', checksum_sha1: 'invalid' }) - thread = double(value: nil) - - expect(Thread).to receive(:new).and_yield.and_return(thread) - expect { large_obj.download_file(path) }.to raise_error(Aws::Errors::ChecksumError) - end - - it 'does not download object when ETAG does not match during multipart get by ranges' do - client.stub_responses(:head_object, content_length: 15 * one_meg, parts_count: nil, etag: 'test-etag') - client.stub_responses(:get_object, lambda { |ctx| - expect(ctx.params[:if_match]).to eq('test-etag') - 'PreconditionFailed' - }) - thread = double(value: nil) - expect(Thread).to receive(:new).and_yield.and_return(thread) - expect { single_obj.download_file(path) }.to raise_error(Aws::S3::Errors::PreconditionFailed) - end - - it 'does not download object when ETAG does not match during multipart get by parts' do - client.stub_responses(:head_object, { content_length: 20 * one_meg, etag: 'test-etag', parts_count: 4 }) - client.stub_responses(:get_object, lambda { |ctx| - expect(ctx.params[:if_match]).to eq('test-etag') - 'PreconditionFailed' - }) - - thread = double(value: nil) - expect(Thread).to receive(:new).and_yield.and_return(thread) - expect { large_obj.download_file(path) }.to raise_error(Aws::S3::Errors::PreconditionFailed) - end - - it 'raises an error if an invalid mode is specified' do - expect { large_obj.download_file(path, mode: 'invalid_mode') } - .to raise_error(ArgumentError, /Invalid mode invalid_mode provided/) - end - - it 'raises an error if choose :get_range without :chunk_size' do - expect { large_obj.download_file(path, mode: 'get_range') } - .to raise_error(ArgumentError, 'In get_range mode, :chunk_size must be provided') - end - - it 'raises an error if :chunk_size is larger than file size' do - expect { large_obj.download_file(path, chunk_size: 50 * one_meg) } - .to raise_error(ArgumentError, ":chunk_size shouldn't exceed total file size.") - end - - it 'raises an error if :on_checksum_validated is not callable' do - expect { large_obj.download_file(path, on_checksum_validated: 'string') } - .to raise_error(ArgumentError, ':on_checksum_validated must be callable') - end - - it 'raises error when range validation fails' do - client.stub_responses(:get_object, { body: 'body', content_range: 'bytes 0-3/4' }) - expect { large_obj.download_file(path, mode: 'get_range', chunk_size: one_meg) } - .to raise_error(Aws::S3::MultipartDownloadError) - end - - it 'does not overwrite existing file when download fails' do - File.write(path, 'existing content') - - client.stub_responses(:get_object, lambda { |context| - responses = { - 'bytes=0-5242879' => { body: 'body', content_range: 'bytes 0-5242879/15728640' }, - 'bytes=5242880-10485759' => { body: 'body', content_range: 'bytes 5242880-10485759/15728640' }, - 'bytes=10485760-15728639' => { body: 'fake-range', content_range: 'bytes 10485800-15728639/15728640' } - } - responses[context.params[:range]] - }) + it 'calls progress callback when given' do + n_calls = 0 + callback = proc { |_b, _p, _t| n_calls += 1 } + expect(client).to receive(:get_object) { |args| args[:on_chunk_received]&.call('chunk', 1024, 1024) } - expect { single_obj.download_file(path, chunk_size: 5 * one_meg, mode: 'get_range') } - .to raise_error(Aws::S3::MultipartDownloadError) - expect(File.exist?(path)).to be(true) - expect(File.read(path)).to eq('existing content') - end + subject.download_file(path, progress_callback: callback) + expect(n_calls).to eq(1) end end end diff --git a/gems/aws-sdk-s3/spec/object/upload_file_spec.rb b/gems/aws-sdk-s3/spec/object/upload_file_spec.rb index 442c65afa49..b16df1f3f0f 100644 --- a/gems/aws-sdk-s3/spec/object/upload_file_spec.rb +++ b/gems/aws-sdk-s3/spec/object/upload_file_spec.rb @@ -6,256 +6,55 @@ module Aws module S3 describe Object do - RSpec::Matchers.define :file_part do |expected| - match do |actual| - actual.source == expected[:source] && - actual.first_byte == expected[:offset] && - actual.last_byte == expected[:offset] + expected[:size] && - actual.size == expected[:size] - end - end - let(:client) { S3::Client.new(stub_responses: true) } + let(:subject) { S3::Object.new(bucket_name: 'bucket', key: 'key', client: client) } describe '#upload_file' do - let(:one_meg) { 1024 * 1024 } - let(:object) { S3::Object.new(bucket_name: 'bucket', key: 'key', client: client) } - let(:one_mb) { '.' * 1024 * 1024 } + let(:mb_size) { 1024 * 1024 } + let(:mb_content) { '.' * mb_size } - let(:one_meg_file) do - Tempfile.new('one-meg-file').tap do |f| - f.write(one_mb) - f.rewind - end - end - - let(:ten_meg_file) do + let(:file) do Tempfile.new('ten-meg-file').tap do |f| - 10.times { f.write(one_mb) } + 10.times { f.write(mb_content) } f.rewind end end - let(:one_hundred_seventeen_meg_file) do + let(:large_file) do Tempfile.new('one-hundred-seventeen-meg-file').tap do |f| - 117.times { f.write(one_mb) } + 117.times { f.write(mb_content) } f.rewind end end - it 'uploads objects with custom options without mutating them' do - options = {}.freeze - expect(client).to receive(:put_object).with({ bucket: 'bucket', key: 'key', body: one_meg_file }) - object.upload_file(one_meg_file, options) + it 'returns true when upload succeeds' do + expect(subject.upload_file(file)).to be(true) + end + + it 'raises when upload errors' do + client.stub_responses(:put_object, 'AccessDenied') + expect { subject.upload_file(file) }.to raise_error(Aws::S3::Errors::AccessDenied) end it 'yields the response to the given block' do - object.upload_file(ten_meg_file) do |response| + subject.upload_file(file) do |response| expect(response).to be_kind_of(Seahorse::Client::Response) expect(response.etag).to eq('ETag') end end - context 'small objects' do - it 'uploads small objects using Client#put_object' do - expect(client).to receive(:put_object).with({ bucket: 'bucket', key: 'key', body: ten_meg_file }) - object.upload_file(ten_meg_file) - end + it 'calls progress callback when given' do + n_calls = 0 + callback = proc { |_b, _t| n_calls += 1 } + expect(client).to receive(:put_object) { |args| args[:on_chunk_sent]&.call('chunk', 1024, 1024) } - it 'reports progress for small objects' do - expect(client) - .to receive(:put_object) - .with({ bucket: 'bucket', key: 'key', body: ten_meg_file, on_chunk_sent: instance_of(Proc) }) do |args| - args[:on_chunk_sent].call(ten_meg_file, ten_meg_file.size, ten_meg_file.size) - end - callback = proc do |bytes, totals| - expect(bytes).to eq([ten_meg_file.size]) - expect(totals).to eq([ten_meg_file.size]) - end - object.upload_file(ten_meg_file, progress_callback: callback) - end - - it 'accepts an alternative multipart file threshold' do - expect(client).to receive(:put_object).with({ bucket: 'bucket', key: 'key', body: one_hundred_seventeen_meg_file }) - object.upload_file(one_hundred_seventeen_meg_file, multipart_threshold: 200 * one_meg) - end - - it 'accepts paths to files to upload' do - file = double('file') - expect(File).to receive(:open).with(ten_meg_file.path, 'rb').and_yield(file) - expect(client).to receive(:put_object).with({ bucket: 'bucket', key: 'key', body: file }) - object.upload_file(ten_meg_file.path) - end - - it 'does not fail when given :thread_count' do - expect(client).to receive(:put_object).with({ bucket: 'bucket', key: 'key', body: ten_meg_file }) - object.upload_file(ten_meg_file, thread_count: 1) - end + subject.upload_file(file, progress_callback: callback) + expect(n_calls).to eq(1) end - context 'large objects' do - it 'uses multipart APIs for objects >= 100MB' do - client.stub_responses(:create_multipart_upload, upload_id: 'id') - client.stub_responses(:upload_part, etag: 'etag', checksum_crc32: 'checksum') - expect(client).to receive(:complete_multipart_upload).with( - bucket: 'bucket', - key: 'key', - upload_id: 'id', - multipart_upload: { - parts: [ - { checksum_crc32: 'checksum', etag: 'etag', part_number: 1 }, - { checksum_crc32: 'checksum', etag: 'etag', part_number: 2 }, - { checksum_crc32: 'checksum', etag: 'etag', part_number: 3 }, - { checksum_crc32: 'checksum', etag: 'etag', part_number: 4 }, - { checksum_crc32: 'checksum', etag: 'etag', part_number: 5 }, - { checksum_crc32: 'checksum', etag: 'etag', part_number: 6 }, - { checksum_crc32: 'checksum', etag: 'etag', part_number: 7 }, - { checksum_crc32: 'checksum', etag: 'etag', part_number: 8 }, - { checksum_crc32: 'checksum', etag: 'etag', part_number: 9 }, - { checksum_crc32: 'checksum', etag: 'etag', part_number: 10 }, - { checksum_crc32: 'checksum', etag: 'etag', part_number: 11 }, - { checksum_crc32: 'checksum', etag: 'etag', part_number: 12 }, - { checksum_crc32: 'checksum', etag: 'etag', part_number: 13 }, - { checksum_crc32: 'checksum', etag: 'etag', part_number: 14 }, - { checksum_crc32: 'checksum', etag: 'etag', part_number: 15 }, - { checksum_crc32: 'checksum', etag: 'etag', part_number: 16 }, - { checksum_crc32: 'checksum', etag: 'etag', part_number: 17 }, - { checksum_crc32: 'checksum', etag: 'etag', part_number: 18 }, - { checksum_crc32: 'checksum', etag: 'etag', part_number: 19 }, - { checksum_crc32: 'checksum', etag: 'etag', part_number: 20 }, - { checksum_crc32: 'checksum', etag: 'etag', part_number: 21 }, - { checksum_crc32: 'checksum', etag: 'etag', part_number: 22 }, - { checksum_crc32: 'checksum', etag: 'etag', part_number: 23 }, - { checksum_crc32: 'checksum', etag: 'etag', part_number: 24 } - ] - }, - mpu_object_size: one_hundred_seventeen_meg_file.size - ) - object.upload_file(one_hundred_seventeen_meg_file, content_type: 'text/plain') - end - - it 'allows for full object checksums' do - expect(client).to receive(:create_multipart_upload) - .with( - { - bucket: 'bucket', - key: 'key', - checksum_algorithm: 'CRC32', - checksum_type: 'FULL_OBJECT', - content_type: 'text/plain' - } - ).and_call_original - expect(client).to receive(:upload_part) - .with(hash_not_including(checksum_crc32: anything)) - .exactly(24).times - .and_call_original - expect(client).to receive(:complete_multipart_upload) - .with(hash_including(checksum_type: 'FULL_OBJECT', checksum_crc32: 'checksum')) - .and_call_original - client.stub_responses(:create_multipart_upload, upload_id: 'id') - client.stub_responses(:upload_part, etag: 'etag', checksum_crc32: 'part') - - object.upload_file(one_hundred_seventeen_meg_file, content_type: 'text/plain', checksum_crc32: 'checksum') - end - - it 'reports progress for multipart uploads' do - thread = double(value: nil) - allow(Thread).to receive(:new).and_yield.and_return(thread) - - client.stub_responses(:create_multipart_upload, upload_id: 'id') - client.stub_responses(:complete_multipart_upload) - expect(client).to receive(:upload_part).exactly(24).times do |args| - args[:on_chunk_sent].call(args[:body], args[:body].size, args[:body].size) - double( - context: double(params: { checksum_algorithm: 'crc32' }), - checksum_crc32: 'checksum', - etag: 'etag' - ) - end - callback = proc do |bytes, totals| - expect(bytes.size).to eq(24) - expect(totals.size).to eq(24) - end - object.upload_file(one_hundred_seventeen_meg_file, content_type: 'text/plain', progress_callback: callback) - end - - it 'defaults to THREAD_COUNT without the thread_count option' do - expect(Thread).to receive(:new).exactly(S3::MultipartFileUploader::THREAD_COUNT).times.and_yield.and_return(double(value: nil)) - client.stub_responses(:create_multipart_upload, upload_id: 'id') - client.stub_responses(:complete_multipart_upload) - object.upload_file(one_hundred_seventeen_meg_file) - end - - it 'respects the thread_count option' do - custom_thread_count = 20 - expect(Thread).to receive(:new).exactly(custom_thread_count).times.and_yield.and_return(double(value: nil)) - client.stub_responses(:create_multipart_upload, upload_id: 'id') - client.stub_responses(:complete_multipart_upload) - object.upload_file(one_hundred_seventeen_meg_file, thread_count: custom_thread_count) - end - - it 'raises an error if the multipart threshold is too small' do - error_msg = 'unable to multipart upload files smaller than 5MB' - expect do - object.upload_file(one_meg_file, multipart_threshold: one_meg) - end.to raise_error(ArgumentError, error_msg) - end - - it 'automatically deletes failed multipart upload on error' do - allow_any_instance_of(FilePart).to receive(:read).and_return(nil) - - client.stub_responses( - :upload_part, - [ - { etag: 'etag-1' }, - { etag: 'etag-2' }, - RuntimeError.new('part 3 failed'), - { etag: 'etag-4' } - ] - ) - - expect(client).to receive(:abort_multipart_upload).with( - bucket: 'bucket', - key: 'key', - upload_id: 'MultipartUploadId' - ) - - expect do - object.upload_file(one_hundred_seventeen_meg_file) - end.to raise_error(/multipart upload failed: part 3 failed/) - end - - it 'reports when it is unable to abort a failed multipart upload' do - allow(Thread).to receive(:new) do |_, &block| - double(value: block.call) - end - - client.stub_responses( - :upload_part, - [ - { etag: 'etag-1' }, - { etag: 'etag-2' }, - { etag: 'etag-3' }, - RuntimeError.new('part failed') - ] - ) - client.stub_responses(:abort_multipart_upload, [RuntimeError.new('network-error')]) - expect { object.upload_file(one_hundred_seventeen_meg_file) }.to raise_error( - S3::MultipartUploadError, - /failed to abort multipart upload: network-error. Multipart upload failed: part failed/ - ) - end - - it 'aborts multipart upload when upload fails to complete' do - client.stub_responses(:complete_multipart_upload, RuntimeError.new('network-error')) - - expect(client).to receive(:abort_multipart_upload).with( - bucket: 'bucket', - key: 'key', - upload_id: 'MultipartUploadId' - ) - expect { object.upload_file(one_hundred_seventeen_meg_file) }.to raise_error(Aws::S3::MultipartUploadError) - end + it 'accepts an alternative multipart file threshold' do + expect(client).to receive(:put_object).with({ bucket: 'bucket', key: 'key', body: large_file }) + subject.upload_file(large_file, multipart_threshold: 200 * mb_size) end end end diff --git a/gems/aws-sdk-s3/spec/object/upload_stream_spec.rb b/gems/aws-sdk-s3/spec/object/upload_stream_spec.rb index 8b4c1b56d1b..269b91746eb 100644 --- a/gems/aws-sdk-s3/spec/object/upload_stream_spec.rb +++ b/gems/aws-sdk-s3/spec/object/upload_stream_spec.rb @@ -7,44 +7,22 @@ module Aws module S3 describe Object do let(:client) { S3::Client.new(stub_responses: true) } + let(:subject) { S3::Object.new(bucket_name: 'bucket', key: 'key', client: client) } describe '#upload_stream', :jruby_flaky do - let(:object) do - S3::Object.new( - bucket_name: 'bucket', - key: 'key', - client: client - ) - end - - let(:zero_mb) { '' } - - let(:one_mb) { '.' * 1024 * 1024 } - - let(:ten_mb) do - one_mb * 10 - end + let(:params) { { bucket: 'bucket', key: 'key' } } + let(:seventeen_mb) { '.' * 1024 * 1024 * 17 } - let(:seventeen_mb) do - one_mb * 17 + it 'returns true when succeeds' do + resp = subject.upload_stream(content_type: 'text/plain') { |write_stream| write_stream << seventeen_mb } + expect(resp).to be(true) end - it 'can upload empty stream' do - client.stub_responses(:create_multipart_upload, upload_id: 'id') - client.stub_responses(:upload_part, etag: 'etag') - expect(client).to receive(:complete_multipart_upload).with( - bucket: 'bucket', - key: 'key', - upload_id: 'id', - multipart_upload: { - parts: [ - { etag: 'etag', part_number: 1 } - ] - } - ).once - object.upload_stream(content_type: 'text/plain') do |write_stream| - write_stream << zero_mb - end + it 'raises when errors' do + client.stub_responses(:upload_part, RuntimeError.new('part failed')) + expect do + subject.upload_stream { |write_stream| write_stream << seventeen_mb } + end.to raise_error(Aws::S3::MultipartUploadError, /part failed/) end it 'respects the thread_count option' do @@ -52,426 +30,19 @@ module S3 expect(Thread).to receive(:new).exactly(custom_thread_count).times.and_return(double(value: nil)) client.stub_responses(:create_multipart_upload, upload_id: 'id') client.stub_responses(:complete_multipart_upload) - object.upload_stream(thread_count: custom_thread_count) { |_write_stream| } - end - - it 'uses multipart APIs' do - client.stub_responses(:create_multipart_upload, upload_id: 'id') - client.stub_responses(:upload_part, etag: 'etag') - expect(client).to receive(:complete_multipart_upload).with( - bucket: 'bucket', - key: 'key', - upload_id: 'id', - multipart_upload: { - parts: [ - { etag: 'etag', part_number: 1 }, - { etag: 'etag', part_number: 2 }, - { etag: 'etag', part_number: 3 }, - { etag: 'etag', part_number: 4 } - ] - } - ).once - object.upload_stream(content_type: 'text/plain') do |write_stream| - write_stream << seventeen_mb - end - end - - it 'uploads the correct parts' do - client.stub_responses(:create_multipart_upload, upload_id: 'id') - client.stub_responses(:complete_multipart_upload) - expect(client).to receive(:upload_part).with({ - bucket: 'bucket', - key: 'key', - upload_id: 'id', - body: instance_of(StringIO), - part_number: 1 - }).once.and_return(double(:upload_part, etag: 'etag')) - expect(client).to receive(:upload_part).with({ - bucket: 'bucket', - key: 'key', - upload_id: 'id', - body: instance_of(StringIO), - part_number: 2 - }).once.and_return(double(:upload_part, etag: 'etag')) - expect(client).to receive(:upload_part).with({ - bucket: 'bucket', - key: 'key', - upload_id: 'id', - body: instance_of(StringIO), - part_number: 3 - }).once.and_return(double(:upload_part, etag: 'etag')) - expect(client).to receive(:upload_part).with({ - bucket: 'bucket', - key: 'key', - upload_id: 'id', - body: instance_of(StringIO), - part_number: 4 - }).once.and_return(double(:upload_part, etag: 'etag')) - object.upload_stream do |write_stream| - write_stream << seventeen_mb - end - end - - it 'uploads the correct parts when input is chunked' do - client.stub_responses(:create_multipart_upload, upload_id: 'id') - client.stub_responses(:complete_multipart_upload) - expect(client).to receive(:upload_part).with({ - bucket: 'bucket', - key: 'key', - upload_id: 'id', - body: instance_of(StringIO), - part_number: 1 - }).once.and_return(double(:upload_part, etag: 'etag')) - expect(client).to receive(:upload_part).with({ - bucket: 'bucket', - key: 'key', - upload_id: 'id', - body: instance_of(StringIO), - part_number: 2 - }).once.and_return(double(:upload_part, etag: 'etag')) - expect(client).to receive(:upload_part).with({ - bucket: 'bucket', - key: 'key', - upload_id: 'id', - body: instance_of(StringIO), - part_number: 3 - }).once.and_return(double(:upload_part, etag: 'etag')) - expect(client).to receive(:upload_part).with({ - bucket: 'bucket', - key: 'key', - upload_id: 'id', - body: instance_of(StringIO), - part_number: 4 - }).once.and_return(double(:upload_part, etag: 'etag')) - object.upload_stream do |write_stream| - 17.times { write_stream << one_mb } - end - end - - it 'uploads correct parts when chunked with custom part_size' do - client.stub_responses(:create_multipart_upload, upload_id: 'id') - client.stub_responses(:complete_multipart_upload) - expect(client).to receive(:upload_part).with({ - bucket: 'bucket', - key: 'key', - upload_id: 'id', - body: instance_of(StringIO), - part_number: 1 - }).once.and_return(double(:upload_part, etag: 'etag')) - expect(client).to receive(:upload_part).with({ - bucket: 'bucket', - key: 'key', - upload_id: 'id', - body: instance_of(StringIO), - part_number: 2 - }).once.and_return(double(:upload_part, etag: 'etag')) - expect(client).to receive(:upload_part).with({ - bucket: 'bucket', - key: 'key', - upload_id: 'id', - body: instance_of(StringIO), - part_number: 3 - }).once.and_return(double(:upload_part, etag: 'etag')) - object.upload_stream(part_size: 7 * 1024 * 1024) do |write_stream| - 17.times { write_stream << one_mb } - end - end - - it 'passes stringios with correct contents with custom part_size' do - client.stub_responses(:create_multipart_upload, upload_id: 'id') - client.stub_responses(:complete_multipart_upload) - result = [] - mutex = Mutex.new - allow(client).to receive(:upload_part) do |part| - mutex.synchronize do - result << [ - part[:part_number], - part[:body].read.size - ] - end - end.and_return(double(:upload_part, etag: 'etag')) - object.upload_stream(part_size: 7 * 1024 * 1024) do |write_stream| - 17.times { write_stream << one_mb } - end - expect(result.sort_by(&:first)).to eq( - [ - [1, 7 * 1024 * 1024], - [2, 7 * 1024 * 1024], - [3, 3 * 1024 * 1024] - ] - ) + subject.upload_stream(thread_count: custom_thread_count) { |_write_stream| } end - it 'passes stringios with correct contents to upload_part' do + it 'respects the tempfile option' do client.stub_responses(:create_multipart_upload, upload_id: 'id') - client.stub_responses(:complete_multipart_upload) - result = [] - mutex = Mutex.new - allow(client).to receive(:upload_part) do |part| - mutex.synchronize do - result << [ - part[:part_number], - part[:body].read.size - ] - end - end.and_return(double(:upload_part, etag: 'etag')) - object.upload_stream do |write_stream| - 17.times { write_stream << one_mb } - end - expect(result.sort_by(&:first)).to eq( - [ - [1, 5 * 1024 * 1024], - [2, 5 * 1024 * 1024], - [3, 5 * 1024 * 1024], - [4, 2 * 1024 * 1024] - ] - ) - end - - it 'automatically deletes failed multipart upload on part processing error' do - client.stub_responses( - :upload_part, [ - { etag: 'etag-1' }, - { etag: 'etag-2' }, - RuntimeError.new('part 3 failed'), - { etag: 'etag-4' } - ] - ) - - expect(client).to receive(:abort_multipart_upload) - .with(bucket: 'bucket', key: 'key', upload_id: 'MultipartUploadId') - - expect do - object.upload_stream do |write_stream| - begin - write_stream << seventeen_mb - rescue Errno::EPIPE - end - end - end.to raise_error('multipart upload failed: part 3 failed') - end - - it 'automatically deletes failed multipart upload on stream read error' do - expect(client).to receive(:abort_multipart_upload) - .with(bucket: 'bucket', key: 'key', upload_id: 'MultipartUploadId') - - expect do - object.upload_stream do |_write_stream| - raise 'something went wrong' - end - end.to raise_error(/something went wrong/) - end - - it 'reports when it is unable to abort a failed multipart upload' do - client.stub_responses( - :upload_part, - [ - { etag: 'etag-1' }, - { etag: 'etag-2' }, - { etag: 'etag-3' }, - RuntimeError.new('part failed') - ] - ) - client.stub_responses( - :abort_multipart_upload, - [ - RuntimeError.new('network-error') - ] - ) - expect do - object.upload_stream do |write_stream| - write_stream << seventeen_mb - end - end.to raise_error( - S3::MultipartUploadError, - /failed to abort multipart upload: network-error/ - ) - end - - context 'with tempfile option' do - it 'uses multipart APIs' do - client.stub_responses(:create_multipart_upload, upload_id: 'id') - client.stub_responses(:upload_part, etag: 'etag') - expect(client).to receive(:complete_multipart_upload).with( - bucket: 'bucket', - key: 'key', - upload_id: 'id', - multipart_upload: { - parts: [ - { etag: 'etag', part_number: 1 }, - { etag: 'etag', part_number: 2 }, - { etag: 'etag', part_number: 3 }, - { etag: 'etag', part_number: 4 } - ] - } - ).once - object.upload_stream( - content_type: 'text/plain', - tempfile: true - ) do |write_stream| - write_stream << seventeen_mb - end - end - - it 'uploads the correct parts' do - client.stub_responses(:create_multipart_upload, upload_id: 'id') - client.stub_responses(:complete_multipart_upload) - expect(client).to receive(:upload_part).with({ - bucket: 'bucket', - key: 'key', - upload_id: 'id', - body: instance_of(Tempfile), - part_number: 1 - }).once.and_return(double(:upload_part, etag: 'etag')) - expect(client).to receive(:upload_part).with({ - bucket: 'bucket', - key: 'key', - upload_id: 'id', - body: instance_of(Tempfile), - part_number: 2 - }).once.and_return(double(:upload_part, etag: 'etag')) - expect(client).to receive(:upload_part).with({ - bucket: 'bucket', - key: 'key', - upload_id: 'id', - body: instance_of(Tempfile), - part_number: 3 - }).once.and_return(double(:upload_part, etag: 'etag')) - expect(client).to receive(:upload_part).with({ - bucket: 'bucket', - key: 'key', - upload_id: 'id', - body: instance_of(Tempfile), - part_number: 4 - }).once.and_return(double(:upload_part, etag: 'etag')) - object.upload_stream(tempfile: true) do |write_stream| - write_stream << seventeen_mb - end - end - - it 'uploads the correct parts when input is chunked' do - client.stub_responses(:create_multipart_upload, upload_id: 'id') - client.stub_responses(:complete_multipart_upload) - expect(client).to receive(:upload_part).with({ - bucket: 'bucket', - key: 'key', - upload_id: 'id', - body: instance_of(Tempfile), - part_number: 1 - }).once.and_return(double(:upload_part, etag: 'etag')) - expect(client).to receive(:upload_part).with({ - bucket: 'bucket', - key: 'key', - upload_id: 'id', - body: instance_of(Tempfile), - part_number: 2 - }).once.and_return(double(:upload_part, etag: 'etag')) - expect(client).to receive(:upload_part).with({ - bucket: 'bucket', - key: 'key', - upload_id: 'id', - body: instance_of(Tempfile), - part_number: 3 - }).once.and_return(double(:upload_part, etag: 'etag')) - expect(client).to receive(:upload_part).with({ - bucket: 'bucket', - key: 'key', - upload_id: 'id', - body: instance_of(Tempfile), - part_number: 4 - }).once.and_return(double(:upload_part, etag: 'etag')) - object.upload_stream(tempfile: true) do |write_stream| - 17.times { write_stream << one_mb } - end - end - - it 'passes tempfiles with correct contents to upload_part' do - client.stub_responses(:create_multipart_upload, upload_id: 'id') - client.stub_responses(:complete_multipart_upload) - result = [] - mutex = Mutex.new - allow(client).to receive(:upload_part) do |part| - mutex.synchronize do - result << [ - part[:part_number], - part[:body].read.size - ] - end - end.and_return(double(:upload_part, etag: 'etag')) - object.upload_stream(tempfile: true) do |write_stream| - 17.times { write_stream << one_mb } - end - expect(result.sort_by(&:first)).to eq( - [ - [1, 5 * 1024 * 1024], - [2, 5 * 1024 * 1024], - [3, 5 * 1024 * 1024], - [4, 2 * 1024 * 1024] - ] - ) - end - - it 'automatically deletes failed multipart upload on part processing error' do - client.stub_responses( - :upload_part, - [ - { etag: 'etag-1' }, - { etag: 'etag-2' }, - RuntimeError.new('part 3 failed'), - { etag: 'etag-4' } - ] - ) - - expect(client).to receive(:abort_multipart_upload).with( - bucket: 'bucket', key: 'key', upload_id: 'MultipartUploadId' - ) - - expect do - object.upload_stream(tempfile: true) do |write_stream| - begin - write_stream << seventeen_mb - rescue Errno::EPIPE - end - end - end.to raise_error('multipart upload failed: part 3 failed') - end - - it 'automatically deletes failed multipart upload on stream read error' do - expect(client).to receive(:abort_multipart_upload).with( - bucket: 'bucket', key: 'key', upload_id: 'MultipartUploadId' - ) - - expect do - object.upload_stream(tempfile: true) do |_write_stream| - raise 'something went wrong' - end - end.to raise_error(/multipart upload failed/, /something went wrong/) - end - - it 'reports when it is unable to abort a failed multipart upload' do - client.stub_responses( - :upload_part, - [ - { etag: 'etag-1' }, - { etag: 'etag-2' }, - { etag: 'etag-3' }, - RuntimeError.new('part failed') - ] - ) - client.stub_responses( - :abort_multipart_upload, - [RuntimeError.new('network-error')] - ) - expect do - object.upload_stream(tempfile: true) do |write_stream| - write_stream << seventeen_mb - end - end.to raise_error( - S3::MultipartUploadError, - 'failed to abort multipart upload: network-error. '\ - 'Multipart upload failed: part failed' - ) + 4.times.each do |p| + expect(client) + .to receive(:upload_part) + .with(params.merge(upload_id: 'id', body: instance_of(Tempfile), part_number: p + 1)) + .once + .and_return(double(:upload_part, etag: 'etag')) end + subject.upload_stream(tempfile: true) { |write_stream| write_stream << seventeen_mb } end end end diff --git a/gems/aws-sdk-s3/spec/transfer_manager_spec.rb b/gems/aws-sdk-s3/spec/transfer_manager_spec.rb new file mode 100644 index 00000000000..a752b35f0f6 --- /dev/null +++ b/gems/aws-sdk-s3/spec/transfer_manager_spec.rb @@ -0,0 +1,119 @@ +# frozen_string_literal: true + +require_relative 'spec_helper' +require 'tempfile' + +module Aws + module S3 + describe TransferManager do + let(:client) { S3::Client.new(stub_responses: true) } + let(:subject) { TransferManager.new(client: client) } + let(:one_mb_size) { 1024 * 1024 } + let(:one_mb_content) { '.' * one_mb_size } + + describe '#initialize' do + it 'constructs a default s3 client when not given' do + client = double('client') + expect(S3::Client).to receive(:new).and_return(client) + + tm = TransferManager.new + expect(tm.client).to be(client) + end + end + + describe '#download_file', :jruby_flaky do + let(:path) { Tempfile.new('destination').path } + + before do + client.stub_responses(:head_object, content_length: one_mb_size, parts_count: nil) + client.stub_responses(:get_object, { body: 'hello-world' }) + end + + it 'returns true when download succeeds' do + expect(subject.download_file(path, bucket: 'bucket', key: 'key')).to be(true) + expect(File.read(path)).to eq('hello-world') + end + + it 'raises when download errors' do + client.stub_responses(:head_object, 'NoSuchKey') + expect { subject.download_file(path, bucket: 'bucket', key: 'missing-key') } + .to raise_error(Aws::S3::Errors::NoSuchKey) + end + + it 'calls progress callback when given' do + n_calls = 0 + callback = proc { |_b, _p, _t| n_calls += 1 } + expect(client).to receive(:get_object) { |args| args[:on_chunk_received]&.call('chunk', 1024, 1024) } + + subject.download_file(path, bucket: 'bucket', key: 'key', progress_callback: callback) + expect(n_calls).to eq(1) + end + end + + describe '#upload_file' do + let(:file) do + Tempfile.new('ten-meg-file').tap do |f| + 10.times { f.write(one_mb_content) } + f.rewind + end + end + + let(:large_file) do + Tempfile.new('one-hundred-seventeen-meg-file').tap do |f| + 117.times { f.write(one_mb_content) } + f.rewind + end + end + + it 'returns true when upload succeeds' do + expect(subject.upload_file(file, bucket: 'bucket', key: 'key')).to be(true) + end + + it 'raises when upload errors' do + client.stub_responses(:put_object, 'AccessDenied') + expect { subject.upload_file(file, bucket: 'forbidden-bucket', key: 'key') } + .to raise_error(Aws::S3::Errors::AccessDenied) + end + + it 'yields response when block given' do + subject.upload_file(file, bucket: 'bucket', key: 'key') do |response| + expect(response).to be_kind_of(Seahorse::Client::Response) + expect(response.etag).to eq('ETag') + end + end + + it 'calls progress callback when given' do + n_calls = 0 + callback = proc { |_b, _t| n_calls += 1 } + expect(client).to receive(:put_object) { |args| args[:on_chunk_sent]&.call('chunk', 1024, 1024) } + + subject.upload_file(file, bucket: 'bucket', key: 'key', progress_callback: callback) + expect(n_calls).to eq(1) + end + + it 'accepts an alternative multipart file threshold' do + expect(client).to receive(:put_object).with({ bucket: 'bucket', key: 'key', body: large_file }) + subject.upload_file(large_file, bucket: 'bucket', key: 'key', multipart_threshold: 200 * one_mb_size) + end + end + + describe '#upload_stream', :jruby_flaky do + let(:seventeen_mb) { one_mb_content * 17 } + + it 'returns true when succeeds' do + resp = subject.upload_stream(bucket: 'bucket', key: 'key', content_type: 'text/plain') do |write_stream| + write_stream << seventeen_mb + end + expect(resp).to be(true) + end + + it 'raises when errors' do + client.stub_responses(:upload_part, RuntimeError.new('part failed')) + expect do + subject.upload_stream(bucket: 'bucket', key: 'key') { |write_stream| write_stream << seventeen_mb } + end.to raise_error(Aws::S3::MultipartUploadError, /part failed/) + end + end + end + end +end